mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
chore(cloud): add the relay fence broker, ops console, Terraform root, scripts, and 24 cloud-* workflows (#18413)
Phase 6 of the relay split: the relay's deploy/operate surface moves under cloud/ with 24 cloud-* workflows gated on ORCA_CLOUD_OPERATIONS_ENABLED, the Cloud SQL rollout lease action, the relay Terraform root (dual-accept identities for both repositories), scripts, docs, CODEOWNERS, and a terraform validate job in Cloud Verify.
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design),
|
||||
// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses
|
||||
// must be identical before and after a state move. This captures that set deterministically.
|
||||
// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded
|
||||
// because the baseline has none.
|
||||
|
||||
const usage =
|
||||
'usage: capture-terraform-plan-baseline.mjs --root <dir> --env <staging|production> --out <dir> [--tag <name>]'
|
||||
|
||||
export function normalizePlan(planJson) {
|
||||
const changes = (planJson.resource_changes ?? [])
|
||||
.filter((entry) => !entry.change.actions.includes('forget'))
|
||||
.map((entry) => ({
|
||||
address: entry.address,
|
||||
actions: entry.change.actions,
|
||||
before: entry.change.before ?? null,
|
||||
after: entry.change.after ?? null,
|
||||
after_unknown: entry.change.after_unknown ?? null
|
||||
}))
|
||||
.sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))
|
||||
return changes
|
||||
}
|
||||
|
||||
export function summarize(changes) {
|
||||
const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 }
|
||||
for (const change of changes) {
|
||||
const key = change.actions.join('-')
|
||||
if (key === 'create') counts.create += 1
|
||||
else if (key === 'update') counts.update += 1
|
||||
else if (key === 'delete') counts.delete += 1
|
||||
else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1
|
||||
else if (key === 'read') counts.read += 1
|
||||
else counts['no-op'] += 1
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
function argument(flag) {
|
||||
const index = process.argv.indexOf(flag)
|
||||
return index >= 0 ? process.argv[index + 1] : undefined
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) {
|
||||
const root = argument('--root')
|
||||
const environment = argument('--env')
|
||||
const out = argument('--out')
|
||||
const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}`
|
||||
if (!root || !['staging', 'production'].includes(environment) || !out) {
|
||||
process.stderr.write(`${usage}\n`)
|
||||
process.exit(2)
|
||||
}
|
||||
// The Cloudflare override only applies to the root that still declares the records; the relay
|
||||
// root dropped them in the carve and errors on a -var for an undeclared variable.
|
||||
const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes(
|
||||
'variable "manage_artifact_dns"'
|
||||
)
|
||||
mkdirSync(out, { recursive: true })
|
||||
const planFile = join(out, `${tag}.tfplan`)
|
||||
execFileSync(
|
||||
'terraform',
|
||||
[
|
||||
`-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []),
|
||||
`-out=${planFile}`
|
||||
],
|
||||
{ stdio: ['ignore', 'inherit', 'inherit'] }
|
||||
)
|
||||
const json = JSON.parse(
|
||||
execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 256 * 1024 * 1024
|
||||
})
|
||||
)
|
||||
const normalized = normalizePlan(json)
|
||||
writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`)
|
||||
process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`)
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs'
|
||||
|
||||
test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => {
|
||||
const changes = normalizePlan({
|
||||
resource_changes: [
|
||||
{ address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } },
|
||||
{ address: 'a.one', change: { actions: ['forget'], before: {}, after: null } },
|
||||
{ address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } }
|
||||
]
|
||||
})
|
||||
assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three'])
|
||||
assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 })
|
||||
})
|
||||
@@ -0,0 +1,109 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
|
||||
function parseArguments(argv) {
|
||||
if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) {
|
||||
throw new Error('missing --capacity-service-account')
|
||||
}
|
||||
return argv[1]
|
||||
}
|
||||
|
||||
export function classifyProductionCapacityDirector(state, capacityServiceAccount) {
|
||||
const currentIdentity = state.currentCapacityServiceAccount
|
||||
if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) {
|
||||
throw new Error('director has an unexpected capacity identity')
|
||||
}
|
||||
const {
|
||||
baseCells,
|
||||
currentCells,
|
||||
capacityCellIds,
|
||||
targetCellId,
|
||||
targetHardCap
|
||||
} = state
|
||||
if (
|
||||
!Array.isArray(baseCells) ||
|
||||
!Array.isArray(currentCells) ||
|
||||
!Array.isArray(capacityCellIds) ||
|
||||
![600, 1000].includes(targetHardCap) ||
|
||||
new Set(capacityCellIds).size !== capacityCellIds.length ||
|
||||
!capacityCellIds.includes(targetCellId) ||
|
||||
baseCells.length !== currentCells.length ||
|
||||
new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length ||
|
||||
new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length
|
||||
) {
|
||||
throw new Error('director topology transition input is invalid')
|
||||
}
|
||||
const capacityCells = new Set(capacityCellIds)
|
||||
const normalizedCurrent = currentCells.map((current, index) => {
|
||||
const base = baseCells[index]
|
||||
if (
|
||||
typeof current?.id !== 'string' ||
|
||||
current.id !== base?.id
|
||||
) {
|
||||
throw new Error('director topology cell identity is invalid')
|
||||
}
|
||||
if (!capacityCells.has(current.id)) {
|
||||
if (!isDeepStrictEqual(current, base)) {
|
||||
throw new Error('director topology changed outside the capacity rollout')
|
||||
}
|
||||
return current
|
||||
}
|
||||
if (
|
||||
base.connectionHardCap !== 1000 ||
|
||||
base.connectionUnobservedBound !== 60 ||
|
||||
![600, 1000].includes(current.connectionHardCap) ||
|
||||
current.connectionUnobservedBound !== 60
|
||||
) {
|
||||
throw new Error('director capacity rollout state is invalid')
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
connectionHardCap: base.connectionHardCap,
|
||||
connectionUnobservedBound: base.connectionUnobservedBound
|
||||
}
|
||||
})
|
||||
if (
|
||||
!isDeepStrictEqual(normalizedCurrent, baseCells) ||
|
||||
capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId))
|
||||
) {
|
||||
throw new Error('director topology is outside the reviewed capacity envelope')
|
||||
}
|
||||
const withTargetCap = (hardCap) => currentCells.map((cell) =>
|
||||
cell.id === targetCellId
|
||||
? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 }
|
||||
: cell
|
||||
)
|
||||
const desiredCells = withTargetCap(targetHardCap)
|
||||
const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600)
|
||||
const topologyPhase = isDeepStrictEqual(currentCells, desiredCells)
|
||||
? 'desired'
|
||||
: isDeepStrictEqual(currentCells, predecessorCells)
|
||||
? 'predecessor'
|
||||
: null
|
||||
if (!topologyPhase) throw new Error('director topology is not a reviewed transition state')
|
||||
return {
|
||||
topologyPhase,
|
||||
directorReady:
|
||||
topologyPhase === 'desired' && currentIdentity === capacityServiceAccount,
|
||||
desiredCells
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
const capacityServiceAccount = parseArguments(argv)
|
||||
const state = JSON.parse(readFileSync(0, 'utf8'))
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
event: 'relay_production_capacity_director_classified',
|
||||
...classifyProductionCapacityDirector(state, capacityServiceAccount)
|
||||
})}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
classifyProductionCapacityDirector
|
||||
} from './classify-relay-production-capacity-director.mjs'
|
||||
|
||||
const capacityServiceAccount =
|
||||
'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com'
|
||||
const capacityCellIds = ['production-gce-c25', 'production-gce-c26']
|
||||
const baseCells = [
|
||||
{ id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 }
|
||||
]
|
||||
const mixedCells = [
|
||||
baseCells[0],
|
||||
{ ...baseCells[1], connectionHardCap: 600 },
|
||||
baseCells[2]
|
||||
]
|
||||
|
||||
function state(overrides = {}) {
|
||||
return {
|
||||
baseCells,
|
||||
currentCells: mixedCells,
|
||||
capacityCellIds,
|
||||
targetCellId: 'production-gce-c25',
|
||||
targetHardCap: 1000,
|
||||
currentCapacityServiceAccount: capacityServiceAccount,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('classifies one target while preserving completed rollout cells', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state(), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: baseCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('skips deployment only for exact topology and identity', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount),
|
||||
{ topologyPhase: 'desired', directorReady: true, desiredCells: baseCells }
|
||||
)
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({
|
||||
currentCells: baseCells,
|
||||
targetHardCap: 600
|
||||
}), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: mixedCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects an unknown topology or capacity identity', () => {
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/rollout state is invalid/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com'
|
||||
}), capacityServiceAccount),
|
||||
/unexpected capacity identity/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the capacity rollout/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the reviewed capacity envelope/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
targetCellId: 'production-gce-c17'
|
||||
}), capacityServiceAccount),
|
||||
/transition input is invalid/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,47 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) {
|
||||
for (const kind of [c2Kind, c3Kind]) {
|
||||
if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid')
|
||||
}
|
||||
for (const admission of [c2Admission, c3Admission]) {
|
||||
if (!['general', 'migration-only'].includes(admission)) {
|
||||
throw new Error('bootstrap admission is not recoverable')
|
||||
}
|
||||
}
|
||||
if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete'
|
||||
if (c2Kind === 'modern') return 'roll-c3'
|
||||
if (c3Kind === 'modern') return 'roll-c2'
|
||||
if (c2Admission === 'general') return 'normalize-and-roll-both'
|
||||
if (c3Admission === 'general') return 'resume-c2-then-c3'
|
||||
throw new Error('bootstrap has no general fallback')
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
return {
|
||||
c2Kind: values['c2-kind'],
|
||||
c2Admission: values['c2-admission'],
|
||||
c3Kind: values['c3-kind'],
|
||||
c3Admission: values['c3-admission']
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs'
|
||||
|
||||
const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({
|
||||
c2Kind,
|
||||
c2Admission,
|
||||
c3Kind,
|
||||
c3Admission
|
||||
})
|
||||
|
||||
test('classifies the fresh legacy bootstrap', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('retries normalization after a failed C3 restart', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 isolation', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')),
|
||||
'resume-c2-then-c3'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 apply or a partial C3 transition', () => {
|
||||
for (const c2Admission of ['general', 'migration-only']) {
|
||||
for (const c3Admission of ['general', 'migration-only']) {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)),
|
||||
'roll-c3'
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('repairs an unexpected modern C3 before rolling legacy C2', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')),
|
||||
'roll-c2'
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts already complete modern cells and rejects no-fallback legacy state', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')),
|
||||
'complete'
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')),
|
||||
/no general fallback/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,305 @@
|
||||
// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL
|
||||
// instance. Hand lists go stale silently; everything here is read back off disk.
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import {
|
||||
RELAY_WORKFLOW_DIRECTORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayWorkflowFile
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY
|
||||
export const SCRIPT_ROOT = new URL('./', import.meta.url)
|
||||
|
||||
export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease'
|
||||
export const PRODUCTION_LEASE = {
|
||||
bucket: 'onorca-cloud-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/production.lock'
|
||||
}
|
||||
export const STAGING_LEASE = {
|
||||
bucket: 'onorca-cloud-staging-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/staging.lock'
|
||||
}
|
||||
|
||||
export const PRODUCTION_GROUP = 'production-cloud-sql-rollout'
|
||||
export const STAGING_GROUP = 'relay-staging-mutation'
|
||||
export const SELECTABLE_GROUP =
|
||||
"${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}"
|
||||
|
||||
const selectable = (production, staging) =>
|
||||
`\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}`
|
||||
|
||||
export const SELECTABLE_LEASE = {
|
||||
bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket),
|
||||
object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object)
|
||||
}
|
||||
|
||||
export const LOCK_GROUPS = new Set([PRODUCTION_GROUP, STAGING_GROUP, SELECTABLE_GROUP])
|
||||
|
||||
export function readWorkflow(file) {
|
||||
return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8')
|
||||
}
|
||||
|
||||
export function workflowFiles() {
|
||||
return readdirSync(WORKFLOW_ROOT)
|
||||
.filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX))
|
||||
.sort()
|
||||
}
|
||||
|
||||
// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) ---
|
||||
|
||||
function indentOf(line) {
|
||||
return line.length - line.trimStart().length
|
||||
}
|
||||
|
||||
function blockAfter(lines, index) {
|
||||
const base = indentOf(lines[index])
|
||||
const body = []
|
||||
for (let i = index + 1; i < lines.length; i += 1) {
|
||||
if (lines[i].trim() === '') {
|
||||
body.push(lines[i])
|
||||
continue
|
||||
}
|
||||
if (indentOf(lines[i]) <= base) break
|
||||
body.push(lines[i])
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
export function concurrencyBlocks(text) {
|
||||
const lines = text.split('\n')
|
||||
const blocks = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== 'concurrency:') return
|
||||
const body = blockAfter(lines, index)
|
||||
blocks.push({
|
||||
group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(),
|
||||
cancelInProgress: body
|
||||
.find((l) => l.trim().startsWith('cancel-in-progress:'))
|
||||
?.trim()
|
||||
.slice('cancel-in-progress:'.length)
|
||||
.trim()
|
||||
})
|
||||
})
|
||||
return blocks
|
||||
}
|
||||
|
||||
export function jobs(text) {
|
||||
const lines = text.split('\n')
|
||||
const start = lines.findIndex((line) => line === 'jobs:')
|
||||
if (start === -1) return []
|
||||
const found = []
|
||||
for (let i = start + 1; i < lines.length; i += 1) {
|
||||
const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i])
|
||||
if (!match) continue
|
||||
found.push({ id: match[1], start: i, body: blockAfter(lines, i) })
|
||||
}
|
||||
return found.map((job) => ({ ...job, text: job.body.join('\n') }))
|
||||
}
|
||||
|
||||
function scalarField(jobText, key) {
|
||||
const lines = jobText.split('\n')
|
||||
const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`))
|
||||
if (index === -1) return undefined
|
||||
const inline = lines[index].trim().slice(`${key}:`.length).trim()
|
||||
if (inline !== '' && inline !== '>-' && inline !== '|') return inline
|
||||
return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
export function jobNeeds(jobText) {
|
||||
const raw = scalarField(jobText, 'needs')
|
||||
if (!raw) return []
|
||||
return raw
|
||||
.replace(/^\[|\]$/g, '')
|
||||
.split(/[,\n]|\s+-\s+/)
|
||||
.map((entry) => entry.replace(/^-/, '').trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function jobIf(jobText) {
|
||||
return scalarField(jobText, 'if') ?? ''
|
||||
}
|
||||
|
||||
export function leaseSteps(text) {
|
||||
const lines = text.split('\n')
|
||||
const steps = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== `- uses: ${LEASE_ACTION}`) return
|
||||
const body = blockAfter(lines, index)
|
||||
const read = (key) =>
|
||||
body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim()
|
||||
steps.push({
|
||||
line: index + 1,
|
||||
bucket: read('bucket'),
|
||||
object: read('object'),
|
||||
release: read('release')
|
||||
})
|
||||
})
|
||||
return steps
|
||||
}
|
||||
|
||||
export function leaseStepsByJob(file) {
|
||||
const text = readWorkflow(file)
|
||||
const steps = leaseSteps(text)
|
||||
return jobs(text).map((job) => ({
|
||||
id: job.id,
|
||||
steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length)
|
||||
}))
|
||||
}
|
||||
|
||||
// --- trigger and reusable-call graph ---
|
||||
|
||||
export function triggers(text) {
|
||||
const lines = text.split('\n')
|
||||
const index = lines.findIndex((line) => line === 'on:')
|
||||
if (index === -1) return []
|
||||
return blockAfter(lines, index)
|
||||
.map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1])
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function isEntrypoint(text) {
|
||||
return triggers(text).some((trigger) => trigger !== 'workflow_call')
|
||||
}
|
||||
|
||||
export function reusableCalls(text) {
|
||||
const counts = new Map()
|
||||
for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) {
|
||||
counts.set(match[1], (counts.get(match[1]) ?? 0) + 1)
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
export function entrypointsFor(file, seen = new Set()) {
|
||||
if (seen.has(file)) return new Set()
|
||||
seen.add(file)
|
||||
if (isEntrypoint(readWorkflow(file))) return new Set([file])
|
||||
const reached = new Set()
|
||||
for (const candidate of workflowFiles()) {
|
||||
if (candidate === file) continue
|
||||
if (!reusableCalls(readWorkflow(candidate)).has(file)) continue
|
||||
for (const entry of entrypointsFor(candidate, seen)) reached.add(entry)
|
||||
}
|
||||
return reached
|
||||
}
|
||||
|
||||
// --- what counts as a Cloud SQL connection-budget rollout ---
|
||||
|
||||
const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/
|
||||
|
||||
function commandLines(text) {
|
||||
return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, ''))
|
||||
}
|
||||
|
||||
export function appliesTerraform(text) {
|
||||
return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line))
|
||||
}
|
||||
|
||||
export function runsCloudRunMutation(text) {
|
||||
return commandLines(text).some((line) =>
|
||||
/^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line)
|
||||
)
|
||||
}
|
||||
|
||||
// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them.
|
||||
export function revisionMintingScripts() {
|
||||
const self = new URL(import.meta.url).pathname.split('/').pop()
|
||||
const names = readdirSync(SCRIPT_ROOT).filter(
|
||||
(name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self
|
||||
)
|
||||
const source = new Map(
|
||||
names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')])
|
||||
)
|
||||
const minting = new Set(
|
||||
names.filter((name) => {
|
||||
const text = source.get(name)
|
||||
return (
|
||||
text.includes("'--no-traffic'") ||
|
||||
/'run',\s*'services',\s*'update'/.test(text) ||
|
||||
/'run',\s*'deploy'/.test(text)
|
||||
)
|
||||
})
|
||||
)
|
||||
for (let changed = true; changed; ) {
|
||||
changed = false
|
||||
for (const name of names) {
|
||||
if (minting.has(name)) continue
|
||||
const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map(
|
||||
(match) => match[1]
|
||||
)
|
||||
if (!imports.some((imported) => minting.has(imported))) continue
|
||||
minting.add(name)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return minting
|
||||
}
|
||||
|
||||
export function mutatesSharedInstance(text, minters = revisionMintingScripts()) {
|
||||
if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates'
|
||||
if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision'
|
||||
for (const script of minters) {
|
||||
if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision`
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// --- the declared contract ---
|
||||
|
||||
const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra })
|
||||
const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra })
|
||||
const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP })
|
||||
|
||||
// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place.
|
||||
const named = (entries) =>
|
||||
Object.fromEntries(
|
||||
entries.map(([file, entry]) => [
|
||||
relayWorkflowFile(file),
|
||||
entry.leaseFiles
|
||||
? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) }
|
||||
: entry
|
||||
])
|
||||
)
|
||||
|
||||
export const LEASED_WORKFLOWS = named([
|
||||
['deploy-relay-fence-broker.yml', production()],
|
||||
['deploy-relay-production.yml', production()],
|
||||
['deploy-relay-production-director.yml', production()],
|
||||
['deploy-relay-production-multi-target.yml', production()],
|
||||
[
|
||||
'deploy-relay-production-capacity.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-capacity-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'deploy-relay-production-same-cap.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-same-cap-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'operate-relay-production-rehome.yml',
|
||||
production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
|
||||
],
|
||||
['deploy-relay-asia-topology.yml', eitherEnvironment()],
|
||||
['operate-relay-asia-admission.yml', eitherEnvironment()],
|
||||
['deploy-relay-staging.yml', staging()],
|
||||
['deploy-relay-staging-gce-candidate.yml', staging()],
|
||||
['bootstrap-relay-staging-capacity.yml', staging()],
|
||||
['power-relay-staging.yml', staging()],
|
||||
['prove-relay-asia-staging.yml', staging()],
|
||||
[
|
||||
'prove-relay-staging-capacity.yml',
|
||||
staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" })
|
||||
],
|
||||
['recover-relay-staging-c4-image.yml', staging()]
|
||||
])
|
||||
|
||||
export const NOT_A_CLOUD_SQL_CANDIDATE = named([
|
||||
[
|
||||
'monitor-relay-production.yml',
|
||||
'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.'
|
||||
]
|
||||
])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,814 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
activeRevision,
|
||||
cloudRunTrafficTag,
|
||||
DIRECTOR_ADMISSION_ENVIRONMENT,
|
||||
DIRECTOR_REGIONAL_PLACEMENT_ENV,
|
||||
DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
DIRECTOR_REHOME_AUDIENCE_ENV,
|
||||
DIRECTOR_REHOME_IDENTITY_ENV,
|
||||
assertRegionalRehomeDisabled,
|
||||
deployDirector,
|
||||
directorDeploymentEnvironment,
|
||||
directorCellSetAddition,
|
||||
directorStartupProbeArguments,
|
||||
directorTopologyChange,
|
||||
environmentUpdateValue,
|
||||
parseArguments,
|
||||
revisionEnvironment,
|
||||
revisionSecretEnvironment,
|
||||
suppliedAdminIdentityToken,
|
||||
taggedRevisionOrigin,
|
||||
taggedTraffic,
|
||||
trafficTags,
|
||||
waitForEvacuationCapacity
|
||||
} from './deploy-relay-blue-green.mjs'
|
||||
|
||||
// Terraform declares these values; audited blue/green stamps the same values without targeting
|
||||
// the drifted service, so this contract must fail before either side can silently diverge.
|
||||
function terraformDirectorEnvironment(names) {
|
||||
const read = (name) =>
|
||||
readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8')
|
||||
const relay = read('relay.tf')
|
||||
const variables = read('variables.tf')
|
||||
const production = read('environments/production.tfvars')
|
||||
return Object.fromEntries(
|
||||
names.map((name) => {
|
||||
const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay)
|
||||
assert.ok(block, `${name} is not set by relay.tf`)
|
||||
const variable = /var\.([a-z_]+)/.exec(block[1])
|
||||
assert.ok(variable, `${name} is not sourced from a Terraform variable`)
|
||||
// An environment override wins over the variable default, as Terraform resolves it.
|
||||
const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production)
|
||||
const fallback = new RegExp(
|
||||
`variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)`
|
||||
).exec(variables)
|
||||
assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`)
|
||||
return [name, String((override ?? fallback)[1]).replace(/"/g, '')]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
test('director admission environment matches what Terraform deploys', () => {
|
||||
const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT)
|
||||
assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT })
|
||||
|
||||
const relay = readFileSync(
|
||||
fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(
|
||||
relay,
|
||||
/name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/
|
||||
)
|
||||
assert.match(relay, /data "external" "relay_serving_regional_placement_version"/)
|
||||
assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/)
|
||||
assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/)
|
||||
})
|
||||
|
||||
test('validates the final stamped-cell admission state', () => {
|
||||
const required = [
|
||||
'--project',
|
||||
'project',
|
||||
'--region',
|
||||
'region',
|
||||
'--service',
|
||||
'service',
|
||||
'--image',
|
||||
'image',
|
||||
'--role',
|
||||
'cell',
|
||||
'--release-id',
|
||||
'release',
|
||||
'--director-origin',
|
||||
'https://relay.example.com',
|
||||
'--admin-audience',
|
||||
'https://relay.example.com/v1/admin/drain'
|
||||
]
|
||||
|
||||
assert.equal(parseArguments(required)['final-admission'], undefined)
|
||||
assert.equal(
|
||||
parseArguments([...required, '--final-admission', 'disabled'])['final-admission'],
|
||||
'disabled'
|
||||
)
|
||||
assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes']))
|
||||
assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0')
|
||||
assert.throws(() => parseArguments([...required, '--min-instances', '-1']))
|
||||
assert.throws(() =>
|
||||
parseArguments([...required, '--capacity-service-account', 'relay@example.com'])
|
||||
)
|
||||
})
|
||||
|
||||
test('validates optional director capacity configuration', () => {
|
||||
const cells = [
|
||||
{
|
||||
id: 'staging-gce-c3',
|
||||
url: 'https://c3.relay-staging.onorca.dev',
|
||||
capacityRequests: 4_000,
|
||||
initiallyEnabled: false,
|
||||
region: 'us-central1',
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
const config = {
|
||||
project: 'onorca-cloud-staging',
|
||||
'capacity-service-account':
|
||||
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
|
||||
'director-cells-json': JSON.stringify(cells)
|
||||
}
|
||||
assert.deepEqual(directorDeploymentEnvironment(config), {
|
||||
...DIRECTOR_ADMISSION_ENVIRONMENT,
|
||||
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
|
||||
ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT:
|
||||
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
|
||||
ORCA_RELAY_CELLS_JSON: JSON.stringify([
|
||||
{
|
||||
id: cells[0].id,
|
||||
url: cells[0].url,
|
||||
capacityRequests: cells[0].capacityRequests,
|
||||
region: cells[0].region,
|
||||
initiallyEnabled: cells[0].initiallyEnabled,
|
||||
connectionHardCap: cells[0].connectionHardCap,
|
||||
connectionUnobservedBound: cells[0].connectionUnobservedBound
|
||||
}
|
||||
])
|
||||
})
|
||||
assert.throws(
|
||||
() =>
|
||||
directorDeploymentEnvironment({
|
||||
...config,
|
||||
'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com'
|
||||
}),
|
||||
/selected project/
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
directorDeploymentEnvironment({
|
||||
...config,
|
||||
'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }])
|
||||
}),
|
||||
/invalid cell/
|
||||
)
|
||||
assert.match(
|
||||
environmentUpdateValue(directorDeploymentEnvironment(config)),
|
||||
/^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/
|
||||
)
|
||||
assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two')
|
||||
assert.deepEqual(
|
||||
directorTopologyChange(
|
||||
JSON.stringify([
|
||||
{
|
||||
capacityRequests: 4_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60,
|
||||
id: 'staging-gce-c3',
|
||||
initiallyEnabled: false,
|
||||
region: 'us-central1',
|
||||
url: 'https://c3.relay-staging.onorca.dev'
|
||||
}
|
||||
]),
|
||||
JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]),
|
||||
'staging-gce-c3'
|
||||
),
|
||||
{
|
||||
changed: true,
|
||||
value: directorDeploymentEnvironment({
|
||||
'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }])
|
||||
}).ORCA_RELAY_CELLS_JSON
|
||||
}
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
directorTopologyChange(
|
||||
JSON.stringify(cells),
|
||||
JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]),
|
||||
'staging-gce-c3'
|
||||
),
|
||||
/outside the reviewed capacity pair/
|
||||
)
|
||||
})
|
||||
|
||||
test('validates exact director runtime and regional rehome identities', () => {
|
||||
const base = [
|
||||
'--project',
|
||||
'onorca-cloud',
|
||||
'--region',
|
||||
'us-central1',
|
||||
'--service',
|
||||
'orca-cloud-relay',
|
||||
'--image',
|
||||
`relay@sha256:${'a'.repeat(64)}`,
|
||||
'--role',
|
||||
'director',
|
||||
'--release-id',
|
||||
'rehome',
|
||||
'--runtime-service-account',
|
||||
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
||||
'--rehome-director-service-account',
|
||||
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
||||
'--rehome-audience',
|
||||
'https://relay.onorca.dev/v1/admin/host-drain',
|
||||
'--expected-rehome-generation',
|
||||
'7',
|
||||
'--rehome-control-origin',
|
||||
'https://relay.onorca.dev',
|
||||
'--admin-audience',
|
||||
'https://relay.onorca.dev/v1/admin/drain'
|
||||
]
|
||||
const config = parseArguments(base)
|
||||
assert.deepEqual(directorDeploymentEnvironment(config), {
|
||||
...DIRECTOR_ADMISSION_ENVIRONMENT,
|
||||
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
|
||||
ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`,
|
||||
[DIRECTOR_REHOME_IDENTITY_ENV]:
|
||||
'relay-director@onorca-cloud.iam.gserviceaccount.com',
|
||||
[DIRECTOR_REHOME_AUDIENCE_ENV]:
|
||||
'https://relay.onorca.dev/v1/admin/host-drain'
|
||||
})
|
||||
const missingAudience = [...base]
|
||||
missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2)
|
||||
assert.throws(() => parseArguments(missingAudience), /configured together/)
|
||||
const invalidOrigin = [...base]
|
||||
invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] =
|
||||
'http://relay.onorca.dev'
|
||||
assert.throws(
|
||||
() => parseArguments(invalidOrigin),
|
||||
/HTTPS origin/
|
||||
)
|
||||
const mutableImage = [...base]
|
||||
mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest'
|
||||
assert.throws(() => parseArguments(mutableImage), /immutable digest/)
|
||||
})
|
||||
|
||||
test('requires durable regional rehome control to be disabled at the exact generation', async () => {
|
||||
const config = {
|
||||
'admin-audience': 'https://relay.onorca.dev/v1/admin/drain',
|
||||
'expected-rehome-generation': '7'
|
||||
}
|
||||
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
|
||||
try {
|
||||
const control = await assertRegionalRehomeDisabled(
|
||||
config,
|
||||
'https://candidate.example.test',
|
||||
async (url, init) => {
|
||||
assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control')
|
||||
assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc')
|
||||
return new Response(JSON.stringify({
|
||||
v: 1,
|
||||
control: { generation: 7, enabled: false }
|
||||
}))
|
||||
}
|
||||
)
|
||||
assert.equal(control.generation, 7)
|
||||
await assert.rejects(
|
||||
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
|
||||
new Response(JSON.stringify({
|
||||
v: 1,
|
||||
control: { generation: 8, enabled: false }
|
||||
}))
|
||||
),
|
||||
/expected generation/
|
||||
)
|
||||
await assert.rejects(
|
||||
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
|
||||
new Response(JSON.stringify({
|
||||
v: 1,
|
||||
control: { generation: 7, enabled: true }
|
||||
}))
|
||||
),
|
||||
/durably disabled/
|
||||
)
|
||||
} finally {
|
||||
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects literal regional placement changes outside the runtime-setting step', () => {
|
||||
const base = {
|
||||
project: 'onorca-cloud',
|
||||
region: 'us-central1',
|
||||
service: 'orca-cloud-relay',
|
||||
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`,
|
||||
role: 'director',
|
||||
'release-id': 'regional-kill-switch'
|
||||
}
|
||||
const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value])
|
||||
|
||||
assert.doesNotThrow(() => parseArguments(args))
|
||||
assert.throws(
|
||||
() => parseArguments([...args, '--regional-placement-enabled', 'false']),
|
||||
/audited runtime-setting step/
|
||||
)
|
||||
})
|
||||
|
||||
test('appends disabled Asia cells without changing the existing director topology', () => {
|
||||
const current = [
|
||||
{
|
||||
id: 'production-gce-c26',
|
||||
url: 'https://c26.relay.onorca.dev',
|
||||
capacityRequests: 4_000,
|
||||
initiallyEnabled: true,
|
||||
connectionHardCap: 1_000,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
const asia = {
|
||||
id: 'production-gce-c27',
|
||||
url: 'https://c27.relay.onorca.dev',
|
||||
region: 'asia-east2',
|
||||
capacityRequests: 6_000,
|
||||
initiallyEnabled: false,
|
||||
connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
|
||||
assert.deepEqual(
|
||||
directorCellSetAddition(
|
||||
JSON.stringify(current),
|
||||
JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
|
||||
),
|
||||
{
|
||||
changed: true,
|
||||
value: directorDeploymentEnvironment({
|
||||
'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
|
||||
}).ORCA_RELAY_CELLS_JSON
|
||||
}
|
||||
)
|
||||
const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia])
|
||||
assert.deepEqual(directorCellSetAddition(exact, exact), {
|
||||
changed: false,
|
||||
value: directorDeploymentEnvironment({ 'director-cells-json': exact })
|
||||
.ORCA_RELAY_CELLS_JSON
|
||||
})
|
||||
assert.throws(
|
||||
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])),
|
||||
/changes an existing cell/
|
||||
)
|
||||
assert.throws(
|
||||
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])),
|
||||
/must start disabled/
|
||||
)
|
||||
})
|
||||
|
||||
test('pins a director startup probe above the bounded reconciliation window', () => {
|
||||
assert.deepEqual(directorStartupProbeArguments('director'), [
|
||||
'--startup-probe',
|
||||
'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
|
||||
])
|
||||
assert.deepEqual(directorStartupProbeArguments('cell'), [])
|
||||
})
|
||||
|
||||
test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => {
|
||||
const service = 'orca-cloud-relay-staging-c1'
|
||||
const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')
|
||||
assert.match(candidate, /^candidate-[a-f0-9]{9}$/)
|
||||
assert.equal(service.length + candidate.length, 46)
|
||||
assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a'))
|
||||
assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a'))
|
||||
assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release'))
|
||||
})
|
||||
|
||||
test('derives and validates a Cloud Run tagged revision origin', () => {
|
||||
assert.equal(
|
||||
taggedRevisionOrigin(
|
||||
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
|
||||
'candidate-123'
|
||||
),
|
||||
'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app'
|
||||
)
|
||||
assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123'))
|
||||
assert.throws(() =>
|
||||
taggedRevisionOrigin(
|
||||
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
|
||||
'123-invalid'
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
test('requires exactly one active revision and reads queried tag metadata', () => {
|
||||
const service = {
|
||||
status: {
|
||||
traffic: [
|
||||
{ percent: 100, revisionName: 'relay-00001-old' },
|
||||
{
|
||||
percent: 0,
|
||||
revisionName: 'relay-00002-new',
|
||||
tag: 'candidate-123',
|
||||
url: 'https://candidate-123---relay-hash-uc.a.run.app'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
assert.equal(activeRevision(service), 'relay-00001-old')
|
||||
assert.deepEqual(taggedTraffic(service, 'candidate-123'), {
|
||||
origin: 'https://candidate-123---relay-hash-uc.a.run.app',
|
||||
revision: 'relay-00002-new'
|
||||
})
|
||||
assert.throws(() =>
|
||||
activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } })
|
||||
)
|
||||
assert.deepEqual(trafficTags(service), ['candidate-123'])
|
||||
})
|
||||
|
||||
function directorHarness({
|
||||
deployFailure,
|
||||
deleteFailure,
|
||||
cleanupReportsFailure = false,
|
||||
cleanupFailsBeforeRemoval = false,
|
||||
servingMinimum = 1,
|
||||
servingMaximum = 5,
|
||||
// Reproduces gcloud dropping minScale from a newly created revision.
|
||||
dropRequestedMinimum = false,
|
||||
servingServiceAccount,
|
||||
servingImageDigest = `sha256:${'f'.repeat(64)}`
|
||||
} = {}) {
|
||||
const state = {
|
||||
activeRevision: 'relay-00001-old',
|
||||
tags: new Map([['candidate-old', 'relay-00000-stale']]),
|
||||
revisions: new Map([
|
||||
['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }],
|
||||
[
|
||||
'relay-00001-old',
|
||||
{
|
||||
env: {
|
||||
ORCA_RELAY_ROLE: 'director'
|
||||
},
|
||||
secrets: {
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
||||
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
version: '1'
|
||||
}
|
||||
},
|
||||
minimum: servingMinimum,
|
||||
maximum: servingMaximum,
|
||||
serviceAccount: servingServiceAccount,
|
||||
image: `relay@${servingImageDigest}`
|
||||
}
|
||||
]
|
||||
]),
|
||||
nextRevision: 2
|
||||
}
|
||||
const removed = []
|
||||
const healthProtocols = []
|
||||
let pendingCleanupFailure = cleanupFailsBeforeRemoval
|
||||
const operations = {
|
||||
describeService: () => ({
|
||||
status: {
|
||||
traffic: [
|
||||
{ percent: 100, revisionName: state.activeRevision },
|
||||
...[...state.tags].map(([tag, revisionName]) => ({
|
||||
tag,
|
||||
revisionName,
|
||||
url: `https://${tag}---relay-hash-uc.a.run.app`
|
||||
}))
|
||||
]
|
||||
}
|
||||
}),
|
||||
describeRevision: (_config, revision) => {
|
||||
const value = state.revisions.get(revision)
|
||||
return {
|
||||
metadata: {
|
||||
annotations: {
|
||||
'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0),
|
||||
'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5)
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: value?.serviceAccount,
|
||||
containers: [
|
||||
{
|
||||
image: value?.image,
|
||||
env: [
|
||||
...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })),
|
||||
...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({
|
||||
name,
|
||||
valueSource: { secretKeyRef }
|
||||
}))
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => {
|
||||
const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new`
|
||||
state.revisions.set(revision, {
|
||||
env: { ORCA_RELAY_ROLE: 'director', ...env },
|
||||
secrets: {
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
||||
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
version: regionalVersion
|
||||
}
|
||||
},
|
||||
minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1),
|
||||
maximum,
|
||||
serviceAccount:
|
||||
config['runtime-service-account'] ??
|
||||
state.revisions.get(state.activeRevision)?.serviceAccount,
|
||||
image: image ?? state.revisions.get(state.activeRevision)?.image
|
||||
})
|
||||
state.tags.set(tag, revision)
|
||||
if (deployFailure) throw deployFailure
|
||||
},
|
||||
listRevisions: () =>
|
||||
[...state.revisions].map(([name]) => ({ metadata: { name } })),
|
||||
deleteRevision: (_config, revision) => {
|
||||
if (deleteFailure) throw deleteFailure
|
||||
state.revisions.delete(revision)
|
||||
},
|
||||
updateTraffic: (_config, args) => {
|
||||
const remove = args.find((argument) => argument.startsWith('--remove-tags='))
|
||||
if (remove) {
|
||||
const tags = remove.slice('--remove-tags='.length).split(',')
|
||||
removed.push(tags)
|
||||
if (pendingCleanupFailure && tags.includes('candidate-new')) {
|
||||
pendingCleanupFailure = false
|
||||
throw new Error('failed to remove promoted candidate tag')
|
||||
}
|
||||
for (const tag of tags) state.tags.delete(tag)
|
||||
if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision')
|
||||
return
|
||||
}
|
||||
const promote = args.find((argument) => argument.startsWith('--to-tags='))
|
||||
assert.ok(promote)
|
||||
const tag = promote.slice('--to-tags='.length).split('=')[0]
|
||||
state.activeRevision = state.tags.get(tag)
|
||||
},
|
||||
waitForHealth: async (_origin, connectionCapacityProtocol) => {
|
||||
healthProtocols.push(connectionCapacityProtocol)
|
||||
}
|
||||
}
|
||||
return { state, removed, healthProtocols, operations }
|
||||
}
|
||||
|
||||
test('director deploy removes stale and promoted Cloud Run tags', async () => {
|
||||
const harness = directorHarness()
|
||||
const config = {
|
||||
project: 'onorca-cloud-staging',
|
||||
'capacity-service-account':
|
||||
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
|
||||
}
|
||||
await deployDirector(config, 'candidate-new', harness.operations)
|
||||
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
|
||||
assert.equal(harness.state.activeRevision, 'relay-00003-new')
|
||||
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
||||
assert.deepEqual([...harness.state.revisions.keys()], [
|
||||
'relay-00000-stale',
|
||||
'relay-00001-old',
|
||||
'relay-00002-new',
|
||||
'relay-00003-new'
|
||||
])
|
||||
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
||||
assert.deepEqual(
|
||||
Object.fromEntries(
|
||||
Object.entries(harness.state.revisions.get(revision).env).filter(([key]) =>
|
||||
key in DIRECTOR_ADMISSION_ENVIRONMENT
|
||||
)
|
||||
),
|
||||
DIRECTOR_ADMISSION_ENVIRONMENT
|
||||
)
|
||||
assert.equal(
|
||||
harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT,
|
||||
config['capacity-service-account']
|
||||
)
|
||||
}
|
||||
assert.deepEqual(harness.healthProtocols, [undefined, undefined])
|
||||
})
|
||||
|
||||
test('director deploy stamps the durable regional placement secret reference', async () => {
|
||||
const harness = directorHarness()
|
||||
await deployDirector({}, 'candidate-new', harness.operations)
|
||||
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
||||
assert.deepEqual(
|
||||
harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV],
|
||||
{ secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' }
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('bootstraps both rollback and candidate onto the distinct director identity', async () => {
|
||||
const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
|
||||
const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
||||
const harness = directorHarness({ servingServiceAccount: predecessor })
|
||||
const inspected = []
|
||||
harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => {
|
||||
inspected.push(origin)
|
||||
}
|
||||
await deployDirector({
|
||||
project: 'onorca-cloud',
|
||||
'runtime-service-account': director,
|
||||
'predecessor-runtime-service-account': predecessor,
|
||||
'predecessor-image-digest': `sha256:${'f'.repeat(64)}`,
|
||||
'bootstrap-runtime-identity': 'true',
|
||||
'expected-rehome-generation': '0',
|
||||
'rehome-control-origin': 'https://relay.onorca.dev'
|
||||
}, 'candidate-new', harness.operations)
|
||||
assert.equal(
|
||||
harness.state.revisions.get(harness.state.activeRevision).serviceAccount,
|
||||
director
|
||||
)
|
||||
assert.equal(
|
||||
harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount,
|
||||
director
|
||||
)
|
||||
assert.deepEqual(inspected, [
|
||||
'https://selector-rollback---relay-hash-uc.a.run.app',
|
||||
'https://candidate-new---relay-hash-uc.a.run.app'
|
||||
])
|
||||
})
|
||||
|
||||
test('steady-state director deploy rejects the predecessor identity', async () => {
|
||||
const harness = directorHarness({
|
||||
servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
|
||||
})
|
||||
await assert.rejects(deployDirector({
|
||||
'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com'
|
||||
}, 'candidate-new', harness.operations), /unexpected runtime service account/)
|
||||
})
|
||||
|
||||
test('director deploy prunes old revisions when requested', async () => {
|
||||
const harness = directorHarness()
|
||||
await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations)
|
||||
assert.deepEqual([...harness.state.revisions.keys()], [
|
||||
'relay-00002-new',
|
||||
'relay-00003-new'
|
||||
])
|
||||
assert.deepEqual(harness.healthProtocols, [2, 2])
|
||||
})
|
||||
|
||||
test('a prune failure preserves the active and rollback traffic pair', async () => {
|
||||
const harness = directorHarness({ deleteFailure: new Error('injected delete failure') })
|
||||
await assert.rejects(
|
||||
deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations),
|
||||
/injected delete failure/
|
||||
)
|
||||
assert.equal(harness.state.activeRevision, 'relay-00003-new')
|
||||
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
||||
})
|
||||
|
||||
test('director deploy removes a candidate tag left by a failed update', async () => {
|
||||
const failure = new Error('candidate failed to become ready')
|
||||
const harness = directorHarness({ deployFailure: failure })
|
||||
await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure)
|
||||
assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']])
|
||||
assert.equal(harness.state.tags.size, 0)
|
||||
})
|
||||
|
||||
test('director candidate inherits the serving warm-instance floor', async () => {
|
||||
const harness = directorHarness({ servingMinimum: 5 })
|
||||
await deployDirector({}, 'candidate-new', harness.operations)
|
||||
const serving = harness.state.revisions.get(harness.state.activeRevision)
|
||||
assert.equal(serving.minimum, 5)
|
||||
// The standby rollback revision must stay cold.
|
||||
const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback'))
|
||||
assert.equal(rollback.minimum, 0)
|
||||
})
|
||||
|
||||
test('director deploy rejects a serving maximum above the checked budget', async () => {
|
||||
const harness = directorHarness({ servingMaximum: 6 })
|
||||
await assert.rejects(
|
||||
deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations),
|
||||
/holds 6 maximum instances, expected 5/
|
||||
)
|
||||
})
|
||||
|
||||
test('an explicit --min-instances still overrides the serving floor', async () => {
|
||||
const harness = directorHarness({ servingMinimum: 5 })
|
||||
await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations)
|
||||
assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0)
|
||||
})
|
||||
|
||||
test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => {
|
||||
const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true })
|
||||
await assert.rejects(
|
||||
deployDirector({}, 'candidate-new', harness.operations),
|
||||
/candidate holds 0 minimum instances, expected 5/
|
||||
)
|
||||
// Traffic never moved, so the original revision still serves.
|
||||
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
||||
})
|
||||
|
||||
test('director deploy rejects unrelated revision-shape drift', async () => {
|
||||
const harness = directorHarness()
|
||||
const describeRevision = harness.operations.describeRevision
|
||||
harness.operations.describeRevision = (config, revision) => {
|
||||
const described = describeRevision(config, revision)
|
||||
described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000
|
||||
return described
|
||||
}
|
||||
await assert.rejects(
|
||||
deployDirector({}, 'candidate-new', harness.operations),
|
||||
/unrelated revision shape/
|
||||
)
|
||||
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
||||
})
|
||||
|
||||
test('director cleanup verifies success when gcloud reports a stale revision failure', async () => {
|
||||
const harness = directorHarness({ cleanupReportsFailure: true })
|
||||
await deployDirector({}, 'candidate-new', harness.operations)
|
||||
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
|
||||
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
||||
})
|
||||
|
||||
test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => {
|
||||
const harness = directorHarness({ cleanupFailsBeforeRemoval: true })
|
||||
await assert.rejects(
|
||||
deployDirector({}, 'candidate-new', harness.operations),
|
||||
/failed to remove promoted candidate tag/
|
||||
)
|
||||
assert.equal(
|
||||
harness.state.activeRevision,
|
||||
harness.state.tags.get('selector-rollback')
|
||||
)
|
||||
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
|
||||
})
|
||||
|
||||
test('reads only literal revision environment values', () => {
|
||||
assert.deepEqual(
|
||||
revisionEnvironment({
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
env: [
|
||||
{ name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' },
|
||||
{ name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' },
|
||||
{ name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}),
|
||||
{ ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' }
|
||||
)
|
||||
})
|
||||
|
||||
test('reads only Secret Manager revision environment references', () => {
|
||||
assert.deepEqual(
|
||||
revisionSecretEnvironment({
|
||||
spec: {
|
||||
containers: [{ env: [
|
||||
{ name: 'LITERAL', value: 'true' },
|
||||
{
|
||||
name: DIRECTOR_REGIONAL_PLACEMENT_ENV,
|
||||
valueSource: { secretKeyRef: {
|
||||
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
version: 'latest'
|
||||
} }
|
||||
},
|
||||
{
|
||||
name: 'GCP_SECRET_SHAPE',
|
||||
valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } }
|
||||
}
|
||||
] }]
|
||||
}
|
||||
}),
|
||||
{
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
|
||||
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
|
||||
version: 'latest'
|
||||
},
|
||||
GCP_SECRET_SHAPE: {
|
||||
secret: 'gcp-secret',
|
||||
version: '2'
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
|
||||
assert.equal(suppliedAdminIdentityToken({}), null)
|
||||
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
|
||||
assert.throws(() =>
|
||||
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
|
||||
)
|
||||
})
|
||||
|
||||
test('waits for authenticated target readiness without hiding other capacity errors', async () => {
|
||||
let attempts = 0
|
||||
const capacity = await waitForEvacuationCapacity(
|
||||
async () => {
|
||||
attempts += 1
|
||||
if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable')
|
||||
return { requiredTargetUnits: 2, availableTargetUnits: 4_000 }
|
||||
},
|
||||
'source',
|
||||
'target',
|
||||
{ pollIntervalMs: 1, timeoutMs: 100 }
|
||||
)
|
||||
assert.equal(attempts, 3)
|
||||
assert.equal(capacity.availableTargetUnits, 4_000)
|
||||
await assert.rejects(
|
||||
waitForEvacuationCapacity(async () => {
|
||||
throw new Error('/v1/admin/evacuation-capacity failed: forbidden')
|
||||
}, 'source', 'target'),
|
||||
/forbidden/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,871 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
inspectAdmissionSelector,
|
||||
selectorCellState,
|
||||
transitionAdmissionSelector
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const DEFAULT_POLL_INTERVAL_MS = 5_000
|
||||
const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000
|
||||
const ADMIN_RETRY_ATTEMPTS = 3
|
||||
const ADMIN_RETRY_BASE_MS = 250
|
||||
const CONNECTION_CONTROL_REBIND_RESERVE = 100
|
||||
const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000])
|
||||
const RETRYABLE_ADMIN_PATHS = new Set([
|
||||
'/v1/admin/runtime-status',
|
||||
'/v1/admin/cell-status',
|
||||
'/v1/admin/evacuation-capacity',
|
||||
'/v1/admin/evacuation-status'
|
||||
])
|
||||
|
||||
function canonicalOrigin(value, name) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') {
|
||||
throw new Error(`${name} must be a canonical HTTPS origin`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function adminAudience(value) {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'https:' ||
|
||||
url.pathname !== '/v1/admin/drain' ||
|
||||
url.search ||
|
||||
url.hash ||
|
||||
url.toString() !== value
|
||||
) {
|
||||
throw new Error('--admin-audience must be the canonical HTTPS director drain URL')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) {
|
||||
throw new Error(`${name} must be a positive integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function nonnegativeInteger(value, name) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a nonnegative integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of [
|
||||
'project',
|
||||
'director-origin',
|
||||
'admin-audience',
|
||||
'topology-file',
|
||||
'source-cell-id',
|
||||
'target-cell-id',
|
||||
'runtime-service-account',
|
||||
'mode'
|
||||
]) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (
|
||||
![
|
||||
'audit',
|
||||
'preflight',
|
||||
'recover-forward',
|
||||
'continue-evacuation',
|
||||
'disable-cell',
|
||||
'execute',
|
||||
'reset-empty-candidate',
|
||||
'enable-empty-cell'
|
||||
].includes(values.mode)
|
||||
) {
|
||||
throw new Error(
|
||||
'--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell'
|
||||
)
|
||||
}
|
||||
return {
|
||||
project: values.project,
|
||||
directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'),
|
||||
adminAudience: adminAudience(values['admin-audience']),
|
||||
topologyFile: values['topology-file'],
|
||||
sourceCellId: values['source-cell-id'],
|
||||
targetCellId: values['target-cell-id'],
|
||||
runtimeServiceAccount: values['runtime-service-account'],
|
||||
mode: values.mode,
|
||||
batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100),
|
||||
drainGraceMs: positiveInteger(
|
||||
values['drain-grace-ms'] ?? 120_000,
|
||||
'--drain-grace-ms',
|
||||
60 * 60 * 1_000
|
||||
),
|
||||
pollIntervalMs: positiveInteger(
|
||||
values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS,
|
||||
'--poll-interval-ms',
|
||||
60_000
|
||||
),
|
||||
timeoutMs: positiveInteger(
|
||||
values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS,
|
||||
'--timeout-ms',
|
||||
60 * 60 * 1_000
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function deployment(value, cellId) {
|
||||
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
|
||||
const expected = {
|
||||
cellId,
|
||||
origin: canonicalOrigin(value.origin, `${cellId} origin`),
|
||||
region: String(value.region ?? 'us-central1'),
|
||||
zone: String(value.zone ?? ''),
|
||||
migName: String(value.mig_name ?? ''),
|
||||
instanceGroup: String(value.instance_group ?? ''),
|
||||
backendName: String(value.backend_name ?? ''),
|
||||
backendId: String(value.backend_id ?? ''),
|
||||
urlMapName: String(value.url_map_name ?? ''),
|
||||
generationIdentity: String(value.generation_identity ?? ''),
|
||||
image: String(value.image ?? ''),
|
||||
imageDigest: String(value.image ?? '').split('@')[1] ?? '',
|
||||
capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`),
|
||||
databasePoolMax: positiveInteger(
|
||||
value.database_pool_max ?? 10,
|
||||
`${cellId} database pool maximum`,
|
||||
100
|
||||
),
|
||||
connectionHardCap:
|
||||
value.connection_hard_cap === null || value.connection_hard_cap === undefined
|
||||
? undefined
|
||||
: positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`),
|
||||
connectionUnobservedBound:
|
||||
value.connection_unobserved_bound === null ||
|
||||
value.connection_unobserved_bound === undefined
|
||||
? undefined
|
||||
: nonnegativeInteger(
|
||||
value.connection_unobserved_bound,
|
||||
`${cellId} unobserved connection bound`
|
||||
),
|
||||
initiallyEnabled: value.initially_enabled,
|
||||
fenced: value.fenced,
|
||||
desiredTargetSize: value.desired_target_size
|
||||
}
|
||||
if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`)
|
||||
if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) {
|
||||
throw new Error(`${cellId} has an invalid region`)
|
||||
}
|
||||
for (const [name, resource] of [
|
||||
['MIG', expected.migName],
|
||||
['instance group', expected.instanceGroup],
|
||||
['backend', expected.backendName],
|
||||
['backend ID', expected.backendId]
|
||||
]) {
|
||||
if (!resource) throw new Error(`${cellId} has no ${name}`)
|
||||
}
|
||||
if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) {
|
||||
throw new Error(`${cellId} image is not digest-pinned`)
|
||||
}
|
||||
if (typeof expected.initiallyEnabled !== 'boolean') {
|
||||
throw new Error(`${cellId} has no initial admission state`)
|
||||
}
|
||||
if (
|
||||
(expected.connectionHardCap === undefined) !==
|
||||
(expected.connectionUnobservedBound === undefined) ||
|
||||
(expected.connectionHardCap !== undefined &&
|
||||
(!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) ||
|
||||
expected.connectionUnobservedBound >=
|
||||
expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE))
|
||||
) {
|
||||
throw new Error(`${cellId} has invalid connection capacity`)
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
export function assertDeploymentConnectionCapacity(expected, runtime, director) {
|
||||
if (expected.connectionHardCap === undefined) {
|
||||
if (runtime !== null || director !== null) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
return
|
||||
}
|
||||
const hardCap = expected.connectionHardCap
|
||||
const unobservedBound = expected.connectionUnobservedBound
|
||||
const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE
|
||||
const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound
|
||||
const matches = (capacity) =>
|
||||
capacity?.hardCap === hardCap &&
|
||||
capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE &&
|
||||
capacity.ordinaryConnectionLimit === ordinaryConnectionLimit &&
|
||||
capacity.unobservedBound === unobservedBound &&
|
||||
capacity.normalAdmissionPause === normalAdmissionPause
|
||||
if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectDeployments(topology, sourceCellId, targetCellId) {
|
||||
if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ')
|
||||
const source = deployment(topology[sourceCellId], sourceCellId)
|
||||
const target = deployment(topology[targetCellId], targetCellId)
|
||||
for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) {
|
||||
if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`)
|
||||
}
|
||||
if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled')
|
||||
return { source, target }
|
||||
}
|
||||
|
||||
export function validateMig(mig, instances, expected) {
|
||||
if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`)
|
||||
const policy = mig.updatePolicy ?? {}
|
||||
if (
|
||||
policy.replacementMethod !== 'RECREATE' ||
|
||||
Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 ||
|
||||
Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1
|
||||
) {
|
||||
throw new Error(`${expected.cellId} MIG replacement policy is unsafe`)
|
||||
}
|
||||
const serving = instances.filter(
|
||||
(entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE'
|
||||
)
|
||||
if (instances.length !== 1 || serving.length !== 1) {
|
||||
throw new Error(`${expected.cellId} MIG must have one running endpoint`)
|
||||
}
|
||||
return serving[0].instance.split('/').at(-1)
|
||||
}
|
||||
|
||||
export function validateInstance(instance, expected, runtimeServiceAccount) {
|
||||
const publicConfigs = (instance.networkInterfaces ?? []).flatMap(
|
||||
(network) => network.accessConfigs ?? []
|
||||
)
|
||||
if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`)
|
||||
const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email)
|
||||
if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) {
|
||||
throw new Error(`${expected.cellId} runtime service account mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateBackend(backend, expected) {
|
||||
if (
|
||||
backend.protocol !== 'HTTP' ||
|
||||
Number(backend.timeoutSec) !== 86_400 ||
|
||||
(backend.backends ?? []).length !== 1 ||
|
||||
backend.backends[0].group !== expected.instanceGroup
|
||||
) {
|
||||
throw new Error(`${expected.cellId} backend topology mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function defaultCommandJson(args) {
|
||||
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
|
||||
}
|
||||
return JSON.parse(result.stdout)
|
||||
}
|
||||
|
||||
export function suppliedAdminIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'admin')
|
||||
}
|
||||
|
||||
export function suppliedFenceMutationIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'fence mutation')
|
||||
}
|
||||
|
||||
function validatedIdentityToken(token, label) {
|
||||
// WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly.
|
||||
if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
||||
throw new Error(`invalid supplied ${label} identity token`)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
export function defaultIdentityToken(audience) {
|
||||
const supplied = suppliedAdminIdentityToken()
|
||||
if (supplied !== null) return supplied
|
||||
const result = spawnSync(
|
||||
'gcloud',
|
||||
['auth', 'print-identity-token', `--audiences=${audience}`],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
|
||||
)
|
||||
if (result.status !== 0) throw new Error('gcloud identity-token command failed')
|
||||
return result.stdout.trim()
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
||||
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
export function createAdminPost(config, deps, token) {
|
||||
return async (origin, path, body) => {
|
||||
const requestToken = typeof token === 'function' ? token(path) : token
|
||||
for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) {
|
||||
let response
|
||||
try {
|
||||
response = await deps.fetch(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${requestToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
} catch (error) {
|
||||
if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error
|
||||
deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' })
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
if (
|
||||
RETRYABLE_ADMIN_PATHS.has(path) &&
|
||||
([502, 503, 504].includes(response.status) ||
|
||||
(path === '/v1/admin/evacuation-status' && response.status === 500)) &&
|
||||
attempt < ADMIN_RETRY_ATTEMPTS
|
||||
) {
|
||||
// These endpoints are read-only or transactionally idempotent, so a
|
||||
// lost response may be retried without widening deployment authority.
|
||||
deps.emit({
|
||||
event: 'candidate_admin_retry',
|
||||
path,
|
||||
attempt,
|
||||
reason: `http_${response.status}`
|
||||
})
|
||||
await response.arrayBuffer().catch(() => undefined)
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
return await responseJson(response, path)
|
||||
}
|
||||
throw new Error(`${path} retry attempts exhausted`)
|
||||
}
|
||||
}
|
||||
|
||||
async function checkHttp(deps, origin, path) {
|
||||
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`)
|
||||
}
|
||||
|
||||
export async function inspectCell(config, deps, adminPost, expected) {
|
||||
const common = ['--project', config.project, '--zone', expected.zone, '--format=json']
|
||||
const mig = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'describe',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instances = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'list-instances',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instanceName = validateMig(mig, instances, expected)
|
||||
const instance = deps.commandJson([
|
||||
'compute',
|
||||
'instances',
|
||||
'describe',
|
||||
instanceName,
|
||||
...common
|
||||
])
|
||||
validateInstance(instance, expected, config.runtimeServiceAccount)
|
||||
const backend = deps.commandJson([
|
||||
'compute',
|
||||
'backend-services',
|
||||
'describe',
|
||||
expected.backendName,
|
||||
'--global',
|
||||
'--project',
|
||||
config.project,
|
||||
'--format=json'
|
||||
])
|
||||
validateBackend(backend, expected)
|
||||
await checkHttp(deps, expected.origin, '/health')
|
||||
await checkHttp(deps, expected.origin, '/ready')
|
||||
const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 })
|
||||
if (
|
||||
runtime.role !== 'cell' ||
|
||||
runtime.cellId !== expected.cellId ||
|
||||
runtime.cellUrl !== expected.origin ||
|
||||
(runtime.region ?? 'us-central1') !== expected.region ||
|
||||
runtime.imageDigest !== expected.imageDigest
|
||||
) {
|
||||
throw new Error(`${expected.cellId} served runtime does not match Terraform topology`)
|
||||
}
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: expected.cellId
|
||||
})
|
||||
if (
|
||||
status.status?.cellUrl !== expected.origin ||
|
||||
(status.status?.region ?? 'us-central1') !== expected.region ||
|
||||
status.status?.runtime?.cellUrl !== expected.origin ||
|
||||
status.status?.runtime?.ready !== true ||
|
||||
status.status?.runtime?.heartbeatFresh !== true
|
||||
) {
|
||||
throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`)
|
||||
}
|
||||
assertDeploymentConnectionCapacity(
|
||||
expected,
|
||||
runtime.connectionCapacity ?? null,
|
||||
status.status.connectionCapacity ?? null
|
||||
)
|
||||
return {
|
||||
...status.status,
|
||||
draining: runtime.draining === true,
|
||||
process: runtime.runtime ?? null,
|
||||
runtimeConnectionCapacity: runtime.connectionCapacity ?? null
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForMigration(config, deps, adminPost, completeReady) {
|
||||
const deadline = deps.now() + config.timeoutMs
|
||||
while (deps.now() < deadline) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: config.sourceCellId,
|
||||
targetCellId: config.targetCellId,
|
||||
completeReady
|
||||
})
|
||||
deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status })
|
||||
if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) {
|
||||
return status
|
||||
}
|
||||
if (
|
||||
completeReady &&
|
||||
status.targetRegistered === status.inProgress &&
|
||||
status.registeredSourceActive === 0 &&
|
||||
status.registeredCompletable === 0 &&
|
||||
status.registeredTargetInactive === status.inProgress
|
||||
) {
|
||||
// CI waiting cannot revive an offline desktop; keep its proven migration
|
||||
// pending until that target control reconnects.
|
||||
return status
|
||||
}
|
||||
await deps.wait(config.pollIntervalMs)
|
||||
}
|
||||
throw new Error('timed out waiting for candidate migration')
|
||||
}
|
||||
|
||||
export async function setCellState(config, adminPost, cellId, enabled) {
|
||||
const post = async (path, body) => await adminPost(config.directorOrigin, path, body)
|
||||
const inspected = await inspectAdmissionSelector(post)
|
||||
if (inspected.selector.generation > 0) {
|
||||
await transitionAdmissionSelector(post, {
|
||||
[cellId]: enabled ? 'general' : 'existing-only'
|
||||
})
|
||||
return
|
||||
}
|
||||
await post('/v1/admin/cell-state', { v: 1, cellId, enabled })
|
||||
}
|
||||
|
||||
function assertNoDurableActivity(status, operation) {
|
||||
const activity = [
|
||||
status.assignments,
|
||||
status.activityLeases,
|
||||
status.reservedRequests,
|
||||
status.outgoingMigrations,
|
||||
status.incomingMigrations
|
||||
]
|
||||
if (activity.some((value) => Number(value) !== 0)) {
|
||||
throw new Error(`${operation} requires zero durable activity`)
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
allowEmptyAdmissionRollback,
|
||||
selectorActive
|
||||
) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
}).catch(() => null)
|
||||
if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
await setCellState(config, adminPost, target.cellId, false).catch(() => undefined)
|
||||
return
|
||||
}
|
||||
if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
deps.emit({
|
||||
event: 'candidate_rollback_waiting_for_lease_expiry',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: status.inProgress
|
||||
})
|
||||
return
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_forward_recovery_required',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
targetRegistered: status?.targetRegistered ?? null
|
||||
})
|
||||
}
|
||||
|
||||
export async function drainSource(
|
||||
config,
|
||||
deps,
|
||||
token,
|
||||
source,
|
||||
graceMs = config.drainGraceMs,
|
||||
traceValue
|
||||
) {
|
||||
const response = await deps.fetch(`${source.origin}/v1/admin/drain`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
...(traceValue ? { 'x-orca-drain-trace': traceValue } : {})
|
||||
},
|
||||
body: JSON.stringify({ v: 1, graceMs }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
await responseJson(response, 'source drain')
|
||||
return {
|
||||
backendStatus: response.status,
|
||||
backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
event,
|
||||
eventName = 'candidate_complete'
|
||||
) {
|
||||
const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: source.cellId
|
||||
})
|
||||
const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: target.cellId
|
||||
})
|
||||
if (
|
||||
finalSource.status.activityLeases !== 0 ||
|
||||
finalSource.status.reservedRequests !== 0 ||
|
||||
finalSource.status.outgoingMigrations !== 0 ||
|
||||
finalSource.status.runtime?.observedRequests !== 0 ||
|
||||
finalTarget.status.incomingMigrations !== 0 ||
|
||||
finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits
|
||||
) {
|
||||
throw new Error('aggregate post-migration counts are not reconciled')
|
||||
}
|
||||
event({
|
||||
event: eventName,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
dormantSourceAssignments: finalSource.status.assignments,
|
||||
targetAssignments: finalTarget.status.assignments,
|
||||
targetActivityLeases: finalTarget.status.activityLeases,
|
||||
targetReservedRequests: finalTarget.status.reservedRequests
|
||||
})
|
||||
}
|
||||
|
||||
export async function runCandidateDeployment(config, overrides = {}) {
|
||||
const deps = {
|
||||
commandJson: overrides.commandJson ?? defaultCommandJson,
|
||||
identityToken: overrides.identityToken ?? defaultIdentityToken,
|
||||
fetch: overrides.fetch ?? fetch,
|
||||
emit:
|
||||
overrides.emit ??
|
||||
((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
|
||||
now: overrides.now ?? Date.now,
|
||||
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
||||
random: overrides.random ?? Math.random
|
||||
}
|
||||
const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8'))
|
||||
const { source, target } = selectDeployments(
|
||||
topology,
|
||||
config.sourceCellId,
|
||||
config.targetCellId
|
||||
)
|
||||
const token = deps.identityToken(config.adminAudience)
|
||||
const adminPost = createAdminPost(config, deps, token)
|
||||
const selectorPost = async (path, body) =>
|
||||
await adminPost(config.directorOrigin, path, body)
|
||||
const selectorInspection = await inspectAdmissionSelector(selectorPost)
|
||||
const selectorActive = selectorInspection.selector.generation > 0
|
||||
const sourceStatus = await inspectCell(config, deps, adminPost, source)
|
||||
const targetStatus = await inspectCell(config, deps, adminPost, target)
|
||||
const sourceAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, source.cellId)
|
||||
: sourceStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
const targetAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, target.cellId)
|
||||
: targetStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
if (config.mode === 'audit') {
|
||||
const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
})
|
||||
// Forward recovery needs durable aggregate evidence without exposing assignment identities.
|
||||
deps.emit({
|
||||
event: 'candidate_audit',
|
||||
source: aggregateCellStatus(sourceStatus),
|
||||
target: aggregateCellStatus(targetStatus),
|
||||
migration
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'recover-forward') {
|
||||
if (
|
||||
selectorActive
|
||||
? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only'
|
||||
: sourceStatus.enabled || !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'forward recovery requires existing-only source and migration-only target'
|
||||
: 'forward recovery requires disabled source and enabled target'
|
||||
)
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
deps.emit({
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: completion.inProgress,
|
||||
registeredSourceActive: completion.registeredSourceActive,
|
||||
registeredCompletable: completion.registeredCompletable,
|
||||
registeredTargetInactive: completion.registeredTargetInactive
|
||||
})
|
||||
throw new Error('forward recovery remains pending for inactive target controls')
|
||||
}
|
||||
await verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
deps.emit,
|
||||
'candidate_forward_recovered'
|
||||
)
|
||||
return
|
||||
}
|
||||
if (config.mode === 'disable-cell') {
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
// Disabling new admission preserves origin-owned sessions and durable recovery work.
|
||||
deps.emit({
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled,
|
||||
assignments: targetStatus.assignments,
|
||||
activityLeases: targetStatus.activityLeases,
|
||||
reservedRequests: targetStatus.reservedRequests,
|
||||
outgoingMigrations: targetStatus.outgoingMigrations,
|
||||
incomingMigrations: targetStatus.incomingMigrations
|
||||
})
|
||||
return
|
||||
}
|
||||
// Repair is safe only before a candidate owns assignments or origin-scoped work.
|
||||
if (config.mode === 'reset-empty-candidate') {
|
||||
assertNoDurableActivity(targetStatus, 'candidate admission reset')
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
deps.emit({
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
assertNoDurableActivity(targetStatus, 'cell admission enable')
|
||||
if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) {
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false })
|
||||
return
|
||||
}
|
||||
}
|
||||
const continuingEvacuation = config.mode === 'continue-evacuation'
|
||||
if (continuingEvacuation) {
|
||||
if (
|
||||
selectorActive
|
||||
? targetAdmission !== 'migration-only'
|
||||
: !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'continued evacuation requires migration-only target'
|
||||
: 'continued evacuation requires enabled target'
|
||||
)
|
||||
}
|
||||
} else if (!selectorActive && targetStatus.enabled) {
|
||||
throw new Error('candidate cell is already enabled')
|
||||
} else if (
|
||||
selectorActive &&
|
||||
!['migration-only', 'existing-only'].includes(targetAdmission)
|
||||
) {
|
||||
throw new Error('candidate cell must not be generally admitted')
|
||||
}
|
||||
const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId
|
||||
})
|
||||
if (capacity.requiredTargetUnits > capacity.availableTargetUnits) {
|
||||
throw new Error('candidate lacks survivor request-unit headroom')
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_preflight',
|
||||
mode: config.mode,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
sourceOrigin: source.origin,
|
||||
targetOrigin: target.origin,
|
||||
sourceMig: source.migName,
|
||||
targetMig: target.migName,
|
||||
sourceBackend: source.backendName,
|
||||
targetBackend: target.backendName,
|
||||
sourceDigest: source.imageDigest,
|
||||
targetDigest: target.imageDigest,
|
||||
sourceAssignments: capacity.sourceAssignments,
|
||||
requiredTargetUnits: capacity.requiredTargetUnits,
|
||||
availableTargetUnits: capacity.availableTargetUnits
|
||||
})
|
||||
if (config.mode === 'preflight') return
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
await setCellState(config, adminPost, target.cellId, true)
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true })
|
||||
return
|
||||
}
|
||||
// Fresh execution starts from source-only admission; continuation preserves its target.
|
||||
if (
|
||||
!continuingEvacuation &&
|
||||
(selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled)
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled'
|
||||
)
|
||||
}
|
||||
if (selectorActive && targetAdmission !== 'migration-only') {
|
||||
throw new Error('target cell is not migration-only')
|
||||
}
|
||||
|
||||
let migrationsStarted = 0
|
||||
try {
|
||||
if (!selectorActive) {
|
||||
if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false)
|
||||
if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true)
|
||||
}
|
||||
for (;;) {
|
||||
const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
limit: config.batchSize
|
||||
})
|
||||
migrationsStarted += result.started
|
||||
deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted })
|
||||
if (result.started === 0) break
|
||||
}
|
||||
} catch (error) {
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
!continuingEvacuation,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
try {
|
||||
if (selectorActive) {
|
||||
const currentSelector = await inspectAdmissionSelector(selectorPost)
|
||||
if (
|
||||
currentSelector.selector.generation !== selectorInspection.selector.generation ||
|
||||
JSON.stringify(currentSelector.selector.membership) !==
|
||||
JSON.stringify(selectorInspection.selector.membership)
|
||||
) {
|
||||
throw new Error('admission selector changed before drain')
|
||||
}
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
throw new Error('candidate migration remains pending for inactive target controls')
|
||||
}
|
||||
} catch (error) {
|
||||
// A completion response can be lost after its transaction commits. Never
|
||||
// reverse admission here merely because no in-progress row remains.
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
false,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
await verifyCandidateCompletion(config, adminPost, source, target, deps.emit)
|
||||
}
|
||||
|
||||
export function aggregateCellStatus(status) {
|
||||
return {
|
||||
cellId: status.cellId,
|
||||
enabled: status.enabled,
|
||||
assignments: status.assignments,
|
||||
activityLeases: status.activityLeases,
|
||||
activityRequestUnits: status.activityRequestUnits,
|
||||
reservedRequests: status.reservedRequests,
|
||||
outgoingMigrations: status.outgoingMigrations,
|
||||
incomingMigrations: status.incomingMigrations,
|
||||
runtimeReady: status.runtime?.ready ?? false,
|
||||
heartbeatFresh: status.runtime?.heartbeatFresh ?? false,
|
||||
observedRequests: status.runtime?.observedRequests ?? null
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
await runCandidateDeployment(parseArguments(argv))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,889 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertDeploymentConnectionCapacity,
|
||||
createAdminPost,
|
||||
deployment,
|
||||
parseArguments,
|
||||
runCandidateDeployment,
|
||||
selectDeployments,
|
||||
suppliedAdminIdentityToken,
|
||||
validateBackend,
|
||||
validateInstance,
|
||||
validateMig
|
||||
} from './deploy-relay-gce-candidate.mjs'
|
||||
|
||||
const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com'
|
||||
const digestA = `sha256:${'a'.repeat(64)}`
|
||||
const digestB = `sha256:${'b'.repeat(64)}`
|
||||
|
||||
test('retries evacuation-status HTTP 500 without widening other admin retries', async () => {
|
||||
const events = []
|
||||
let calls = 0
|
||||
const adminPost = createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return calls === 1
|
||||
? new Response(JSON.stringify({ error: 'transient' }), { status: 500 })
|
||||
: new Response(JSON.stringify({ ok: true }), { status: 200 })
|
||||
},
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')
|
||||
assert.deepEqual(
|
||||
await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}),
|
||||
{ ok: true }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(events, [{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'http_500'
|
||||
}])
|
||||
|
||||
calls = 0
|
||||
await assert.rejects(
|
||||
createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 })
|
||||
},
|
||||
emit: () => {},
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')('https://relay.example', '/v1/admin/runtime-status', {}),
|
||||
/runtime-status failed: persistent/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('verifies a 1,000-cap cell against both runtime and director telemetry', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 1_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 1_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 840
|
||||
}
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
hardCap: 600,
|
||||
heartbeatFresh: true
|
||||
}),
|
||||
/differs from Terraform/
|
||||
)
|
||||
})
|
||||
|
||||
test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 3_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 2_900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 2_840
|
||||
}
|
||||
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
function topology() {
|
||||
return {
|
||||
source: {
|
||||
origin: 'https://c1.relay.example.com',
|
||||
zone: 'us-central1-b',
|
||||
mig_name: 'relay-c1',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c1',
|
||||
backend_name: 'relay-c1',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c1',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: true
|
||||
},
|
||||
target: {
|
||||
origin: 'https://c2.relay.example.com',
|
||||
zone: 'us-central1-c',
|
||||
mig_name: 'relay-c2',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c2',
|
||||
backend_name: 'relay-c2',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c2',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function withTopology(operation) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-'))
|
||||
const file = join(directory, 'topology.json')
|
||||
writeFileSync(file, JSON.stringify(topology()))
|
||||
return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true }))
|
||||
}
|
||||
|
||||
function config(topologyFile, mode = 'preflight') {
|
||||
return {
|
||||
project: 'test-project',
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
adminAudience: 'https://relay.example.com/v1/admin/drain',
|
||||
topologyFile,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
runtimeServiceAccount,
|
||||
mode,
|
||||
batchSize: 100,
|
||||
drainGraceMs: 120_000,
|
||||
pollIntervalMs: 1,
|
||||
timeoutMs: 1_000
|
||||
}
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function migrationResponse(body) {
|
||||
return response({
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0,
|
||||
...body
|
||||
})
|
||||
}
|
||||
|
||||
function fakeCommand(args) {
|
||||
const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1]
|
||||
if (args.includes('list-instances')) {
|
||||
return [
|
||||
{
|
||||
instance: `https://compute.example/instances/${name}-vm`,
|
||||
instanceStatus: 'RUNNING',
|
||||
currentAction: 'NONE'
|
||||
}
|
||||
]
|
||||
}
|
||||
if (args.includes('instance-groups')) {
|
||||
return {
|
||||
targetSize: 1,
|
||||
updatePolicy: {
|
||||
replacementMethod: 'RECREATE',
|
||||
maxSurge: { fixed: 0 },
|
||||
maxUnavailable: { fixed: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (args.includes('instances')) {
|
||||
return {
|
||||
networkInterfaces: [{ networkIP: '10.42.0.2' }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
}
|
||||
}
|
||||
const cell = name === 'relay-c1' ? topology().source : topology().target
|
||||
return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] }
|
||||
}
|
||||
|
||||
function harness({
|
||||
failTargetEnable = false,
|
||||
failDrain = false,
|
||||
failAfterRegistration = false,
|
||||
failCompletionResponse = false,
|
||||
sourceEnabled = true,
|
||||
targetEnabled = false,
|
||||
targetAssignments = 0,
|
||||
migrationInProgress = 0,
|
||||
migrationTargetRegistered = 0,
|
||||
migrationTargetInactive = 0,
|
||||
transientCompletionFailures = 0,
|
||||
dormantSourceAssignments = 0,
|
||||
selectorGeneration = 0
|
||||
} = {}) {
|
||||
const state = {
|
||||
source: {
|
||||
enabled: selectorGeneration > 0 ? false : sourceEnabled,
|
||||
assignments: 2,
|
||||
activityLeases: 2
|
||||
},
|
||||
target: {
|
||||
enabled: selectorGeneration > 0 ? true : targetEnabled,
|
||||
assignments: targetAssignments,
|
||||
activityLeases: targetAssignments
|
||||
}
|
||||
}
|
||||
const events = []
|
||||
const stateChanges = []
|
||||
let batch = 0
|
||||
let migrationCompleted = false
|
||||
let remainingTransientCompletionFailures = transientCompletionFailures
|
||||
const fetch = async (url, options = {}) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true })
|
||||
const body = JSON.parse(options.body ?? '{}')
|
||||
if (parsed.pathname === '/v1/admin/runtime-status') {
|
||||
const target = parsed.origin.includes('c2.')
|
||||
return response({
|
||||
v: 1,
|
||||
role: 'cell',
|
||||
cellId: target ? 'target' : 'source',
|
||||
cellUrl: parsed.origin,
|
||||
imageDigest: target ? digestB : digestA
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: selectorGeneration,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly:
|
||||
selectorGeneration > 0
|
||||
? ['source']
|
||||
: Object.keys(state).filter((cellId) => !state[cellId].enabled),
|
||||
migrationOnly: selectorGeneration > 0 ? ['target'] : [],
|
||||
general:
|
||||
selectorGeneration > 0
|
||||
? []
|
||||
: Object.keys(state).filter((cellId) => state[cellId].enabled)
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
const cell = state[body.cellId]
|
||||
return response({
|
||||
v: 1,
|
||||
status: {
|
||||
cellId: body.cellId,
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
enabled: cell.enabled,
|
||||
admissionState:
|
||||
selectorGeneration > 0
|
||||
? body.cellId === 'source'
|
||||
? 'existing-only'
|
||||
: 'migration-only'
|
||||
: cell.enabled
|
||||
? 'general'
|
||||
: 'existing-only',
|
||||
assignments: cell.assignments,
|
||||
activityLeases: cell.activityLeases,
|
||||
activityRequestUnits: cell.activityLeases,
|
||||
reservedRequests: cell.activityLeases,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtime: {
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: cell.activityLeases
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-capacity') {
|
||||
return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-state') {
|
||||
stateChanges.push([body.cellId, body.enabled])
|
||||
if (failTargetEnable && body.cellId === 'target' && body.enabled) {
|
||||
return response({ error: 'injected_enable_failure' }, 409)
|
||||
}
|
||||
state[body.cellId].enabled = body.enabled
|
||||
return response({ ok: true })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuate-cell') {
|
||||
if (failAfterRegistration && batch > 0) {
|
||||
return response({ error: 'injected_batch_failure' }, 503)
|
||||
}
|
||||
const started = batch++ === 0 ? 2 : 0
|
||||
return response({ v: 1, started })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-status') {
|
||||
if (failAfterRegistration) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 1,
|
||||
registeredSourceActive: 1,
|
||||
completed: 0,
|
||||
blocked: 1
|
||||
})
|
||||
}
|
||||
if (failDrain) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (body.completeReady) {
|
||||
state.source.assignments = dormantSourceAssignments
|
||||
state.source.activityLeases = 0
|
||||
state.target.assignments = 2
|
||||
state.target.activityLeases = 2
|
||||
if (remainingTransientCompletionFailures > 0) {
|
||||
remainingTransientCompletionFailures--
|
||||
throw new TypeError('injected transient fetch failure')
|
||||
}
|
||||
if (migrationTargetInactive > 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationTargetInactive,
|
||||
targetRegistered: migrationTargetInactive,
|
||||
registeredTargetInactive: migrationTargetInactive,
|
||||
completed: 0,
|
||||
blocked: migrationTargetInactive
|
||||
})
|
||||
}
|
||||
migrationCompleted = true
|
||||
if (failCompletionResponse) {
|
||||
return response({ error: 'injected_completion_response_failure' }, 503)
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 2,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (migrationCompleted) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (batch === 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationInProgress,
|
||||
targetRegistered: migrationTargetRegistered,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredCompletable: 2,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/drain') {
|
||||
return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true })
|
||||
}
|
||||
return response({ error: 'unexpected_request' }, 500)
|
||||
}
|
||||
return {
|
||||
overrides: {
|
||||
commandJson: fakeCommand,
|
||||
identityToken: () => 'secret-token-never-emitted',
|
||||
fetch,
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => undefined,
|
||||
random: () => 0
|
||||
},
|
||||
events,
|
||||
stateChanges,
|
||||
state
|
||||
}
|
||||
}
|
||||
|
||||
test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => {
|
||||
assert.throws(() => parseArguments([]), /missing --project/)
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArguments([
|
||||
'--project',
|
||||
'project',
|
||||
'--director-origin',
|
||||
'https://relay.example.com',
|
||||
'--admin-audience',
|
||||
'https://relay.example.com/not-drain',
|
||||
'--topology-file',
|
||||
'topology.json',
|
||||
'--source-cell-id',
|
||||
'source',
|
||||
'--target-cell-id',
|
||||
'target',
|
||||
'--runtime-service-account',
|
||||
runtimeServiceAccount,
|
||||
'--mode',
|
||||
'preflight'
|
||||
]),
|
||||
/director drain URL/
|
||||
)
|
||||
assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/)
|
||||
const overlapping = topology()
|
||||
overlapping.target.backend_id = overlapping.source.backend_id
|
||||
assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/)
|
||||
const enabled = topology()
|
||||
enabled.target.initially_enabled = true
|
||||
assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/)
|
||||
})
|
||||
|
||||
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
|
||||
assert.equal(suppliedAdminIdentityToken({}), null)
|
||||
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
|
||||
assert.throws(() =>
|
||||
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => {
|
||||
const expected = selectDeployments(topology(), 'source', 'target').target
|
||||
assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/)
|
||||
assert.throws(
|
||||
() =>
|
||||
validateInstance(
|
||||
{
|
||||
networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
},
|
||||
expected,
|
||||
runtimeServiceAccount
|
||||
),
|
||||
/public IP/
|
||||
)
|
||||
assert.throws(
|
||||
() => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected),
|
||||
/topology mismatch/
|
||||
)
|
||||
})
|
||||
|
||||
test('preflights exact served digests and survivor headroom without mutating admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events[0].event, 'candidate_preflight')
|
||||
assert.equal(events[0].targetDigest, digestB)
|
||||
assert.equal(JSON.stringify(events).includes('secret-token'), false)
|
||||
})
|
||||
})
|
||||
|
||||
test('audits a partially committed migration without changing admission or completing rows', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'audit'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events, [
|
||||
{
|
||||
event: 'candidate_audit',
|
||||
source: {
|
||||
cellId: 'source',
|
||||
enabled: false,
|
||||
assignments: 2,
|
||||
activityLeases: 2,
|
||||
activityRequestUnits: 2,
|
||||
reservedRequests: 2,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 2
|
||||
},
|
||||
target: {
|
||||
cellId: 'target',
|
||||
enabled: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
activityRequestUnits: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 1
|
||||
},
|
||||
migration: {
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
completed: 0,
|
||||
blocked: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0
|
||||
}
|
||||
}
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('preflights with source admission disabled but still refuses execution', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events.at(-1).event, 'candidate_preflight')
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/source cell is not enabled/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly resets only an empty declared candidate to disabled admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to reset candidate admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('disables only new admission while preserving durable candidate activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'disable-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: 'target',
|
||||
changed: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly enables only an empty preflighted cell for admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', true]])
|
||||
assert.equal(events.at(-2).event, 'candidate_preflight')
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_enabled',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to enable cell admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'enable-empty-cell'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('executes target-first evacuation and verifies aggregate drained counts', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file, 'execute'), overrides)
|
||||
assert.deepEqual(stateChanges.slice(0, 2), [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
assert.equal(events.at(-1).targetAssignments, 2)
|
||||
})
|
||||
})
|
||||
|
||||
test('executes within selector membership without legacy admission writes', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({ selectorGeneration: 1 })
|
||||
await runCandidateDeployment(config(file, 'execute'), testHarness.overrides)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
assert.equal(testHarness.state.target.enabled, true)
|
||||
})
|
||||
})
|
||||
|
||||
test('never restores legacy general admission after selector-era failure', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({
|
||||
selectorGeneration: 1,
|
||||
failAfterRegistration: true
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), testHarness.overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
})
|
||||
})
|
||||
|
||||
test('continues a partial evacuation without resetting target admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'continue-evacuation'), overrides)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses continued evacuation unless target admission is already enabled', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/requires enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves partial target admission when continued batching fails', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
failAfterRegistration: true,
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
})
|
||||
})
|
||||
|
||||
test('resumes only a committed forward migration and preserves dormant source assignments', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
dormantSourceAssignments: 7
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_recovered',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
dormantSourceAssignments: 7,
|
||||
targetAssignments: 2,
|
||||
targetActivityLeases: 2,
|
||||
targetReservedRequests: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('retries a transient idempotent completion request without reversing admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
transientCompletionFailures: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(
|
||||
events.filter(({ event }) => event === 'candidate_admin_retry'),
|
||||
[
|
||||
{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'transport'
|
||||
}
|
||||
]
|
||||
)
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovered')
|
||||
})
|
||||
})
|
||||
|
||||
test('stops forward recovery promptly when only registered offline targets remain', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
migrationTargetInactive: 2
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/pending for inactive target controls/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
inProgress: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses forward recovery unless source and target admission match committed direction', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/requires disabled source and enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables an intact source when candidate admission fails before migration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ failTargetEnable: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_enable_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true],
|
||||
['source', true],
|
||||
['target', false]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables the source and waits for lease rollback when no target registered', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failDrain: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_drain_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges.slice(-1), [['source', true]])
|
||||
assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry')
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves both routes for forward recovery after a target registration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failAfterRegistration: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 1)
|
||||
})
|
||||
})
|
||||
|
||||
test('does not reverse admission after completion commits but its response is lost', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failCompletionResponse: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_completion_response_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 0)
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,133 @@
|
||||
const REQUEST_TIMEOUT_MS = 15_000
|
||||
const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
|
||||
|
||||
export async function requestGitHubSmokeTokens(
|
||||
authOrigin,
|
||||
fetchImpl = fetch,
|
||||
environment = process.env,
|
||||
options = {}
|
||||
) {
|
||||
const origin = canonicalHttpsOrigin(authOrigin)
|
||||
const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable')
|
||||
const audience = `${origin}/v1/internal/github-smoke-token`
|
||||
const oidcUrl = new URL(requestUrl)
|
||||
oidcUrl.searchParams.set('audience', audience)
|
||||
const oidcResponse = await fetchImpl(oidcUrl, {
|
||||
headers: { authorization: `Bearer ${requestToken}` },
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const oidc = await readJson(oidcResponse, 'GitHub OIDC request')
|
||||
if (!oidcResponse.ok || !validJwt(oidc.value)) {
|
||||
throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`)
|
||||
}
|
||||
const exchangeResponse = await fetchImpl(audience, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${oidc.value}`,
|
||||
...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {})
|
||||
},
|
||||
...(options.relayAsiaLoad
|
||||
? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) }
|
||||
: {}),
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange')
|
||||
if (!exchangeResponse.ok) {
|
||||
throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`)
|
||||
}
|
||||
return {
|
||||
...parseAccessTokens(exchange.accessTokens),
|
||||
...(options.relayAsiaLoad
|
||||
? {
|
||||
relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals(
|
||||
exchange.relayAsiaLoadPrincipals,
|
||||
options.relayAsiaLoad.principalCount
|
||||
)
|
||||
}
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
function parseRelayAsiaLoadOptions(value) {
|
||||
if (
|
||||
!value || typeof value !== 'object' ||
|
||||
!Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex > 3 ||
|
||||
!Number.isSafeInteger(value.principalCount) || value.principalCount < 1 ||
|
||||
value.principalCount > 32
|
||||
) throw new Error('Relay Asia load principal request is invalid')
|
||||
return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount }
|
||||
}
|
||||
|
||||
function canonicalHttpsOrigin(value) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) {
|
||||
throw new Error('auth origin must be canonical HTTPS')
|
||||
}
|
||||
return url.origin
|
||||
}
|
||||
|
||||
function validJwt(value) {
|
||||
return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value)
|
||||
}
|
||||
|
||||
function parseAccessTokens(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Orca smoke identity response is malformed')
|
||||
}
|
||||
const expected = ['outsider', 'owner', 'recipient']
|
||||
if (Object.keys(value).sort().join(',') !== expected.join(',')) {
|
||||
throw new Error('Orca smoke identity response principals are invalid')
|
||||
}
|
||||
return Object.fromEntries(
|
||||
expected.map((name) => {
|
||||
const principal = value[name]
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' ||
|
||||
!validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' ||
|
||||
principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) {
|
||||
throw new Error('Orca smoke identity response principal is malformed')
|
||||
}
|
||||
return [name, principal]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function parseRelayAsiaLoadPrincipals(value, expectedCount) {
|
||||
if (!Array.isArray(value) || value.length !== expectedCount) {
|
||||
throw new Error('Relay Asia load principal response is invalid')
|
||||
}
|
||||
const userIds = new Set()
|
||||
return value.map((principal, principalIndex) => {
|
||||
if (
|
||||
!principal || typeof principal !== 'object' ||
|
||||
principal.principalIndex !== principalIndex ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) ||
|
||||
typeof principal.profileId !== 'string' ||
|
||||
principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') ||
|
||||
userIds.has(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) throw new Error('Relay Asia load principal response is malformed')
|
||||
userIds.add(principal.userId)
|
||||
return principal
|
||||
})
|
||||
}
|
||||
|
||||
async function readJson(response, label) {
|
||||
try {
|
||||
return await response.json()
|
||||
} catch {
|
||||
throw new Error(`${label} did not return JSON`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
|
||||
|
||||
const jwt = (value) => `${value}.${value}.${value}`
|
||||
|
||||
test('exchanges the runner OIDC token without returning request credentials', async () => {
|
||||
const requests = []
|
||||
const fetchImpl = async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
if (requests.length === 1) return Response.json({ value: jwt('github') })
|
||||
return Response.json({
|
||||
accessTokens: Object.fromEntries(
|
||||
['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])
|
||||
)
|
||||
})
|
||||
}
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
fetchImpl,
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
)
|
||||
assert.equal(result.owner.userId, 'usr_owner')
|
||||
assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/)
|
||||
assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token')
|
||||
assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`)
|
||||
})
|
||||
|
||||
test('fails with bounded errors and never includes credentials', async () => {
|
||||
await assert.rejects(
|
||||
requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => new Response('denied', { status: 403 }),
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token'
|
||||
}
|
||||
),
|
||||
(error) => {
|
||||
assert.doesNotMatch(String(error), /private-request-token|denied/)
|
||||
return true
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('requests and validates an exact Relay Asia principal batch', async () => {
|
||||
const requests = []
|
||||
const principals = Array.from({ length: 32 }, (_, principalIndex) => {
|
||||
const suffix = String(principalIndex).padStart(32, 'a')
|
||||
return {
|
||||
principalIndex,
|
||||
userId: `usr_relay_asia_load_${suffix}`,
|
||||
profileId: `prof_relay_asia_load_${suffix}`,
|
||||
accessToken: jwt(`load${principalIndex}`),
|
||||
expiresAt: Date.now() + 600_000
|
||||
}
|
||||
})
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
return requests.length === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: principals
|
||||
})
|
||||
},
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
},
|
||||
{ relayAsiaLoad: { shardIndex: 3, principalCount: 32 } }
|
||||
)
|
||||
assert.equal(result.relayAsiaLoadPrincipals.length, 32)
|
||||
assert.deepEqual(JSON.parse(requests[1].init.body), {
|
||||
relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 }
|
||||
})
|
||||
assert.equal(requests[1].init.headers['content-type'], 'application/json')
|
||||
})
|
||||
|
||||
test('rejects malformed or duplicate Relay Asia principal batches', async () => {
|
||||
const environment = {
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
let request = 0
|
||||
await assert.rejects(requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => ++request === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: []
|
||||
}),
|
||||
environment,
|
||||
{ relayAsiaLoad: { shardIndex: 0, principalCount: 32 } }
|
||||
), /principal response is invalid/)
|
||||
})
|
||||
@@ -0,0 +1,146 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs'
|
||||
|
||||
// The relay root keeps its historical path so every existing caller — 9 workflows, the fence
|
||||
// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps
|
||||
// roots stay in the private repository with the services they own.
|
||||
const ROOT_DIRECTORIES = {
|
||||
relay: join('infra', 'terraform')
|
||||
}
|
||||
|
||||
const command = process.argv[2]
|
||||
const environment = readEnvironment(process.argv.slice(3))
|
||||
const root = readRoot(process.argv.slice(3))
|
||||
const tool = process.env.IAC_TOOL || findTool()
|
||||
|
||||
if (!command || !['init', 'plan', 'apply'].includes(command)) {
|
||||
exitWithUsage()
|
||||
}
|
||||
|
||||
if (!environment) {
|
||||
exitWithUsage('Missing --env staging|production')
|
||||
}
|
||||
|
||||
if (!root) {
|
||||
exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`)
|
||||
}
|
||||
|
||||
const rootDirectory = ROOT_DIRECTORIES[root]
|
||||
const terraformDir = join(process.cwd(), rootDirectory)
|
||||
const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`)
|
||||
const varFile = join(terraformDir, 'environments', `${environment}.tfvars`)
|
||||
|
||||
if (!existsSync(backendConfig)) {
|
||||
throw new Error(`Backend config not found: ${backendConfig}`)
|
||||
}
|
||||
|
||||
if (!existsSync(varFile)) {
|
||||
throw new Error(`Variable file not found: ${varFile}`)
|
||||
}
|
||||
|
||||
const chdir = `-chdir=${rootDirectory}`
|
||||
|
||||
if (command === 'init') {
|
||||
run([chdir, 'init', `-backend-config=backend/${environment}.hcl`])
|
||||
} else if (command === 'plan') {
|
||||
run([
|
||||
chdir,
|
||||
'plan',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
`-out=${environment}.tfplan`
|
||||
])
|
||||
} else {
|
||||
// A normal staging apply must not implicitly wake or partially mutate a sleeping data plane.
|
||||
// Only the relay root can touch that data plane; the guard would refuse app work for no reason.
|
||||
if (environment === 'staging' && root === 'relay') assertStagingRelayAwake()
|
||||
run([chdir, 'apply', `${environment}.tfplan`])
|
||||
}
|
||||
|
||||
function readEnvironment(args) {
|
||||
const envIndex = args.indexOf('--env')
|
||||
if (envIndex >= 0) {
|
||||
return args[envIndex + 1]
|
||||
}
|
||||
|
||||
return process.env.ORCA_CLOUD_ENV
|
||||
}
|
||||
|
||||
function readRoot(args) {
|
||||
const rootIndex = args.indexOf('--root')
|
||||
const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay'
|
||||
return requested in ROOT_DIRECTORIES ? requested : undefined
|
||||
}
|
||||
|
||||
function findTool() {
|
||||
for (const candidate of ['tofu', 'terraform']) {
|
||||
try {
|
||||
execFileSync(candidate, ['version'], { stdio: 'ignore' })
|
||||
return candidate
|
||||
} catch {
|
||||
// Try the next compatible IaC binary.
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.')
|
||||
}
|
||||
|
||||
function run(args) {
|
||||
execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' })
|
||||
}
|
||||
|
||||
function terraformEnv() {
|
||||
if (
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GOOGLE_CREDENTIALS ||
|
||||
process.env.GOOGLE_OAUTH_ACCESS_TOKEN
|
||||
) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
const token = readGcloudAccessToken()
|
||||
if (!token) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
// Local convenience: Terraform uses ADC, while engineers often only have
|
||||
// gcloud CLI auth. CI should use Workload Identity instead.
|
||||
return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token }
|
||||
}
|
||||
|
||||
function readGcloudAccessToken() {
|
||||
for (const candidate of gcloudCandidates()) {
|
||||
try {
|
||||
return execFileSync(candidate, ['auth', 'print-access-token'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
}).trim()
|
||||
} catch {
|
||||
// Try the next common gcloud location.
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function gcloudCandidates() {
|
||||
return [
|
||||
process.env.GCLOUD_PATH,
|
||||
'gcloud',
|
||||
join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'),
|
||||
join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud')
|
||||
].filter(Boolean)
|
||||
}
|
||||
|
||||
function exitWithUsage(message) {
|
||||
if (message) {
|
||||
console.error(message)
|
||||
}
|
||||
|
||||
console.error(
|
||||
'Usage: pnpm infra:<init|plan|apply> --env staging|production [--root relay]'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const repository = fileURLToPath(new URL('../../', import.meta.url))
|
||||
const script = 'dev/scripts/infra.mjs'
|
||||
|
||||
// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects
|
||||
// is observable without running Terraform.
|
||||
function invoke(args) {
|
||||
return execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
}).trim()
|
||||
}
|
||||
|
||||
function rejects(args) {
|
||||
try {
|
||||
execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
})
|
||||
} catch (error) {
|
||||
return error.stderr
|
||||
}
|
||||
throw new Error(`expected ${args.join(' ')} to exit non-zero`)
|
||||
}
|
||||
|
||||
// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this
|
||||
// without --root. If the default ever moves off infra/terraform they break silently at the plan.
|
||||
test('omitting --root keeps every existing caller on the relay root', () => {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment]),
|
||||
`-chdir=infra/terraform init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
assert.equal(
|
||||
invoke(['plan', '--env', environment]),
|
||||
`-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// Only the relay root ships here; the foundation and apps roots stay in the private repository.
|
||||
test('each root name selects exactly its own directory', () => {
|
||||
const directories = { relay: 'infra/terraform' }
|
||||
for (const [root, directory] of Object.entries(directories)) {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment, '--root', root]),
|
||||
`-chdir=${directory} init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('an unknown root fails closed rather than falling back to the relay root', () => {
|
||||
const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay'])
|
||||
assert.match(stderr, /Unknown --root/)
|
||||
assert.doesNotMatch(stderr, /infra\/terraform /)
|
||||
})
|
||||
|
||||
test('a missing environment still fails before any root is resolved', () => {
|
||||
assert.match(rejects(['plan']), /Missing --env/)
|
||||
})
|
||||
|
||||
// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the
|
||||
// app or foundation roots would block work that never touches that data plane.
|
||||
test('the sleeping staging relay guard is scoped to the relay root', () => {
|
||||
const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8')
|
||||
assert.match(source, /environment === 'staging' && root === 'relay'/)
|
||||
})
|
||||
@@ -0,0 +1,570 @@
|
||||
import { createHash, createPrivateKey, createPublicKey } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { RelayLoadControlPeer } from './relay-load-control-peer.mjs'
|
||||
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
|
||||
import { relayLoadFailureReason } from './relay-load-connection-failure.mjs'
|
||||
import {
|
||||
assertRelayLoadDirectorCapacityToken,
|
||||
waitForRelayLoadDirectorCapacity,
|
||||
waitForRelayLoadRequestUnits
|
||||
} from './relay-load-director-capacity-gate.mjs'
|
||||
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
|
||||
import {
|
||||
proveRelayLoadPlacementBoundary,
|
||||
proveRelayLoadRegionalFallback
|
||||
} from './relay-load-placement-boundary.mjs'
|
||||
import {
|
||||
proveRelayLoadRebindBoundary,
|
||||
waitForRelayLoadRebindGate
|
||||
} from './relay-load-rebind-boundary.mjs'
|
||||
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
|
||||
import {
|
||||
openRelayLoadInviteOffers,
|
||||
proveRelayLoadRequestUnitBoundary
|
||||
} from './relay-load-request-unit-boundary.mjs'
|
||||
import {
|
||||
assertRelayLoadRampAccepted,
|
||||
relayLoadRunHasDisallowedFailures,
|
||||
runRelayLoadWithShutdown
|
||||
} from './relay-load-run-lifecycle.mjs'
|
||||
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
|
||||
import {
|
||||
parseRelayLoadArguments,
|
||||
relayLoadPrincipalIndex,
|
||||
relayLoadReaderEvidenceError,
|
||||
relayLoadSpliceIndexes,
|
||||
relayLoadSpliceProfile,
|
||||
relayLoadSpliceStartDelayMs
|
||||
} from './relay-load-profile.mjs'
|
||||
|
||||
function signingKey(path) {
|
||||
if (!path) return {}
|
||||
const key = createPrivateKey(readFileSync(path, 'utf8'))
|
||||
const signingKeyId = createHash('sha256')
|
||||
.update(createPublicKey(key).export({ type: 'spki', format: 'der' }))
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
return { signingKey: key, signingKeyId }
|
||||
}
|
||||
|
||||
function report(state, final = false) {
|
||||
const elapsedSeconds = Math.max(1, (Date.now() - state.startedAt) / 1000)
|
||||
const memory = process.memoryUsage()
|
||||
const cpu = process.cpuUsage(state.generatorBaselineCpu)
|
||||
const rssMiB = memory.rss / 1_048_576
|
||||
state.generatorPeakRssMiB = Math.max(state.generatorPeakRssMiB, rssMiB)
|
||||
const readerQueueEvidence = state.readerEvidence?.snapshot() ?? []
|
||||
const output = {
|
||||
event: final ? 'relay_load_complete' : 'relay_load_progress',
|
||||
controls: state.controls,
|
||||
shardCount: state.shardCount,
|
||||
shardIndex: state.shardIndex,
|
||||
configuredRampSeconds: state.rampMs / 1000,
|
||||
configuredSteadySeconds: state.durationMs / 1000,
|
||||
configuredSpliceHoldSeconds: state.spliceHoldMs / 1000,
|
||||
requiredLeaseHorizons: state.requiredLeaseHorizons,
|
||||
configuredSplices: state.splices,
|
||||
configuredSlowReaderSplices: state.slowReaderSplices,
|
||||
configuredWedgedReaderSplices: state.wedgedReaderSplices,
|
||||
active: state.active.size,
|
||||
peakActive: state.peakActive,
|
||||
steadyMinimumActive: state.steadyMinimumActive,
|
||||
connected: state.connected,
|
||||
connectionFailures: state.connectionFailures,
|
||||
rampConnectionFailures: state.rampConnectionFailures,
|
||||
steadyConnectionFailures: state.steadyConnectionFailures,
|
||||
transitionConnectionFailures: state.transitionConnectionFailures,
|
||||
connectionFailuresByReason: state.connectionFailuresByReason,
|
||||
closes: state.closes,
|
||||
unexpectedCloses: state.unexpectedCloses,
|
||||
unexpectedClosesByCode: state.unexpectedClosesByCode,
|
||||
drains: state.drains,
|
||||
pings: state.pings,
|
||||
pingRate: Number((state.pings / elapsedSeconds).toFixed(2)),
|
||||
tokens: state.tokens,
|
||||
tokenRate: Number((state.tokens / elapsedSeconds).toFixed(2)),
|
||||
refreshes: state.refreshes,
|
||||
refreshErrors: state.refreshErrors,
|
||||
protocolErrors: state.protocolErrors,
|
||||
socketErrors: state.socketErrors,
|
||||
rebindProbesOpened: state.rebindProbesOpened,
|
||||
rebindOverflowReason: state.rebindOverflowReason,
|
||||
placementOverflowReason: state.placementOverflowReason,
|
||||
regionalFallbacksProved: state.regionalFallbacksProved,
|
||||
oldClientUsFirstProved: state.oldClientUsFirstProved,
|
||||
stickyAssignmentProved: state.stickyAssignmentProved,
|
||||
requestUnitInvitesOpened: state.requestUnitInvitesOpened,
|
||||
requestUnitPrincipalCount: state.requestUnitPrincipalCount,
|
||||
relayAsiaLoadPrincipalCount: state.relayAsiaLoadPrincipalCount,
|
||||
requestUnitOverflowReason: state.requestUnitOverflowReason,
|
||||
requestUnitCleanupProved: state.requestUnitCleanupProved,
|
||||
phaseBarrierPassed: state.phaseBarrierPassed,
|
||||
activeSplices: state.activeSplices,
|
||||
peakActiveSplices: state.peakActiveSplices,
|
||||
completedSplices: state.completedSplices,
|
||||
failedSplices: state.failedSplices,
|
||||
slowReaderSplicesCompleted: state.slowReaderSplicesCompleted,
|
||||
wedgedReaderSplicesClosed: state.wedgedReaderSplicesClosed,
|
||||
readerQueueEvidence,
|
||||
readerQueuedBytesPeak: Math.max(
|
||||
0,
|
||||
...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes)
|
||||
),
|
||||
readerClosesByCode: state.readerClosesByCode,
|
||||
controlHeadroom: Math.max(0, state.controls - state.active.size),
|
||||
generatorRssMiB: Number(rssMiB.toFixed(1)),
|
||||
generatorPeakRssMiB: Number(state.generatorPeakRssMiB.toFixed(1)),
|
||||
generatorRssGrowthMiB: Number(
|
||||
Math.max(0, state.generatorPeakRssMiB - state.generatorBaselineRssMiB).toFixed(1)
|
||||
),
|
||||
generatorHeapUsedMiB: Number((memory.heapUsed / 1_048_576).toFixed(1)),
|
||||
generatorCpuPercent: Number(
|
||||
(((cpu.user + cpu.system) / 1_000_000 / elapsedSeconds) * 100).toFixed(1)
|
||||
),
|
||||
generatorEventLoopP99Ms: Number((state.eventLoopDelay.percentile(99) / 1_000_000).toFixed(2)),
|
||||
shutdownEvidence: final ? state.shutdownEvidence : undefined,
|
||||
elapsedSeconds: Number(elapsedSeconds.toFixed(1))
|
||||
}
|
||||
console.log(JSON.stringify(output))
|
||||
return output
|
||||
}
|
||||
|
||||
const config = parseRelayLoadArguments(process.argv.slice(2))
|
||||
let accessToken = process.env.ORCA_RELAY_LOAD_ACCESS_TOKEN
|
||||
let accessTokenProviderForIndex
|
||||
const adminToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (
|
||||
config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0 ||
|
||||
config.slowReaderSplices + config.wedgedReaderSplices > 0 ||
|
||||
config.requestUnitOverflowProbes > 0 || config.requestUnitCleanupTimeoutMs > 0
|
||||
) {
|
||||
assertRelayLoadDirectorCapacityToken({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken
|
||||
}, Date.now,
|
||||
config.rampMs + config.durationMs + config.wedgedReaderHoldMs +
|
||||
(config.phaseBarrierDir ? 2 * config.phaseBarrierTimeoutMs : 0) +
|
||||
config.spliceRampMs + config.requestUnitCleanupTimeoutMs + 120_000)
|
||||
}
|
||||
const key = signingKey(config.signingKeyFile)
|
||||
if (!accessToken && !key.signingKey && process.env.ACTIONS_ID_TOKEN_REQUEST_URL &&
|
||||
process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) {
|
||||
let tokens
|
||||
let refresh
|
||||
const loadOptions = config.relayAsiaLoadPrincipalCount > 0
|
||||
? {
|
||||
relayAsiaLoad: {
|
||||
shardIndex: config.shardIndex,
|
||||
principalCount: config.relayAsiaLoadPrincipalCount
|
||||
}
|
||||
}
|
||||
: undefined
|
||||
const smokeTokens = async () => {
|
||||
const expiresAt = config.relayAsiaLoadPrincipalCount > 0
|
||||
? tokens?.relayAsiaLoadPrincipals?.[0]?.expiresAt
|
||||
: tokens?.owner?.expiresAt
|
||||
if (expiresAt > Date.now() + 60_000) return tokens
|
||||
refresh ??= requestGitHubSmokeTokens(
|
||||
config.authOrigin,
|
||||
fetch,
|
||||
process.env,
|
||||
loadOptions
|
||||
)
|
||||
try {
|
||||
tokens = await refresh
|
||||
return tokens
|
||||
} finally {
|
||||
refresh = undefined
|
||||
}
|
||||
}
|
||||
accessTokenProviderForIndex = (index) => async () => {
|
||||
const current = await smokeTokens()
|
||||
return config.relayAsiaLoadPrincipalCount > 0
|
||||
? current.relayAsiaLoadPrincipals[
|
||||
relayLoadPrincipalIndex(
|
||||
index,
|
||||
config.shardCount,
|
||||
current.relayAsiaLoadPrincipals.length
|
||||
)
|
||||
].accessToken
|
||||
: current.owner.accessToken
|
||||
}
|
||||
await smokeTokens()
|
||||
}
|
||||
if (!accessToken && !accessTokenProviderForIndex && !key.signingKey) {
|
||||
throw new Error('provide GitHub OIDC, ORCA_RELAY_LOAD_ACCESS_TOKEN, or --signing-key-file')
|
||||
}
|
||||
const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 })
|
||||
eventLoopDelay.enable()
|
||||
const generatorBaselineRssMiB = process.memoryUsage().rss / 1_048_576
|
||||
const generatorBaselineCpu = process.cpuUsage()
|
||||
const state = {
|
||||
...config,
|
||||
startedAt: Date.now(),
|
||||
active: new Set(),
|
||||
peakActive: 0,
|
||||
steadyMinimumActive: null,
|
||||
steadyStarted: false,
|
||||
connected: 0,
|
||||
connectionFailures: 0,
|
||||
rampConnectionFailures: 0,
|
||||
steadyConnectionFailures: 0,
|
||||
transitionConnectionFailures: 0,
|
||||
connectionFailuresByReason: {},
|
||||
closes: 0,
|
||||
unexpectedCloses: 0,
|
||||
unexpectedClosesByCode: {},
|
||||
drains: 0,
|
||||
pings: 0,
|
||||
tokens: 0,
|
||||
refreshes: 0,
|
||||
refreshErrors: 0,
|
||||
protocolErrors: 0,
|
||||
socketErrors: 0,
|
||||
rebindProbesOpened: 0,
|
||||
rebindOverflowReason: null,
|
||||
placementOverflowReason: null,
|
||||
regionalFallbacksProved: 0,
|
||||
oldClientUsFirstProved: 0,
|
||||
stickyAssignmentProved: 0,
|
||||
requestUnitInvitesOpened: 0,
|
||||
requestUnitOverflowReason: null,
|
||||
requestUnitCleanupProved: 0,
|
||||
phaseBarrierPassed: false,
|
||||
activeSplices: 0,
|
||||
peakActiveSplices: 0,
|
||||
completedSplices: 0,
|
||||
failedSplices: 0,
|
||||
slowReaderSplicesCompleted: 0,
|
||||
wedgedReaderSplicesClosed: 0,
|
||||
readerEvidence: null,
|
||||
readerClosesByCode: {},
|
||||
generatorBaselineRssMiB,
|
||||
generatorBaselineCpu,
|
||||
generatorPeakRssMiB: generatorBaselineRssMiB,
|
||||
peerShutdowns: 0,
|
||||
shutdownEvidence: null,
|
||||
eventLoopDelay,
|
||||
stopping: false,
|
||||
transitionWindow: false
|
||||
}
|
||||
const peers = new Map()
|
||||
const reconnectTimers = new Set()
|
||||
|
||||
async function readRuntimeQueuedBytes(origin) {
|
||||
const response = await fetch(`${origin}/v1/admin/runtime-status`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${adminToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(5_000)
|
||||
})
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new Error('reader evidence identity was rejected')
|
||||
}
|
||||
if (!response.ok) throw new Error(`reader runtime status returned ${response.status}`)
|
||||
const status = await response.json()
|
||||
const queuedBytes = status?.runtime?.queuedBytes
|
||||
if (!Number.isSafeInteger(queuedBytes) || queuedBytes < 0) {
|
||||
throw new Error('reader runtime queued bytes are invalid')
|
||||
}
|
||||
return queuedBytes
|
||||
}
|
||||
|
||||
async function observeReaderPressure(input) {
|
||||
if (!state.readerEvidence) throw new Error('reader evidence baseline is unavailable')
|
||||
await state.readerEvidence.observe(input)
|
||||
state.generatorPeakRssMiB = Math.max(
|
||||
state.generatorPeakRssMiB,
|
||||
process.memoryUsage().rss / 1_048_576
|
||||
)
|
||||
}
|
||||
|
||||
function recordSteadyMinimum() {
|
||||
if (!state.steadyStarted || state.stopping) return
|
||||
state.steadyMinimumActive = Math.min(state.steadyMinimumActive, state.active.size)
|
||||
}
|
||||
|
||||
function scheduleReconnect(peer) {
|
||||
if (state.stopping) return
|
||||
const timeout = setTimeout(() => {
|
||||
reconnectTimers.delete(timeout)
|
||||
void connect(peer)
|
||||
}, Math.floor(Math.random() * (config.reconnectMaxMs + 1)))
|
||||
reconnectTimers.add(timeout)
|
||||
}
|
||||
|
||||
function observe(type, detail) {
|
||||
if (type === 'connected') {
|
||||
state.active.add(detail.index)
|
||||
state.connected++
|
||||
state.peakActive = Math.max(state.peakActive, state.active.size)
|
||||
recordSteadyMinimum()
|
||||
} else if (type === 'closed') {
|
||||
state.active.delete(detail.index)
|
||||
state.closes++
|
||||
if (!detail.stopped && !detail.expectedDrain) {
|
||||
state.unexpectedCloses++
|
||||
const code = String(detail.code)
|
||||
state.unexpectedClosesByCode[code] = (state.unexpectedClosesByCode[code] ?? 0) + 1
|
||||
}
|
||||
if (!detail.stopped) scheduleReconnect(peers.get(detail.index))
|
||||
recordSteadyMinimum()
|
||||
} else if (type === 'drain') state.drains++
|
||||
else if (type === 'ping') state.pings++
|
||||
else if (type === 'token') state.tokens++
|
||||
else if (type === 'refresh') state.refreshes++
|
||||
else if (type === 'refreshError') state.refreshErrors++
|
||||
else if (type === 'protocolError') state.protocolErrors++
|
||||
else if (type === 'socketError') state.socketErrors++
|
||||
else if (type === 'spliceOpened') {
|
||||
state.activeSplices++
|
||||
state.peakActiveSplices = Math.max(state.peakActiveSplices, state.activeSplices)
|
||||
} else if (type === 'spliceCompleted') {
|
||||
state.completedSplices++
|
||||
if (detail.readerMode === 'slow') state.slowReaderSplicesCompleted++
|
||||
} else if (type === 'spliceWedged') {
|
||||
state.wedgedReaderSplicesClosed++
|
||||
const code = String(detail.code)
|
||||
state.readerClosesByCode[code] = (state.readerClosesByCode[code] ?? 0) + 1
|
||||
} else if (type === 'spliceClosed') state.activeSplices--
|
||||
else if (type === 'spliceFailed') state.failedSplices++
|
||||
else if (type === 'shutdown') state.peerShutdowns++
|
||||
}
|
||||
|
||||
async function connect(peer) {
|
||||
try {
|
||||
await peer.connect()
|
||||
} catch (error) {
|
||||
state.connectionFailures++
|
||||
if (state.steadyStarted) state.steadyConnectionFailures++
|
||||
else if (state.transitionWindow) state.transitionConnectionFailures++
|
||||
else state.rampConnectionFailures++
|
||||
const reason = relayLoadFailureReason(error)
|
||||
state.connectionFailuresByReason[reason] =
|
||||
(state.connectionFailuresByReason[reason] ?? 0) + 1
|
||||
scheduleReconnect(peer)
|
||||
}
|
||||
}
|
||||
|
||||
const peerOptions = (index, overrides = {}) => ({
|
||||
...config,
|
||||
...key,
|
||||
accessToken,
|
||||
...(accessTokenProviderForIndex
|
||||
? { accessTokenProvider: accessTokenProviderForIndex(index) }
|
||||
: {}),
|
||||
seed: 0x4f524341 ^ config.shardIndex,
|
||||
...overrides
|
||||
})
|
||||
if (config.regionBehaviorProbes > 0) {
|
||||
const proofIndex = config.controls * config.shardCount + 10_000
|
||||
const regionProof = await proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: new RelayLoadControlPeer(
|
||||
proofIndex,
|
||||
peerOptions(proofIndex, { preferredRegion: undefined }),
|
||||
() => undefined
|
||||
),
|
||||
stickyPeer: new RelayLoadControlPeer(
|
||||
proofIndex + 1,
|
||||
peerOptions(proofIndex + 1, { preferredRegion: 'asia-east2' }),
|
||||
() => undefined
|
||||
),
|
||||
asiaOrigin: config.capacityCellOrigin
|
||||
})
|
||||
state.oldClientUsFirstProved = regionProof.oldClientUsFirst ? 1 : 0
|
||||
state.stickyAssignmentProved = regionProof.stickyAssignmentPreserved ? 1 : 0
|
||||
}
|
||||
const initialConnections = []
|
||||
for (let localIndex = 0; localIndex < config.controls; localIndex++) {
|
||||
const globalIndex = localIndex * config.shardCount + config.shardIndex
|
||||
const peer = new RelayLoadControlPeer(globalIndex, peerOptions(globalIndex), observe)
|
||||
peers.set(globalIndex, peer)
|
||||
const rampOffset =
|
||||
config.controls === 1 ? 0 : Math.floor((localIndex / (config.controls - 1)) * config.rampMs)
|
||||
const offset = config.rampStartDelayMs + rampOffset
|
||||
initialConnections.push(delay(offset).then(() => connect(peer)))
|
||||
}
|
||||
const progressTimer = setInterval(() => report(state), 10_000)
|
||||
progressTimer.unref()
|
||||
await runRelayLoadWithShutdown(async () => {
|
||||
await Promise.all(initialConnections)
|
||||
assertRelayLoadRampAccepted(state.rampConnectionFailures, config.maxRampConnectionFailures)
|
||||
if (
|
||||
config.rebindProbes > 0 || config.placementOverflowProbes > 0 ||
|
||||
config.regionalFallbackProbes > 0
|
||||
) {
|
||||
state.transitionWindow = config.rebindDelayMs > 0
|
||||
await waitForRelayLoadRebindGate({
|
||||
delay,
|
||||
delayMs: config.rebindDelayMs,
|
||||
activeCount: () => state.active.size,
|
||||
requiredCount: config.controls
|
||||
})
|
||||
state.transitionWindow = false
|
||||
}
|
||||
if (config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0) {
|
||||
const closesBeforeBoundary = state.closes
|
||||
await waitForRelayLoadDirectorCapacity({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
hardCap: config.capacityHardCap,
|
||||
unobservedBound: config.capacityUnobservedBound,
|
||||
requiredConnections: config.aggregateControls
|
||||
})
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed during the director capacity gate')
|
||||
}
|
||||
const overflowIndex = config.controls * config.shardCount + config.shardIndex
|
||||
if (config.placementOverflowProbes > 0) {
|
||||
state.placementOverflowReason = await proveRelayLoadPlacementBoundary({
|
||||
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), observe),
|
||||
failureReason: relayLoadFailureReason
|
||||
})
|
||||
}
|
||||
if (config.regionalFallbackProbes > 0) {
|
||||
await proveRelayLoadRegionalFallback({
|
||||
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), () => undefined),
|
||||
blockedOrigin: config.capacityCellOrigin
|
||||
})
|
||||
state.regionalFallbacksProved = 1
|
||||
}
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed during the placement boundary probe')
|
||||
}
|
||||
await waitForRelayLoadDirectorCapacity({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
hardCap: config.capacityHardCap,
|
||||
unobservedBound: config.capacityUnobservedBound,
|
||||
requiredConnections: config.aggregateControls,
|
||||
requiredSamples: 1
|
||||
})
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed before post-probe capacity verification')
|
||||
}
|
||||
}
|
||||
const rebindResult = await proveRelayLoadRebindBoundary({
|
||||
peers: [...state.active].map((index) => peers.get(index)),
|
||||
probeCount: config.rebindProbes,
|
||||
holdMs: config.rebindHoldMs,
|
||||
delay,
|
||||
failureReason: relayLoadFailureReason,
|
||||
requireOverflow: config.requireRebindOverflow
|
||||
})
|
||||
state.rebindProbesOpened = rebindResult.opened
|
||||
state.rebindOverflowReason = rebindResult.overflowReason
|
||||
if (config.requestUnitInvites > 0) {
|
||||
state.requestUnitInvitesOpened = await openRelayLoadInviteOffers({
|
||||
peers: [...state.active].sort((left, right) => left - right).map((index) => peers.get(index)),
|
||||
count: config.requestUnitInvites,
|
||||
ratePerSecond: config.requestUnitInvitesPerSecond
|
||||
})
|
||||
}
|
||||
if (config.requestUnitOverflowProbes > 0) {
|
||||
await waitForRelayLoadRequestUnits({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
capacityRequests: config.requestUnitCapacity,
|
||||
expectedRequestUnits: config.requestUnitCapacity,
|
||||
expectedActivityLeases: config.requestUnitCapacity
|
||||
})
|
||||
state.requestUnitOverflowReason = await proveRelayLoadRequestUnitBoundary(
|
||||
peers.get([...state.active][0])
|
||||
)
|
||||
}
|
||||
if (config.phaseBarrierDir) {
|
||||
await waitForRelayLoadPhaseBarrier({
|
||||
directory: config.phaseBarrierDir,
|
||||
shardCount: config.shardCount,
|
||||
shardIndex: config.shardIndex,
|
||||
timeoutMs: config.phaseBarrierTimeoutMs
|
||||
})
|
||||
state.phaseBarrierPassed = true
|
||||
}
|
||||
state.steadyStarted = true
|
||||
state.steadyMinimumActive = state.active.size
|
||||
const spliceIndexes = relayLoadSpliceIndexes(config)
|
||||
const readerOrigins = spliceIndexes.flatMap((index, spliceIndex) =>
|
||||
relayLoadSpliceProfile(config, spliceIndex).readerMode === 'normal'
|
||||
? []
|
||||
: [peers.get(index).lastAssignment.cellUrl]
|
||||
)
|
||||
state.readerEvidence = await createRelayLoadReaderEvidence(readerOrigins, {
|
||||
readQueuedBytes: readRuntimeQueuedBytes,
|
||||
delay
|
||||
})
|
||||
if (config.phaseBarrierDir) {
|
||||
await waitForRelayLoadPhaseBarrier({
|
||||
directory: `${config.phaseBarrierDir}-splices`,
|
||||
shardCount: config.shardCount,
|
||||
shardIndex: config.shardIndex,
|
||||
timeoutMs: config.phaseBarrierTimeoutMs
|
||||
})
|
||||
}
|
||||
const splicePromises = spliceIndexes.map((index, spliceIndex) =>
|
||||
delay(relayLoadSpliceStartDelayMs(config, spliceIndex)).then(() =>
|
||||
peers.get(index).openSplice({
|
||||
payloadBytes: config.splicePayloadBytes,
|
||||
...relayLoadSpliceProfile(config, spliceIndex),
|
||||
observeReaderPressure,
|
||||
holdMs: config.spliceHoldMs
|
||||
})
|
||||
)
|
||||
)
|
||||
await Promise.all([...splicePromises, delay(config.durationMs)])
|
||||
}, async () => {
|
||||
state.stopping = true
|
||||
clearInterval(progressTimer)
|
||||
for (const timeout of reconnectTimers) clearTimeout(timeout)
|
||||
reconnectTimers.clear()
|
||||
await Promise.all([...peers.values()].map((peer) => peer.shutdown()))
|
||||
eventLoopDelay.disable()
|
||||
state.shutdownEvidence = {
|
||||
peerShutdowns: state.peerShutdowns,
|
||||
activeControls: state.active.size,
|
||||
activeSplices: state.activeSplices,
|
||||
reconnectTimers: reconnectTimers.size
|
||||
}
|
||||
})
|
||||
if (config.requestUnitCleanupTimeoutMs > 0) {
|
||||
await waitForRelayLoadRequestUnits({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
capacityRequests: config.requestUnitCapacity,
|
||||
expectedRequestUnits: 0,
|
||||
expectedActivityLeases: 0,
|
||||
timeoutMs: config.requestUnitCleanupTimeoutMs
|
||||
})
|
||||
state.requestUnitCleanupProved = 1
|
||||
}
|
||||
const result = report(state, true)
|
||||
const minimumPeak = config.allowPartial ? 1 : Math.ceil(config.controls * 0.95)
|
||||
if (result.peakActive < minimumPeak) {
|
||||
throw new Error(`peak active controls ${result.peakActive} below required ${minimumPeak}`)
|
||||
}
|
||||
if (result.steadyMinimumActive < minimumPeak) {
|
||||
throw new Error(
|
||||
`steady minimum active controls ${result.steadyMinimumActive} below required ${minimumPeak}`
|
||||
)
|
||||
}
|
||||
if (relayLoadRunHasDisallowedFailures(result, config)) {
|
||||
throw new Error('relay load run observed connection, protocol, refresh, or socket errors')
|
||||
}
|
||||
const readerEvidenceError = relayLoadReaderEvidenceError(result, config)
|
||||
if (readerEvidenceError) throw new Error(readerEvidenceError)
|
||||
if (
|
||||
result.failedSplices > 0 ||
|
||||
result.completedSplices + result.wedgedReaderSplicesClosed !== config.splices ||
|
||||
result.shutdownEvidence.peerShutdowns !== config.controls ||
|
||||
result.shutdownEvidence.activeControls !== 0 ||
|
||||
result.shutdownEvidence.activeSplices !== 0 ||
|
||||
result.shutdownEvidence.reconnectTimers !== 0
|
||||
) {
|
||||
throw new Error('relay load run did not complete splices or shut down cleanly')
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
addExactMigrationCells,
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const SHAPES = {
|
||||
staging: {
|
||||
directorOrigin: 'https://relay-staging.onorca.dev',
|
||||
domain: 'relay-staging.onorca.dev',
|
||||
allCells: ['staging-gce-c4']
|
||||
},
|
||||
production: {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
domain: 'relay.onorca.dev',
|
||||
allCells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
|
||||
}
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['environment', 'mode', 'cell-ids', 'image-digest']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(values['image-digest'])) {
|
||||
throw new Error('--image-digest is invalid')
|
||||
}
|
||||
if (![
|
||||
'inspect', 'initialize', 'verify', 'registered', 'register',
|
||||
'promote', 'recover-promotion', 'rollback'
|
||||
].includes(values.mode)) {
|
||||
throw new Error('--mode is invalid')
|
||||
}
|
||||
const expectedGeneration = values.mode === 'inspect'
|
||||
? undefined
|
||||
: Number(values['expected-generation'])
|
||||
if (
|
||||
values.mode !== 'inspect' &&
|
||||
(!Number.isSafeInteger(expectedGeneration) || expectedGeneration < 0)
|
||||
) {
|
||||
throw new Error('--expected-generation is invalid')
|
||||
}
|
||||
const shape = SHAPES[values.environment]
|
||||
if (!shape) throw new Error('--environment is invalid')
|
||||
const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean)
|
||||
const distinct = new Set(cells)
|
||||
if (distinct.size !== cells.length || cells.some((cell) => !shape.allCells.includes(cell))) {
|
||||
throw new Error('--cell-ids are invalid')
|
||||
}
|
||||
const exact = (expected) => JSON.stringify([...cells].sort()) === JSON.stringify([...expected].sort())
|
||||
if (
|
||||
(['inspect', 'initialize', 'register', 'registered', 'verify'].includes(values.mode) &&
|
||||
!exact(shape.allCells)) ||
|
||||
(['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'production' &&
|
||||
!exact(['production-gce-c27']) && !exact(['production-gce-c28', 'production-gce-c29'])) ||
|
||||
(['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'staging' && !exact(shape.allCells)) ||
|
||||
(values.mode === 'rollback' && cells.length === 0)
|
||||
) throw new Error('--cell-ids do not match the reviewed admission wave')
|
||||
const attemptId = values['attempt-id']
|
||||
if (!['inspect', 'verify', 'registered'].includes(values.mode) &&
|
||||
!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
|
||||
throw new Error('--attempt-id is invalid')
|
||||
}
|
||||
return {
|
||||
environment: values.environment,
|
||||
mode: values.mode,
|
||||
cells,
|
||||
expectedGeneration,
|
||||
expectedMembershipSha256: values['expected-membership-sha256'],
|
||||
imageDigest: values['image-digest'],
|
||||
attemptId,
|
||||
token: process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? ''
|
||||
}
|
||||
}
|
||||
|
||||
function hostname(cellId) {
|
||||
return cellId.split('-').at(-1)
|
||||
}
|
||||
|
||||
function cellOrigin(shape, cellId) {
|
||||
return `https://${hostname(cellId)}.${shape.domain}`
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function defaultPost(fetchImpl, token) {
|
||||
return async (url, body) => await responseJson(await fetchImpl(url, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}), new URL(url).pathname)
|
||||
}
|
||||
|
||||
async function verifyRuntime(fetchImpl, post, shape, cellId, imageDigest, requireDirector) {
|
||||
const origin = cellOrigin(shape, cellId)
|
||||
const [health, ready, runtime] = await Promise.all([
|
||||
fetchImpl(`${origin}/health`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
|
||||
fetchImpl(`${origin}/ready`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
|
||||
post(`${origin}/v1/admin/runtime-status`, { v: 1 })
|
||||
])
|
||||
if (!health.ok || !ready.ok) throw new Error(`${cellId} is not ready`)
|
||||
if (
|
||||
runtime.cellId !== cellId ||
|
||||
runtime.cellUrl !== origin ||
|
||||
runtime.region !== 'asia-east2' ||
|
||||
runtime.imageDigest !== imageDigest ||
|
||||
runtime.draining !== false ||
|
||||
runtime.connectionCapacity?.hardCap !== 3_000 ||
|
||||
runtime.connectionCapacity?.unobservedBound !== 60
|
||||
) throw new Error(`${cellId} runtime does not match the reviewed Asia shape`)
|
||||
if (requireDirector) {
|
||||
const result = await post(`${shape.directorOrigin}/v1/admin/cell-status`, { v: 1, cellId })
|
||||
if (
|
||||
result.status?.cellUrl !== origin ||
|
||||
result.status?.runtime?.heartbeatFresh !== true ||
|
||||
result.status?.runtime?.ready !== true
|
||||
) throw new Error(`${cellId} has no fresh ready director heartbeat`)
|
||||
}
|
||||
}
|
||||
|
||||
function membershipStates(selector, cells) {
|
||||
return Object.fromEntries(cells.map((cellId) => [cellId, selectorCellState(selector, cellId)]))
|
||||
}
|
||||
|
||||
function inspectedMembershipStates(selector, cells) {
|
||||
const known = new Set([
|
||||
...selector.membership.existingOnly,
|
||||
...selector.membership.migrationOnly,
|
||||
...selector.membership.general
|
||||
])
|
||||
return Object.fromEntries(cells.map((cellId) => [
|
||||
cellId,
|
||||
known.has(cellId) ? selectorCellState(selector, cellId) : 'absent'
|
||||
]))
|
||||
}
|
||||
|
||||
function sameMembership(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right)
|
||||
}
|
||||
|
||||
function membershipSha256(membership) {
|
||||
return createHash('sha256').update(JSON.stringify(membership)).digest('hex')
|
||||
}
|
||||
|
||||
async function initializeAdmissionBoundary(post, selectorPost, shape, config, current) {
|
||||
if (config.expectedGeneration !== 0) {
|
||||
throw new Error('admission boundary initialization requires generation 0')
|
||||
}
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(config.expectedMembershipSha256 ?? '') ||
|
||||
membershipSha256(current.selector.membership) !== config.expectedMembershipSha256
|
||||
) {
|
||||
throw new Error('admission membership changed before boundary initialization')
|
||||
}
|
||||
const targetStates = inspectedMembershipStates(current.selector, config.cells)
|
||||
if (Object.values(targetStates).some((state) => state !== 'absent')) {
|
||||
throw new Error('Asia cell exists before admission boundary initialization')
|
||||
}
|
||||
const intendedMembership = current.intent?.previousMembership ?? current.selector.membership
|
||||
const exactCommitted = (inspection) =>
|
||||
inspection.intent?.state === 'committed' &&
|
||||
inspection.intent.expectedGeneration === 0 &&
|
||||
inspection.selector.generation === 1 &&
|
||||
inspection.selector.attemptId === config.attemptId &&
|
||||
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
|
||||
sameMembership(inspection.intent.membership, intendedMembership) &&
|
||||
sameMembership(inspection.selector.membership, intendedMembership)
|
||||
const exactUnchanged = (inspection) =>
|
||||
inspection.intent?.state === 'unchanged' &&
|
||||
inspection.intent.expectedGeneration === 0 &&
|
||||
inspection.selector.generation === 0 &&
|
||||
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
|
||||
sameMembership(inspection.intent.membership, intendedMembership) &&
|
||||
sameMembership(inspection.selector.membership, intendedMembership)
|
||||
if (exactCommitted(current)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: inspectedMembershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
if (current.intent && !exactUnchanged(current)) {
|
||||
throw new Error('admission boundary initialization attempt diverged')
|
||||
}
|
||||
const request = {
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: config.expectedMembershipSha256,
|
||||
membership: intendedMembership
|
||||
}
|
||||
let applyError
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
await post(`${shape.directorOrigin}/v1/admin/admission-selector/apply`, request)
|
||||
} catch (error) {
|
||||
applyError = error
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(selectorPost, config.attemptId)
|
||||
if (exactCommitted(verified)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: verified.selector.generation,
|
||||
states: targetStates,
|
||||
recovered: current.intent !== null || applyError !== undefined || attempt > 0
|
||||
}
|
||||
}
|
||||
if (!exactUnchanged(verified)) {
|
||||
throw new Error('admission boundary initialization did not commit exactly', {
|
||||
cause: applyError
|
||||
})
|
||||
}
|
||||
}
|
||||
throw new Error('admission boundary initialization remained unchanged after retry', {
|
||||
cause: applyError
|
||||
})
|
||||
}
|
||||
|
||||
export async function operateRelayAsiaAdmission(config, dependencies = {}) {
|
||||
const shape = SHAPES[config.environment]
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const post = dependencies.post ?? defaultPost(fetchImpl, config.token)
|
||||
const selectorPost = (path, body) => {
|
||||
if (config.environment !== 'staging' || path !== '/v1/admin/admission-selector/apply') {
|
||||
return post(`${shape.directorOrigin}${path}`, body)
|
||||
}
|
||||
const state = selectorCellState({ membership: body.membership }, 'staging-gce-c4')
|
||||
if (!['general', 'migration-only'].includes(state)) {
|
||||
throw new Error('staging proof can only transition C4 between reviewed states')
|
||||
}
|
||||
return post(`${shape.directorOrigin}/v1/admin/admission-selector/apply-staging-asia-proof`, {
|
||||
v: 1,
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
state
|
||||
})
|
||||
}
|
||||
const current = await inspectAdmissionSelector(
|
||||
selectorPost,
|
||||
['inspect', 'verify', 'registered'].includes(config.mode) ? undefined : config.attemptId
|
||||
)
|
||||
if (config.mode === 'inspect') {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
membership: current.selector.membership,
|
||||
membershipSha256: membershipSha256(current.selector.membership),
|
||||
states: inspectedMembershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (
|
||||
!current.intent &&
|
||||
current.selector.generation !== config.expectedGeneration
|
||||
) {
|
||||
throw new Error('admission selector generation changed')
|
||||
}
|
||||
if (current.intent && current.intent.expectedGeneration !== config.expectedGeneration) {
|
||||
throw new Error('admission attempt generation does not match')
|
||||
}
|
||||
if (config.mode === 'initialize') {
|
||||
return await initializeAdmissionBoundary(post, selectorPost, shape, config, current)
|
||||
}
|
||||
if (config.mode === 'recover-promotion') {
|
||||
if (config.cells.every(
|
||||
(cellId) => selectorCellState(current.selector, cellId) === 'migration-only'
|
||||
)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: false,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (!current.intent) {
|
||||
if (
|
||||
current.selector.generation !== config.expectedGeneration
|
||||
) throw new Error('promotion state changed without the reviewed attempt')
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: false,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
const expectedMembership = membershipWithStates(
|
||||
{ membership: current.intent.previousMembership },
|
||||
Object.fromEntries(config.cells.map((cellId) => [cellId, 'general']))
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
|
||||
config.cells.some((cellId) => selectorCellState(current.selector, cellId) !== 'general')
|
||||
) throw new Error('promotion attempt is not the current general state')
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: true,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (config.mode === 'rollback') {
|
||||
for (const cellId of config.cells) {
|
||||
if (!['general', 'migration-only'].includes(selectorCellState(current.selector, cellId))) {
|
||||
throw new Error(`${cellId} cannot roll back to migration-only`)
|
||||
}
|
||||
}
|
||||
} else if (config.mode === 'register') {
|
||||
const known = new Set([
|
||||
...current.selector.membership.existingOnly,
|
||||
...current.selector.membership.migrationOnly,
|
||||
...current.selector.membership.general
|
||||
])
|
||||
if (!current.intent && config.cells.some((cellId) => known.has(cellId))) {
|
||||
throw new Error('Asia cell is already registered')
|
||||
}
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
|
||||
))
|
||||
} else if (config.mode !== 'registered') {
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, true)
|
||||
))
|
||||
}
|
||||
if (config.mode === 'registered') {
|
||||
if (config.cells.some(
|
||||
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
|
||||
)) throw new Error('Asia cells are not registered migration-only')
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
|
||||
))
|
||||
}
|
||||
if (['verify', 'registered'].includes(config.mode)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (config.mode === 'register') {
|
||||
if (current.intent) {
|
||||
const expectedCells = new Set(config.cells)
|
||||
const addedCells = current.intent.membership.migrationOnly.filter(
|
||||
(cellId) => !current.intent.previousMembership.migrationOnly.includes(cellId)
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
addedCells.length !== expectedCells.size ||
|
||||
addedCells.some((cellId) => !expectedCells.has(cellId)) ||
|
||||
current.selector.generation !== config.expectedGeneration + 1 ||
|
||||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
|
||||
) {
|
||||
throw new Error('admission attempt does not match the requested Asia registration')
|
||||
}
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
const result = await addExactMigrationCells(
|
||||
selectorPost,
|
||||
{
|
||||
attemptId: config.attemptId,
|
||||
cells: config.cells.map((cellId) => ({
|
||||
cellId,
|
||||
cellUrl: cellOrigin(shape, cellId),
|
||||
region: 'asia-east2',
|
||||
capacityRequests: 6_000,
|
||||
connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
}))
|
||||
},
|
||||
{ expectedCurrentSelector: current.selector }
|
||||
)
|
||||
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
|
||||
}
|
||||
const desiredState = config.mode === 'promote' ? 'general' : 'migration-only'
|
||||
if (current.intent) {
|
||||
const expectedMembership = membershipWithStates(
|
||||
{ membership: current.intent.previousMembership },
|
||||
Object.fromEntries(config.cells.map((cellId) => [cellId, desiredState]))
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
|
||||
current.selector.generation !== config.expectedGeneration + 1 ||
|
||||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
|
||||
) {
|
||||
throw new Error('admission attempt does not match the requested Asia transition')
|
||||
}
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
if (config.mode === 'promote' && config.cells.some(
|
||||
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
|
||||
)) throw new Error('Asia promotion requires migration-only cells')
|
||||
if (
|
||||
config.mode === 'promote' &&
|
||||
config.environment === 'production' &&
|
||||
config.cells.includes('production-gce-c28') &&
|
||||
selectorCellState(current.selector, 'production-gce-c27') !== 'general'
|
||||
) {
|
||||
throw new Error('Asia expansion requires the C27 canary to be general')
|
||||
}
|
||||
const result = await applyExactAdmissionSelector(
|
||||
selectorPost,
|
||||
membershipWithStates(current.selector, Object.fromEntries(
|
||||
config.cells.map((cellId) => [cellId, desiredState])
|
||||
)),
|
||||
{ attemptId: config.attemptId, expectedCurrentSelector: current.selector }
|
||||
)
|
||||
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const config = parseArguments(process.argv.slice(2))
|
||||
if (!config.token) throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required')
|
||||
console.log(JSON.stringify(await operateRelayAsiaAdmission(config)))
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { test } from 'node:test'
|
||||
import { operateRelayAsiaAdmission } from './operate-relay-asia-admission.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const membershipDigest = (membership) =>
|
||||
createHash('sha256').update(JSON.stringify(membership)).digest('hex')
|
||||
|
||||
function harness(initialSelector) {
|
||||
const initialMembership = structuredClone(initialSelector.membership)
|
||||
let selector = structuredClone(initialSelector)
|
||||
const intents = new Map()
|
||||
const requests = []
|
||||
let fetches = 0
|
||||
let failAfterIntent = false
|
||||
const post = async (url, body) => {
|
||||
const parsed = new URL(url)
|
||||
requests.push({ path: parsed.pathname, body })
|
||||
if (parsed.pathname === '/v1/admin/runtime-status') {
|
||||
const cell = parsed.hostname.split('.')[0]
|
||||
return {
|
||||
cellId: `production-gce-${cell}`,
|
||||
cellUrl: parsed.origin,
|
||||
region: 'asia-east2',
|
||||
imageDigest: digest,
|
||||
draining: false,
|
||||
connectionCapacity: { hardCap: 3_000, unobservedBound: 60 }
|
||||
}
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
return {
|
||||
status: {
|
||||
cellUrl: `https://${body.cellId.split('-').at(-1)}.relay.onorca.dev`,
|
||||
runtime: { heartbeatFresh: true, ready: true }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (parsed.pathname.endsWith('/status')) {
|
||||
return { selector, intent: body.attemptId ? intents.get(body.attemptId) ?? null : null }
|
||||
}
|
||||
if (parsed.pathname.endsWith('/add-migration-cells')) {
|
||||
selector = {
|
||||
generation: selector.generation + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: {
|
||||
...selector.membership,
|
||||
migrationOnly: [...selector.membership.migrationOnly, ...body.cells.map((cell) => cell.cellId)].sort()
|
||||
}
|
||||
}
|
||||
} else if (parsed.pathname.endsWith('/apply-staging-asia-proof')) {
|
||||
const membership = structuredClone(selector.membership)
|
||||
membership.migrationOnly = membership.migrationOnly.filter((cell) => cell !== 'staging-gce-c4')
|
||||
membership.general = membership.general.filter((cell) => cell !== 'staging-gce-c4')
|
||||
membership[body.state === 'general' ? 'general' : 'migrationOnly'].push('staging-gce-c4')
|
||||
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership }
|
||||
} else if (parsed.pathname.endsWith('/apply')) {
|
||||
if (
|
||||
body.expectedMembershipSha256 &&
|
||||
body.expectedMembershipSha256 !== membershipDigest(selector.membership)
|
||||
) throw new Error('admission_selector_membership_mismatch')
|
||||
if (failAfterIntent) {
|
||||
failAfterIntent = false
|
||||
intents.set(body.attemptId, {
|
||||
state: 'unchanged',
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
previousMembership: structuredClone(selector.membership),
|
||||
membership: structuredClone(body.membership)
|
||||
})
|
||||
throw new Error('failure after intent persistence')
|
||||
}
|
||||
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership: body.membership }
|
||||
} else throw new Error(`unexpected ${parsed.pathname}`)
|
||||
intents.set(body.attemptId, {
|
||||
state: 'committed', expectedGeneration: body.expectedGeneration,
|
||||
previousMembership: initialSelector.membership,
|
||||
membership: selector.membership
|
||||
})
|
||||
return { changed: true, selector }
|
||||
}
|
||||
const fetch = async () => {
|
||||
fetches++
|
||||
return new Response(null, { status: 200 })
|
||||
}
|
||||
const commitWithoutResponse = async (path, body) => {
|
||||
await post(`https://relay.onorca.dev${path}`, body)
|
||||
throw new Error('response lost after commit')
|
||||
}
|
||||
return {
|
||||
post, fetch, requests, commitWithoutResponse,
|
||||
failNextApplyAfterIntent: () => (failAfterIntent = true),
|
||||
apply: async (attemptId, membership) => await post(
|
||||
'https://relay.onorca.dev/v1/admin/admission-selector/apply',
|
||||
{ attemptId, expectedGeneration: selector.generation, membership }
|
||||
),
|
||||
fetchCount: () => fetches, selector: () => selector
|
||||
}
|
||||
}
|
||||
|
||||
const baseSelector = {
|
||||
generation: 7,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26'] }
|
||||
}
|
||||
|
||||
test('inspects generation zero without requiring target registration or making a mutation', async () => {
|
||||
const subject = harness({
|
||||
generation: 0,
|
||||
membership: {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'inspect', cells: ['staging-gce-c4'],
|
||||
imageDigest: digest, token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(result.generation, 0)
|
||||
assert.equal(result.states['staging-gce-c4'], 'absent')
|
||||
assert.deepEqual(result.membership, subject.selector().membership)
|
||||
assert.equal(result.membershipSha256, membershipDigest(subject.selector().membership))
|
||||
assert.deepEqual(subject.requests.map(({ path }) => path), [
|
||||
'/v1/admin/admission-selector/status'
|
||||
])
|
||||
})
|
||||
|
||||
test('initializes generation zero without changing membership', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_0', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/apply'))
|
||||
assert.deepEqual(request.body, {
|
||||
v: 1,
|
||||
attemptId: 'asia_boundary_0',
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
membership
|
||||
})
|
||||
assert.equal(result.generation, 1)
|
||||
assert.equal(result.states['staging-gce-c4'], 'absent')
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('retries the same fingerprint-bound initialization after intent persistence', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
subject.failNextApplyAfterIntent()
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_intent_retry', token: 'not-logged'
|
||||
}, subject)
|
||||
const applies = subject.requests.filter(({ path }) => path.endsWith('/apply'))
|
||||
assert.equal(applies.length, 2)
|
||||
assert.deepEqual(applies[1].body, applies[0].body)
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(result.generation, 1)
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('recovers a committed generation-zero initialization', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
const config = {
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_retry', token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
membership
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 1)
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('rejects generation-zero membership drift after inspect', async () => {
|
||||
const inspected = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const changed = {
|
||||
existingOnly: ['staging-gce-c2', 'staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership: changed })
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(inspected),
|
||||
attemptId: 'asia_boundary_drift', token: 'not-logged'
|
||||
}, subject), /membership changed/)
|
||||
assert.equal(subject.requests.some(({ path }) => path.endsWith('/apply')), false)
|
||||
})
|
||||
|
||||
test('registers all three Asia cells atomically with region and exact limits', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_7', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.equal(request.body.cells.length, 3)
|
||||
assert.ok(request.body.cells.every((cell) =>
|
||||
cell.region === 'asia-east2' && cell.capacityRequests === 6_000 &&
|
||||
cell.connectionHardCap === 3_000 && cell.connectionUnobservedBound === 60
|
||||
))
|
||||
assert.equal(result.generation, 8)
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
|
||||
})
|
||||
|
||||
test('promotes the canary only after runtime and director-heartbeat checks', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: { existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_8', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 2)
|
||||
assert.equal(result.states['production-gce-c27'], 'general')
|
||||
})
|
||||
|
||||
test('checks registered migration-only cells before director configuration without requiring heartbeat', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'registered',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 8, imageDigest: digest, token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 6)
|
||||
assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 0)
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
|
||||
})
|
||||
|
||||
test('rolls back admission without requiring an unhealthy runtime to answer', async () => {
|
||||
const subject = harness({
|
||||
generation: 9,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_9', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 0)
|
||||
assert.equal(result.states['production-gce-c27'], 'migration-only')
|
||||
})
|
||||
|
||||
test('uses the server-enforced C4-only route for staging proof transitions', async () => {
|
||||
const subject = harness({
|
||||
generation: 3,
|
||||
membership: {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2', 'staging-gce-c4']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'rollback', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 3, imageDigest: digest, attemptId: 'asia_staging_rollback',
|
||||
token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(
|
||||
({ path }) => path.endsWith('/apply-staging-asia-proof')
|
||||
)
|
||||
assert.deepEqual(request.body, {
|
||||
v: 1,
|
||||
attemptId: 'asia_staging_rollback',
|
||||
expectedGeneration: 3,
|
||||
state: 'migration-only'
|
||||
})
|
||||
assert.deepEqual(subject.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: ['staging-gce-c4'],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
assert.equal(result.states['staging-gce-c4'], 'migration-only')
|
||||
})
|
||||
|
||||
test('fails closed when the exact selector generation moved', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 6, imageDigest: digest, attemptId: 'asia_register_6', token: 'not-logged'
|
||||
}, subject), /generation changed/)
|
||||
assert.equal(subject.fetchCount(), 0)
|
||||
})
|
||||
|
||||
test('recovers a committed registration when the workflow retries the original generation', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
const config = {
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/add-migration-cells',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
cells: config.cells.map((cellId) => ({ cellId }))
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 8)
|
||||
})
|
||||
|
||||
test('recovers a committed promotion when the workflow retries the original generation', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [],
|
||||
general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 9)
|
||||
assert.equal(recovered.states['production-gce-c27'], 'general')
|
||||
})
|
||||
|
||||
test('inspects an ambiguous promotion without creating a new transition', async () => {
|
||||
const untouched = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_promote',
|
||||
token: 'not-logged'
|
||||
}
|
||||
const absent = await operateRelayAsiaAdmission(config, untouched)
|
||||
assert.equal(absent.promoted, false)
|
||||
assert.equal(untouched.requests.some(({ path }) => path.endsWith('/apply')), false)
|
||||
|
||||
const committed = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
await assert.rejects(committed.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, committed)
|
||||
assert.equal(recovered.promoted, true)
|
||||
assert.equal(recovered.generation, 9)
|
||||
})
|
||||
|
||||
test('treats an already rolled-back promotion as recovered', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_after_rollback',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
await subject.apply('later_rollback', {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
})
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.promoted, false)
|
||||
assert.equal(recovered.generation, 10)
|
||||
})
|
||||
|
||||
test('recovers a committed rollback when the workflow retries the original generation', async () => {
|
||||
const subject = harness({
|
||||
generation: 9,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 10)
|
||||
assert.equal(recovered.states['production-gce-c27'], 'migration-only')
|
||||
})
|
||||
|
||||
test('rejects a committed transition retry after a later selector change', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_stale_promote',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
await subject.apply('later_rollback', {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
})
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission(config, subject),
|
||||
/does not match the requested Asia transition/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires the C27 canary before promoting C28 and C29', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote',
|
||||
cells: ['production-gce-c28', 'production-gce-c29'], expectedGeneration: 8,
|
||||
imageDigest: digest, attemptId: 'asia_wave_before_canary', token: 'not-logged'
|
||||
}, subject), /C27 canary/)
|
||||
})
|
||||
@@ -0,0 +1,312 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { inspectAdmissionSelector } from './relay-admission-selector.mjs'
|
||||
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
const MODES = new Set(['inspect', 'enable', 'pause', 'disable', 'recover-enable'])
|
||||
|
||||
function canonicalCells(value) {
|
||||
if (value === 'none') return []
|
||||
const cells = value.split(',').map((cell) => cell.trim()).filter(Boolean).sort()
|
||||
if (
|
||||
cells.length === 0 ||
|
||||
new Set(cells).size !== cells.length ||
|
||||
cells.some((cell) => !/^production-gce-c(?:[1-9]|[12][0-9])$/.test(cell))
|
||||
) throw new Error('selector membership is invalid')
|
||||
return cells
|
||||
}
|
||||
|
||||
function integer(value, name, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new Error(`${name} is invalid`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['mode', 'director-origin', 'expected-control-generation']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (!MODES.has(values.mode)) throw new Error('--mode is invalid')
|
||||
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
|
||||
throw new Error('--director-origin must be the production Relay origin')
|
||||
}
|
||||
const recovery = values.mode === 'recover-enable'
|
||||
const mutation = values.mode !== 'inspect' && !recovery
|
||||
const mutationKeys = [
|
||||
'not-before',
|
||||
'rate-per-minute',
|
||||
'preference-max-age-ms',
|
||||
'drain-grace-ms',
|
||||
'confirmation'
|
||||
]
|
||||
if (mutation && mutationKeys.some((key) => values[key] === undefined)) {
|
||||
throw new Error('mutations require the complete durable control shape')
|
||||
}
|
||||
if (!mutation && !recovery && mutationKeys.some((key) => values[key] !== undefined)) {
|
||||
throw new Error('inspect cannot carry mutation arguments')
|
||||
}
|
||||
const selectorKeys = [
|
||||
'expected-selector-generation',
|
||||
'expected-existing-only-cells',
|
||||
'expected-migration-only-cells',
|
||||
'expected-general-cells'
|
||||
]
|
||||
if (!recovery && selectorKeys.some((key) => values[key] === undefined)) {
|
||||
throw new Error('operation requires exact selector state')
|
||||
}
|
||||
if (recovery && selectorKeys.some((key) => values[key] !== undefined)) {
|
||||
throw new Error('enable recovery cannot depend on selector diagnostics')
|
||||
}
|
||||
const expectedConfirmation = {
|
||||
enable: 'ENABLE_REGIONAL_REHOMING',
|
||||
pause: 'PAUSE_REGIONAL_REHOMING',
|
||||
disable: 'DISABLE_REGIONAL_REHOMING'
|
||||
}[values.mode]
|
||||
if (mutation && values.confirmation !== expectedConfirmation) {
|
||||
throw new Error('confirmation does not match the requested control action')
|
||||
}
|
||||
if (recovery && values.confirmation !== 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE') {
|
||||
throw new Error('confirmation does not authorize failed-enable recovery')
|
||||
}
|
||||
if (
|
||||
recovery &&
|
||||
mutationKeys
|
||||
.filter((key) => key !== 'confirmation')
|
||||
.some((key) => values[key] !== undefined)
|
||||
) throw new Error('enable recovery cannot carry durable control shape arguments')
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
return {
|
||||
mode: values.mode,
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
...(!recovery
|
||||
? {
|
||||
expectedSelectorGeneration: integer(
|
||||
values['expected-selector-generation'],
|
||||
'--expected-selector-generation'
|
||||
),
|
||||
expectedMembership: {
|
||||
existingOnly: canonicalCells(values['expected-existing-only-cells']),
|
||||
migrationOnly: canonicalCells(values['expected-migration-only-cells']),
|
||||
general: canonicalCells(values['expected-general-cells'])
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
expectedControlGeneration: integer(
|
||||
values['expected-control-generation'],
|
||||
'--expected-control-generation'
|
||||
),
|
||||
...(mutation
|
||||
? {
|
||||
notBefore: integer(values['not-before'], '--not-before'),
|
||||
ratePerMinute: integer(values['rate-per-minute'], '--rate-per-minute', {
|
||||
minimum: 1,
|
||||
maximum: 120
|
||||
}),
|
||||
preferenceMaxAgeMs: integer(
|
||||
values['preference-max-age-ms'],
|
||||
'--preference-max-age-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
|
||||
minimum: 60_000,
|
||||
maximum: 60 * 60_000
|
||||
})
|
||||
}
|
||||
: {}),
|
||||
token
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}: ${body.error ?? 'unknown'}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function exactMembership(actual, expected) {
|
||||
return ['existingOnly', 'migrationOnly', 'general'].every(
|
||||
(key) => JSON.stringify(actual[key]) === JSON.stringify(expected[key])
|
||||
)
|
||||
}
|
||||
|
||||
function assertControl(control, expected) {
|
||||
if (
|
||||
(expected.generation !== undefined && control?.generation !== expected.generation) ||
|
||||
typeof control.enabled !== 'boolean' ||
|
||||
!Number.isSafeInteger(control.observationStartedAt) ||
|
||||
!Number.isSafeInteger(control.notBefore) ||
|
||||
!Number.isSafeInteger(control.ratePerMinute) ||
|
||||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
|
||||
!Number.isSafeInteger(control.drainGraceMs)
|
||||
) throw new Error('director returned an invalid regional rehome control')
|
||||
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
|
||||
throw new Error('regional rehome enabled state does not match')
|
||||
}
|
||||
return control
|
||||
}
|
||||
|
||||
async function verifiedDisabledControl(post, generation) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, { generation, enabled: false })
|
||||
}
|
||||
|
||||
async function applyDisabledControl(post, before) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: before.generation,
|
||||
enabled: false,
|
||||
notBefore: before.notBefore,
|
||||
ratePerMinute: before.ratePerMinute,
|
||||
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
|
||||
drainGraceMs: before.drainGraceMs,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})).control, { generation: before.generation + 1, enabled: false })
|
||||
}
|
||||
|
||||
async function resolveAmbiguousDisable(post, before, firstError) {
|
||||
const observed = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {})
|
||||
if (observed.generation === before.generation + 1 && !observed.enabled) {
|
||||
return observed
|
||||
}
|
||||
if (observed.generation !== before.generation || !observed.enabled) {
|
||||
throw new AggregateError(
|
||||
[firstError],
|
||||
'failed-enable recovery reached an unexpected control generation'
|
||||
)
|
||||
}
|
||||
try {
|
||||
return await applyDisabledControl(post, before)
|
||||
} catch (retryError) {
|
||||
try {
|
||||
return await verifiedDisabledControl(post, before.generation + 1)
|
||||
} catch (readbackError) {
|
||||
throw new AggregateError(
|
||||
[firstError, retryError, readbackError],
|
||||
'failed-enable recovery exhausted two bounded CAS attempts'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function recoverRegionalRehomeEnable(config, post) {
|
||||
const before = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {})
|
||||
if (
|
||||
before.generation < config.expectedControlGeneration ||
|
||||
(before.generation === config.expectedControlGeneration && before.enabled)
|
||||
) throw new Error('durable control cannot belong to the failed enable attempt')
|
||||
if (!before.enabled) {
|
||||
const verified = await verifiedDisabledControl(post, before.generation)
|
||||
return { mode: config.mode, recovered: false, control: verified }
|
||||
}
|
||||
let applied
|
||||
try {
|
||||
applied = await applyDisabledControl(post, before)
|
||||
} catch (error) {
|
||||
applied = await resolveAmbiguousDisable(post, before, error)
|
||||
}
|
||||
const verified = await verifiedDisabledControl(post, applied.generation)
|
||||
return { mode: config.mode, recovered: true, control: verified }
|
||||
}
|
||||
|
||||
export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const post = dependencies.post ?? (async (path, body) => await responseJson(
|
||||
await fetchImpl(`${config.directorOrigin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.token}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
))
|
||||
if (config.mode === 'recover-enable') {
|
||||
return await recoverRegionalRehomeEnable(config, post)
|
||||
}
|
||||
const selector = (await inspectAdmissionSelector(post)).selector
|
||||
if (
|
||||
selector.generation !== config.expectedSelectorGeneration ||
|
||||
!exactMembership(selector.membership, config.expectedMembership)
|
||||
) throw new Error('admission selector does not match the reviewed generation and membership')
|
||||
|
||||
const inspected = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})
|
||||
const before = assertControl(inspected.control, {
|
||||
generation: config.expectedControlGeneration
|
||||
})
|
||||
if (config.mode === 'inspect') return { mode: config.mode, selector, control: before }
|
||||
if (config.mode === 'enable' && before.enabled) {
|
||||
throw new Error('regional rehome is already enabled; inspect before changing its rate')
|
||||
}
|
||||
if (config.mode === 'pause' && !before.enabled) {
|
||||
throw new Error('regional rehome is already paused')
|
||||
}
|
||||
const enabled = config.mode === 'enable'
|
||||
const applied = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: config.expectedControlGeneration,
|
||||
enabled,
|
||||
notBefore: config.notBefore,
|
||||
ratePerMinute: config.ratePerMinute,
|
||||
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
|
||||
drainGraceMs: config.drainGraceMs,
|
||||
confirmation: enabled
|
||||
? 'ENABLE_REGIONAL_REHOMING'
|
||||
: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
const after = assertControl(applied.control, {
|
||||
generation: config.expectedControlGeneration + 1,
|
||||
enabled
|
||||
})
|
||||
const verified = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {
|
||||
generation: after.generation,
|
||||
enabled
|
||||
})
|
||||
return { mode: config.mode, selector, control: verified }
|
||||
}
|
||||
|
||||
export async function main(
|
||||
argv = process.argv.slice(2),
|
||||
environment = process.env,
|
||||
dependencies = {},
|
||||
write = (value) => process.stdout.write(value)
|
||||
) {
|
||||
const result = await operateRegionalRehome(
|
||||
parseRegionalRehomeArguments(argv, environment),
|
||||
dependencies
|
||||
)
|
||||
write(`${JSON.stringify({ event: 'relay_regional_rehome_control', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
main,
|
||||
operateRegionalRehome,
|
||||
parseRegionalRehomeArguments,
|
||||
recoverRegionalRehomeEnable
|
||||
} from './operate-relay-regional-rehome.mjs'
|
||||
|
||||
const membership = {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c2'],
|
||||
general: ['production-gce-c7', 'production-gce-c27']
|
||||
}
|
||||
|
||||
function argumentsFor(mode, confirmation) {
|
||||
return [
|
||||
'--mode', mode,
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-selector-generation', '11',
|
||||
'--expected-existing-only-cells', membership.existingOnly.join(','),
|
||||
'--expected-migration-only-cells', membership.migrationOnly.join(','),
|
||||
'--expected-general-cells', membership.general.join(','),
|
||||
'--expected-control-generation', '4',
|
||||
...(mode === 'inspect' ? [] : [
|
||||
'--not-before', '2000000000000',
|
||||
'--rate-per-minute', '10',
|
||||
'--preference-max-age-ms', '86400000',
|
||||
'--drain-grace-ms', '60000',
|
||||
'--confirmation', confirmation
|
||||
])
|
||||
]
|
||||
}
|
||||
|
||||
function control(generation, enabled) {
|
||||
return {
|
||||
generation,
|
||||
enabled,
|
||||
observationStartedAt: 1,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
drainGraceMs: 60_000
|
||||
}
|
||||
}
|
||||
|
||||
test('parses exact selector and typed control confirmation', () => {
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
assert.equal(parsed.expectedSelectorGeneration, 11)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.ratePerMinute, 10)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/confirmation/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect').concat('--rate-per-minute', '10'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/inspect cannot/
|
||||
)
|
||||
})
|
||||
|
||||
test('binds enable to exact selector and durable control generations', async () => {
|
||||
const requests = []
|
||||
const controls = [
|
||||
{ generation: 4, enabled: false },
|
||||
{ generation: 5, enabled: true },
|
||||
{ generation: 5, enabled: true }
|
||||
].map((control) => ({
|
||||
observationStartedAt: 1,
|
||||
notBefore: 0,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
drainGraceMs: 60_000,
|
||||
...control
|
||||
}))
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
const result = await operateRegionalRehome(config, {
|
||||
post: async (path, body) => {
|
||||
requests.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return { selector: { generation: 11, membership } }
|
||||
}
|
||||
return { v: 1, control: controls.shift() }
|
||||
}
|
||||
})
|
||||
assert.equal(result.control.generation, 5)
|
||||
assert.deepEqual(requests[2].body, {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: 4,
|
||||
enabled: true,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('fails closed on selector drift before reading or mutating control', async () => {
|
||||
let calls = 0
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('disable', 'DISABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, {
|
||||
post: async () => {
|
||||
calls += 1
|
||||
return { selector: { generation: 12, membership } }
|
||||
}
|
||||
}),
|
||||
/selector/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('failed-enable recovery CAS-disables an advanced enabled generation', async () => {
|
||||
const requests = []
|
||||
let current = control(7, true)
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
assert.equal(body.expectedGeneration, 7)
|
||||
current = control(8, false)
|
||||
throw new Error('enable recovery response was lost')
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.deepEqual(result.control, control(8, false))
|
||||
assert.deepEqual(requests[1], {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: 7,
|
||||
enabled: false,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('failed-enable recovery retries once when the first CAS never commits', async () => {
|
||||
let current = control(7, true)
|
||||
const applyRequests = []
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
applyRequests.push(body)
|
||||
if (applyRequests.length === 1) {
|
||||
throw new Error('disable request was lost before commit')
|
||||
}
|
||||
current = control(8, false)
|
||||
throw new Error('retry response was lost after commit')
|
||||
})
|
||||
assert.equal(applyRequests.length, 2)
|
||||
assert.deepEqual(applyRequests[1], applyRequests[0])
|
||||
assert.equal(result.recovered, true)
|
||||
assert.deepEqual(result.control, control(8, false))
|
||||
})
|
||||
|
||||
test('failed-enable recovery stops after two uncommitted CAS attempts', async () => {
|
||||
let applyCalls = 0
|
||||
await assert.rejects(
|
||||
recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
if (body.action === 'inspect') return { control: control(7, true) }
|
||||
applyCalls += 1
|
||||
throw new Error(`disable attempt ${applyCalls} was lost before commit`)
|
||||
}),
|
||||
/exhausted two bounded CAS attempts/
|
||||
)
|
||||
assert.equal(applyCalls, 2)
|
||||
})
|
||||
|
||||
test('failed-enable recovery is a verified no-op before enable and after cleanup', async () => {
|
||||
for (const current of [control(4, false), control(8, false)]) {
|
||||
const requests = []
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
return { control: current }
|
||||
})
|
||||
assert.equal(result.recovered, false)
|
||||
assert.equal(result.control.enabled, false)
|
||||
assert.deepEqual(requests.map(({ action }) => action), ['inspect', 'inspect'])
|
||||
}
|
||||
})
|
||||
|
||||
test('failed-enable recovery rejects an unchanged pre-existing enabled state', async () => {
|
||||
await assert.rejects(
|
||||
recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async () => ({ control: control(4, true) })),
|
||||
/cannot belong to the failed enable attempt/
|
||||
)
|
||||
})
|
||||
|
||||
test('parses recovery without depending on selector diagnostics', () => {
|
||||
const recoveryArguments = [
|
||||
'--mode', 'recover-enable',
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-control-generation', '4',
|
||||
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
|
||||
]
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
recoveryArguments,
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.expectedMembership, undefined)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
recoveryArguments.concat('--not-before', '2000000000000'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/cannot carry durable control shape/
|
||||
)
|
||||
})
|
||||
|
||||
test('main executes recovery mode and emits verified disabled control', async () => {
|
||||
let current = control(5, true)
|
||||
let output = ''
|
||||
await main([
|
||||
'--mode', 'recover-enable',
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-control-generation', '4',
|
||||
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
|
||||
], { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }, {
|
||||
post: async (_path, body) => {
|
||||
if (body.action === 'apply') current = control(6, false)
|
||||
return { control: current }
|
||||
}
|
||||
}, (value) => {
|
||||
output += value
|
||||
})
|
||||
assert.deepEqual(JSON.parse(output), {
|
||||
event: 'relay_regional_rehome_control',
|
||||
mode: 'recover-enable',
|
||||
recovered: true,
|
||||
control: control(6, false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,487 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
inspectAdmissionSelector,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const PROJECT = 'onorca-cloud-staging'
|
||||
const REGION = 'us-central1'
|
||||
const DIRECTOR_ORIGIN = 'https://relay-staging.onorca.dev'
|
||||
const ADMIN_AUDIENCE = `${DIRECTOR_ORIGIN}/v1/admin/drain`
|
||||
const SQL_INSTANCE = 'orca-cloud-staging-auth-db'
|
||||
const CLOUD_RUN_SERVICES = [
|
||||
{ name: 'orca-cloud-relay-staging', healthOrigin: DIRECTOR_ORIGIN },
|
||||
{ name: 'orca-cloud-auth-staging', healthOrigin: 'https://auth-staging.onorca.dev' }
|
||||
]
|
||||
const POLL_INTERVAL_MS = 5_000
|
||||
const WAKE_TIMEOUT_MS = 12 * 60 * 1_000
|
||||
|
||||
export function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
||||
const name = key.slice(2)
|
||||
if (!['mode', 'wake-cells', 'topology-file'].includes(name)) {
|
||||
throw new Error(`unsupported argument --${name}`)
|
||||
}
|
||||
values[name] = value
|
||||
}
|
||||
if (!['status', 'sleep', 'wake'].includes(values.mode)) {
|
||||
throw new Error('--mode must be status, sleep, or wake')
|
||||
}
|
||||
if (!['configured', 'all'].includes(values['wake-cells'])) {
|
||||
throw new Error('--wake-cells must be configured or all')
|
||||
}
|
||||
if (!values['topology-file']) throw new Error('missing --topology-file')
|
||||
return {
|
||||
mode: values.mode,
|
||||
wakeCells: values['wake-cells'],
|
||||
topologyFile: values['topology-file']
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalStagingCell(cellId, value) {
|
||||
if (!/^staging-gce-[a-z0-9-]+$/.test(cellId)) throw new Error(`unsafe staging cell ID ${cellId}`)
|
||||
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
|
||||
const cell = {
|
||||
cellId,
|
||||
migName: String(value.mig_name ?? ''),
|
||||
zone: String(value.zone ?? ''),
|
||||
origin: String(value.origin ?? ''),
|
||||
initiallyEnabled: value.initially_enabled
|
||||
}
|
||||
if (!/^orca-cloud-staging-relay-gce-[a-z0-9-]+$/.test(cell.migName)) {
|
||||
throw new Error(`${cellId} has an unsafe MIG name`)
|
||||
}
|
||||
if (!/^(?:us-central1|asia-east2)-[a-z]$/.test(cell.zone)) {
|
||||
throw new Error(`${cellId} has an unsafe zone`)
|
||||
}
|
||||
const origin = new URL(cell.origin)
|
||||
if (
|
||||
origin.protocol !== 'https:' ||
|
||||
origin.origin !== cell.origin ||
|
||||
!origin.hostname.endsWith('.relay-staging.onorca.dev')
|
||||
) {
|
||||
throw new Error(`${cellId} has an unsafe origin`)
|
||||
}
|
||||
if (typeof cell.initiallyEnabled !== 'boolean') {
|
||||
throw new Error(`${cellId} has no initial admission state`)
|
||||
}
|
||||
return cell
|
||||
}
|
||||
|
||||
export function readStagingTopology(file) {
|
||||
const parsed = JSON.parse(readFileSync(file, 'utf8'))
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error('staging topology must be an object')
|
||||
}
|
||||
const cells = Object.entries(parsed)
|
||||
.map(([cellId, value]) => canonicalStagingCell(cellId, value))
|
||||
.sort((left, right) => left.cellId.localeCompare(right.cellId))
|
||||
if (cells.length < 2 || cells.length > 10) {
|
||||
throw new Error('staging topology must contain 2..10 cells')
|
||||
}
|
||||
if (cells.filter((cell) => cell.initiallyEnabled).length < 2) {
|
||||
throw new Error('staging topology must retain two configured admission cells')
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function defaultCommand(args, json) {
|
||||
const result = spawnSync('gcloud', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`)
|
||||
}
|
||||
return json ? JSON.parse(result.stdout) : result.stdout.trim()
|
||||
}
|
||||
|
||||
function suppliedAdminToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
||||
throw new Error('workflow did not supply a valid masked staging admin token')
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
||||
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function createAdminPost(deps) {
|
||||
const token = deps.adminToken()
|
||||
return async (origin, path, body) =>
|
||||
await responseJson(
|
||||
await deps.fetch(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
)
|
||||
}
|
||||
|
||||
async function waitUntil(deps, label, operation, timeoutMs = WAKE_TIMEOUT_MS) {
|
||||
const deadline = deps.now() + timeoutMs
|
||||
let lastError
|
||||
while (deps.now() < deadline) {
|
||||
try {
|
||||
const result = await operation()
|
||||
if (result) return result
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
}
|
||||
await deps.wait(POLL_INTERVAL_MS)
|
||||
}
|
||||
const detail = lastError instanceof Error ? `: ${lastError.message}` : ''
|
||||
throw new Error(`timed out waiting for ${label}${detail}`)
|
||||
}
|
||||
|
||||
async function checkHealth(deps, origin, path) {
|
||||
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
|
||||
const body = await response.json().catch(() => ({}))
|
||||
return response.ok && body.ok === true
|
||||
}
|
||||
|
||||
function sqlActivationPolicy(instance) {
|
||||
return String(instance.settings?.activationPolicy ?? '')
|
||||
}
|
||||
|
||||
function describeSql(deps) {
|
||||
return deps.commandJson([
|
||||
'sql',
|
||||
'instances',
|
||||
'describe',
|
||||
SQL_INSTANCE,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
async function ensureSqlPolicy(deps, policy) {
|
||||
if (sqlActivationPolicy(describeSql(deps)) === policy) return false
|
||||
deps.command([
|
||||
'sql',
|
||||
'instances',
|
||||
'patch',
|
||||
SQL_INSTANCE,
|
||||
'--project',
|
||||
PROJECT,
|
||||
`--activation-policy=${policy}`,
|
||||
'--quiet'
|
||||
])
|
||||
await waitUntil(deps, `Cloud SQL activation policy ${policy}`, () =>
|
||||
sqlActivationPolicy(describeSql(deps)) === policy
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
function describeMig(deps, cell) {
|
||||
return deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'describe',
|
||||
cell.migName,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--zone',
|
||||
cell.zone,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
async function setMigSize(deps, cell, size) {
|
||||
const before = describeMig(deps, cell)
|
||||
if (Number(before.targetSize) !== size) {
|
||||
deps.command([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'resize',
|
||||
cell.migName,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--zone',
|
||||
cell.zone,
|
||||
`--size=${size}`,
|
||||
'--quiet'
|
||||
])
|
||||
}
|
||||
await waitUntil(deps, `${cell.cellId} size ${size}`, () => {
|
||||
const current = describeMig(deps, cell)
|
||||
return Number(current.targetSize) === size && current.status?.isStable === true
|
||||
})
|
||||
}
|
||||
|
||||
function activeRevisionName(service) {
|
||||
const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0)
|
||||
if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) {
|
||||
throw new Error('Cloud Run service must have exactly one active revision')
|
||||
}
|
||||
return active[0].revisionName
|
||||
}
|
||||
|
||||
function describeRunService(deps, name) {
|
||||
return deps.commandJson([
|
||||
'run',
|
||||
'services',
|
||||
'describe',
|
||||
name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
function describeRunRevision(deps, name) {
|
||||
return deps.commandJson([
|
||||
'run',
|
||||
'revisions',
|
||||
'describe',
|
||||
name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
function revisionMinimum(revision) {
|
||||
return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0)
|
||||
}
|
||||
|
||||
async function ensureCloudRunScaleToZero(deps, service) {
|
||||
const before = describeRunService(deps, service.name)
|
||||
const activeRevision = describeRunRevision(deps, activeRevisionName(before))
|
||||
if (revisionMinimum(activeRevision) === 0) return false
|
||||
|
||||
const latestName = before.status?.latestReadyRevisionName
|
||||
const latest = latestName ? describeRunRevision(deps, latestName) : null
|
||||
if (!latest || revisionMinimum(latest) !== 0) {
|
||||
deps.command([
|
||||
'run',
|
||||
'services',
|
||||
'update',
|
||||
service.name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--min-instances=0',
|
||||
'--quiet'
|
||||
])
|
||||
}
|
||||
deps.command([
|
||||
'run',
|
||||
'services',
|
||||
'update-traffic',
|
||||
service.name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--to-latest',
|
||||
'--quiet'
|
||||
])
|
||||
await waitUntil(deps, `${service.name} scale-to-zero revision`, async () => {
|
||||
const current = describeRunService(deps, service.name)
|
||||
const revision = describeRunRevision(deps, activeRevisionName(current))
|
||||
return revisionMinimum(revision) === 0 && (await checkHealth(deps, service.healthOrigin, '/health'))
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
async function cellStatus(adminPost, cell) {
|
||||
const response = await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: cell.cellId
|
||||
})
|
||||
if (!response.status || response.status.cellId !== cell.cellId) {
|
||||
throw new Error(`${cell.cellId} returned an invalid status`)
|
||||
}
|
||||
return response.status
|
||||
}
|
||||
|
||||
function assertQuiescent(status) {
|
||||
const active = {
|
||||
activityLeases: status.activityLeases,
|
||||
activityRequestUnits: status.activityRequestUnits,
|
||||
outgoingMigrations: status.outgoingMigrations,
|
||||
incomingMigrations: status.incomingMigrations,
|
||||
observedRequests: status.runtime?.observedRequests ?? 0
|
||||
}
|
||||
if (Object.values(active).some((value) => Number(value) !== 0)) {
|
||||
throw new Error(`${status.cellId} still has active Relay work: ${JSON.stringify(active)}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function setCellState(adminPost, cell, enabled) {
|
||||
await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-state', {
|
||||
v: 1,
|
||||
cellId: cell.cellId,
|
||||
enabled
|
||||
})
|
||||
}
|
||||
|
||||
async function stagingStatus(deps, cells) {
|
||||
return {
|
||||
event: 'staging_relay_power_status',
|
||||
project: PROJECT,
|
||||
sqlActivationPolicy: sqlActivationPolicy(describeSql(deps)),
|
||||
cells: cells.map((cell) => ({
|
||||
cellId: cell.cellId,
|
||||
initiallyEnabled: cell.initiallyEnabled,
|
||||
targetSize: Number(describeMig(deps, cell).targetSize)
|
||||
})),
|
||||
cloudRun: CLOUD_RUN_SERVICES.map((service) => {
|
||||
const described = describeRunService(deps, service.name)
|
||||
const revision = describeRunRevision(deps, activeRevisionName(described))
|
||||
return { service: service.name, activeRevisionMinimum: revisionMinimum(revision) }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function sleepStaging(deps, cells) {
|
||||
const sqlPolicy = sqlActivationPolicy(describeSql(deps))
|
||||
if (sqlPolicy === 'NEVER') {
|
||||
const runningCells = cells.filter((cell) => Number(describeMig(deps, cell).targetSize) !== 0)
|
||||
if (runningCells.length > 0) {
|
||||
// SQL-off plus running workers is an unknown partial state; never kill those workers blindly.
|
||||
throw new Error(
|
||||
`staging is partially asleep with running cells: ${runningCells.map((cell) => cell.cellId).join(', ')}`
|
||||
)
|
||||
}
|
||||
deps.emit({ event: 'staging_relay_sleep_reconciled', alreadyAsleep: true })
|
||||
return
|
||||
}
|
||||
|
||||
const adminPost = createAdminPost(deps)
|
||||
const initial = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of initial) assertQuiescent(status)
|
||||
const selector = await inspectAdmissionSelector(
|
||||
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
|
||||
)
|
||||
if (selector.selector.generation > 0) {
|
||||
throw new Error(
|
||||
'staging sleep cannot reverse the monotonic admission selector; keep staging awake'
|
||||
)
|
||||
}
|
||||
const previouslyEnabled = new Set(initial.filter((status) => status.enabled).map((status) => status.cellId))
|
||||
|
||||
for (const cell of cells) await setCellState(adminPost, cell, false)
|
||||
await deps.wait(15_000)
|
||||
try {
|
||||
const disabled = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of disabled) {
|
||||
if (status.enabled) throw new Error(`${status.cellId} admission did not disable`)
|
||||
assertQuiescent(status)
|
||||
}
|
||||
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
|
||||
const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of final) assertQuiescent(status)
|
||||
} catch (error) {
|
||||
for (const cell of cells.filter((candidate) => previouslyEnabled.has(candidate.cellId))) {
|
||||
await setCellState(adminPost, cell, true).catch(() => undefined)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0)))
|
||||
await ensureSqlPolicy(deps, 'NEVER')
|
||||
deps.emit({ event: 'staging_relay_slept', stoppedCells: cells.map((cell) => cell.cellId) })
|
||||
}
|
||||
|
||||
async function wakeStaging(deps, cells, wakeCells) {
|
||||
await ensureSqlPolicy(deps, 'ALWAYS')
|
||||
await waitUntil(deps, 'staging director health', () => checkHealth(deps, DIRECTOR_ORIGIN, '/health'))
|
||||
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
|
||||
|
||||
const adminPost = createAdminPost(deps)
|
||||
const selector = await inspectAdmissionSelector(
|
||||
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
|
||||
)
|
||||
const selectorActive = selector.selector.generation > 0
|
||||
// Existing-only cells may still own live or dormant assignments, so a
|
||||
// selector-era wake restores the complete retained topology.
|
||||
const selected = selectorActive
|
||||
? cells
|
||||
: cells.filter((cell) => wakeCells === 'all' || cell.initiallyEnabled)
|
||||
await Promise.all(
|
||||
cells.map((cell) => setMigSize(deps, cell, selected.includes(cell) ? 1 : 0))
|
||||
)
|
||||
for (const cell of selected) {
|
||||
await waitUntil(deps, `${cell.cellId} health`, () => checkHealth(deps, cell.origin, '/health'))
|
||||
await waitUntil(deps, `${cell.cellId} readiness`, () => checkHealth(deps, cell.origin, '/ready'))
|
||||
}
|
||||
|
||||
for (const cell of cells) {
|
||||
if (selectorActive) {
|
||||
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
|
||||
const current = await cellStatus(adminPost, cell)
|
||||
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
|
||||
})
|
||||
if (status.admissionState !== selectorCellState(selector.selector, cell.cellId)) {
|
||||
throw new Error(`${cell.cellId} admission does not match selector`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!selected.includes(cell)) {
|
||||
await setCellState(adminPost, cell, false)
|
||||
continue
|
||||
}
|
||||
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
|
||||
const current = await cellStatus(adminPost, cell)
|
||||
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
|
||||
})
|
||||
if (cell.initiallyEnabled && !status.enabled) await setCellState(adminPost, cell, true)
|
||||
if (!cell.initiallyEnabled && status.enabled) await setCellState(adminPost, cell, false)
|
||||
}
|
||||
deps.emit({
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: selected.map((cell) => cell.cellId),
|
||||
admissionCells: selectorActive
|
||||
? selector.selector.membership.general
|
||||
: selected.filter((cell) => cell.initiallyEnabled).map((cell) => cell.cellId)
|
||||
})
|
||||
}
|
||||
|
||||
export async function runStagingRelayPower(config, overrides = {}) {
|
||||
const deps = {
|
||||
command: overrides.command ?? ((args) => defaultCommand(args, false)),
|
||||
commandJson: overrides.commandJson ?? ((args) => defaultCommand(args, true)),
|
||||
fetch: overrides.fetch ?? fetch,
|
||||
adminToken: overrides.adminToken ?? suppliedAdminToken,
|
||||
emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
|
||||
now: overrides.now ?? Date.now,
|
||||
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
}
|
||||
const cells = readStagingTopology(config.topologyFile)
|
||||
if (config.mode === 'status') deps.emit(await stagingStatus(deps, cells))
|
||||
else if (config.mode === 'sleep') await sleepStaging(deps, cells)
|
||||
else await wakeStaging(deps, cells, config.wakeCells)
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
await runStagingRelayPower(parseArguments(argv))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseArguments,
|
||||
readStagingTopology,
|
||||
runStagingRelayPower
|
||||
} from './power-staging-relay.mjs'
|
||||
|
||||
function topologyFile(overrides = {}) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'staging-relay-power-'))
|
||||
const topology = {
|
||||
'staging-gce-c1': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c1',
|
||||
zone: 'us-central1-b',
|
||||
origin: 'https://c1.relay-staging.onorca.dev',
|
||||
initially_enabled: true
|
||||
},
|
||||
'staging-gce-c2': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c2',
|
||||
zone: 'us-central1-c',
|
||||
origin: 'https://c2.relay-staging.onorca.dev',
|
||||
initially_enabled: true
|
||||
},
|
||||
'staging-gce-c3': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c3',
|
||||
zone: 'us-central1-a',
|
||||
origin: 'https://c3.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
},
|
||||
'staging-gce-c4': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c4',
|
||||
zone: 'asia-east2-a',
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
},
|
||||
...overrides
|
||||
}
|
||||
const file = join(directory, 'topology.json')
|
||||
writeFileSync(file, JSON.stringify(topology))
|
||||
return file
|
||||
}
|
||||
|
||||
function argumentConfig(file, mode, wakeCells = 'configured') {
|
||||
return parseArguments([
|
||||
'--mode',
|
||||
mode,
|
||||
'--wake-cells',
|
||||
wakeCells,
|
||||
'--topology-file',
|
||||
file
|
||||
])
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function harness({
|
||||
sqlPolicy = 'ALWAYS',
|
||||
migSize = 1,
|
||||
observedRequests = 0,
|
||||
selectorGeneration = 0
|
||||
} = {}) {
|
||||
const cells = new Map(
|
||||
['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'].map((cellId, index) => [
|
||||
cellId,
|
||||
{
|
||||
enabled: index < 2,
|
||||
targetSize: migSize,
|
||||
observedRequests,
|
||||
initiallyEnabled: index < 2
|
||||
}
|
||||
])
|
||||
)
|
||||
const revisions = new Map([
|
||||
['orca-cloud-relay-staging', { active: 'relay-00001', latest: 'relay-00001', min: 1 }],
|
||||
['orca-cloud-auth-staging', { active: 'auth-00001', latest: 'auth-00001', min: 1 }]
|
||||
])
|
||||
const revisionMinimums = new Map([
|
||||
['relay-00001', 1],
|
||||
['auth-00001', 1]
|
||||
])
|
||||
const commands = []
|
||||
const events = []
|
||||
let activationPolicy = sqlPolicy
|
||||
let clock = 0
|
||||
|
||||
function cellForMig(name) {
|
||||
return [...cells.entries()].find(([, value], index) => {
|
||||
const suffix = `c${index + 1}`
|
||||
return name.endsWith(suffix) && value
|
||||
})
|
||||
}
|
||||
|
||||
function commandJson(args) {
|
||||
if (args[0] === 'sql') return { settings: { activationPolicy } }
|
||||
if (args[0] === 'compute') {
|
||||
const entry = cellForMig(args[4])
|
||||
return { targetSize: entry[1].targetSize, status: { isStable: true } }
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services') {
|
||||
const state = revisions.get(args[3])
|
||||
return {
|
||||
status: {
|
||||
latestReadyRevisionName: state.latest,
|
||||
traffic: [{ percent: 100, revisionName: state.active }]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'revisions') {
|
||||
return {
|
||||
metadata: {
|
||||
annotations: {
|
||||
'autoscaling.knative.dev/minScale': String(revisionMinimums.get(args[3]) ?? 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`unexpected JSON command ${args.join(' ')}`)
|
||||
}
|
||||
|
||||
function command(args) {
|
||||
commands.push(args)
|
||||
if (args[0] === 'sql') {
|
||||
activationPolicy = args.find((arg) => arg.startsWith('--activation-policy='))?.split('=')[1]
|
||||
return
|
||||
}
|
||||
if (args[0] === 'compute') {
|
||||
const entry = cellForMig(args[4])
|
||||
entry[1].targetSize = Number(args.find((arg) => arg.startsWith('--size='))?.split('=')[1])
|
||||
return
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update') {
|
||||
const state = revisions.get(args[3])
|
||||
state.latest = `${args[3]}-power`
|
||||
revisionMinimums.set(state.latest, 0)
|
||||
return
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update-traffic') {
|
||||
const state = revisions.get(args[3])
|
||||
state.active = state.latest
|
||||
return
|
||||
}
|
||||
throw new Error(`unexpected command ${args.join(' ')}`)
|
||||
}
|
||||
|
||||
async function fetchImpl(url, options = {}) {
|
||||
const parsed = new URL(url)
|
||||
if (!options.method) return response({ ok: true })
|
||||
const body = JSON.parse(options.body)
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
const state = cells.get(body.cellId)
|
||||
const index = [...cells.keys()].indexOf(body.cellId)
|
||||
return response({
|
||||
v: 1,
|
||||
status: {
|
||||
cellId: body.cellId,
|
||||
enabled: state.enabled,
|
||||
admissionState:
|
||||
selectorGeneration > 0
|
||||
? index === 0
|
||||
? 'existing-only'
|
||||
: index === 1
|
||||
? 'migration-only'
|
||||
: index === 2
|
||||
? 'general'
|
||||
: 'migration-only'
|
||||
: state.enabled
|
||||
? 'general'
|
||||
: 'existing-only',
|
||||
assignments: 0,
|
||||
reservedRequests: 0,
|
||||
activityLeases: 0,
|
||||
activityRequestUnits: 0,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtime: {
|
||||
ready: state.targetSize === 1,
|
||||
heartbeatFresh: state.targetSize === 1,
|
||||
observedRequests: state.observedRequests
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: selectorGeneration,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly:
|
||||
selectorGeneration > 0
|
||||
? ['staging-gce-c1']
|
||||
: [...cells]
|
||||
.filter(([, state]) => !state.enabled)
|
||||
.map(([cellId]) => cellId),
|
||||
migrationOnly:
|
||||
selectorGeneration > 0 ? ['staging-gce-c2', 'staging-gce-c4'] : [],
|
||||
general:
|
||||
selectorGeneration > 0
|
||||
? ['staging-gce-c3']
|
||||
: [...cells]
|
||||
.filter(([, state]) => state.enabled)
|
||||
.map(([cellId]) => cellId)
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-state') {
|
||||
cells.get(body.cellId).enabled = body.enabled
|
||||
return response({ ok: true })
|
||||
}
|
||||
throw new Error(`unexpected fetch ${parsed.pathname}`)
|
||||
}
|
||||
|
||||
return {
|
||||
cells,
|
||||
commands,
|
||||
events,
|
||||
deps: {
|
||||
command,
|
||||
commandJson,
|
||||
fetch: fetchImpl,
|
||||
adminToken: () => 'header.payload.signature',
|
||||
emit: (event) => events.push(event),
|
||||
now: () => clock,
|
||||
wait: async (ms) => {
|
||||
clock += ms
|
||||
}
|
||||
},
|
||||
sqlPolicy: () => activationPolicy
|
||||
}
|
||||
}
|
||||
|
||||
test('accepts only explicit staging power arguments and topology', () => {
|
||||
const file = topologyFile()
|
||||
assert.equal(argumentConfig(file, 'status').mode, 'status')
|
||||
assert.equal(readStagingTopology(file).at(-1).zone, 'asia-east2-a')
|
||||
assert.throws(() => argumentConfig(file, 'destroy'))
|
||||
assert.throws(() => parseArguments(['--mode', 'sleep', '--wake-cells', 'configured']))
|
||||
|
||||
const unsafe = topologyFile({
|
||||
'staging-gce-c1': {
|
||||
mig_name: 'orca-cloud-relay-gce-c1',
|
||||
zone: 'us-central1-a',
|
||||
origin: 'https://c1.relay.onorca.dev',
|
||||
initially_enabled: true
|
||||
}
|
||||
})
|
||||
assert.throws(() => readStagingTopology(unsafe), /unsafe/)
|
||||
|
||||
const unreviewedRegion = topologyFile({
|
||||
'staging-gce-c4': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c4',
|
||||
zone: 'europe-west1-b',
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
}
|
||||
})
|
||||
assert.throws(() => readStagingTopology(unreviewedRegion), /unsafe zone/)
|
||||
})
|
||||
|
||||
test('refuses sleep before changing admission when a cell has active requests', async () => {
|
||||
const testHarness = harness({ observedRequests: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
|
||||
/still has active Relay work/
|
||||
)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.equal(testHarness.cells.get('staging-gce-c1').enabled, true)
|
||||
})
|
||||
|
||||
test('sleeps only after disabling admission and proving zero active work', async () => {
|
||||
const testHarness = harness()
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'NEVER')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0])
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [false, false, false, false])
|
||||
assert.equal(testHarness.events.at(-1).event, 'staging_relay_slept')
|
||||
assert.equal(
|
||||
testHarness.commands.filter((args) => args[0] === 'run' && args[2] === 'update').length,
|
||||
2
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses staging sleep after the monotonic selector boundary', async () => {
|
||||
const testHarness = harness({ selectorGeneration: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
|
||||
/cannot reverse the monotonic admission selector/
|
||||
)
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
})
|
||||
|
||||
test('refuses to terminate workers from an unknown partially asleep state', async () => {
|
||||
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
|
||||
/partially asleep with running cells/
|
||||
)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
})
|
||||
|
||||
test('wakes SQL and configured cells while leaving the candidate off and disabled', async () => {
|
||||
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 0 })
|
||||
for (const cell of testHarness.cells.values()) cell.enabled = false
|
||||
for (const state of testHarness.cells.values()) state.observedRequests = 0
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'ALWAYS')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 0, 0])
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [true, true, false, false])
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: ['staging-gce-c1', 'staging-gce-c2'],
|
||||
admissionCells: ['staging-gce-c1', 'staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('wakes every retained cell without rewriting selector-era admission', async () => {
|
||||
const testHarness = harness({
|
||||
sqlPolicy: 'NEVER',
|
||||
migSize: 0,
|
||||
selectorGeneration: 1
|
||||
})
|
||||
const states = [...testHarness.cells.values()]
|
||||
states[0].enabled = false
|
||||
states[1].enabled = true
|
||||
states[2].enabled = true
|
||||
states[3].enabled = true
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
|
||||
|
||||
assert.deepEqual(states.map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
assert.deepEqual(states.map((cell) => cell.enabled), [false, true, true, true])
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'],
|
||||
admissionCells: ['staging-gce-c3']
|
||||
})
|
||||
})
|
||||
|
||||
test('status is read-only and reports the current billable floor controls', async () => {
|
||||
const testHarness = harness()
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'status'), testHarness.deps)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.equal(testHarness.events[0].project, 'onorca-cloud-staging')
|
||||
assert.equal(testHarness.events[0].sqlActivationPolicy, 'ALWAYS')
|
||||
assert.equal(testHarness.events[0].cells.length, 4)
|
||||
})
|
||||
@@ -0,0 +1,78 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
function argumentsFrom(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['current-json', 'topology-json', 'output', 'cell-ids', 'image-digest']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
export function prepareRelayAsiaDirectorCells({ currentCells, topology, cellIds, imageDigest }) {
|
||||
if (!Array.isArray(currentCells) || !topology || Array.isArray(topology)) {
|
||||
throw new Error('director inputs are invalid')
|
||||
}
|
||||
const additions = cellIds.split(',').map((value) => value.trim()).filter(Boolean)
|
||||
if (
|
||||
additions.length === 0 ||
|
||||
new Set(additions).size !== additions.length
|
||||
) throw new Error('director Asia additions are invalid')
|
||||
const currentIds = new Set(currentCells.map((cell) => cell.id))
|
||||
if (currentIds.size !== currentCells.length) throw new Error('current director cells contain duplicates')
|
||||
const normalizedCurrent = currentCells.map((cell) => ({
|
||||
...cell,
|
||||
region: cell.region ?? 'us-central1'
|
||||
}))
|
||||
const desiredCells = additions.map((cellId) => {
|
||||
const cell = topology[cellId]
|
||||
if (
|
||||
!cell ||
|
||||
cell.region !== 'asia-east2' ||
|
||||
cell.capacity_requests !== 6_000 ||
|
||||
cell.database_pool_max !== 10 ||
|
||||
cell.connection_hard_cap !== 3_000 ||
|
||||
cell.connection_unobserved_bound !== 60 ||
|
||||
cell.initially_enabled !== false ||
|
||||
cell.image?.split('@')[1] !== imageDigest
|
||||
) throw new Error(`${cellId} state output does not match the reviewed Asia shape`)
|
||||
return {
|
||||
id: cellId,
|
||||
url: cell.origin,
|
||||
capacityRequests: cell.capacity_requests,
|
||||
region: cell.region,
|
||||
initiallyEnabled: false,
|
||||
connectionHardCap: cell.connection_hard_cap,
|
||||
connectionUnobservedBound: cell.connection_unobserved_bound
|
||||
}
|
||||
})
|
||||
const desiredById = new Map(desiredCells.map((cell) => [cell.id, cell]))
|
||||
for (const current of normalizedCurrent) {
|
||||
const desired = desiredById.get(current.id)
|
||||
if (!desired) continue
|
||||
for (const [key, value] of Object.entries(desired)) {
|
||||
if (current[key] !== value) {
|
||||
throw new Error(`${current.id} director configuration differs from the reviewed Asia shape`)
|
||||
}
|
||||
}
|
||||
}
|
||||
const configured = new Set(normalizedCurrent.map((cell) => cell.id))
|
||||
return [...normalizedCurrent, ...desiredCells.filter((cell) => !configured.has(cell.id))]
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const values = argumentsFrom(process.argv.slice(2))
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: JSON.parse(readFileSync(values['current-json'], 'utf8')),
|
||||
topology: JSON.parse(readFileSync(values['topology-json'], 'utf8')),
|
||||
cellIds: values['cell-ids'],
|
||||
imageDigest: values['image-digest']
|
||||
})
|
||||
writeFileSync(values.output, `${JSON.stringify(result)}\n`, { mode: 0o600 })
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { prepareRelayAsiaDirectorCells } from './prepare-relay-asia-director-cells.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const topologyCell = (ordinal, zone) => ({
|
||||
origin: `https://c${ordinal}.relay.onorca.dev`, region: 'asia-east2', zone,
|
||||
capacity_requests: 6_000, database_pool_max: 10,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60,
|
||||
initially_enabled: false,
|
||||
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${digest}`
|
||||
})
|
||||
|
||||
test('preserves current order, defaults predecessor regions, and appends exact Asia cells', () => {
|
||||
const current = [{
|
||||
id: 'production-gce-c1', url: 'https://c1.relay.onorca.dev',
|
||||
capacityRequests: 4_000, initiallyEnabled: false
|
||||
}]
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: current,
|
||||
topology: {
|
||||
'production-gce-c27': topologyCell(27, 'asia-east2-a'),
|
||||
'production-gce-c28': topologyCell(28, 'asia-east2-b'),
|
||||
'production-gce-c29': topologyCell(29, 'asia-east2-c')
|
||||
},
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
imageDigest: digest
|
||||
})
|
||||
assert.equal(result[0].region, 'us-central1')
|
||||
assert.deepEqual(result.slice(1).map(({ id }) => id), [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
|
||||
])
|
||||
assert.ok(result.slice(1).every((cell) =>
|
||||
cell.region === 'asia-east2' && cell.initiallyEnabled === false &&
|
||||
cell.connectionHardCap === 3_000
|
||||
))
|
||||
})
|
||||
|
||||
test('is idempotent for an exact existing Asia cell and rejects director drift', () => {
|
||||
const topology = { 'production-gce-c27': topologyCell(27, 'asia-east2-a') }
|
||||
const current = prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
})
|
||||
assert.deepEqual(prepareRelayAsiaDirectorCells({
|
||||
currentCells: current, topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), current)
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [{ ...current[0], capacityRequests: 5_999 }],
|
||||
topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), /director configuration differs/)
|
||||
})
|
||||
|
||||
test('rejects a mismatching topology state output', () => {
|
||||
const wrong = topologyCell(27, 'asia-east2-a')
|
||||
wrong.database_pool_max = 20
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology: { 'production-gce-c27': wrong },
|
||||
cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), /does not match/)
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const BOOT_IMAGE = 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21'
|
||||
const SHAPES = {
|
||||
staging: { project: 'onorca-cloud-staging', cells: { 'staging-gce-c4': 'asia-east2-a' } },
|
||||
production: {
|
||||
project: 'onorca-cloud',
|
||||
cells: {
|
||||
'production-gce-c27': 'asia-east2-a',
|
||||
'production-gce-c28': 'asia-east2-b',
|
||||
'production-gce-c29': 'asia-east2-c'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function argumentsFrom(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['existing-json', 'environment', 'cell-ids', 'image']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
function canonical(value) {
|
||||
if (Array.isArray(value)) return value.map(canonical)
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]))
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function prepareRelayAsiaTopologyInput({
|
||||
existingCells,
|
||||
existingAdditionalRegions,
|
||||
environment,
|
||||
cellIds,
|
||||
image
|
||||
}) {
|
||||
const shape = SHAPES[environment]
|
||||
if (!shape) throw new Error('invalid environment')
|
||||
const requested = cellIds.split(',').map((value) => value.trim()).filter(Boolean).sort()
|
||||
const expected = Object.keys(shape.cells).sort()
|
||||
if (new Set(requested).size !== requested.length || JSON.stringify(requested) !== JSON.stringify(expected)) {
|
||||
throw new Error('cell IDs do not match the reviewed Asia topology')
|
||||
}
|
||||
const prefix = `us-central1-docker.pkg.dev/${shape.project}/orca-cloud/relay@sha256:`
|
||||
if (!image.startsWith(prefix) || !/sha256:[a-f0-9]{64}$/.test(image)) {
|
||||
throw new Error('image is not the environment Relay image pinned by digest')
|
||||
}
|
||||
if (!existingCells || Array.isArray(existingCells) || typeof existingCells !== 'object') {
|
||||
throw new Error('existing Relay cells must be an object')
|
||||
}
|
||||
if (
|
||||
!existingAdditionalRegions ||
|
||||
Array.isArray(existingAdditionalRegions) ||
|
||||
typeof existingAdditionalRegions !== 'object' ||
|
||||
JSON.stringify(canonical(existingAdditionalRegions)) !==
|
||||
JSON.stringify(canonical({ 'asia-east2': '10.42.1.0/24' }))
|
||||
) {
|
||||
throw new Error('Asia subnet must be committed before topology planning')
|
||||
}
|
||||
const additions = Object.fromEntries(expected.map((cellId) => {
|
||||
const hostname = cellId.split('-').at(-1)
|
||||
return [cellId, {
|
||||
hostname,
|
||||
region: 'asia-east2',
|
||||
zone: shape.cells[cellId],
|
||||
machine_type: 'e2-standard-4',
|
||||
boot_disk_gb: 30,
|
||||
boot_image: BOOT_IMAGE,
|
||||
capacity_requests: 6_000,
|
||||
database_pool_max: 10,
|
||||
image,
|
||||
initially_enabled: false,
|
||||
connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
}]
|
||||
}))
|
||||
for (const cellId of expected) {
|
||||
if (!existingCells[cellId]) {
|
||||
throw new Error('Asia cells must be committed before topology planning')
|
||||
}
|
||||
if (
|
||||
JSON.stringify(canonical(existingCells[cellId])) !==
|
||||
JSON.stringify(canonical(additions[cellId]))
|
||||
) {
|
||||
throw new Error('committed Asia cell differs from the reviewed topology')
|
||||
}
|
||||
}
|
||||
return {
|
||||
relay_gce_additional_region_subnetwork_cidrs: existingAdditionalRegions,
|
||||
relay_gce_cells: existingCells
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const values = argumentsFrom(process.argv.slice(2))
|
||||
const existing = JSON.parse(readFileSync(values['existing-json'], 'utf8'))
|
||||
prepareRelayAsiaTopologyInput({
|
||||
existingCells: existing.relay_gce_cells,
|
||||
existingAdditionalRegions: existing.relay_gce_additional_region_subnetwork_cidrs,
|
||||
environment: values.environment,
|
||||
cellIds: values['cell-ids'],
|
||||
image: values.image
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { prepareRelayAsiaTopologyInput } from './prepare-relay-asia-topology-input.mjs'
|
||||
|
||||
const image = `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`
|
||||
|
||||
const additionalRegions = { 'asia-east2': '10.42.1.0/24' }
|
||||
|
||||
const productionCells = () => Object.fromEntries([
|
||||
[27, 'asia-east2-a'],
|
||||
[28, 'asia-east2-b'],
|
||||
[29, 'asia-east2-c']
|
||||
].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, {
|
||||
hostname: `c${ordinal}`, region: 'asia-east2', zone,
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
|
||||
capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60
|
||||
}]))
|
||||
|
||||
test('accepts the exact production topology only after it is durably committed', () => {
|
||||
const existing = {
|
||||
'production-gce-c26': { hostname: 'c26', image: 'existing' },
|
||||
...productionCells()
|
||||
}
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: existing,
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image })
|
||||
assert.equal(result.relay_gce_cells, existing)
|
||||
assert.equal(result.relay_gce_additional_region_subnetwork_cidrs, additionalRegions)
|
||||
assert.deepEqual(existing['production-gce-c27'], {
|
||||
hostname: 'c27', region: 'asia-east2', zone: 'asia-east2-a',
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
|
||||
capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60
|
||||
})
|
||||
})
|
||||
|
||||
test('accepts the one exact committed staging Asia cell', () => {
|
||||
const stagingImage = image.replace('onorca-cloud/', 'onorca-cloud-staging/')
|
||||
const stagingCell = {
|
||||
hostname: 'c4', region: 'asia-east2', zone: 'asia-east2-a',
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
|
||||
capacity_requests: 6_000, database_pool_max: 10, image: stagingImage,
|
||||
initially_enabled: false, connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
}
|
||||
const result = prepareRelayAsiaTopologyInput({
|
||||
existingCells: { 'staging-gce-c3': { hostname: 'c3' }, 'staging-gce-c4': stagingCell },
|
||||
existingAdditionalRegions: additionalRegions,
|
||||
environment: 'staging',
|
||||
cellIds: 'staging-gce-c4',
|
||||
image: stagingImage
|
||||
})
|
||||
assert.equal(result.relay_gce_cells['staging-gce-c4'].zone, 'asia-east2-a')
|
||||
assert.equal(result.relay_gce_cells['staging-gce-c4'].image, stagingImage)
|
||||
})
|
||||
|
||||
test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drift', () => {
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production', cellIds: 'production-gce-c27', image
|
||||
}), /cell IDs/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
image: image.replace('onorca-cloud/', 'other-project/')
|
||||
}), /environment Relay image/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: {}, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /subnet must be committed/)
|
||||
const missing = productionCells()
|
||||
delete missing['production-gce-c29']
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: missing, existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /cells must be committed/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: { ...productionCells(), 'production-gce-c27': {} },
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /differs from the reviewed topology/)
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-id', 'mode']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
const origin = new URL(values['director-origin'])
|
||||
if (origin.protocol !== 'https:' || origin.origin !== values['director-origin']) {
|
||||
throw new Error('--director-origin must be a canonical HTTPS origin')
|
||||
}
|
||||
if (!['isolate', 'activate', 'restore-fallback', 'restore'].includes(values.mode)) {
|
||||
throw new Error('--mode must be isolate, activate, restore-fallback, or restore')
|
||||
}
|
||||
const cellOrigin = values['cell-origin'] ? new URL(values['cell-origin']) : null
|
||||
if (
|
||||
values.mode === 'isolate' &&
|
||||
(!cellOrigin || cellOrigin.protocol !== 'https:' || cellOrigin.origin !== values['cell-origin'])
|
||||
) {
|
||||
throw new Error('--cell-origin must be a canonical HTTPS origin for isolate mode')
|
||||
}
|
||||
const restoreGeneralCellIds = values['general-cell-ids']?.split(',').filter(Boolean) ?? []
|
||||
if (
|
||||
['restore-fallback', 'restore'].includes(values.mode) &&
|
||||
restoreGeneralCellIds.length === 0
|
||||
) {
|
||||
throw new Error('--general-cell-ids is required for restore modes')
|
||||
}
|
||||
if (values.mode === 'restore' && !restoreGeneralCellIds.includes(values['cell-id'])) {
|
||||
throw new Error('--general-cell-ids must include the canary for restore mode')
|
||||
}
|
||||
if (values.mode === 'restore-fallback' && restoreGeneralCellIds.includes(values['cell-id'])) {
|
||||
throw new Error('--general-cell-ids cannot include the canary for fallback restore')
|
||||
}
|
||||
if (new Set(restoreGeneralCellIds).size !== restoreGeneralCellIds.length) {
|
||||
throw new Error('--general-cell-ids must be distinct')
|
||||
}
|
||||
return {
|
||||
directorOrigin: origin.origin,
|
||||
cellOrigin: cellOrigin?.origin,
|
||||
cellId: values['cell-id'],
|
||||
mode: values.mode,
|
||||
restoreGeneralCellIds
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function sameMembership(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right)
|
||||
}
|
||||
|
||||
async function legacyCellState(post, cellId) {
|
||||
const result = await post('/v1/admin/cell-status', { v: 1, cellId })
|
||||
if (result.status?.cellId !== cellId) throw new Error('legacy admission status is invalid')
|
||||
const state = result.status.admissionState
|
||||
if (!['existing-only', 'migration-only', 'general'].includes(state)) {
|
||||
throw new Error('legacy admission status is invalid')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
async function applyLegacyStates(post, before, states, order) {
|
||||
const expected = membershipWithStates(before.selector, states)
|
||||
let changed = false
|
||||
for (const cellId of order) {
|
||||
const desired = states[cellId]
|
||||
const current = await legacyCellState(post, cellId)
|
||||
if (current === 'existing-only' && desired !== current) {
|
||||
throw new Error(`legacy admission cannot re-enable existing-only cell ${cellId}`)
|
||||
}
|
||||
if (current === desired) continue
|
||||
let cause
|
||||
try {
|
||||
await post('/v1/admin/cell-state', { v: 1, cellId, state: desired })
|
||||
} catch (error) {
|
||||
cause = error
|
||||
}
|
||||
if ((await legacyCellState(post, cellId)) !== desired) {
|
||||
const detail = cause instanceof Error ? `: ${cause.message}` : ''
|
||||
throw new Error(`legacy admission did not commit ${cellId} exactly${detail}`, { cause })
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post)
|
||||
if (verified.selector.generation !== 0 ||
|
||||
!sameMembership(verified.selector.membership, expected)) {
|
||||
throw new Error('legacy admission membership changed unexpectedly')
|
||||
}
|
||||
return { changed, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function prepareCapacityCanary(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const postAt = async (origin, path, body) =>
|
||||
await responseJson(
|
||||
await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
)
|
||||
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const state = selectorCellState(before.selector, config.cellId)
|
||||
if (state === 'existing-only' && config.mode !== 'restore-fallback') {
|
||||
throw new Error('capacity canary cannot restore existing-only admission')
|
||||
}
|
||||
const states =
|
||||
config.mode === 'isolate'
|
||||
? { [config.cellId]: 'migration-only' }
|
||||
: config.mode === 'activate'
|
||||
? Object.fromEntries([
|
||||
...before.selector.membership.general.map((cellId) => [
|
||||
cellId,
|
||||
cellId === config.cellId ? 'general' : 'migration-only'
|
||||
]),
|
||||
[config.cellId, 'general']
|
||||
])
|
||||
: Object.fromEntries([
|
||||
...config.restoreGeneralCellIds.map((cellId) => [cellId, 'general']),
|
||||
...(config.mode === 'restore-fallback'
|
||||
? [[config.cellId, state === 'existing-only' ? 'existing-only' : 'migration-only']]
|
||||
: [])
|
||||
])
|
||||
const membership = membershipWithStates(before.selector, states)
|
||||
const legacyOrder =
|
||||
config.mode === 'activate'
|
||||
? [config.cellId, ...before.selector.membership.general.filter((id) => id !== config.cellId)]
|
||||
: config.mode === 'restore-fallback'
|
||||
? [...config.restoreGeneralCellIds, config.cellId]
|
||||
: Object.keys(states)
|
||||
const result = before.selector.generation === 0
|
||||
? await applyLegacyStates(post, before, states, legacyOrder)
|
||||
: sameMembership(membership, before.selector.membership)
|
||||
? { changed: false, selector: before.selector }
|
||||
: await applyExactAdmissionSelector(post, membership, {
|
||||
expectedCurrentSelector: before.selector
|
||||
})
|
||||
if (config.mode === 'isolate') {
|
||||
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
|
||||
}
|
||||
return {
|
||||
changed: result.changed,
|
||||
generation: result.selector.generation,
|
||||
...(config.mode === 'isolate' ? { drained: true } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const config = parseArguments(argv)
|
||||
const result = await prepareCapacityCanary(config)
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ event: 'relay_capacity_canary_admission', cellId: config.cellId, mode: config.mode, ...result })}\n`
|
||||
)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
import { prepareCapacityCanary } from './prepare-relay-capacity-canary.mjs'
|
||||
|
||||
function harness(initialState, options = {}) {
|
||||
const {
|
||||
generation = 4,
|
||||
ambiguousCellState = false,
|
||||
rejectCellState = false,
|
||||
fallbackState = 'general',
|
||||
extraGeneralCellIds = []
|
||||
} = options
|
||||
let selector = {
|
||||
generation,
|
||||
attemptId: 'initial',
|
||||
membership: {
|
||||
existingOnly: [
|
||||
'staging-gce-c1',
|
||||
...(initialState === 'existing-only' ? ['staging-gce-c3'] : [])
|
||||
],
|
||||
migrationOnly: [
|
||||
...(fallbackState === 'migration-only' ? ['staging-gce-c2'] : []),
|
||||
...(initialState === 'migration-only' ? ['staging-gce-c3'] : [])
|
||||
],
|
||||
general: [
|
||||
...extraGeneralCellIds,
|
||||
...(fallbackState === 'general' ? ['staging-gce-c2'] : []),
|
||||
...(initialState === 'general' ? ['staging-gce-c3'] : [])
|
||||
].sort()
|
||||
}
|
||||
}
|
||||
let intent = null
|
||||
let applies = 0
|
||||
let drains = 0
|
||||
const cellStateChanges = []
|
||||
const fetch = async (url, options) => {
|
||||
const path = new URL(url).pathname
|
||||
const body = JSON.parse(options.body)
|
||||
if (path === '/v1/admin/drain') {
|
||||
assert.deepEqual(body, { v: 1, graceMs: 0 })
|
||||
drains++
|
||||
return Response.json({ ok: true })
|
||||
}
|
||||
if (path === '/v1/admin/cell-status') {
|
||||
const state = selector.membership.existingOnly.includes(body.cellId)
|
||||
? 'existing-only'
|
||||
: selector.membership.migrationOnly.includes(body.cellId)
|
||||
? 'migration-only'
|
||||
: 'general'
|
||||
return Response.json({ status: { cellId: body.cellId, admissionState: state } })
|
||||
}
|
||||
if (path === '/v1/admin/cell-state') {
|
||||
assert.equal(selector.generation, 0)
|
||||
if (rejectCellState) {
|
||||
return Response.json({ error: 'invalid_token' }, { status: 401 })
|
||||
}
|
||||
const keys = {
|
||||
'existing-only': 'existingOnly',
|
||||
'migration-only': 'migrationOnly',
|
||||
general: 'general'
|
||||
}
|
||||
for (const cells of Object.values(selector.membership)) {
|
||||
const index = cells.indexOf(body.cellId)
|
||||
if (index !== -1) cells.splice(index, 1)
|
||||
}
|
||||
selector.membership[keys[body.state]].push(body.cellId)
|
||||
for (const cells of Object.values(selector.membership)) cells.sort()
|
||||
cellStateChanges.push({ cellId: body.cellId, state: body.state })
|
||||
if (ambiguousCellState) throw new Error('response lost')
|
||||
return Response.json({ ok: true })
|
||||
}
|
||||
if (path.endsWith('/status')) return Response.json({ selector, intent })
|
||||
applies++
|
||||
selector = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
intent = {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership,
|
||||
state: 'committed'
|
||||
}
|
||||
return Response.json({ changed: true, selector })
|
||||
}
|
||||
return {
|
||||
fetch,
|
||||
selector: () => selector,
|
||||
applies: () => applies,
|
||||
drains: () => drains,
|
||||
cellStateChanges
|
||||
}
|
||||
}
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
cellOrigin: 'https://c3.relay.example.com',
|
||||
cellId: 'staging-gce-c3',
|
||||
mode: 'isolate',
|
||||
restoreGeneralCellIds: []
|
||||
}
|
||||
|
||||
test('isolates a general canary as migration-only', async () => {
|
||||
const testHarness = harness('general')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
|
||||
{ changed: true, generation: 5, drained: true }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership.migrationOnly, [config.cellId])
|
||||
assert.equal(testHarness.applies(), 1)
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('activates the canary as the only general cell', async () => {
|
||||
const testHarness = harness('migration-only')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: true, generation: 5 }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: ['staging-gce-c2'],
|
||||
general: [config.cellId]
|
||||
})
|
||||
assert.equal(testHarness.applies(), 1)
|
||||
})
|
||||
|
||||
test('restores the reviewed staging general membership', async () => {
|
||||
const testHarness = harness('migration-only')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore',
|
||||
restoreGeneralCellIds: ['staging-gce-c2', config.cellId]
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: true, generation: 5 }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2', config.cellId]
|
||||
})
|
||||
})
|
||||
|
||||
test('restores the fallback without promoting a possibly drained canary', async () => {
|
||||
const testHarness = harness('general')
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [config.cellId],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('restores the fallback while preserving an irreversible canary', async () => {
|
||||
const testHarness = harness('existing-only')
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1', config.cellId],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('uses exact legacy admission writes before the selector boundary', async () => {
|
||||
const testHarness = harness('general', { generation: 0 })
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
|
||||
{ changed: true, generation: 0, drained: true }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'migration-only' }
|
||||
])
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('promotes a legacy canary before demoting its fallback', async () => {
|
||||
const testHarness = harness('migration-only', { generation: 0 })
|
||||
await prepareCapacityCanary(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'general' },
|
||||
{ cellId: 'staging-gce-c2', state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('makes the canary sole general with the live legacy membership shape', async () => {
|
||||
const extraGeneralCellIds = ['combined', 'staging-c1', 'staging-c2']
|
||||
const testHarness = harness('migration-only', { generation: 0, extraGeneralCellIds })
|
||||
const activate = { ...config, mode: 'activate' }
|
||||
await prepareCapacityCanary(activate, { fetch: testHarness.fetch, token: 'masked' })
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'general' },
|
||||
{ cellId: 'combined', state: 'migration-only' },
|
||||
{ cellId: 'staging-c1', state: 'migration-only' },
|
||||
{ cellId: 'staging-c2', state: 'migration-only' },
|
||||
{ cellId: 'staging-gce-c2', state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('restores a legacy fallback before demoting the target', async () => {
|
||||
const testHarness = harness('general', {
|
||||
generation: 0,
|
||||
fallbackState: 'migration-only'
|
||||
})
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: 'staging-gce-c2', state: 'general' },
|
||||
{ cellId: config.cellId, state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('keeps an already restored legacy fallback unchanged', async () => {
|
||||
const testHarness = harness('migration-only', { generation: 0 })
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: false, generation: 0 }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [])
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [config.cellId],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('recovers an ambiguous legacy admission response by exact readback', async () => {
|
||||
const testHarness = harness('general', { generation: 0, ambiguousCellState: true })
|
||||
await assert.doesNotReject(
|
||||
prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
|
||||
)
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('reports a rejected legacy admission write without draining', async () => {
|
||||
const testHarness = harness('general', { generation: 0, rejectCellState: true })
|
||||
const operation = prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
|
||||
await assert.rejects(operation, /cell-state returned 401/)
|
||||
assert.equal(testHarness.drains(), 0)
|
||||
})
|
||||
|
||||
test('the staging workflow supplies every required capacity transition argument', () => {
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const verifyCalls = workflow.match(
|
||||
/node dev\/scripts\/verify-relay-capacity-transition\.mjs[\s\S]*?(?=\n\s*\n|\n\s*- name:)/g
|
||||
)
|
||||
assert.ok(verifyCalls?.length >= 5)
|
||||
for (const call of verifyCalls) {
|
||||
for (const flag of ['--cell-origin', '--heartbeat', '--admission', '--draining', '--activity']) {
|
||||
assert.match(call, new RegExp(flag))
|
||||
}
|
||||
}
|
||||
const isolate = workflow.match(
|
||||
/node dev\/scripts\/prepare-relay-capacity-canary\.mjs[\s\S]*?--mode isolate/
|
||||
)?.[0]
|
||||
assert.match(isolate, /--cell-origin/)
|
||||
})
|
||||
@@ -0,0 +1,116 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
export const PRODUCTION_CAPACITY_CELL_IDS = [
|
||||
'production-gce-c7',
|
||||
'production-gce-c8',
|
||||
'production-gce-c9',
|
||||
'production-gce-c10',
|
||||
'production-gce-c13',
|
||||
'production-gce-c14',
|
||||
'production-gce-c15',
|
||||
'production-gce-c16',
|
||||
'production-gce-c19',
|
||||
'production-gce-c20',
|
||||
'production-gce-c21',
|
||||
'production-gce-c22',
|
||||
'production-gce-c23',
|
||||
'production-gce-c24',
|
||||
'production-gce-c25',
|
||||
'production-gce-c26'
|
||||
]
|
||||
|
||||
function cellOrigin(cellId) {
|
||||
return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev`
|
||||
}
|
||||
|
||||
export function parseProductionCapacityCellArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
if (!['isolate', 'drain', 'activate'].includes(values.mode)) {
|
||||
throw new Error('--mode must be isolate, drain, or activate')
|
||||
}
|
||||
const cellId = values['cell-id']
|
||||
if (!PRODUCTION_CAPACITY_CELL_IDS.includes(cellId)) {
|
||||
throw new Error('production capacity target is not approved')
|
||||
}
|
||||
const expectedCellOrigin = cellOrigin(cellId)
|
||||
if (
|
||||
values['director-origin'] !== DIRECTOR_ORIGIN ||
|
||||
values['cell-origin'] !== expectedCellOrigin
|
||||
) {
|
||||
throw new Error('production capacity target origin is not exact')
|
||||
}
|
||||
return {
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
cellOrigin: expectedCellOrigin,
|
||||
cellId,
|
||||
mode: values.mode
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
export async function prepareProductionCapacityCell(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const postAt = async (origin, path, body) =>
|
||||
await responseJson(
|
||||
await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
)
|
||||
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
|
||||
if (config.mode === 'drain') {
|
||||
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
|
||||
return { changed: false, drained: true }
|
||||
}
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const state = selectorCellState(before.selector, config.cellId)
|
||||
if (state === 'existing-only') throw new Error('production capacity target is irreversible')
|
||||
const desiredState = config.mode === 'isolate' ? 'migration-only' : 'general'
|
||||
const membership = membershipWithStates(before.selector, { [config.cellId]: desiredState })
|
||||
const result = await applyExactAdmissionSelector(post, membership, {
|
||||
expectedCurrentSelector: before.selector
|
||||
})
|
||||
return {
|
||||
changed: result.changed,
|
||||
generation: result.selector.generation,
|
||||
admissionState: desiredState
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const config = parseProductionCapacityCellArguments(argv)
|
||||
const result = await prepareProductionCapacityCell(config)
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ event: 'relay_production_capacity_canary', cellId: config.cellId, mode: config.mode, ...result })}\n`
|
||||
)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import {
|
||||
parseProductionCapacityCellArguments,
|
||||
prepareProductionCapacityCell,
|
||||
PRODUCTION_CAPACITY_CELL_IDS
|
||||
} from './prepare-relay-production-capacity-canary.mjs'
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
cellOrigin: 'https://c26.relay.onorca.dev',
|
||||
cellId: 'production-gce-c26'
|
||||
}
|
||||
|
||||
const membership = {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17'],
|
||||
general: ['production-gce-c25', 'production-gce-c26']
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function canaryFetch() {
|
||||
let selector = { generation: 20, attemptId: null, membership }
|
||||
const calls = []
|
||||
const fetch = async (url, init) => {
|
||||
const path = new URL(url).pathname
|
||||
const body = JSON.parse(init.body)
|
||||
calls.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector,
|
||||
intent: body.attemptId
|
||||
? {
|
||||
attemptId: body.attemptId,
|
||||
state: 'committed',
|
||||
expectedGeneration: selector.generation - 1,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership
|
||||
}
|
||||
: null
|
||||
})
|
||||
}
|
||||
if (path === '/v1/admin/admission-selector/apply') {
|
||||
selector = {
|
||||
generation: selector.generation + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
return response({ v: 1, changed: true, selector })
|
||||
}
|
||||
if (path === '/v1/admin/drain') return response({ v: 1, draining: true })
|
||||
throw new Error(`unexpected ${path}`)
|
||||
}
|
||||
return { calls, fetch, selector: () => selector }
|
||||
}
|
||||
|
||||
describe('production Relay capacity cell admission', () => {
|
||||
it('allows only the serving rollout cells', () => {
|
||||
assert.deepEqual(PRODUCTION_CAPACITY_CELL_IDS, [
|
||||
'production-gce-c7',
|
||||
'production-gce-c8',
|
||||
'production-gce-c9',
|
||||
'production-gce-c10',
|
||||
'production-gce-c13',
|
||||
'production-gce-c14',
|
||||
'production-gce-c15',
|
||||
'production-gce-c16',
|
||||
'production-gce-c19',
|
||||
'production-gce-c20',
|
||||
'production-gce-c21',
|
||||
'production-gce-c22',
|
||||
'production-gce-c23',
|
||||
'production-gce-c24',
|
||||
'production-gce-c25',
|
||||
'production-gce-c26'
|
||||
])
|
||||
assert.deepEqual(parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c7.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--mode', 'isolate'
|
||||
]), {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
cellOrigin: 'https://c7.relay.onorca.dev',
|
||||
cellId: 'production-gce-c7',
|
||||
mode: 'isolate'
|
||||
})
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c17.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c17',
|
||||
'--mode', 'isolate'
|
||||
]), /not approved/)
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c8.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--mode', 'isolate'
|
||||
]), /origin is not exact/)
|
||||
})
|
||||
|
||||
it('isolates only the selected cell without depending on its runtime', async () => {
|
||||
const fake = canaryFetch()
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.equal(result.admissionState, 'migration-only')
|
||||
assert.deepEqual(fake.selector().membership, {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c26'],
|
||||
general: ['production-gce-c25']
|
||||
})
|
||||
assert.doesNotMatch(fake.calls.map(({ path }) => path).join(','), /\/v1\/admin\/drain/)
|
||||
})
|
||||
|
||||
it('drains the selected cell independently after durable isolation', async () => {
|
||||
const fake = canaryFetch()
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.deepEqual(result, { changed: false, drained: true })
|
||||
assert.deepEqual(fake.calls, [{
|
||||
path: '/v1/admin/drain',
|
||||
body: { v: 1, graceMs: 0 }
|
||||
}])
|
||||
})
|
||||
|
||||
it('restores only the selected cell to general admission', async () => {
|
||||
const fake = canaryFetch()
|
||||
await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.equal(result.admissionState, 'general')
|
||||
assert.deepEqual(fake.selector().membership, membership)
|
||||
})
|
||||
|
||||
it('refuses an irreversible existing-only target', async () => {
|
||||
const fetch = async () => response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: 20,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly: ['production-gce-c26'],
|
||||
migrationOnly: ['production-gce-c17'],
|
||||
general: ['production-gce-c25']
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
await assert.rejects(
|
||||
prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch, token: 'token' }
|
||||
),
|
||||
/irreversible/
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,73 @@
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const WRONG_CELL = 4409
|
||||
const DRAINING = 4503
|
||||
|
||||
function once(socket, event, listener) {
|
||||
if (typeof socket.once === 'function') {
|
||||
socket.once(event, listener)
|
||||
return
|
||||
}
|
||||
if (typeof socket.addEventListener !== 'function') {
|
||||
throw new Error('WebSocket event API is unavailable')
|
||||
}
|
||||
socket.addEventListener(event, (value) => {
|
||||
if (event === 'close') listener(value.code)
|
||||
else if (event === 'error') listener(value.error ?? new Error(value.message))
|
||||
else listener()
|
||||
}, { once: true })
|
||||
}
|
||||
|
||||
export function parseLegacyAdmissionProbeArguments(argv) {
|
||||
if (argv.length !== 2 || argv[0] !== '--cell-origin') throw new Error('invalid arguments')
|
||||
const origin = new URL(argv[1])
|
||||
if (origin.protocol !== 'https:' || origin.origin !== argv[1]) {
|
||||
throw new Error('--cell-origin must be a canonical HTTPS origin')
|
||||
}
|
||||
return { cellOrigin: origin.origin }
|
||||
}
|
||||
|
||||
export async function probeLegacyAdmission(config, overrides = {}) {
|
||||
const Socket = overrides.WebSocket ?? globalThis.WebSocket
|
||||
if (typeof Socket !== 'function') throw new Error('WebSocket is unavailable')
|
||||
const random = overrides.randomBytes ?? randomBytes
|
||||
const timeoutMs = overrides.timeoutMs ?? 15_000
|
||||
const hostId = random(12).toString('base64url')
|
||||
const credential = random(32).toString('base64url')
|
||||
const url = `${config.cellOrigin.replace('https://', 'wss://')}/v1/connect/${hostId}`
|
||||
await new Promise((resolve, reject) => {
|
||||
const socket = new Socket(url)
|
||||
const timer = setTimeout(() => {
|
||||
if (typeof socket.terminate === 'function') socket.terminate()
|
||||
else socket.close()
|
||||
reject(new Error('legacy admission probe timed out'))
|
||||
}, timeoutMs)
|
||||
once(socket, 'open', () => {
|
||||
socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
|
||||
})
|
||||
once(socket, 'close', (code) => {
|
||||
clearTimeout(timer)
|
||||
if (code === WRONG_CELL) resolve()
|
||||
else if (code === DRAINING) reject(new Error('legacy cell is draining'))
|
||||
else reject(new Error(`legacy admission probe closed with ${code}`))
|
||||
})
|
||||
once(socket, 'error', (error) => {
|
||||
clearTimeout(timer)
|
||||
reject(new Error(`legacy admission probe failed: ${error.message}`))
|
||||
})
|
||||
})
|
||||
return { accepting: true }
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await probeLegacyAdmission(parseLegacyAdmissionProbeArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_admission_verified', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseLegacyAdmissionProbeArguments,
|
||||
probeLegacyAdmission
|
||||
} from './probe-relay-legacy-admission.mjs'
|
||||
|
||||
function socketClosingWith(code, observed) {
|
||||
return class extends EventEmitter {
|
||||
constructor(url) {
|
||||
super()
|
||||
observed.url = url
|
||||
queueMicrotask(() => this.emit('open'))
|
||||
}
|
||||
|
||||
send(payload) {
|
||||
observed.payload = JSON.parse(payload)
|
||||
queueMicrotask(() => this.emit('close', code))
|
||||
}
|
||||
|
||||
terminate() {}
|
||||
}
|
||||
}
|
||||
|
||||
function nativeSocketClosingWith(code) {
|
||||
return class extends EventTarget {
|
||||
constructor() {
|
||||
super()
|
||||
queueMicrotask(() => this.dispatchEvent(new Event('open')))
|
||||
}
|
||||
|
||||
send() {
|
||||
const event = new Event('close')
|
||||
Object.defineProperty(event, 'code', { value: code })
|
||||
queueMicrotask(() => this.dispatchEvent(event))
|
||||
}
|
||||
|
||||
close() {}
|
||||
}
|
||||
}
|
||||
|
||||
const config = { cellOrigin: 'https://c2.relay.example.com' }
|
||||
const random = (length) => Buffer.alloc(length, length)
|
||||
|
||||
test('accepts only a canonical cell origin', () => {
|
||||
assert.deepEqual(
|
||||
parseLegacyAdmissionProbeArguments(['--cell-origin', config.cellOrigin]),
|
||||
config
|
||||
)
|
||||
assert.throws(
|
||||
() => parseLegacyAdmissionProbeArguments(['--cell-origin', `${config.cellOrigin}/path`]),
|
||||
/canonical/
|
||||
)
|
||||
})
|
||||
|
||||
test('proves admission with a synthetic invalid credential and exposes no identifier', async () => {
|
||||
const observed = {}
|
||||
assert.deepEqual(
|
||||
await probeLegacyAdmission(config, {
|
||||
WebSocket: socketClosingWith(4409, observed),
|
||||
randomBytes: random
|
||||
}),
|
||||
{ accepting: true }
|
||||
)
|
||||
assert.match(observed.url, /^wss:\/\/c2\.relay\.example\.com\/v1\/connect\/[A-Za-z0-9_-]{16}$/)
|
||||
assert.deepEqual(Object.keys(observed.payload).sort(), ['credential', 'mode', 'type', 'v'])
|
||||
})
|
||||
|
||||
test('uses the dependency-free Node WebSocket event API', async () => {
|
||||
await assert.doesNotReject(
|
||||
probeLegacyAdmission(config, {
|
||||
WebSocket: nativeSocketClosingWith(4409),
|
||||
randomBytes: random
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects the legacy draining close and any unknown outcome', async () => {
|
||||
for (const [code, message] of [[4503, /draining/], [4401, /closed with 4401/]]) {
|
||||
await assert.rejects(
|
||||
probeLegacyAdmission(config, {
|
||||
WebSocket: socketClosingWith(code, {}),
|
||||
randomBytes: random
|
||||
}),
|
||||
message
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,80 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
|
||||
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
|
||||
throw new Error('--director-origin must be the production Relay origin')
|
||||
}
|
||||
if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved')
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
||||
values['cell-incarnation']
|
||||
)) throw new Error('--cell-incarnation is invalid')
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) {
|
||||
throw new Error('admin identity token is unavailable')
|
||||
}
|
||||
return {
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
cellId: values['cell-id'],
|
||||
cellIncarnation: values['cell-incarnation'],
|
||||
token
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeRehomeTrust(config, dependencies = {}) {
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const response = await fetchImpl(
|
||||
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.token}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
sourceCellId: config.cellId,
|
||||
sourceCellIncarnation: config.cellIncarnation
|
||||
}),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}
|
||||
)
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) {
|
||||
throw new Error(`application-mediated rehome trust probe returned ${response.status}`)
|
||||
}
|
||||
if (
|
||||
body.v !== 1 ||
|
||||
body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' ||
|
||||
body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' ||
|
||||
body.dedicatedIdentity?.accepted !== true ||
|
||||
body.dedicatedIdentity?.idempotent !== true ||
|
||||
body.sharedRuntimeIdentityRejected !== true ||
|
||||
body.proven !== true
|
||||
) throw new Error('application-mediated rehome trust proof is incomplete')
|
||||
return body
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseRehomeTrustProbeArguments,
|
||||
probeRehomeTrust
|
||||
} from './probe-relay-rehome-trust.mjs'
|
||||
|
||||
const argv = [
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--cell-incarnation', '11111111-1111-4111-8111-111111111111'
|
||||
]
|
||||
const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }
|
||||
|
||||
test('binds the application-mediated probe to an exact approved cell incarnation', () => {
|
||||
assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7')
|
||||
assert.throws(() => parseRehomeTrustProbeArguments(
|
||||
argv.with(1, 'https://other.example.test'),
|
||||
environment
|
||||
))
|
||||
assert.throws(() => parseRehomeTrustProbeArguments(argv, {
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token'
|
||||
}))
|
||||
})
|
||||
|
||||
test('requires complete aggregate application-mediated trust proof', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
const result = await probeRehomeTrust(config, {
|
||||
fetch: async (url, init) => {
|
||||
assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe')
|
||||
assert.deepEqual(JSON.parse(init.body), {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
return Response.json({
|
||||
v: 1,
|
||||
dedicatedIdentity: {
|
||||
firstOutcome: 'host-not-connected',
|
||||
secondOutcome: 'host-not-connected',
|
||||
accepted: true,
|
||||
idempotent: true
|
||||
},
|
||||
sharedRuntimeIdentityRejected: true,
|
||||
proven: true
|
||||
})
|
||||
}
|
||||
})
|
||||
assert.equal(result.proven, true)
|
||||
})
|
||||
|
||||
test('rejects partial or mismatched proof', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
await assert.rejects(
|
||||
probeRehomeTrust(config, {
|
||||
fetch: async () => Response.json({
|
||||
v: 1,
|
||||
dedicatedIdentity: {
|
||||
firstOutcome: 'host-not-connected',
|
||||
secondOutcome: 'host-not-connected',
|
||||
accepted: true,
|
||||
idempotent: true
|
||||
},
|
||||
sharedRuntimeIdentityRejected: false,
|
||||
proven: false
|
||||
})
|
||||
}),
|
||||
/incomplete/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const production = source('infra/terraform/environments/production.tfvars')
|
||||
const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml')
|
||||
const jobWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
|
||||
|
||||
const RELAY_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
|
||||
|
||||
function source(path) {
|
||||
return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
// Slice each "<cell-id>" = { ... } entry out of relay_gce_cells.
|
||||
function productionCells() {
|
||||
const block = production.slice(production.indexOf('relay_gce_cells = {'))
|
||||
const cells = new Map()
|
||||
for (const match of block.matchAll(/"(production-gce-c\d+)" = \{([\s\S]*?)\n {2}\}/g)) {
|
||||
cells.set(match[1], match[2])
|
||||
}
|
||||
assert.ok(cells.size > 0, 'relay_gce_cells parsed empty')
|
||||
return cells
|
||||
}
|
||||
|
||||
function hardCap(body) {
|
||||
const match = body.match(/connection_hard_cap\s*=\s*(\d+)/)
|
||||
return match ? Number(match[1]) : undefined
|
||||
}
|
||||
|
||||
function imageDigest(body) {
|
||||
const match = body.match(/^\s*image\s*=\s*"([^"]+)"/m)
|
||||
assert.ok(match, 'cell entry has no image')
|
||||
const [repository, digest] = match[1].split('@')
|
||||
assert.equal(repository, RELAY_REPOSITORY)
|
||||
assert.match(digest, /^sha256:[0-9a-f]{64}$/)
|
||||
return digest
|
||||
}
|
||||
|
||||
function workflowPin(workflow, name) {
|
||||
const match = workflow.match(new RegExp(`${name}: (sha256:[0-9a-f]{64})`))
|
||||
assert.ok(match, `${name} is missing or not a full digest`)
|
||||
return match[1]
|
||||
}
|
||||
|
||||
test('every production cell pins a full relay image digest', () => {
|
||||
for (const [cellId, body] of productionCells()) {
|
||||
assert.match(imageDigest(body), /^sha256:[0-9a-f]{64}$/, `${cellId} image digest`)
|
||||
}
|
||||
})
|
||||
|
||||
test('the 1,000-cap cells are exactly the canonical capacity set', () => {
|
||||
const thousandCap = [...productionCells()]
|
||||
.filter(([, body]) => hardCap(body) === 1000)
|
||||
.map(([cellId]) => cellId)
|
||||
assert.deepEqual([...thousandCap].sort(), [...PRODUCTION_CAPACITY_CELL_IDS].sort())
|
||||
})
|
||||
|
||||
test('the 1,000-cap cells all serve one image digest', () => {
|
||||
const digests = new Map()
|
||||
for (const [cellId, body] of productionCells()) {
|
||||
if (hardCap(body) !== 1000) continue
|
||||
const digest = imageDigest(body)
|
||||
if (!digests.has(digest)) digests.set(digest, [])
|
||||
digests.get(digest).push(cellId)
|
||||
}
|
||||
assert.equal(
|
||||
digests.size,
|
||||
1,
|
||||
`1,000-cap cells split across digests: ${JSON.stringify([...digests])}`
|
||||
)
|
||||
assert.equal([...digests.values()][0].length, PRODUCTION_CAPACITY_CELL_IDS.length)
|
||||
})
|
||||
|
||||
// COMPATIBLE_CELL_IMAGE_DIGEST is one half of a reviewed (director, cell) skew pair, not a
|
||||
// claim about what the fleet serves; it is re-derived by hand for each capacity wave. So it
|
||||
// is deliberately NOT tied to the tfvars digest — only to its twin in the dispatch workflow.
|
||||
test('both capacity workflows declare the same reviewed image pins', () => {
|
||||
for (const name of ['PREDECESSOR_IMAGE_DIGEST', 'COMPATIBLE_CELL_IMAGE_DIGEST']) {
|
||||
assert.equal(workflowPin(dispatchWorkflow, name), workflowPin(jobWorkflow, name), name)
|
||||
}
|
||||
workflowPin(jobWorkflow, 'COMPATIBLE_DIRECTOR_IMAGE_DIGEST')
|
||||
})
|
||||
@@ -0,0 +1,210 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
LEASED_WORKFLOWS,
|
||||
LOCK_GROUPS,
|
||||
NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
PRODUCTION_LEASE,
|
||||
SELECTABLE_LEASE,
|
||||
STAGING_LEASE,
|
||||
concurrencyBlocks,
|
||||
entrypointsFor,
|
||||
jobIf,
|
||||
jobNeeds,
|
||||
jobs,
|
||||
leaseSteps,
|
||||
leaseStepsByJob,
|
||||
mutatesSharedInstance,
|
||||
readWorkflow,
|
||||
reusableCalls,
|
||||
revisionMintingScripts,
|
||||
workflowFiles
|
||||
} from './cloud-sql-rollout-lock-census.mjs'
|
||||
import { relayWorkflowFile } from './relay-repository.mjs'
|
||||
|
||||
const expectedLease = { production: PRODUCTION_LEASE, staging: STAGING_LEASE, selectable: SELECTABLE_LEASE }
|
||||
const leasedFiles = Object.keys(LEASED_WORKFLOWS)
|
||||
|
||||
function contractFiles(file) {
|
||||
return [file, ...(LEASED_WORKFLOWS[file].leaseFiles ?? [])]
|
||||
}
|
||||
|
||||
test('every locked workflow declares exactly one lock group that never cancels', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const blocks = concurrencyBlocks(readWorkflow(file))
|
||||
assert.equal(blocks.length, 1, `${file} must declare exactly one concurrency block`)
|
||||
assert.equal(blocks[0].group, LEASED_WORKFLOWS[file].group, file)
|
||||
assert.equal(blocks[0].cancelInProgress, 'false', file)
|
||||
}
|
||||
})
|
||||
|
||||
test('every locked workflow takes the lease for its own environment', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const lease = expectedLease[LEASED_WORKFLOWS[file].env]
|
||||
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
|
||||
assert.ok(steps.length > 0, `${file} must use the Cloud SQL rollout lease action`)
|
||||
for (const step of steps) {
|
||||
assert.equal(step.bucket, lease.bucket, file)
|
||||
assert.equal(step.object, lease.object, file)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('the lease step runs after the credential that authorizes it', () => {
|
||||
for (const file of leasedFiles) {
|
||||
for (const member of contractFiles(file)) {
|
||||
const text = readWorkflow(member)
|
||||
const lines = text.split('\n')
|
||||
for (const step of leaseSteps(text)) {
|
||||
const before = lines.slice(0, step.line - 1)
|
||||
const gcloud = before.lastIndexOf(' - uses: google-github-actions/setup-gcloud@v2')
|
||||
assert.notEqual(gcloud, -1, `${member}: lease step at line ${step.line} has no setup-gcloud before it`)
|
||||
assert.ok(
|
||||
before.lastIndexOf(' - uses: actions/checkout@v4') !== -1,
|
||||
`${member}: lease step at line ${step.line} runs before the local action is checked out`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('multi-wave workflows hold one lease per run and free it exactly once', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const entry = LEASED_WORKFLOWS[file]
|
||||
const waveFiles = entry.leaseFiles ?? []
|
||||
const callCount = waveFiles.reduce(
|
||||
(total, member) => total + (reusableCalls(readWorkflow(file)).get(member) ?? 0),
|
||||
0
|
||||
)
|
||||
if (!entry.reentrant) {
|
||||
assert.ok(callCount <= 1, `${file} calls a leased reusable job ${callCount} times; it needs a release job`)
|
||||
for (const member of contractFiles(file)) {
|
||||
for (const step of leaseSteps(readWorkflow(member))) {
|
||||
assert.equal(step.release, undefined, `${member} must leave release at its default`)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
assert.ok(callCount > 1, `${file} no longer calls its reusable job more than once`)
|
||||
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
|
||||
const released = steps.filter((step) => step.release === "'true'")
|
||||
assert.equal(released.length, 1, `${file} must free the run lease exactly once`)
|
||||
for (const step of steps) {
|
||||
if (step === released[0]) continue
|
||||
assert.equal(step.release, "'false'", `${file} wave jobs must hold the lease`)
|
||||
}
|
||||
|
||||
const callerJobs = jobs(readWorkflow(file))
|
||||
const releaseJob = leaseStepsByJob(file).find((job) =>
|
||||
job.steps.some((step) => step.release === "'true'")
|
||||
)
|
||||
assert.ok(releaseJob, `${file} must free the lease from its own job`)
|
||||
const guard = jobIf(callerJobs.find((job) => job.id === releaseJob.id).text)
|
||||
assert.match(guard, /always\(\)/, `${file}: the release job must run on failure and cancellation`)
|
||||
|
||||
const holders = callerJobs
|
||||
.filter(
|
||||
(job) =>
|
||||
job.id !== releaseJob.id &&
|
||||
(waveFiles.some((member) => job.text.includes(`uses: ./.github/workflows/${member}`)) ||
|
||||
leaseStepsByJob(file).find((entry) => entry.id === job.id)?.steps.length > 0)
|
||||
)
|
||||
.map((job) => job.id)
|
||||
const needs = jobNeeds(callerJobs.find((job) => job.id === releaseJob.id).text)
|
||||
for (const holder of holders) {
|
||||
assert.ok(needs.includes(holder), `${file}: the release job must need ${holder}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('workflows with two lease-holding jobs can never run them together', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const entry = LEASED_WORKFLOWS[file]
|
||||
if (entry.reentrant) continue
|
||||
const holding = leaseStepsByJob(file).filter((job) => job.steps.length > 0)
|
||||
if (holding.length <= 1) continue
|
||||
assert.ok(entry.exclusiveBy, `${file} has ${holding.length} lease-holding jobs and no exclusivity guard`)
|
||||
const guards = holding.map((job) => jobIf(jobs(readWorkflow(file)).find((j) => j.id === job.id).text))
|
||||
assert.equal(
|
||||
guards.filter((guard) => guard.includes(entry.exclusiveBy)).length,
|
||||
1,
|
||||
`${file}: exactly one job may run when ${entry.exclusiveBy}`
|
||||
)
|
||||
assert.equal(
|
||||
guards.filter((guard) => guard.includes(entry.exclusiveBy.replace('==', '!='))).length,
|
||||
guards.length - 1,
|
||||
`${file}: every other lease-holding job must be excluded when ${entry.exclusiveBy}`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('census: no workflow rolls out against the shared instance outside the lease', () => {
|
||||
const minters = revisionMintingScripts()
|
||||
assert.ok(minters.size > 0, 'the revision-minting script scan found nothing and is vacuous')
|
||||
const flagged = new Map()
|
||||
for (const file of workflowFiles()) {
|
||||
const reason = mutatesSharedInstance(readWorkflow(file), minters)
|
||||
if (reason) flagged.set(file, reason)
|
||||
}
|
||||
assert.ok(flagged.size > 0, 'the rollout census found no candidates and is vacuous')
|
||||
|
||||
for (const [file, reason] of flagged) {
|
||||
for (const entrypoint of entrypointsFor(file)) {
|
||||
assert.ok(
|
||||
entrypoint in LEASED_WORKFLOWS || entrypoint in NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
`${entrypoint} reaches ${file} (${reason}) but is neither leased nor recorded as a non-candidate`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of workflowFiles()) {
|
||||
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
|
||||
if (!groups.some((group) => LOCK_GROUPS.has(group))) continue
|
||||
assert.ok(
|
||||
file in LEASED_WORKFLOWS || file in NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
`${file} sits in a Cloud SQL lock group but is neither leased nor recorded as a non-candidate`
|
||||
)
|
||||
}
|
||||
|
||||
for (const [file, reason] of Object.entries(NOT_A_CLOUD_SQL_CANDIDATE)) {
|
||||
assert.ok(typeof reason === 'string' && reason.length > 40, `${file} needs a real reason`)
|
||||
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
|
||||
assert.ok(
|
||||
flagged.has(file) || groups.some((group) => LOCK_GROUPS.has(group)),
|
||||
`${file} is recorded as a non-candidate but nothing would have flagged it`
|
||||
)
|
||||
}
|
||||
|
||||
for (const file of leasedFiles) {
|
||||
assert.doesNotThrow(() => readWorkflow(file), `${file} is leased but does not exist`)
|
||||
assert.ok(!(file in NOT_A_CLOUD_SQL_CANDIDATE), `${file} cannot be both leased and a non-candidate`)
|
||||
}
|
||||
})
|
||||
|
||||
// The API and auth deploy scripts share this contract but stay in the private repository.
|
||||
const serviceCapScripts = ['dev/scripts/deploy-relay-blue-green.mjs']
|
||||
|
||||
test('budgets tagged Cloud Run candidates outside the service-wide instance cap', () => {
|
||||
for (const file of serviceCapScripts) {
|
||||
const script = readFileSync(new URL(`../../${file}`, import.meta.url), 'utf8')
|
||||
assert.match(script, /'--no-traffic'/, file)
|
||||
assert.match(script, /'--max'/, file)
|
||||
}
|
||||
const budget = readFileSync(
|
||||
new URL('../../dev/scripts/relay-cloud-sql-connection-budget.mjs', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(budget, /directly addressable tagged revisions outside service-level caps/)
|
||||
assert.match(
|
||||
budget,
|
||||
/apiCandidate: retainedDirectorRollback \+ inputs\.apiInstances \* inputs\.apiPoolMax/
|
||||
)
|
||||
const director = readWorkflow(relayWorkflowFile('deploy-relay-production-director.yml'))
|
||||
const capacity = readWorkflow(relayWorkflowFile('deploy-relay-production-capacity-job.yml'))
|
||||
const asia = readWorkflow(relayWorkflowFile('operate-relay-asia-admission.yml'))
|
||||
assert.match(director, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
|
||||
assert.match(capacity, /--max-instances 5/)
|
||||
assert.match(asia, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
|
||||
})
|
||||
@@ -0,0 +1,31 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const CAPACITY_IDENTITY_NAME = 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT'
|
||||
|
||||
export function readProductionCapacityIdentity(revision) {
|
||||
const env = revision?.spec?.containers?.[0]?.env
|
||||
if (!Array.isArray(env)) throw new Error('director revision environment is missing')
|
||||
const matches = env.filter((entry) => entry?.name === CAPACITY_IDENTITY_NAME)
|
||||
if (matches.length === 0) return null
|
||||
if (matches.length !== 1) throw new Error('duplicate capacity identity')
|
||||
const value = matches[0]?.value
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new Error('capacity identity is not a literal string')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function main() {
|
||||
const revision = JSON.parse(readFileSync(0, 'utf8'))
|
||||
process.stdout.write(`${JSON.stringify(readProductionCapacityIdentity(revision))}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { readProductionCapacityIdentity } from './read-relay-production-capacity-identity.mjs'
|
||||
|
||||
function revision(env) {
|
||||
return { spec: { containers: [{ env }] } }
|
||||
}
|
||||
|
||||
test('reads absent, exact, and foreign literal capacity identities', () => {
|
||||
assert.equal(readProductionCapacityIdentity(revision([])), null)
|
||||
assert.equal(
|
||||
readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'capacity@example.test' }
|
||||
])),
|
||||
'capacity@example.test'
|
||||
)
|
||||
assert.equal(
|
||||
readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'foreign@example.test' }
|
||||
])),
|
||||
'foreign@example.test'
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects malformed or duplicate capacity identity entries', () => {
|
||||
for (const entry of [
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: null },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: '' },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', valueSource: { secretKeyRef: {} } }
|
||||
]) {
|
||||
assert.throws(
|
||||
() => readProductionCapacityIdentity(revision([entry])),
|
||||
/not a literal string/
|
||||
)
|
||||
}
|
||||
assert.throws(
|
||||
() => readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'one@example.test' },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'two@example.test' }
|
||||
])),
|
||||
/duplicate capacity identity/
|
||||
)
|
||||
assert.throws(() => readProductionCapacityIdentity({}), /environment is missing/)
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const SECRET = 'orca-cloud-relay-regional-placement-enabled'
|
||||
|
||||
function validate(input) {
|
||||
for (const key of ['project', 'region', 'service', 'bootstrap_version']) {
|
||||
if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) {
|
||||
throw new Error(`invalid ${key}`)
|
||||
}
|
||||
}
|
||||
if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service')
|
||||
if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) {
|
||||
throw new Error('invalid bootstrap_version')
|
||||
}
|
||||
}
|
||||
|
||||
export function classifyRelayServiceDescribeFailure(args, stderr) {
|
||||
const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe'
|
||||
if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) {
|
||||
return 'NOT_FOUND'
|
||||
}
|
||||
return 'GCLOUD_FAILED'
|
||||
}
|
||||
|
||||
function defaultRun(args) {
|
||||
const result = spawnSync('gcloud', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
maxBuffer: 10 * 1024 * 1024
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
const error = new Error('gcloud read failed')
|
||||
error.code = classifyRelayServiceDescribeFailure(args, result.stderr)
|
||||
throw error
|
||||
}
|
||||
return JSON.parse(result.stdout)
|
||||
}
|
||||
|
||||
function gcloudArguments(kind, input, revision) {
|
||||
return [
|
||||
'run', kind, 'describe', revision ?? input.service,
|
||||
'--project', input.project,
|
||||
'--region', input.region,
|
||||
'--format=json'
|
||||
]
|
||||
}
|
||||
|
||||
export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) {
|
||||
validate(input)
|
||||
const run = dependencies.run ?? defaultRun
|
||||
let service
|
||||
try {
|
||||
service = run(gcloudArguments('services', input))
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
|
||||
throw error
|
||||
}
|
||||
const serving = (service.status?.traffic ?? []).filter(
|
||||
(entry) => Number(entry.percent ?? 0) > 0
|
||||
)
|
||||
if (
|
||||
serving.length !== 1 ||
|
||||
Number(serving[0].percent) !== 100 ||
|
||||
typeof serving[0].revisionName !== 'string'
|
||||
) {
|
||||
throw new Error('Relay director must have exactly one revision serving 100% traffic')
|
||||
}
|
||||
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
|
||||
const references = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter(
|
||||
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
)
|
||||
)
|
||||
if (references.length === 0) return { version: input.bootstrap_version }
|
||||
const reference = normalizeSecretReference(references[0])
|
||||
if (
|
||||
references.length !== 1 ||
|
||||
reference?.secret !== SECRET ||
|
||||
!/^[1-9][0-9]*$/.test(reference?.version ?? '')
|
||||
) {
|
||||
throw new Error('serving regional placement secret reference is invalid')
|
||||
}
|
||||
return { version: reference.version }
|
||||
}
|
||||
|
||||
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
|
||||
// `gcloud run revisions describe --format=json` emits the Knative v1 shape
|
||||
// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path.
|
||||
// A `key` of "latest" is deliberately left invalid: the director's serving
|
||||
// version must be a pinned integer for this data source to mean anything.
|
||||
export function normalizeSecretReference(environment) {
|
||||
const v2 = environment?.valueSource?.secretKeyRef
|
||||
if (v2) return { secret: v2.secret, version: v2.version }
|
||||
const v1 = environment?.valueFrom?.secretKeyRef
|
||||
if (!v1) return undefined
|
||||
const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined
|
||||
return { secret, version: v1.key }
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const chunks = []
|
||||
for await (const chunk of process.stdin) chunks.push(chunk)
|
||||
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
classifyRelayServiceDescribeFailure,
|
||||
readRelayServingRegionalPlacementVersion
|
||||
} from './read-relay-serving-regional-placement-version.mjs'
|
||||
|
||||
const input = {
|
||||
project: 'onorca-cloud',
|
||||
region: 'us-central1',
|
||||
service: 'orca-cloud-relay',
|
||||
bootstrap_version: '7'
|
||||
}
|
||||
|
||||
function revision(version = '11') {
|
||||
return {
|
||||
spec: {
|
||||
containers: [{
|
||||
env: [{
|
||||
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
||||
valueSource: {
|
||||
secretKeyRef: {
|
||||
secret: 'orca-cloud-relay-regional-placement-enabled',
|
||||
version
|
||||
}
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the
|
||||
// secret lives in `name` and the version in `key`, and `name` may be the full resource path.
|
||||
function v1Revision(name, key) {
|
||||
return {
|
||||
spec: {
|
||||
containers: [{
|
||||
env: [{
|
||||
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
||||
valueFrom: { secretKeyRef: { name, key } }
|
||||
}]
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serving() {
|
||||
return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
}
|
||||
|
||||
test('reads the exact version from the sole traffic-serving revision', () => {
|
||||
const calls = []
|
||||
const result = readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => {
|
||||
calls.push(args)
|
||||
return calls.length === 1
|
||||
? {
|
||||
status: {
|
||||
traffic: [
|
||||
{ revisionName: 'relay-failed-latest', tag: 'candidate' },
|
||||
{ revisionName: 'relay-serving', percent: 100 }
|
||||
]
|
||||
}
|
||||
}
|
||||
: revision()
|
||||
}
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { version: '11' })
|
||||
assert.equal(calls[1][3], 'relay-serving')
|
||||
})
|
||||
|
||||
test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => {
|
||||
for (const name of [
|
||||
'orca-cloud-relay-regional-placement-enabled',
|
||||
'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled'
|
||||
]) {
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
|
||||
}), { version: '1' })
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects a v1 reference that names another secret or a floating version', () => {
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? serving()
|
||||
: v1Revision('projects/120364513935/secrets/some-other-secret', '1')
|
||||
}), /secret reference is invalid/)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? serving()
|
||||
: v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest')
|
||||
}), /secret reference is invalid/)
|
||||
})
|
||||
|
||||
test('falls back only when the service or setting is absent', () => {
|
||||
const notFound = new Error('not found')
|
||||
notFound.code = 'NOT_FOUND'
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw notFound }
|
||||
}), { version: '7' })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: { spec: { containers: [{ env: [] }] } }
|
||||
}), { version: '7' })
|
||||
})
|
||||
|
||||
test('classifies real absent-service stderr without weakening revision failures', () => {
|
||||
const serviceArgs = ['run', 'services', 'describe', 'missing-service']
|
||||
const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]'
|
||||
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND')
|
||||
assert.equal(
|
||||
classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr),
|
||||
'GCLOUD_FAILED'
|
||||
)
|
||||
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED')
|
||||
})
|
||||
|
||||
test('rejects ambiguous traffic, malformed references, and read failures', () => {
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => ({
|
||||
status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] }
|
||||
})
|
||||
}), /exactly one revision/)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: revision('latest')
|
||||
}), /secret reference is invalid/)
|
||||
const denied = new Error('denied')
|
||||
denied.code = 'GCLOUD_FAILED'
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw denied }
|
||||
}), denied)
|
||||
})
|
||||
@@ -0,0 +1,277 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
const STATES = ['existing-only', 'migration-only', 'general']
|
||||
|
||||
function normalizeMembership(input) {
|
||||
const membership = {
|
||||
existingOnly: [...input.existingOnly].sort(),
|
||||
migrationOnly: [...input.migrationOnly].sort(),
|
||||
general: [...input.general].sort()
|
||||
}
|
||||
const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general]
|
||||
if (new Set(all).size !== all.length) throw new Error('selector membership contains duplicates')
|
||||
return membership
|
||||
}
|
||||
|
||||
function encodedMembership(membership) {
|
||||
return JSON.stringify(normalizeMembership(membership))
|
||||
}
|
||||
|
||||
function membershipSha256(membership) {
|
||||
return createHash('sha256').update(encodedMembership(membership)).digest('hex')
|
||||
}
|
||||
|
||||
function normalizeMigrationCells(input) {
|
||||
const cells = [...input]
|
||||
.map((cell) => ({
|
||||
cellId: cell.cellId,
|
||||
cellUrl: cell.cellUrl,
|
||||
capacityRequests: cell.capacityRequests,
|
||||
...(cell.region ? { region: cell.region } : {}),
|
||||
connectionHardCap: cell.connectionHardCap,
|
||||
connectionUnobservedBound: cell.connectionUnobservedBound
|
||||
}))
|
||||
.sort((left, right) => left.cellId.localeCompare(right.cellId))
|
||||
if (
|
||||
cells.length === 0 ||
|
||||
new Set(cells.map(({ cellId }) => cellId)).size !== cells.length ||
|
||||
new Set(cells.map(({ cellUrl }) => cellUrl)).size !== cells.length
|
||||
) {
|
||||
throw new Error('migration cell registration must contain distinct cells')
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function membershipWithMigrationCells(membership, cells) {
|
||||
const known = new Set([
|
||||
...membership.existingOnly,
|
||||
...membership.migrationOnly,
|
||||
...membership.general
|
||||
])
|
||||
if (cells.some(({ cellId }) => known.has(cellId))) {
|
||||
throw new Error('migration cell registration contains an existing selector cell')
|
||||
}
|
||||
return normalizeMembership({
|
||||
existingOnly: membership.existingOnly,
|
||||
migrationOnly: [...membership.migrationOnly, ...cells.map(({ cellId }) => cellId)],
|
||||
general: membership.general
|
||||
})
|
||||
}
|
||||
|
||||
function assertSelector(value) {
|
||||
if (
|
||||
!value ||
|
||||
!Number.isSafeInteger(value.generation) ||
|
||||
value.generation < 0 ||
|
||||
!value.membership
|
||||
) {
|
||||
throw new Error('director returned an invalid admission selector')
|
||||
}
|
||||
return {
|
||||
generation: value.generation,
|
||||
attemptId: value.attemptId ?? null,
|
||||
membership: normalizeMembership(value.membership)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectorAttemptId(expectedGeneration, membership) {
|
||||
const digest = createHash('sha256')
|
||||
.update(`${expectedGeneration}:${encodedMembership(membership)}`)
|
||||
.digest('hex')
|
||||
.slice(0, 24)
|
||||
return `selector_${expectedGeneration}_${digest}`
|
||||
}
|
||||
|
||||
export function membershipWithStates(selector, states) {
|
||||
const byCell = new Map()
|
||||
for (const [state, key] of [
|
||||
['existing-only', 'existingOnly'],
|
||||
['migration-only', 'migrationOnly'],
|
||||
['general', 'general']
|
||||
]) {
|
||||
for (const cellId of selector.membership[key]) byCell.set(cellId, state)
|
||||
}
|
||||
for (const [cellId, state] of Object.entries(states)) {
|
||||
if (!byCell.has(cellId)) throw new Error(`selector does not contain ${cellId}`)
|
||||
if (!STATES.includes(state)) throw new Error(`invalid admission state for ${cellId}`)
|
||||
if (byCell.get(cellId) === 'existing-only' && state !== 'existing-only') {
|
||||
throw new Error(`selector cannot re-enable existing-only cell ${cellId}`)
|
||||
}
|
||||
byCell.set(cellId, state)
|
||||
}
|
||||
return normalizeMembership({
|
||||
existingOnly: [...byCell].filter(([, state]) => state === 'existing-only').map(([id]) => id),
|
||||
migrationOnly: [...byCell].filter(([, state]) => state === 'migration-only').map(([id]) => id),
|
||||
general: [...byCell].filter(([, state]) => state === 'general').map(([id]) => id)
|
||||
})
|
||||
}
|
||||
|
||||
export async function inspectAdmissionSelector(post, attemptId) {
|
||||
const result = await post('/v1/admin/admission-selector/status', {
|
||||
v: 1,
|
||||
...(attemptId ? { attemptId } : {})
|
||||
})
|
||||
return {
|
||||
selector: assertSelector(result.selector),
|
||||
intent: result.intent
|
||||
? {
|
||||
...result.intent,
|
||||
previousMembership: result.intent.previousMembership
|
||||
? normalizeMembership(result.intent.previousMembership)
|
||||
: undefined,
|
||||
membership: normalizeMembership(result.intent.membership)
|
||||
}
|
||||
: null
|
||||
}
|
||||
}
|
||||
|
||||
function exactSelector(actual, expected) {
|
||||
return (
|
||||
actual.generation === expected.generation &&
|
||||
encodedMembership(actual.membership) === encodedMembership(expected.membership)
|
||||
)
|
||||
}
|
||||
|
||||
export async function applyExactAdmissionSelector(post, membership, options = {}) {
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const desired = normalizeMembership(membership)
|
||||
if (
|
||||
options.expectedCurrentSelector &&
|
||||
!exactSelector(before.selector, options.expectedCurrentSelector)
|
||||
) {
|
||||
throw new Error('admission selector changed before exact apply')
|
||||
}
|
||||
if (options.requireBoundary !== false && before.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
if (encodedMembership(before.selector.membership) === encodedMembership(desired)) {
|
||||
return { changed: false, selector: before.selector }
|
||||
}
|
||||
const attemptId =
|
||||
options.attemptId ?? selectorAttemptId(before.selector.generation, desired)
|
||||
const expected = {
|
||||
generation: before.selector.generation + 1,
|
||||
membership: desired
|
||||
}
|
||||
let result
|
||||
try {
|
||||
result = await post('/v1/admin/admission-selector/apply', {
|
||||
v: 1,
|
||||
attemptId,
|
||||
expectedGeneration: before.selector.generation,
|
||||
...(before.selector.generation === 0
|
||||
? { expectedMembershipSha256: membershipSha256(before.selector.membership) }
|
||||
: {}),
|
||||
membership: desired
|
||||
})
|
||||
} catch (error) {
|
||||
const inspected = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
inspected.intent?.state === 'committed' &&
|
||||
exactSelector(inspected.selector, expected)
|
||||
) {
|
||||
return { changed: true, selector: inspected.selector, recovered: true }
|
||||
}
|
||||
if (
|
||||
inspected.intent?.state === 'unchanged' &&
|
||||
exactSelector(inspected.selector, before.selector)
|
||||
) {
|
||||
throw new Error('admission selector apply remained unchanged after an ambiguous response', {
|
||||
cause: error
|
||||
})
|
||||
}
|
||||
throw new Error('admission selector apply diverged after an ambiguous response', {
|
||||
cause: error
|
||||
})
|
||||
}
|
||||
const applied = assertSelector(result.selector)
|
||||
if (!exactSelector(applied, expected)) {
|
||||
throw new Error('admission selector apply returned unexpected membership')
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
verified.intent?.state !== 'committed' ||
|
||||
!exactSelector(verified.selector, expected)
|
||||
) {
|
||||
throw new Error('admission selector commit could not be verified')
|
||||
}
|
||||
return { changed: result.changed === true, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function addExactMigrationCells(post, input, options = {}) {
|
||||
const cells = normalizeMigrationCells(input.cells)
|
||||
const attemptId = input.attemptId
|
||||
if (!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
|
||||
throw new Error('migration cell registration requires an exact attempt ID')
|
||||
}
|
||||
const before = await inspectAdmissionSelector(post, attemptId)
|
||||
let expectedGeneration
|
||||
let expectedMembership
|
||||
if (before.intent) {
|
||||
expectedGeneration = before.intent.expectedGeneration
|
||||
expectedMembership = normalizeMembership(before.intent.membership)
|
||||
} else {
|
||||
if (before.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
if (
|
||||
options.expectedCurrentSelector &&
|
||||
!exactSelector(before.selector, options.expectedCurrentSelector)
|
||||
) {
|
||||
throw new Error('admission selector changed before cell registration')
|
||||
}
|
||||
expectedGeneration = before.selector.generation
|
||||
expectedMembership = membershipWithMigrationCells(before.selector.membership, cells)
|
||||
}
|
||||
const expected = {
|
||||
generation: expectedGeneration + 1,
|
||||
membership: expectedMembership
|
||||
}
|
||||
let result
|
||||
try {
|
||||
result = await post('/v1/admin/admission-selector/add-migration-cells', {
|
||||
v: 1,
|
||||
attemptId,
|
||||
expectedGeneration,
|
||||
cells
|
||||
})
|
||||
} catch (error) {
|
||||
const inspected = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
!before.intent &&
|
||||
inspected.intent?.state === 'committed' &&
|
||||
exactSelector(inspected.selector, expected)
|
||||
) {
|
||||
return { changed: true, selector: inspected.selector, recovered: true }
|
||||
}
|
||||
throw new Error('migration cell registration did not commit exactly', { cause: error })
|
||||
}
|
||||
const applied = assertSelector(result.selector)
|
||||
if (!exactSelector(applied, expected)) {
|
||||
throw new Error('migration cell registration returned unexpected membership')
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post, attemptId)
|
||||
if (verified.intent?.state !== 'committed' || !exactSelector(verified.selector, expected)) {
|
||||
throw new Error('migration cell registration commit could not be verified')
|
||||
}
|
||||
return { changed: result.changed === true, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function transitionAdmissionSelector(post, states, options = {}) {
|
||||
const current = await inspectAdmissionSelector(post)
|
||||
if (current.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
return await applyExactAdmissionSelector(
|
||||
post,
|
||||
membershipWithStates(current.selector, states),
|
||||
options
|
||||
)
|
||||
}
|
||||
|
||||
export function selectorCellState(selector, cellId) {
|
||||
if (selector.membership.existingOnly.includes(cellId)) return 'existing-only'
|
||||
if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only'
|
||||
if (selector.membership.general.includes(cellId)) return 'general'
|
||||
throw new Error(`selector does not contain ${cellId}`)
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
addExactMigrationCells,
|
||||
applyExactAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorAttemptId,
|
||||
transitionAdmissionSelector
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const initialMembership = {
|
||||
existingOnly: ['legacy'],
|
||||
migrationOnly: ['target'],
|
||||
general: ['general']
|
||||
}
|
||||
|
||||
function selectorHarness({ ambiguous = null, generation = 1 } = {}) {
|
||||
let selector = { generation, attemptId: 'initial', membership: initialMembership }
|
||||
const intents = new Map()
|
||||
const requests = []
|
||||
let applies = 0
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
|
||||
}
|
||||
}
|
||||
applies++
|
||||
requests.push(body)
|
||||
const before = selector
|
||||
const committed = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
intents.set(body.attemptId, {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: committed.generation,
|
||||
previousMembership: before.membership,
|
||||
membership: body.membership,
|
||||
state: ambiguous === 'unchanged' ? 'unchanged' : 'committed'
|
||||
})
|
||||
if (ambiguous !== 'unchanged') selector = committed
|
||||
if (ambiguous) throw new Error('lost selector response')
|
||||
return { changed: true, selector }
|
||||
}
|
||||
return { post, selector: () => selector, applies: () => applies, requests }
|
||||
}
|
||||
|
||||
test('derives deterministic attempts and applies exact selector transitions', async () => {
|
||||
const harness = selectorHarness()
|
||||
const desired = membershipWithStates(harness.selector(), { target: 'general' })
|
||||
assert.equal(
|
||||
selectorAttemptId(1, desired),
|
||||
selectorAttemptId(1, {
|
||||
existingOnly: ['legacy'],
|
||||
migrationOnly: [],
|
||||
general: ['target', 'general']
|
||||
})
|
||||
)
|
||||
const result = await transitionAdmissionSelector(harness.post, { target: 'general' })
|
||||
assert.equal(result.selector.generation, 2)
|
||||
assert.deepEqual(result.selector.membership.general, ['general', 'target'])
|
||||
})
|
||||
|
||||
test('accepts only an exact committed result after an ambiguous response', async () => {
|
||||
const harness = selectorHarness({ ambiguous: 'committed' })
|
||||
const result = await applyExactAdmissionSelector(harness.post, {
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(harness.applies(), 1)
|
||||
assert.equal(result.selector.generation, 2)
|
||||
})
|
||||
|
||||
test('binds a generation-zero cutover to the inspected membership', async () => {
|
||||
const harness = selectorHarness({ generation: 0 })
|
||||
await applyExactAdmissionSelector(
|
||||
harness.post,
|
||||
{
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
},
|
||||
{ requireBoundary: false }
|
||||
)
|
||||
assert.equal(
|
||||
harness.requests[0].expectedMembershipSha256,
|
||||
createHash('sha256').update(JSON.stringify(initialMembership)).digest('hex')
|
||||
)
|
||||
})
|
||||
|
||||
test('stops on an unchanged ambiguous result without replaying', async () => {
|
||||
const harness = selectorHarness({ ambiguous: 'unchanged' })
|
||||
await assert.rejects(
|
||||
applyExactAdmissionSelector(harness.post, {
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
}),
|
||||
/remained unchanged/
|
||||
)
|
||||
assert.equal(harness.applies(), 1)
|
||||
assert.equal(harness.selector().generation, 1)
|
||||
})
|
||||
|
||||
test('never restores an existing-only cell', () => {
|
||||
assert.throws(
|
||||
() => membershipWithStates({ membership: initialMembership }, { legacy: 'general' }),
|
||||
/cannot re-enable/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses an exact apply after the inspected selector changes', async () => {
|
||||
const harness = selectorHarness()
|
||||
await assert.rejects(
|
||||
applyExactAdmissionSelector(
|
||||
harness.post,
|
||||
{
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
},
|
||||
{
|
||||
expectedCurrentSelector: {
|
||||
generation: 0,
|
||||
membership: {
|
||||
existingOnly: ['target'],
|
||||
migrationOnly: [],
|
||||
general: ['general', 'legacy']
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
/changed before exact apply/
|
||||
)
|
||||
assert.equal(harness.applies(), 0)
|
||||
})
|
||||
|
||||
test('adds exact migration cells and recovers a committed response loss', async () => {
|
||||
let selector = { generation: 1, attemptId: 'initial', membership: initialMembership }
|
||||
const intents = new Map()
|
||||
let additions = 0
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
|
||||
}
|
||||
}
|
||||
additions++
|
||||
selector = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: {
|
||||
...selector.membership,
|
||||
migrationOnly: [
|
||||
...selector.membership.migrationOnly,
|
||||
...body.cells.map(({ cellId }) => cellId)
|
||||
].sort()
|
||||
}
|
||||
}
|
||||
intents.set(body.attemptId, {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership,
|
||||
state: 'committed'
|
||||
})
|
||||
throw new Error('lost cell registration response')
|
||||
}
|
||||
const result = await addExactMigrationCells(post, {
|
||||
attemptId: 'add_cells_exact',
|
||||
cells: [
|
||||
{
|
||||
cellId: 'target-2',
|
||||
cellUrl: 'https://target-2.example.com',
|
||||
capacityRequests: 4_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(additions, 1)
|
||||
assert.deepEqual(result.selector.membership.migrationOnly, ['target', 'target-2'])
|
||||
})
|
||||
|
||||
test('does not recover an attempt owned by another selector operation', async () => {
|
||||
const selector = {
|
||||
generation: 2,
|
||||
attemptId: 'selector_collision',
|
||||
membership: initialMembership
|
||||
}
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId
|
||||
? {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: 1,
|
||||
intendedGeneration: 2,
|
||||
membership: initialMembership,
|
||||
state: 'committed'
|
||||
}
|
||||
: null
|
||||
}
|
||||
}
|
||||
throw new Error('admission_selector_attempt_mismatch')
|
||||
}
|
||||
await assert.rejects(
|
||||
addExactMigrationCells(post, {
|
||||
attemptId: 'selector_collision',
|
||||
cells: [
|
||||
{
|
||||
cellId: 'target-2',
|
||||
cellUrl: 'https://target-2.example.com',
|
||||
capacityRequests: 4_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
}),
|
||||
/did not commit exactly/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,323 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('operate-relay-asia-admission.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const iam = readFileSync(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), 'utf8')
|
||||
const stagingProof = readFileSync(
|
||||
relayWorkflowUrl('prove-relay-asia-staging.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const directorWorkflow = readFileSync(
|
||||
relayWorkflowUrl('deploy-relay-production-director.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const terraformReadme = readFileSync(
|
||||
new URL('../../infra/terraform/README.md', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const proofIam = readFileSync(
|
||||
new URL('../../infra/terraform/relay-asia-proof-iam.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const relayTerraform = readFileSync(
|
||||
new URL('../../infra/terraform/relay.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const rolloutEvidence = readFileSync(
|
||||
new URL('./relay-asia-rollout-evidence.mjs', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const admissionBudgets = readFileSync(
|
||||
new URL('../../packages/relay-contract/src/admission-budgets.ts', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
test('offers the exact audited admission modes under the shared deployment lock', () => {
|
||||
for (const mode of [
|
||||
'inspect', 'initialize', 'verify', 'register', 'configure', 'promote', 'rollback'
|
||||
]) {
|
||||
assert.match(workflow, new RegExp(`\\b${mode}\\b`))
|
||||
}
|
||||
assert.match(workflow, /production-cloud-sql-rollout/)
|
||||
assert.match(workflow, /relay-staging-mutation/)
|
||||
assert.match(workflow, /selector-generation/)
|
||||
assert.match(workflow, /selector-attempt-id/)
|
||||
})
|
||||
|
||||
test('requires exact confirmations and uses the existing admin identity', () => {
|
||||
assert.match(workflow, /INITIALIZE_ADMISSION_SELECTOR/)
|
||||
assert.match(workflow, /REGISTER_ASIA_MIGRATION_ONLY/)
|
||||
assert.match(workflow, /PROMOTE_ASIA_GENERAL/)
|
||||
assert.match(workflow, /ROLLBACK_ASIA_MIGRATION_ONLY/)
|
||||
assert.match(workflow, /CONFIGURE_ASIA_DIRECTOR/)
|
||||
assert.match(workflow, /GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.match(workflow, /id_token_audience: \$\{\{ env\.DIRECTOR_ORIGIN \}\}\/v1\/admin\/drain/)
|
||||
assert.match(iam, /"operate-relay-asia-admission\.yml"/)
|
||||
})
|
||||
|
||||
test('discovers generation read-only and explicitly initializes only generation zero', () => {
|
||||
assert.match(workflow, /leave empty only for inspect/)
|
||||
assert.match(workflow, /test -z "\$\{EXPECTED_SELECTOR_GENERATION\}"/)
|
||||
assert.match(workflow, /test "\$\{EXPECTED_SELECTOR_GENERATION\}" = 0/)
|
||||
assert.match(workflow, /\^\(0\|\[1-9\]\[0-9\]\*\)\$/)
|
||||
assert.match(workflow, /selector-membership-sha256/)
|
||||
assert.match(workflow, /\^\[a-f0-9\]\{64\}\$/)
|
||||
assert.match(workflow, /director-image-digest/)
|
||||
assert.match(workflow, /\.spec\.containers\[0\]\.image == \$image/)
|
||||
})
|
||||
|
||||
test('uploads one sanitized machine-readable admission result', () => {
|
||||
assert.match(workflow, /sanitize-relay-asia-admission-result\.mjs/)
|
||||
const upload = /- name: Upload sanitized admission result\n([\s\S]*?)(?=\n - name:)/
|
||||
.exec(workflow)?.[1]
|
||||
assert.ok(upload)
|
||||
assert.match(
|
||||
upload,
|
||||
/if: \$\{\{ inputs\.mode != 'configure' && steps\.admission-operation\.outcome == 'success' \}\}/
|
||||
)
|
||||
assert.match(upload, /uses: actions\/upload-artifact@v4/)
|
||||
assert.match(
|
||||
upload,
|
||||
/relay-asia-admission-result-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/
|
||||
)
|
||||
assert.match(upload, /path: \$\{\{ runner\.temp \}\}\/relay-asia-admission-result\/result\.json/)
|
||||
assert.match(upload, /if-no-files-found: error/)
|
||||
assert.match(upload, /retention-days: 7/)
|
||||
assert.ok(
|
||||
workflow.indexOf('Upload sanitized admission result') >
|
||||
workflow.indexOf('Upload immutable C27 canary evidence')
|
||||
)
|
||||
})
|
||||
|
||||
test('binds selector operations and director configuration to reviewed implementations', () => {
|
||||
assert.match(workflow, /operate-relay-asia-admission\.mjs/)
|
||||
assert.match(workflow, /prepare-relay-asia-director-cells\.mjs/)
|
||||
assert.match(workflow, /deploy-relay-blue-green\.mjs/)
|
||||
assert.match(workflow, /--prune-revisions false/)
|
||||
assert.doesNotMatch(workflow, /gcloud secrets versions add/)
|
||||
assert.match(workflow, /orca-cloud-relay-regional-placement-enabled/)
|
||||
assert.match(workflow, /\.valueSource\.secretKeyRef/)
|
||||
assert.match(workflow, /jq -er --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/)
|
||||
assert.doesNotMatch(workflow, /jq -e --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/)
|
||||
assert.doesNotMatch(workflow, /--regional-placement-enabled/)
|
||||
assert.doesNotMatch(workflow, /"\$\{\{ inputs\./)
|
||||
assert.doesNotMatch(workflow, /dns/i)
|
||||
})
|
||||
|
||||
test('requires immutable staged evidence and a timed C27 canary before expansion', () => {
|
||||
assert.match(workflow, /actions: read/)
|
||||
assert.match(workflow, /actions\/download-artifact@v4/)
|
||||
assert.match(workflow, /relay-asia-staging-\$\{EVIDENCE_RUN_ID\}-\$\{EVIDENCE_RUN_ATTEMPT\}/)
|
||||
assert.match(workflow, /evidence_kind=staging/)
|
||||
assert.match(workflow, /load-relay-controls\.mjs/)
|
||||
assert.match(workflow, /--controls 1/)
|
||||
assert.match(workflow, /--splices 1/)
|
||||
assert.match(workflow, /--splice-hold-seconds 60/)
|
||||
assert.match(workflow, /--relay-asia-load-principals 1/)
|
||||
assert.match(workflow, /--duration-seconds 300/)
|
||||
assert.match(workflow, /--required-lease-horizons 2/)
|
||||
assert.match(workflow, /pnpm\/action-setup@v4/)
|
||||
assert.match(workflow, /Install exact C27 canary dependencies/)
|
||||
assert.match(workflow, /pnpm install --frozen-lockfile/)
|
||||
assert.match(workflow, /pnpm --filter @orca-cloud\/relay-contract build/)
|
||||
assert.ok(
|
||||
workflow.indexOf('Build the C27 canary Relay contract') <
|
||||
workflow.indexOf('Run a real five-minute C27 control and splice canary')
|
||||
)
|
||||
assert.match(workflow, /--load-report "\$\{RUNNER_TEMP\}\/relay-asia-c27-load\.json"/)
|
||||
assert.match(workflow, /states\["production-gce-c28"\].*= migration-only/)
|
||||
assert.match(workflow, /states\["production-gce-c29"\].*= migration-only/)
|
||||
assert.match(workflow, /relay-asia-c27-canary-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
|
||||
assert.match(workflow, /id: c27-evidence-upload/)
|
||||
assert.match(workflow, /Return an unproven C27 canary to migration-only/)
|
||||
assert.match(workflow, /steps\.c27-evidence-upload\.outcome != 'success'/)
|
||||
assert.match(workflow, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}"/)
|
||||
assert.match(workflow, /--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}-rollback"/)
|
||||
assert.match(workflow, /evidence_kind=c27/)
|
||||
assert.match(workflow, /orca_relay_runtime_metrics/)
|
||||
assert.match(workflow, /relay-asia-rollout-evidence\.mjs create-c27/)
|
||||
assert.match(workflow, /retention-days: 7/)
|
||||
assert.match(workflow, /Require the exact director image before promotion/)
|
||||
assert.match(workflow, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/)
|
||||
assert.match(workflow, /\.imageDigest.*IMAGE_DIGEST/)
|
||||
const provenance = /- name: Verify evidence provenance and rollout binding before authentication\n([\s\S]*?)(?=\n - id: auth)/
|
||||
.exec(workflow)?.[1]
|
||||
assert.ok(provenance)
|
||||
assert.match(provenance, /\.head_sha \| select\(type == "string" and test\("\^\[a-f0-9\]\{40\}\$"\)\)/)
|
||||
assert.match(provenance, /--commit-sha "\$\{evidence_commit_sha\}"/)
|
||||
assert.doesNotMatch(provenance, /--commit-sha "\$\{GITHUB_SHA\}"/)
|
||||
})
|
||||
|
||||
test('creates staging evidence only after the bounded launch-path load and rollback', () => {
|
||||
assert.match(stagingProof, /runs-on: \[self-hosted, linux, x64, relay-asia-east2-load\]/)
|
||||
assert.doesNotMatch(stagingProof, /group: relay-asia-east2-load/)
|
||||
assert.match(stagingProof, /pnpm\/action-setup@v4/)
|
||||
assert.match(stagingProof, /pnpm install --frozen-lockfile/)
|
||||
assert.match(stagingProof, /pnpm --filter @orca-cloud\/relay-contract build/)
|
||||
assert.match(stagingProof, /run_phase launch 5 5/)
|
||||
assert.doesNotMatch(stagingProof, /run_phase control|run_phase mixed/)
|
||||
assert.match(stagingProof, /--aggregate-controls "\$\(\(controls \* 4\)\)"/)
|
||||
assert.match(stagingProof, /--aggregate-splices "\$\(\(splices \* 4\)\)"/)
|
||||
assert.match(stagingProof, /--required-lease-horizons 2/)
|
||||
assert.match(stagingProof, /--splice-ramp-seconds 120/)
|
||||
assert.match(stagingProof, /--max-generator-rss-growth-mib 512/)
|
||||
assert.match(stagingProof, /--relay-asia-load-principals 32/)
|
||||
assert.match(stagingProof, /ulimit -n/)
|
||||
assert.match(stagingProof, /--region-behavior-probes 1/)
|
||||
assert.match(stagingProof, /--capacity-cell-origin https:\/\/c4\.relay-staging\.onorca\.dev/)
|
||||
assert.match(stagingProof, /--rebind-probes 2/)
|
||||
assert.match(stagingProof, /--skip-rebind-overflow-check/)
|
||||
assert.doesNotMatch(stagingProof, /--request-unit-invites|--regional-fallback-probes/)
|
||||
assert.match(stagingProof, /--aggregate-reader-splices.*echo 5/)
|
||||
assert.match(stagingProof, /--aggregate-reader-bytes.*echo 12582912/)
|
||||
assert.match(stagingProof, /--phase-barrier-dir "\$\{proof_dir\}\/\$\{phase\}-barrier"/)
|
||||
assert.match(stagingProof, /--duration-seconds 210/)
|
||||
assert.match(stagingProof, /trap stop_shards EXIT/)
|
||||
assert.match(stagingProof, /if ! wait "\$\{pid\}"; then failed=1; break; fi/)
|
||||
assert.match(stagingProof, /connectionFailuresByReason/)
|
||||
assert.match(stagingProof, /--launch-report "\$\{proof_dir\}\/launch\.json"/)
|
||||
assert.match(stagingProof, /id-token: write/)
|
||||
assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(stagingProof, /STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(stagingProof, /STAGING_RELAY_LOAD_ACCESS_TOKEN/)
|
||||
assert.doesNotMatch(stagingProof, /secrets versions access|signing-key-file/)
|
||||
assert.match(stagingProof, /relay-asia-rollout-evidence\.mjs create-staging/)
|
||||
assert.match(stagingProof, /Require the exact staging director image before promotion/)
|
||||
assert.match(stagingProof, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/)
|
||||
assert.match(stagingProof, /\.imageDigest.*IMAGE_DIGEST/)
|
||||
assert.match(stagingProof, /Return staging C4 to migration-only/)
|
||||
assert.match(stagingProof, /steps\.promote\.outcome != 'skipped'/)
|
||||
assert.match(stagingProof, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{PROMOTE_ATTEMPT_ID\}"/)
|
||||
assert.match(stagingProof, /--mode rollback[\s\S]*?--expected-generation "\$\{promoted_generation\}"/)
|
||||
assert.match(stagingProof, /if: \$\{\{ success\(\) \}\}/)
|
||||
assert.match(
|
||||
stagingProof,
|
||||
/recover:\n if: \$\{\{ always\(\) && github\.ref == 'refs\/heads\/main' \}\}/
|
||||
)
|
||||
assert.match(stagingProof, /needs: prove/)
|
||||
assert.match(stagingProof, /Recover staging C4 with a fresh identity/)
|
||||
assert.equal((stagingProof.match(/google-github-actions\/auth@v2/g) ?? []).length, 2)
|
||||
assert.equal((stagingProof.match(/--mode recover-promotion/g) ?? []).length, 2)
|
||||
assert.equal((stagingProof.match(/--mode rollback/g) ?? []).length, 2)
|
||||
})
|
||||
|
||||
test('keeps the private runner below its 64-port Cloud NAT allocation', () => {
|
||||
const profile = /run_phase launch (\d+) (\d+)/.exec(stagingProof)
|
||||
const controlsPerShard = Number(profile?.[1])
|
||||
const splicesPerShard = Number(profile?.[2])
|
||||
const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1])
|
||||
const runtimeStatusSockets = 1
|
||||
assert.ok(
|
||||
controlsPerShard * 4 + splicesPerShard * 4 * 2 + rebindProbes + runtimeStatusSockets < 64
|
||||
)
|
||||
})
|
||||
|
||||
test('paces one-source staging upgrades below the Relay anti-abuse ceiling', () => {
|
||||
const splicesPerShard = Number(/run_phase launch \d+ (\d+)/.exec(stagingProof)?.[1])
|
||||
const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1])
|
||||
const spliceRampMs = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1]) * 1000
|
||||
const ceiling = Number(
|
||||
/maxPreAuthAttemptsPerSourcePerMinute: (\d+)/.exec(admissionBudgets)?.[1]
|
||||
)
|
||||
const totalSplices = splicesPerShard * 4
|
||||
const attempts = Array.from({ length: totalSplices }, (_, ordinal) =>
|
||||
Math.floor(ordinal * spliceRampMs / (totalSplices - 1))
|
||||
).flatMap((startedAt) => [startedAt, startedAt])
|
||||
attempts.push(...Array.from({ length: 4 + rebindProbes }, () => 0))
|
||||
const busiestMinute = Math.max(...attempts.map((startedAt) =>
|
||||
attempts.filter((attempt) => attempt >= startedAt && attempt < startedAt + 60_000).length
|
||||
))
|
||||
assert.ok(busiestMinute < ceiling)
|
||||
})
|
||||
|
||||
test('reserves rollback time beyond the complete bounded staging proof envelope', () => {
|
||||
const timeoutMinutes = Number(/timeout-minutes: (\d+)/.exec(stagingProof)?.[1])
|
||||
assert.equal(timeoutMinutes, 75)
|
||||
const spliceRampSeconds = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1])
|
||||
const launchSeconds = 180 + spliceRampSeconds + 210 + 60
|
||||
const setupEvidenceAndRollbackSeconds = 10 * 60
|
||||
const envelopeMinutes = Math.ceil((launchSeconds + setupEvidenceAndRollbackSeconds) / 60)
|
||||
assert.ok(timeoutMinutes - envelopeMinutes >= 30)
|
||||
assert.match(stagingProof, /--ramp-seconds 180/)
|
||||
assert.match(stagingProof, /--duration-seconds 210/)
|
||||
})
|
||||
|
||||
test('binds the staging proof to one least-privilege Google identity', () => {
|
||||
assert.match(
|
||||
proofIam,
|
||||
/github_relay_asia_proof_workflow_file = "prove-relay-asia-staging\.yml"/
|
||||
)
|
||||
assert.match(
|
||||
proofIam,
|
||||
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_proof_workflow_file\}@refs\/heads\/main'/
|
||||
)
|
||||
assert.match(proofIam, /assertion\.environment == 'staging'/)
|
||||
assert.match(proofIam, /assertion\.event_name == 'workflow_dispatch'/)
|
||||
assert.match(proofIam, /roles\/logging\.viewer/)
|
||||
assert.match(proofIam, /roles\/monitoring\.viewer/)
|
||||
assert.match(rolloutEvidence, /readCloudSqlBackends/)
|
||||
assert.match(rolloutEvidence, /cloudSql: await readCloudSqlBackends/)
|
||||
assert.doesNotMatch(proofIam, /compute\.|cloudsql\.|secretmanager\.|roles\/editor|roles\/run\./)
|
||||
})
|
||||
|
||||
test('keeps the production US-first switch in durable Secret Manager state', () => {
|
||||
assert.match(directorWorkflow, /options: \[preserve, enable, disable\]/)
|
||||
assert.match(directorWorkflow, /default: preserve/)
|
||||
assert.match(directorWorkflow, /gcloud secrets versions add/)
|
||||
assert.match(directorWorkflow, /preserve\) desired="\$\{current\}"/)
|
||||
assert.match(directorWorkflow, /--regional-placement-secret-version "\$\{target_version\}"/)
|
||||
assert.match(directorWorkflow, /test "\$\{CEILING\}" = "\$\{DIRECTOR_MAX_INSTANCES\}"/)
|
||||
assert.match(directorWorkflow, /orca-cloud-relay-regional-placement-enabled/)
|
||||
assert.match(directorWorkflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/)
|
||||
assert.match(directorWorkflow, /\.version \/\/ \.key/)
|
||||
assert.match(directorWorkflow, /\.secret \/\/ \.name/)
|
||||
assert.match(workflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/)
|
||||
assert.doesNotMatch(directorWorkflow, /--regional-placement-enabled/)
|
||||
assert.doesNotMatch(workflow, /inputs\.regional-placement-enabled/)
|
||||
})
|
||||
|
||||
test('prunes incompatible production revisions only when explicitly confirmed', () => {
|
||||
assert.match(
|
||||
directorWorkflow,
|
||||
/prune-incompatible-revisions:[\s\S]*?default: false[\s\S]*?type: boolean/
|
||||
)
|
||||
assert.match(directorWorkflow, /PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/)
|
||||
assert.match(
|
||||
directorWorkflow,
|
||||
/test "\$\{REGIONAL_PLACEMENT_MODE\}" = preserve[\s\S]*?test "\$\{CONFIRMATION\}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/
|
||||
)
|
||||
assert.match(
|
||||
directorWorkflow,
|
||||
/--prune-revisions "\$\{PRUNE_INCOMPATIBLE_REVISIONS\}"/
|
||||
)
|
||||
})
|
||||
|
||||
test('documents the exact regional-placement secret bootstrap before director rollout', () => {
|
||||
for (const address of [
|
||||
'google_secret_manager_secret.relay_regional_placement_enabled',
|
||||
'google_secret_manager_secret_version.relay_regional_placement_enabled',
|
||||
'google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor',
|
||||
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]',
|
||||
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]',
|
||||
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]'
|
||||
]) {
|
||||
assert.match(terraformReadme, new RegExp(address.replaceAll(/[.[\]]/g, '\\$&')))
|
||||
}
|
||||
assert.match(terraformReadme, /Before the first director deployment/)
|
||||
assert.match(terraformReadme, /Pass the exact environment tfvars/)
|
||||
// The Cloudflare records left with the apps root; a -var for a variable this root no longer
|
||||
// declares is a hard error, so no relay procedure may still tell an operator to pass it.
|
||||
assert.doesNotMatch(terraformReadme, /manage_artifact_dns/)
|
||||
assert.match(terraformReadme, /exactly these six additions/)
|
||||
assert.match(terraformReadme, /version metadata/)
|
||||
assert.match(
|
||||
relayTerraform,
|
||||
/resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_viewer"[\s\S]*?role\s+= "roles\/secretmanager\.viewer"/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,33 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
function accessToken() {
|
||||
const result = spawnSync('gcloud', ['auth', 'print-access-token'], {
|
||||
encoding: 'utf8', timeout: 30_000
|
||||
})
|
||||
const token = result.stdout.trim()
|
||||
if (result.status !== 0 || token.length < 20) {
|
||||
throw new Error('Google access token is unavailable')
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
export async function readCloudSqlBackends(environment, startedAt, endedAt) {
|
||||
const production = environment === 'production'
|
||||
if (!production && environment !== 'staging') throw new Error('Cloud SQL environment is invalid')
|
||||
const project = production ? 'onorca-cloud' : 'onorca-cloud-staging'
|
||||
const instance = production ? 'orca-cloud-auth-db' : 'orca-cloud-staging-auth-db'
|
||||
const url = new URL(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries`)
|
||||
url.searchParams.set('filter', `metric.type = "cloudsql.googleapis.com/database/postgresql/num_backends" AND resource.labels.database_id = "${project}:${instance}"`)
|
||||
url.searchParams.set('interval.startTime', startedAt)
|
||||
url.searchParams.set('interval.endTime', endedAt)
|
||||
url.searchParams.set('aggregation.alignmentPeriod', '60s')
|
||||
url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_MAX')
|
||||
url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_MAX')
|
||||
url.searchParams.set('view', 'FULL')
|
||||
const response = await fetch(url, {
|
||||
headers: { authorization: `Bearer ${accessToken()}` },
|
||||
redirect: 'error', signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Cloud SQL metric query returned ${response.status}`)
|
||||
return await response.json()
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readCloudSqlBackends } from './relay-asia-cloud-sql-metrics.mjs'
|
||||
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
|
||||
|
||||
const REPOSITORY = RELAY_GITHUB_REPOSITORY
|
||||
const ADMISSION_WORKFLOW = relayWorkflowPath('operate-relay-asia-admission.yml')
|
||||
const STAGING_WORKFLOW = relayWorkflowPath('prove-relay-asia-staging.yml')
|
||||
const STAGING_CELL = 'staging-gce-c4'
|
||||
const C27 = 'production-gce-c27'
|
||||
const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/
|
||||
const SHA_PATTERN = /^[a-f0-9]{40}$/
|
||||
const MAX_LOG_EDGE_GAP_MS = 120_000
|
||||
const MAX_LOG_SAMPLE_GAP_MS = 120_000
|
||||
const CLOUD_SQL_LIMIT = 320
|
||||
const C27_CANARY_MINIMUM_MS = 5 * 60_000
|
||||
const GENERATOR_CPU_PERCENT_LIMIT = 80
|
||||
const GENERATOR_EVENT_LOOP_P99_MS_LIMIT = 100
|
||||
const GENERATOR_RSS_GROWTH_MIB_LIMIT = 512
|
||||
const DATABASE_POOL_TRANSIENT_WAITERS_MAX = 4
|
||||
const DATABASE_POOL_TRANSIENT_WAIT_MS_MAX = 50
|
||||
|
||||
function object(value, label) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveInteger(value, label) {
|
||||
const number = Number(value)
|
||||
if (!Number.isSafeInteger(number) || number < 1) throw new Error(`${label} is invalid`)
|
||||
return number
|
||||
}
|
||||
|
||||
function instant(value, label) {
|
||||
const date = new Date(value)
|
||||
if (!Number.isFinite(date.valueOf())) throw new Error(`${label} is invalid`)
|
||||
return date
|
||||
}
|
||||
|
||||
function exactCells(actual, expected) {
|
||||
return Array.isArray(actual) &&
|
||||
JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort())
|
||||
}
|
||||
|
||||
function source(input, environment, workflow) {
|
||||
if (input.repository !== REPOSITORY) throw new Error('repository is invalid')
|
||||
if (!SHA_PATTERN.test(input.commitSha)) throw new Error('commit SHA is invalid')
|
||||
return {
|
||||
repository: input.repository,
|
||||
workflow,
|
||||
environment,
|
||||
runId: positiveInteger(input.runId, 'run ID'),
|
||||
runAttempt: positiveInteger(input.runAttempt, 'run attempt'),
|
||||
commitSha: input.commitSha
|
||||
}
|
||||
}
|
||||
|
||||
function baseEvidence(input, environment, cells, workflow = ADMISSION_WORKFLOW) {
|
||||
if (!DIGEST_PATTERN.test(input.imageDigest)) throw new Error('image digest is invalid')
|
||||
return {
|
||||
version: 1,
|
||||
source: source(input, environment, workflow),
|
||||
imageDigest: input.imageDigest,
|
||||
topology: {
|
||||
cellIds: cells,
|
||||
selectorGeneration: positiveInteger(input.selectorGeneration, 'selector generation')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildStagingEvidence(input) {
|
||||
const start = instant(input.startedAt, 'staging proof start')
|
||||
const end = instant(input.endedAt, 'staging proof end')
|
||||
const launch = loadReports(input.launchReport, 'launch load report')
|
||||
const expectedLoad = {
|
||||
controls: 20, splices: 20, slowReaders: 4, wedgedReaders: 1,
|
||||
minimumSeconds: 210
|
||||
}
|
||||
assertLoadReports(launch, expectedLoad)
|
||||
const metrics = runtimeMetrics(input.logs, start, end, STAGING_CELL)
|
||||
assertPassingRuntimeMetrics(metrics, 'staging proof', 1)
|
||||
const minimumConcurrentSplices = expectedLoad.splices - expectedLoad.wedgedReaders
|
||||
if (
|
||||
metrics.targetControlsMax < expectedLoad.controls ||
|
||||
metrics.targetSplicesMax < minimumConcurrentSplices
|
||||
) {
|
||||
throw new Error('staging launch load did not reach C4 at the reviewed levels')
|
||||
}
|
||||
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
|
||||
if (metrics.cloudSqlBackendsMax >= CLOUD_SQL_LIMIT) {
|
||||
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
|
||||
}
|
||||
return {
|
||||
...baseEvidence(input, 'staging', [STAGING_CELL], STAGING_WORKFLOW),
|
||||
kind: 'staging-asia-readiness',
|
||||
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
|
||||
load: { launch: loadSummary(launch) },
|
||||
metrics
|
||||
}
|
||||
}
|
||||
|
||||
function loadReports(value, label) {
|
||||
if (!Array.isArray(value) || value.length < 2) throw new Error(`${label} must be sharded`)
|
||||
return value.map((report) => object(report, label))
|
||||
}
|
||||
|
||||
function total(reports, key) {
|
||||
return reports.reduce((sum, report) => sum + number(report[key], key), 0)
|
||||
}
|
||||
|
||||
function loadSummary(reports) {
|
||||
return {
|
||||
shards: reports.length,
|
||||
controls: total(reports, 'controls'),
|
||||
peakActive: total(reports, 'peakActive'),
|
||||
steadyMinimumActive: total(reports, 'steadyMinimumActive'),
|
||||
peakActiveSplices: total(reports, 'peakActiveSplices'),
|
||||
completedSplices: total(reports, 'completedSplices'),
|
||||
slowReaderSplicesCompleted: total(reports, 'slowReaderSplicesCompleted'),
|
||||
wedgedReaderSplicesClosed: total(reports, 'wedgedReaderSplicesClosed'),
|
||||
regionalFallbacksProved: total(reports, 'regionalFallbacksProved'),
|
||||
oldClientUsFirstProved: total(reports, 'oldClientUsFirstProved'),
|
||||
stickyAssignmentProved: total(reports, 'stickyAssignmentProved'),
|
||||
requestUnitInvitesOpened: total(reports, 'requestUnitInvitesOpened'),
|
||||
requestUnitPrincipalCounts: reports.map((report) => report.requestUnitPrincipalCount),
|
||||
requestUnitOverflowReasons:
|
||||
reports.map((report) => report.requestUnitOverflowReason).filter(Boolean),
|
||||
requestUnitCleanupProved: total(reports, 'requestUnitCleanupProved'),
|
||||
phaseBarrierPassed: reports.every((report) => report.phaseBarrierPassed === true),
|
||||
rebindProbesOpened: total(reports, 'rebindProbesOpened'),
|
||||
rebindOverflowReasons: reports.map((report) => report.rebindOverflowReason).filter(Boolean),
|
||||
readerQueueEvidence: reports.flatMap((report) => report.readerQueueEvidence),
|
||||
readerQueuedBytesPeak: Math.max(...reports.map((report) => report.readerQueuedBytesPeak)),
|
||||
generatorCpuPercentMax: Math.max(...reports.map((report) => report.generatorCpuPercent)),
|
||||
generatorEventLoopP99MsMax: Math.max(
|
||||
...reports.map((report) => report.generatorEventLoopP99Ms)
|
||||
),
|
||||
generatorRssGrowthMiBMax: Math.max(...reports.map((report) => report.generatorRssGrowthMiB)),
|
||||
configuredSteadySeconds: Math.min(...reports.map((report) => report.configuredSteadySeconds))
|
||||
}
|
||||
}
|
||||
|
||||
function assertLoadReports(reports, expected) {
|
||||
const shardCount = reports.length
|
||||
if (
|
||||
reports.some((report, index) =>
|
||||
report.event !== 'relay_load_complete' ||
|
||||
report.shardCount !== shardCount || report.shardIndex !== index ||
|
||||
report.relayAsiaLoadPrincipalCount !== 32 ||
|
||||
number(report.configuredSteadySeconds, 'staging steady seconds') < expected.minimumSeconds ||
|
||||
number(report.configuredSpliceHoldSeconds, 'staging splice hold seconds') <
|
||||
expected.minimumSeconds ||
|
||||
report.requiredLeaseHorizons !== 2 || report.phaseBarrierPassed !== true
|
||||
) ||
|
||||
total(reports, 'controls') !== expected.controls
|
||||
) {
|
||||
throw new Error('staging load profile does not match')
|
||||
}
|
||||
if (
|
||||
total(reports, 'peakActive') !== expected.controls ||
|
||||
total(reports, 'steadyMinimumActive') !== expected.controls
|
||||
) {
|
||||
throw new Error('staging load did not sustain the required controls')
|
||||
}
|
||||
for (const key of [
|
||||
'rampConnectionFailures', 'steadyConnectionFailures', 'transitionConnectionFailures',
|
||||
'unexpectedCloses', 'protocolErrors', 'refreshErrors', 'socketErrors', 'failedSplices'
|
||||
]) {
|
||||
if (total(reports, key) !== 0) throw new Error(`staging load ${key} must be zero`)
|
||||
}
|
||||
const peakActiveSplices = total(reports, 'peakActiveSplices')
|
||||
if (
|
||||
total(reports, 'configuredSplices') !== expected.splices ||
|
||||
total(reports, 'configuredSlowReaderSplices') !== expected.slowReaders ||
|
||||
total(reports, 'configuredWedgedReaderSplices') !== expected.wedgedReaders ||
|
||||
peakActiveSplices < expected.splices - expected.wedgedReaders ||
|
||||
peakActiveSplices > expected.splices ||
|
||||
total(reports, 'completedSplices') !== expected.splices - expected.wedgedReaders ||
|
||||
total(reports, 'slowReaderSplicesCompleted') !== expected.slowReaders ||
|
||||
total(reports, 'wedgedReaderSplicesClosed') !== expected.wedgedReaders
|
||||
) throw new Error('staging mixed load evidence does not match')
|
||||
if (
|
||||
total(reports, 'regionalFallbacksProved') !== 0 ||
|
||||
total(reports, 'oldClientUsFirstProved') !== 1 ||
|
||||
total(reports, 'stickyAssignmentProved') !== 1 ||
|
||||
total(reports, 'requestUnitInvitesOpened') !== 0 ||
|
||||
reports.some((report) => report.requestUnitPrincipalCount !== 0) ||
|
||||
total(reports, 'requestUnitCleanupProved') !== 0 ||
|
||||
reports.some((report) => report.requestUnitOverflowReason !== null) ||
|
||||
total(reports, 'rebindProbesOpened') !== 2 ||
|
||||
reports.some((report) => report.rebindOverflowReason !== null)
|
||||
) throw new Error('staging launch-path evidence does not match')
|
||||
const readerReports = reports.filter(
|
||||
(report) => report.configuredSlowReaderSplices + report.configuredWedgedReaderSplices > 0
|
||||
)
|
||||
if (expected.slowReaders > 0 && readerReports.length !== 1) {
|
||||
throw new Error('staging reader pressure must have one causal owner')
|
||||
}
|
||||
for (const report of reports) {
|
||||
const queue = report.readerQueueEvidence
|
||||
if (!Array.isArray(queue)) throw new Error('staging reader queue evidence is invalid')
|
||||
if (expected.slowReaders === 0 && queue.length !== 0) {
|
||||
throw new Error('control load unexpectedly contains reader queue evidence')
|
||||
}
|
||||
const ownsReaderPressure = readerReports.includes(report)
|
||||
if (expected.slowReaders > 0 && ownsReaderPressure && (
|
||||
queue.length !== 1 ||
|
||||
queue[0]?.origin !== 'https://c4.relay-staging.onorca.dev' ||
|
||||
number(queue[0]?.baselineBytes, 'reader queue baseline') >
|
||||
number(queue[0]?.peakBytes, 'reader queue peak') ||
|
||||
number(queue[0]?.increaseBytes, 'reader queue increase') <= 0 ||
|
||||
queue[0].peakBytes - queue[0].baselineBytes !== queue[0].increaseBytes ||
|
||||
report.readerQueuedBytesPeak !== queue[0].increaseBytes
|
||||
)) throw new Error('staging reader queue evidence is not causal')
|
||||
if (expected.slowReaders > 0 && !ownsReaderPressure && (
|
||||
queue.length !== 0 || report.readerQueuedBytesPeak !== 0
|
||||
)) throw new Error('non-owner shard contains reader queue evidence')
|
||||
}
|
||||
for (const report of reports) {
|
||||
if (
|
||||
number(report.generatorCpuPercent, 'generator CPU') >= GENERATOR_CPU_PERCENT_LIMIT ||
|
||||
number(report.generatorEventLoopP99Ms, 'generator event loop') >=
|
||||
GENERATOR_EVENT_LOOP_P99_MS_LIMIT ||
|
||||
number(report.generatorRssGrowthMiB, 'generator RSS growth') >=
|
||||
GENERATOR_RSS_GROWTH_MIB_LIMIT
|
||||
) throw new Error('staging load generator has insufficient headroom')
|
||||
const shutdown = object(report.shutdownEvidence, 'load shutdown evidence')
|
||||
if (
|
||||
shutdown.peerShutdowns !== report.controls || shutdown.activeControls !== 0 ||
|
||||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
|
||||
) throw new Error('staging load cleanup is incomplete')
|
||||
}
|
||||
}
|
||||
|
||||
function number(value, label) {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function sumMap(value, label) {
|
||||
const entries = Object.entries(object(value ?? {}, label))
|
||||
return entries.reduce((total, [key, count]) => {
|
||||
if (!key) throw new Error(`${label} is invalid`)
|
||||
return total + number(count, label)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function assertCoverage(timestamps, start, end, label) {
|
||||
if (timestamps.length === 0) throw new Error(`${label} has no samples`)
|
||||
const ordered = timestamps.map((value) => instant(value, `${label} timestamp`).valueOf())
|
||||
.sort((left, right) => left - right)
|
||||
if (
|
||||
ordered[0] > start.valueOf() + MAX_LOG_EDGE_GAP_MS ||
|
||||
ordered.at(-1) < end.valueOf() - MAX_LOG_EDGE_GAP_MS
|
||||
) throw new Error(`${label} does not cover the canary window`)
|
||||
if (ordered.some((timestamp, index) => index > 0 && timestamp - ordered[index - 1] > MAX_LOG_SAMPLE_GAP_MS)) {
|
||||
throw new Error(`${label} has a sampling gap`)
|
||||
}
|
||||
}
|
||||
|
||||
function pointValue(point) {
|
||||
const value = point?.value
|
||||
if (typeof value?.doubleValue === 'number') return value.doubleValue
|
||||
if (typeof value?.int64Value === 'string') return Number(value.int64Value)
|
||||
return NaN
|
||||
}
|
||||
|
||||
function cloudSqlMaximum(response, start, end) {
|
||||
const points = (object(response, 'Cloud SQL response').timeSeries ?? [])
|
||||
.flatMap((series) => series.points ?? [])
|
||||
assertCoverage(points.map((point) => point.interval?.endTime), start, end, 'Cloud SQL metrics')
|
||||
const values = points.map(pointValue)
|
||||
if (values.some((value) => !Number.isFinite(value) || value < 0)) {
|
||||
throw new Error('Cloud SQL backend metric is invalid')
|
||||
}
|
||||
return Math.max(...values)
|
||||
}
|
||||
|
||||
export function buildC27CanaryEvidence(input) {
|
||||
const start = instant(input.startedAt, 'canary start')
|
||||
const end = instant(input.endedAt, 'canary end')
|
||||
if (end.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) {
|
||||
throw new Error('C27 canary window is shorter than 5 minutes')
|
||||
}
|
||||
const load = object(input.loadReport, 'C27 load report')
|
||||
assertC27CanaryLoad(load)
|
||||
const metrics = runtimeMetrics(input.logs, start, end, C27)
|
||||
assertPassingRuntimeMetrics(metrics, 'C27 canary')
|
||||
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
|
||||
assertPassingCanary(metrics)
|
||||
return {
|
||||
...baseEvidence(input, 'production', [C27]),
|
||||
kind: 'production-c27-canary',
|
||||
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
|
||||
load,
|
||||
metrics
|
||||
}
|
||||
}
|
||||
|
||||
function assertC27CanaryLoad(report) {
|
||||
if (
|
||||
report.event !== 'relay_load_complete' ||
|
||||
report.controls !== 1 || report.shardCount !== 1 || report.shardIndex !== 0 ||
|
||||
report.relayAsiaLoadPrincipalCount !== 1 ||
|
||||
number(report.configuredSteadySeconds, 'canary steady seconds') < 300 ||
|
||||
number(report.configuredSpliceHoldSeconds, 'canary splice hold seconds') < 60 ||
|
||||
number(report.requiredLeaseHorizons, 'canary lease horizons') < 2 ||
|
||||
report.peakActive !== 1 || report.steadyMinimumActive !== 1 ||
|
||||
report.configuredSplices !== 1 || report.peakActiveSplices !== 1 ||
|
||||
report.completedSplices !== 1 || report.failedSplices !== 0
|
||||
) throw new Error('C27 control and splice canary did not match')
|
||||
for (const key of [
|
||||
'connectionFailures', 'unexpectedCloses', 'protocolErrors',
|
||||
'refreshErrors', 'socketErrors'
|
||||
]) {
|
||||
if (number(report[key], key) !== 0) throw new Error(`C27 canary ${key} must be zero`)
|
||||
}
|
||||
const shutdown = object(report.shutdownEvidence, 'C27 load shutdown evidence')
|
||||
if (
|
||||
shutdown.peerShutdowns !== 1 || shutdown.activeControls !== 0 ||
|
||||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
|
||||
) throw new Error('C27 load cleanup is incomplete')
|
||||
}
|
||||
|
||||
function runtimeMetrics(logs, start, end, targetCellId) {
|
||||
const entries = logs.map((entry) => object(entry, 'runtime metric entry'))
|
||||
const directorEntries = entries.filter((entry) => entry.jsonPayload?.role === 'director')
|
||||
const cellEntries = entries.filter((entry) =>
|
||||
entry.jsonPayload?.role === 'cell' &&
|
||||
entry.jsonPayload?.cellId === targetCellId &&
|
||||
entry.jsonPayload?.region === 'asia-east2'
|
||||
)
|
||||
assertCoverage(directorEntries.map((entry) => entry.timestamp), start, end, 'director metrics')
|
||||
assertCoverage(cellEntries.map((entry) => entry.timestamp), start, end, `${targetCellId} metrics`)
|
||||
const identifiedDirectors = Map.groupBy(
|
||||
directorEntries.filter((entry) => entry.resource?.labels?.instance_id),
|
||||
(entry) => entry.resource.labels.instance_id
|
||||
)
|
||||
for (const [instanceId, instanceEntries] of identifiedDirectors) {
|
||||
assertCoverage(instanceEntries.map((entry) => entry.timestamp), start, end, `director ${instanceId}`)
|
||||
}
|
||||
const directorPayloads = directorEntries.map((entry) => entry.jsonPayload)
|
||||
const cellPayloads = cellEntries.map((entry) => entry.jsonPayload)
|
||||
const payloads = [...directorPayloads, ...cellPayloads]
|
||||
return {
|
||||
asiaSelections: directorPayloads.reduce(
|
||||
(total, payload) => total + number(payload.selectedRegionsDelta?.['asia-east2'] ?? 0, 'Asia selections'), 0
|
||||
),
|
||||
regionFallbacks: directorPayloads.reduce(
|
||||
(total, payload) => total + sumMap(payload.regionFallbacksDelta, 'region fallbacks'), 0
|
||||
),
|
||||
usRegionFallbacks: directorPayloads.reduce(
|
||||
(total, payload) => total + number(
|
||||
payload.regionFallbacksDelta?.['us-central1'] ?? 0,
|
||||
'US region fallbacks'
|
||||
), 0
|
||||
),
|
||||
unavailableRegions: directorPayloads.reduce(
|
||||
(total, payload) => total + sumMap(payload.unavailableRegionsDelta, 'unavailable regions'), 0
|
||||
),
|
||||
relaySqlFailures: payloads.reduce(
|
||||
(total, payload) => total + number(payload.sqlFailuresDelta, 'Relay SQL failures'), 0
|
||||
),
|
||||
databasePoolWaitingMax: Math.max(...payloads.map(
|
||||
(payload) => number(payload.databasePoolWaiting, 'database pool waiting')
|
||||
)),
|
||||
databasePoolWaitersMax: Math.max(...payloads.map(
|
||||
(payload) => number(payload.databasePoolWaitersMax, 'database pool waiters')
|
||||
)),
|
||||
databasePoolWaitMsMax: Math.max(...payloads.map(
|
||||
(payload) => number(payload.databasePoolWaitMsMax, 'database pool wait time')
|
||||
)),
|
||||
targetControlsMax: Math.max(...cellPayloads.map(
|
||||
(payload) => number(payload.controls, 'target controls')
|
||||
)),
|
||||
targetSplicesMax: Math.max(...cellPayloads.map(
|
||||
(payload) => number(payload.splices, 'target splices')
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
function assertPassingRuntimeMetrics(metrics, label, expectedRegionFallbacks = 0) {
|
||||
if (number(metrics.asiaSelections, 'Asia selections') < 1) {
|
||||
throw new Error(`${label} observed no Asia selections`)
|
||||
}
|
||||
if (
|
||||
number(metrics.regionFallbacks, 'regionFallbacks') !== expectedRegionFallbacks ||
|
||||
number(metrics.usRegionFallbacks, 'usRegionFallbacks') !== expectedRegionFallbacks
|
||||
) {
|
||||
throw new Error(`${label} regionFallbacks did not match the intentional probes`)
|
||||
}
|
||||
for (const key of [
|
||||
'unavailableRegions', 'relaySqlFailures', 'databasePoolWaitingMax'
|
||||
]) {
|
||||
if (number(metrics[key], key) !== 0) throw new Error(`${label} ${key} must be zero`)
|
||||
}
|
||||
if (
|
||||
number(metrics.databasePoolWaitersMax, 'databasePoolWaitersMax') >
|
||||
DATABASE_POOL_TRANSIENT_WAITERS_MAX ||
|
||||
number(metrics.databasePoolWaitMsMax, 'databasePoolWaitMsMax') >
|
||||
DATABASE_POOL_TRANSIENT_WAIT_MS_MAX
|
||||
) throw new Error(`${label} transient database pool pressure exceeded its bound`)
|
||||
}
|
||||
|
||||
function assertPassingCanary(metrics) {
|
||||
if (
|
||||
number(metrics.targetControlsMax, 'C27 controls') < 1 ||
|
||||
number(metrics.targetSplicesMax, 'C27 splices') < 1
|
||||
) throw new Error('C27 canary traffic did not reach C27')
|
||||
if (number(metrics.cloudSqlBackendsMax, 'Cloud SQL backends') >= CLOUD_SQL_LIMIT) {
|
||||
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
|
||||
}
|
||||
}
|
||||
|
||||
function assertProvenance(evidence, run, expected) {
|
||||
const evidenceSource = object(evidence.source, 'evidence source')
|
||||
if (
|
||||
run.id !== evidenceSource.runId ||
|
||||
run.run_attempt !== evidenceSource.runAttempt ||
|
||||
run.conclusion !== 'success' ||
|
||||
run.event !== 'workflow_dispatch' ||
|
||||
run.head_branch !== 'main' ||
|
||||
run.head_sha !== evidenceSource.commitSha ||
|
||||
run.repository?.full_name !== evidenceSource.repository ||
|
||||
run.path?.split('@')[0] !== expected.workflow ||
|
||||
evidenceSource.workflow !== expected.workflow ||
|
||||
evidenceSource.repository !== REPOSITORY ||
|
||||
expected.repository !== REPOSITORY ||
|
||||
evidenceSource.environment !== expected.environment ||
|
||||
evidenceSource.commitSha !== expected.commitSha
|
||||
) throw new Error('evidence workflow provenance does not match')
|
||||
}
|
||||
|
||||
export function verifyRolloutEvidence(evidence, run, expected) {
|
||||
object(evidence, 'evidence')
|
||||
object(run, 'workflow run')
|
||||
if (evidence.version !== 1 || evidence.kind !== expected.kind) {
|
||||
throw new Error('evidence kind is invalid')
|
||||
}
|
||||
assertProvenance(evidence, run, expected)
|
||||
if (evidence.imageDigest !== expected.imageDigest) throw new Error('evidence image digest does not match')
|
||||
if (!exactCells(evidence.topology?.cellIds, expected.cellIds)) {
|
||||
throw new Error('evidence topology does not match')
|
||||
}
|
||||
positiveInteger(evidence.topology?.selectorGeneration, 'evidence selector generation')
|
||||
if (
|
||||
expected.selectorGeneration !== undefined &&
|
||||
evidence.topology.selectorGeneration !== expected.selectorGeneration
|
||||
) throw new Error('evidence selector generation does not match')
|
||||
const now = instant(expected.now, 'verification time')
|
||||
const proofTime = instant(evidence.window?.endedAt, 'evidence time')
|
||||
const maxAgeMs = expected.kind === 'staging-asia-readiness' ? 24 * 60 * 60_000 : 6 * 60 * 60_000
|
||||
if (proofTime > now || now.valueOf() - proofTime.valueOf() > maxAgeMs) {
|
||||
throw new Error('rollout evidence is stale')
|
||||
}
|
||||
if (expected.kind === 'production-c27-canary') {
|
||||
const start = instant(evidence.window?.startedAt, 'canary start')
|
||||
if (proofTime.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) {
|
||||
throw new Error('C27 canary window is shorter than 5 minutes')
|
||||
}
|
||||
assertC27CanaryLoad(object(evidence.load, 'canary load'))
|
||||
assertPassingCanary(object(evidence.metrics, 'canary metrics'))
|
||||
}
|
||||
return evidence
|
||||
}
|
||||
|
||||
function argumentsMap(argv) {
|
||||
const values = new Map()
|
||||
for (let index = 1; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined || values.has(key.slice(2))) {
|
||||
throw new Error('invalid rollout evidence arguments')
|
||||
}
|
||||
values.set(key.slice(2), value)
|
||||
}
|
||||
return { command: argv[0], values }
|
||||
}
|
||||
|
||||
function required(values, key) {
|
||||
const value = values.get(key)
|
||||
if (!value) throw new Error(`missing --${key}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function commonInput(values) {
|
||||
return {
|
||||
repository: required(values, 'repository'), runId: required(values, 'run-id'),
|
||||
runAttempt: required(values, 'run-attempt'), commitSha: required(values, 'commit-sha'),
|
||||
imageDigest: required(values, 'image-digest'),
|
||||
selectorGeneration: required(values, 'selector-generation')
|
||||
}
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const { command, values } = argumentsMap(argv)
|
||||
const output = required(values, 'output')
|
||||
if (command === 'create-staging') {
|
||||
writeFileSync(output, `${JSON.stringify(buildStagingEvidence({
|
||||
...commonInput(values), startedAt: required(values, 'started-at'),
|
||||
endedAt: required(values, 'ended-at'),
|
||||
launchReport: JSON.parse(readFileSync(required(values, 'launch-report'), 'utf8')),
|
||||
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
|
||||
cloudSql: await readCloudSqlBackends(
|
||||
'staging', required(values, 'started-at'), required(values, 'ended-at')
|
||||
)
|
||||
}), null, 2)}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'create-c27') {
|
||||
const startedAt = required(values, 'started-at')
|
||||
const endedAt = required(values, 'ended-at')
|
||||
writeFileSync(output, `${JSON.stringify(buildC27CanaryEvidence({
|
||||
...commonInput(values), startedAt, endedAt,
|
||||
loadReport: JSON.parse(readFileSync(required(values, 'load-report'), 'utf8')),
|
||||
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
|
||||
cloudSql: await readCloudSqlBackends('production', startedAt, endedAt)
|
||||
}), null, 2)}\n`)
|
||||
return
|
||||
}
|
||||
if (!['verify-staging', 'verify-c27'].includes(command)) throw new Error('invalid evidence command')
|
||||
const kind = command === 'verify-staging' ? 'staging-asia-readiness' : 'production-c27-canary'
|
||||
verifyRolloutEvidence(
|
||||
JSON.parse(readFileSync(required(values, 'evidence'), 'utf8')),
|
||||
JSON.parse(readFileSync(required(values, 'run-json'), 'utf8')),
|
||||
{
|
||||
kind, repository: REPOSITORY,
|
||||
workflow: kind === 'staging-asia-readiness' ? STAGING_WORKFLOW : ADMISSION_WORKFLOW,
|
||||
environment: kind === 'staging-asia-readiness' ? 'staging' : 'production',
|
||||
commitSha: required(values, 'commit-sha'), imageDigest: required(values, 'image-digest'),
|
||||
cellIds: [kind === 'staging-asia-readiness' ? STAGING_CELL : C27],
|
||||
selectorGeneration: values.has('selector-generation')
|
||||
? positiveInteger(values.get('selector-generation'), 'selector generation') : undefined,
|
||||
now: required(values, 'now')
|
||||
}
|
||||
)
|
||||
writeFileSync(output, 'verified\n')
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(2))
|
||||
@@ -0,0 +1,357 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
|
||||
import {
|
||||
buildC27CanaryEvidence,
|
||||
buildStagingEvidence,
|
||||
verifyRolloutEvidence
|
||||
} from './relay-asia-rollout-evidence.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const commitSha = 'b'.repeat(40)
|
||||
const repository = RELAY_GITHUB_REPOSITORY
|
||||
const start = new Date('2026-08-13T12:00:00.000Z')
|
||||
const end = new Date(start.valueOf() + 15 * 60_000)
|
||||
const canaryEnd = new Date(start.valueOf() + 5 * 60_000)
|
||||
|
||||
function sourceInput(overrides = {}) {
|
||||
return {
|
||||
repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest,
|
||||
selectorGeneration: 9, ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function metricLog(timestamp, role, cellId, overrides = {}) {
|
||||
return {
|
||||
timestamp: timestamp.toISOString(),
|
||||
resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} },
|
||||
jsonPayload: {
|
||||
role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1',
|
||||
controls: role === 'cell' ? 2_840 : 0,
|
||||
splices: role === 'cell' ? 120 : 0,
|
||||
selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {},
|
||||
regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0,
|
||||
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function completeLogs(cellId = 'production-gce-c27', windowEnd = end) {
|
||||
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1
|
||||
return Array.from({ length: samples }, (_, minute) => {
|
||||
const timestamp = new Date(start.valueOf() + minute * 60_000)
|
||||
return [
|
||||
metricLog(timestamp, 'director', 'production-director'),
|
||||
metricLog(timestamp, 'cell', cellId)
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function cloudSql(max = 319, windowEnd = end) {
|
||||
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000
|
||||
return {
|
||||
timeSeries: [{
|
||||
points: Array.from({ length: samples }, (_, minute) => ({
|
||||
interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() },
|
||||
value: { int64Value: String(minute === samples - 1 ? max : 300) }
|
||||
}))
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
function loadReport({
|
||||
controls, shardIndex, splices = 0, slow = 0, wedged = 0,
|
||||
launch = false
|
||||
}) {
|
||||
return {
|
||||
event: 'relay_load_complete', controls, shardCount: 4, shardIndex,
|
||||
configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2,
|
||||
configuredSpliceHoldSeconds: 210,
|
||||
configuredSplices: splices, configuredSlowReaderSplices: slow,
|
||||
configuredWedgedReaderSplices: wedged,
|
||||
peakActive: controls, steadyMinimumActive: controls,
|
||||
connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0,
|
||||
transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0,
|
||||
refreshErrors: 0, socketErrors: 0, failedSplices: 0,
|
||||
regionalFallbacksProved: 0,
|
||||
oldClientUsFirstProved: launch ? 1 : 0,
|
||||
stickyAssignmentProved: launch ? 1 : 0,
|
||||
requestUnitInvitesOpened: 0,
|
||||
requestUnitPrincipalCount: 0,
|
||||
relayAsiaLoadPrincipalCount: 32,
|
||||
requestUnitOverflowReason: null,
|
||||
requestUnitCleanupProved: 0,
|
||||
phaseBarrierPassed: true,
|
||||
rebindProbesOpened: launch ? 2 : 0,
|
||||
rebindOverflowReason: null,
|
||||
peakActiveSplices: splices, completedSplices: splices - wedged,
|
||||
slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged,
|
||||
readerQueuedBytesPeak: slow > 0 ? 1_024 : 0,
|
||||
generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10,
|
||||
readerQueueEvidence: slow > 0 ? [{
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
baselineBytes: 128,
|
||||
peakBytes: 1_152,
|
||||
increaseBytes: 1_024
|
||||
}] : [],
|
||||
shutdownEvidence: {
|
||||
peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stagingInput(overrides = {}) {
|
||||
const logs = completeLogs('staging-gce-c4')
|
||||
logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 }
|
||||
return {
|
||||
...sourceInput({ selectorGeneration: 4 }),
|
||||
startedAt: start.toISOString(), endedAt: end.toISOString(),
|
||||
launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({
|
||||
controls: 5, shardIndex, splices: 5, launch: shardIndex === 0,
|
||||
slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0
|
||||
})),
|
||||
logs, cloudSql: cloudSql(), ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function canaryInput(overrides = {}) {
|
||||
const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 })
|
||||
Object.assign(load, {
|
||||
shardCount: 1,
|
||||
configuredSteadySeconds: 300,
|
||||
configuredSpliceHoldSeconds: 60,
|
||||
relayAsiaLoadPrincipalCount: 1
|
||||
})
|
||||
return {
|
||||
...sourceInput(), startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(),
|
||||
loadReport: load,
|
||||
logs: completeLogs('production-gce-c27', canaryEnd), cloudSql: cloudSql(319, canaryEnd),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function workflowRun(evidence, overrides = {}) {
|
||||
return {
|
||||
id: evidence.source.runId, run_attempt: evidence.source.runAttempt,
|
||||
conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main',
|
||||
head_sha: evidence.source.commitSha,
|
||||
repository: { full_name: evidence.source.repository }, path: evidence.source.workflow,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function verifyExpected(kind, overrides = {}) {
|
||||
const staging = kind === 'staging-asia-readiness'
|
||||
return {
|
||||
kind, repository,
|
||||
workflow: staging
|
||||
? relayWorkflowPath('prove-relay-asia-staging.yml')
|
||||
: relayWorkflowPath('operate-relay-asia-admission.yml'),
|
||||
environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest,
|
||||
cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'],
|
||||
now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('accepts the exact sharded staging load, telemetry, and provenance proof', () => {
|
||||
const evidence = buildStagingEvidence(stagingInput())
|
||||
assert.equal(evidence.load.launch.controls, 20)
|
||||
assert.equal(evidence.load.launch.peakActiveSplices, 20)
|
||||
assert.equal(evidence.load.launch.readerQueueEvidence.length, 1)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness')
|
||||
), evidence)
|
||||
})
|
||||
|
||||
test('ignores unrelated legacy cell metrics outside the Asia proof', () => {
|
||||
const input = stagingInput()
|
||||
const legacy = metricLog(start, 'cell', 'staging-gce-c1', {
|
||||
region: undefined,
|
||||
sqlFailuresDelta: 1
|
||||
})
|
||||
delete legacy.jsonPayload.databasePoolWaiting
|
||||
delete legacy.jsonPayload.databasePoolWaitersMax
|
||||
delete legacy.jsonPayload.databasePoolWaitMsMax
|
||||
input.logs.push(legacy)
|
||||
|
||||
assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0)
|
||||
})
|
||||
|
||||
test('accepts bounded transient pool waits without a sampled queue', () => {
|
||||
const input = stagingInput()
|
||||
input.logs[0].jsonPayload.databasePoolWaitersMax = 4
|
||||
input.logs[0].jsonPayload.databasePoolWaitMsMax = 50
|
||||
|
||||
const metrics = buildStagingEvidence(input).metrics
|
||||
assert.equal(metrics.databasePoolWaitingMax, 0)
|
||||
assert.equal(metrics.databasePoolWaitersMax, 4)
|
||||
assert.equal(metrics.databasePoolWaitMsMax, 50)
|
||||
})
|
||||
|
||||
test('requires exactly the intentional sticky-assignment fallback in staging', () => {
|
||||
const missing = stagingInput()
|
||||
missing.logs[0].jsonPayload.regionFallbacksDelta = {}
|
||||
assert.throws(() => buildStagingEvidence(missing), /intentional probes/)
|
||||
|
||||
const extra = stagingInput()
|
||||
extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
|
||||
assert.throws(() => buildStagingEvidence(extra), /intentional probes/)
|
||||
|
||||
const substituted = stagingInput()
|
||||
substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
|
||||
assert.throws(() => buildStagingEvidence(substituted), /intentional probes/)
|
||||
})
|
||||
|
||||
test('accepts the wedged splice outside the sustained non-wedged peak', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].peakActiveSplices = 4
|
||||
input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.splices = 19 })
|
||||
const evidence = buildStagingEvidence(input)
|
||||
assert.equal(evidence.load.launch.peakActiveSplices, 19)
|
||||
|
||||
input.launchReport[0].peakActiveSplices = 3
|
||||
assert.throws(() => buildStagingEvidence(input), /mixed load evidence/)
|
||||
})
|
||||
|
||||
test('rejects staging evidence with incomplete load or cleanup', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].steadyMinimumActive--
|
||||
assert.throws(() => buildStagingEvidence(input), /required controls/)
|
||||
const cleanup = stagingInput()
|
||||
cleanup.launchReport[0].shutdownEvidence.activeControls = 1
|
||||
assert.throws(() => buildStagingEvidence(cleanup), /cleanup/)
|
||||
const nonCausal = stagingInput()
|
||||
nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0
|
||||
assert.throws(() => buildStagingEvidence(nonCausal), /not causal/)
|
||||
const multipleOwners = stagingInput()
|
||||
multipleOwners.launchReport[0].configuredSlowReaderSplices--
|
||||
multipleOwners.launchReport[0].slowReaderSplicesCompleted--
|
||||
multipleOwners.launchReport[1] = loadReport({
|
||||
controls: 5, shardIndex: 1, splices: 5, slow: 1
|
||||
})
|
||||
assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/)
|
||||
const overloaded = stagingInput()
|
||||
overloaded.launchReport[0].generatorCpuPercent = 80
|
||||
assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/)
|
||||
const missingLaunchProof = stagingInput()
|
||||
missingLaunchProof.launchReport[0].rebindProbesOpened = 0
|
||||
assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/)
|
||||
const offTarget = stagingInput()
|
||||
offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.controls = 19 })
|
||||
assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/)
|
||||
})
|
||||
|
||||
test('rejects staging evidence with a shortened splice hold', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].configuredSpliceHoldSeconds = 60
|
||||
assert.throws(() => buildStagingEvidence(input), /load profile does not match/)
|
||||
})
|
||||
|
||||
for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) {
|
||||
test(`rejects staging evidence with a ${label} splice hold`, () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].configuredSpliceHoldSeconds = value
|
||||
assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/)
|
||||
})
|
||||
}
|
||||
|
||||
for (const [label, mutate, message] of [
|
||||
['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/],
|
||||
['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/],
|
||||
['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/]
|
||||
]) {
|
||||
test(`rejects staging evidence without ${label} proof`, () => {
|
||||
const input = stagingInput()
|
||||
mutate(input)
|
||||
assert.throws(() => buildStagingEvidence(input), message)
|
||||
})
|
||||
}
|
||||
|
||||
test('rejects staging evidence from a non-canonical repository', () => {
|
||||
assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/)
|
||||
})
|
||||
|
||||
test('rejects mismatched staging provenance, digest, topology, or age', () => {
|
||||
const evidence = buildStagingEvidence(stagingInput())
|
||||
const expected = verifyExpected('staging-asia-readiness')
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, {
|
||||
head_sha: 'c'.repeat(40)
|
||||
}), expected), /provenance/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, commitSha: 'c'.repeat(40)
|
||||
}), /provenance/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, imageDigest: `sha256:${'c'.repeat(64)}`
|
||||
}), /image digest/)
|
||||
assert.throws(() => verifyRolloutEvidence({
|
||||
...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] }
|
||||
}, workflowRun(evidence), expected), /topology/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString()
|
||||
}), /stale/)
|
||||
})
|
||||
|
||||
test('builds and verifies a passing continuous 5-minute C27 canary', () => {
|
||||
const evidence = buildC27CanaryEvidence(canaryInput())
|
||||
assert.equal(evidence.load.completedSplices, 1)
|
||||
assert.equal(evidence.metrics.asiaSelections, 6)
|
||||
assert.equal(evidence.metrics.cloudSqlBackendsMax, 319)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c27-canary', { selectorGeneration: 9 })
|
||||
), evidence)
|
||||
})
|
||||
|
||||
test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => {
|
||||
assert.throws(() => buildC27CanaryEvidence(canaryInput({
|
||||
endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
|
||||
})), /shorter than 5 minutes/)
|
||||
const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) =>
|
||||
entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString()
|
||||
)
|
||||
assert.throws(() => buildC27CanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/)
|
||||
assert.throws(() => buildC27CanaryEvidence(canaryInput({
|
||||
logs: completeLogs('production-gce-c26', canaryEnd)
|
||||
})), /production-gce-c27 metrics has no samples/)
|
||||
})
|
||||
|
||||
test('rejects a C27 canary without a real control and splice', () => {
|
||||
const input = canaryInput()
|
||||
input.loadReport.completedSplices = 0
|
||||
assert.throws(() => buildC27CanaryEvidence(input), /did not match/)
|
||||
const shortHold = canaryInput()
|
||||
shortHold.loadReport.configuredSpliceHoldSeconds = 59
|
||||
assert.throws(() => buildC27CanaryEvidence(shortHold), /did not match/)
|
||||
})
|
||||
|
||||
for (const [label, mutation, message] of [
|
||||
['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/],
|
||||
['region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /regionFallbacks/],
|
||||
['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/],
|
||||
['Relay SQL failures', (input) => { input.logs[0].jsonPayload.sqlFailuresDelta = 1 }, /relaySqlFailures/],
|
||||
['pool waiting', (input) => { input.logs[0].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/],
|
||||
['pool waiters', (input) => { input.logs[0].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/],
|
||||
['pool wait time', (input) => { input.logs[0].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/],
|
||||
['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/],
|
||||
['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/],
|
||||
['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/]
|
||||
]) {
|
||||
test(`rejects C27 evidence with ${label}`, () => {
|
||||
const input = canaryInput()
|
||||
mutation(input)
|
||||
assert.throws(() => buildC27CanaryEvidence(input), message)
|
||||
})
|
||||
}
|
||||
|
||||
test('rejects C27 evidence from a different selector generation', () => {
|
||||
const evidence = buildC27CanaryEvidence(canaryInput())
|
||||
assert.throws(() => verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c27-canary', { selectorGeneration: 10 })
|
||||
), /selector generation/)
|
||||
})
|
||||
@@ -0,0 +1,124 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('deploy-relay-asia-topology.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const iam = readFileSync(
|
||||
new URL('../../infra/terraform/relay-asia-topology-iam.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const cells = readFileSync(
|
||||
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const variables = readFileSync(
|
||||
new URL('../../infra/terraform/variables.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
test('uses only its exact workflow-bound topology identity', () => {
|
||||
assert.match(workflow, /production-cloud-sql-rollout/)
|
||||
assert.match(workflow, /relay-staging-mutation/)
|
||||
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(workflow, /GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.match(
|
||||
iam,
|
||||
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_topology_workflow_file\}@refs\/heads\/main'/
|
||||
)
|
||||
assert.match(iam, /assertion\.ref == 'refs\/heads\/main'/)
|
||||
assert.match(iam, /assertion\.event_name == 'workflow_dispatch'/)
|
||||
assert.match(iam, /assertion\.environment == '\$\{var\.environment\}'/)
|
||||
})
|
||||
|
||||
test('plans only additive Asia topology and applies the saved plan', () => {
|
||||
assert.equal((workflow.match(/manage_artifact_dns=false/g) ?? []).length, 2)
|
||||
for (const target of [
|
||||
'relay_gce_additional',
|
||||
'google_compute_instance_template.relay_gce_cell',
|
||||
'google_compute_instance_group_manager.relay_gce_cell',
|
||||
'google_compute_backend_service.relay_gce_cell',
|
||||
'google_compute_url_map.relay_gce'
|
||||
]) assert.match(workflow, new RegExp(target.replaceAll('.', '\\.')))
|
||||
assert.match(workflow, /apply -input=false -auto-approve "\$\{\{ steps\.plan\.outputs\.plan \}\}"/)
|
||||
assert.match(workflow, /prepare-relay-asia-topology-input\.mjs/)
|
||||
assert.doesNotMatch(workflow, /steps\.variables\.outputs\.file/)
|
||||
assert.match(workflow, /\.variables\.relay_gce_cells\.value/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/\.variables\.relay_gce_additional_region_subnetwork_cidrs\.value/
|
||||
)
|
||||
assert.doesNotMatch(workflow, /terraform -chdir=infra\/terraform console/)
|
||||
assert.equal((workflow.match(/-var-file="\$\{TF_VARS\}"/g) ?? []).length, 2)
|
||||
assert.doesNotMatch(workflow, /terraform[^\n]*apply[^\n]*-target/)
|
||||
assert.doesNotMatch(workflow, /google_(?:sql|cloudflare|dns|certificate_manager)/)
|
||||
})
|
||||
|
||||
test('validates before apply and proves convergence afterward', () => {
|
||||
assert.equal((workflow.match(/validate-relay-asia-topology-plan\.mjs/g) ?? []).length, 2)
|
||||
assert.match(workflow, /APPLY_RELAY_ASIA_TOPOLOGY/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.changes'/)
|
||||
assert.match(workflow, /Register the exact new cells atomically as migration-only/)
|
||||
})
|
||||
|
||||
test('checks the connection budget and production live ceiling before planning', () => {
|
||||
assert.match(workflow, /relay-cloud-sql-connection-budget\.mjs/)
|
||||
assert.match(workflow, /gcloud sql instances describe "\$\{CLOUD_SQL_INSTANCE\}"/)
|
||||
assert.match(workflow, /select\(\.name == "max_connections"\)/)
|
||||
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/)
|
||||
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/)
|
||||
assert.match(workflow, /live_source=verified-shape-default/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/)
|
||||
assert.match(workflow, /test "\$\{live_max\}" = "\$\{checked_max\}"/)
|
||||
assert.ok(
|
||||
workflow.indexOf('relay-cloud-sql-connection-budget.mjs') <
|
||||
workflow.indexOf('terraform -chdir=infra/terraform plan')
|
||||
)
|
||||
})
|
||||
|
||||
test('binds computed Asia references to the matching Terraform cell resources', () => {
|
||||
assert.match(cells, /instance_template = google_compute_instance_template\.relay_gce_cell\[each\.key\]\.self_link/)
|
||||
assert.match(cells, /group\s+= google_compute_instance_group_manager\.relay_gce_cell\[each\.key\]\.instance_group/)
|
||||
assert.match(cells, /default_service = google_compute_backend_service\.relay_gce_cell\[cell\.key\]\.id/)
|
||||
assert.match(cells, /subnetwork = local\.relay_gce_subnetworks\[each\.value\.region\]/)
|
||||
})
|
||||
|
||||
test('keeps cross-variable region constraints in Terraform 1.5 check blocks', () => {
|
||||
assert.doesNotMatch(variables, /region != var\.region/)
|
||||
assert.doesNotMatch(variables, /cell\.region == var\.region/)
|
||||
assert.match(cells, /check "relay_gce_fixed_one_topology"[\s\S]*?region != var\.region/)
|
||||
assert.match(cells, /cell\.region == var\.region[\s\S]*?configured subnetwork/)
|
||||
})
|
||||
|
||||
test('the custom role cannot delete topology or mutate SQL and DNS', () => {
|
||||
assert.doesNotMatch(iam, /compute\.[A-Za-z]+\.delete/)
|
||||
assert.doesNotMatch(
|
||||
iam,
|
||||
/roles\/viewer|cloudsql\.instances\.(?:update|delete)|dns\.|certificatemanager|cloudflare/i
|
||||
)
|
||||
assert.match(iam, /resource "google_project_iam_custom_role" "github_relay_asia_topology_read"/)
|
||||
assert.match(iam, /"cloudsql\.instances\.get"/)
|
||||
assert.match(iam, /"run\.revisions\.get"/)
|
||||
assert.match(iam, /"run\.services\.get"/)
|
||||
assert.match(iam, /"serviceusage\.services\.list"/)
|
||||
assert.match(iam, /"compute\.networks\.updatePolicy"/)
|
||||
assert.match(iam, /"compute\.healthChecks\.useReadOnly"/)
|
||||
assert.match(iam, /"compute\.instanceGroups\.create"/)
|
||||
assert.match(iam, /"compute\.instances\.use"/)
|
||||
assert.match(iam, /roles\/storage\.objectAdmin/)
|
||||
assert.match(iam, /default\.tfstate/)
|
||||
assert.match(iam, /default\.tflock/)
|
||||
assert.match(
|
||||
iam,
|
||||
/resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list"[\s\S]*?permissions = \["storage\.objects\.list"\]/
|
||||
)
|
||||
assert.match(
|
||||
iam,
|
||||
/resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list"[\s\S]*?role\s+= google_project_iam_custom_role\.github_relay_asia_topology_state_list\[0\]\.id/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,148 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const DEFAULT_PATHS = {
|
||||
productionTfvars: new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
terraformVariables: new URL('../../infra/terraform/variables.tf', import.meta.url),
|
||||
relayConfig: new URL('../../apps/relay/src/config.ts', import.meta.url)
|
||||
}
|
||||
|
||||
// Auth and API live outside the Relay tree, so their consumption is published as a contract
|
||||
// rather than parsed from their source. production-cloud-sql-app-consumers.test.mjs binds it back.
|
||||
const APP_CONSUMERS_CONTRACT = new URL(
|
||||
'../contracts/production-cloud-sql-app-consumers.json',
|
||||
import.meta.url
|
||||
)
|
||||
|
||||
const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections']
|
||||
|
||||
export function readProductionCloudSqlAppConsumers(contract) {
|
||||
const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8'))
|
||||
for (const field of APP_CONSUMER_FIELDS) {
|
||||
const value = parsed[field]
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`could not read ${field} from the app consumer contract`)
|
||||
}
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function requiredInteger(source, pattern, label) {
|
||||
const value = Number(source.match(pattern)?.[1])
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`could not read ${label}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function productionCells(source, defaultPoolMax) {
|
||||
const fencedMatch = source.match(/relay_gce_fenced_cells\s*=\s*\[([^\]]*)\]/)
|
||||
if (!fencedMatch) throw new Error('could not read fenced Relay cells')
|
||||
const fenced = new Set([...fencedMatch[1].matchAll(/"([^"]+)"/g)].map((match) => match[1]))
|
||||
const cells = [...source.matchAll(/"(production-gce-[^"]+)"\s*=\s*\{([\s\S]*?)\n\s*\}/g)].map(
|
||||
([, id, body]) => ({
|
||||
id,
|
||||
fenced: fenced.has(id),
|
||||
region: body.match(/\bregion\s*=\s*"([^"]+)"/)?.[1] ?? 'us-central1',
|
||||
poolMax: body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)
|
||||
? Number(body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)[1])
|
||||
: defaultPoolMax
|
||||
})
|
||||
)
|
||||
if (cells.length === 0) throw new Error('could not read production Relay cells')
|
||||
return cells
|
||||
}
|
||||
|
||||
export function calculateRelayCloudSqlConnectionBudget(inputs) {
|
||||
const consumers = {
|
||||
cells: inputs.cellPoolTotal + inputs.asiaCellCount * inputs.asiaPoolMax,
|
||||
directors: inputs.directorInstances * inputs.directorPoolMax,
|
||||
auth: inputs.authInstances * inputs.authPoolMax,
|
||||
api: inputs.apiInstances * inputs.apiPoolMax
|
||||
}
|
||||
const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0)
|
||||
const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax
|
||||
const candidateOverlap = {
|
||||
relayDirectorCandidate: retainedDirectorRollback * 2,
|
||||
apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax,
|
||||
authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax,
|
||||
relayCells: retainedDirectorRollback
|
||||
}
|
||||
const rolloutOverlap = Math.max(...Object.values(candidateOverlap))
|
||||
const operatingMaximum = configuredMaximum + rolloutOverlap + inputs.maintenanceAdminAllowance
|
||||
const usableCeiling = inputs.maxConnections - inputs.explicitReserve
|
||||
const budgetedTotal = operatingMaximum + inputs.explicitReserve
|
||||
return {
|
||||
maxConnections: inputs.maxConnections,
|
||||
consumers,
|
||||
asia: { cells: inputs.asiaCellCount, poolMax: inputs.asiaPoolMax },
|
||||
configuredMaximum,
|
||||
rolloutOverlap: {
|
||||
...candidateOverlap,
|
||||
retainedDirectorRollback,
|
||||
maximum: rolloutOverlap,
|
||||
reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps'
|
||||
},
|
||||
maintenanceAdminAllowance: inputs.maintenanceAdminAllowance,
|
||||
maintenanceAdminAllowanceReason: 'covers bounded work outside configured services',
|
||||
explicitReserve: inputs.explicitReserve,
|
||||
explicitReserveReason: 'remains unavailable to configured services and planned rollouts',
|
||||
usableCeiling,
|
||||
operatingMaximum,
|
||||
remainingWithinUsableCeiling: usableCeiling - operatingMaximum,
|
||||
budgetedTotal,
|
||||
unallocated: inputs.maxConnections - budgetedTotal,
|
||||
withinBudget: operatingMaximum <= usableCeiling && budgetedTotal < inputs.maxConnections
|
||||
}
|
||||
}
|
||||
|
||||
export function readRelayCloudSqlConnectionBudget({
|
||||
sources,
|
||||
appConsumers,
|
||||
proposedAsiaCellCount = 3,
|
||||
asiaPoolMax = 10,
|
||||
maxConnections,
|
||||
maintenanceAdminAllowance = 5,
|
||||
explicitReserve = 10
|
||||
} = {}) {
|
||||
const read = (name) => sources?.[name] ?? readFileSync(DEFAULT_PATHS[name], 'utf8')
|
||||
const apps = readProductionCloudSqlAppConsumers(appConsumers)
|
||||
const productionTfvars = read('productionTfvars')
|
||||
const terraformVariables = read('terraformVariables')
|
||||
const relayConfig = read('relayConfig')
|
||||
const cells = productionCells(
|
||||
productionTfvars,
|
||||
requiredInteger(relayConfig, /RELAY_DATABASE_POOL_MAX\s*=\s*(\d+)/, 'Relay pool maximum')
|
||||
)
|
||||
const poweredCells = cells.filter(({ fenced }) => !fenced)
|
||||
const configuredAsiaCells = poweredCells.filter(({ region }) => region === 'asia-east2')
|
||||
const nonAsiaCells = poweredCells.filter(({ region }) => region !== 'asia-east2')
|
||||
const cellPoolTotal = nonAsiaCells.reduce((total, cell) => total + cell.poolMax, 0)
|
||||
const asiaCellCount = configuredAsiaCells.length || proposedAsiaCellCount
|
||||
const configuredAsiaPoolMax = configuredAsiaCells[0]?.poolMax ?? asiaPoolMax
|
||||
if (configuredAsiaCells.some(({ poolMax }) => poolMax !== configuredAsiaPoolMax)) {
|
||||
throw new Error('Asia Relay cells must use one checked pool maximum')
|
||||
}
|
||||
return calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal,
|
||||
asiaCellCount,
|
||||
asiaPoolMax: configuredAsiaPoolMax,
|
||||
directorInstances: requiredInteger(productionTfvars, /relay_max_instances\s*=\s*(\d+)/, 'director instances'),
|
||||
directorPoolMax: requiredInteger(
|
||||
terraformVariables,
|
||||
/variable\s+"relay_director_database_pool_max"[\s\S]*?default\s*=\s*(\d+)/,
|
||||
'director pool maximum'
|
||||
),
|
||||
authInstances: apps.authInstances,
|
||||
authPoolMax: apps.authPoolMax,
|
||||
apiInstances: apps.apiInstances,
|
||||
apiPoolMax: apps.apiPoolMax,
|
||||
maxConnections: maxConnections ?? apps.maxConnections,
|
||||
maintenanceAdminAllowance,
|
||||
explicitReserve
|
||||
})
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
console.log(JSON.stringify({ event: 'relay_cloud_sql_connection_budget', ...report }, null, 2))
|
||||
if (!report.withinBudget) process.exitCode = 1
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
calculateRelayCloudSqlConnectionBudget,
|
||||
readRelayCloudSqlConnectionBudget
|
||||
} from './relay-cloud-sql-connection-budget.mjs'
|
||||
|
||||
test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
|
||||
assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 })
|
||||
assert.deepEqual(report.asia, { cells: 3, poolMax: 10 })
|
||||
assert.equal(report.configuredMaximum, 315)
|
||||
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
|
||||
assert.equal(report.rolloutOverlap.apiCandidate, 65)
|
||||
assert.equal(report.rolloutOverlap.authCandidate, 35)
|
||||
assert.equal(report.rolloutOverlap.relayCells, 15)
|
||||
assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15)
|
||||
assert.equal(report.rolloutOverlap.maximum, 65)
|
||||
assert.equal(report.maintenanceAdminAllowance, 5)
|
||||
assert.equal(report.explicitReserve, 10)
|
||||
assert.equal(report.usableCeiling, 390)
|
||||
assert.equal(report.operatingMaximum, 385)
|
||||
assert.equal(report.remainingWithinUsableCeiling, 5)
|
||||
assert.equal(report.budgetedTotal, 395)
|
||||
assert.equal(report.unallocated, 5)
|
||||
assert.equal(report.withinBudget, true)
|
||||
})
|
||||
|
||||
test('fails closed when pool growth consumes the explicit reserve', () => {
|
||||
const report = calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal: 200,
|
||||
asiaCellCount: 3,
|
||||
asiaPoolMax: 20,
|
||||
directorInstances: 5,
|
||||
directorPoolMax: 3,
|
||||
authInstances: 2,
|
||||
authPoolMax: 10,
|
||||
apiInstances: 20,
|
||||
apiPoolMax: 5,
|
||||
maxConnections: 400,
|
||||
maintenanceAdminAllowance: 5,
|
||||
explicitReserve: 10
|
||||
})
|
||||
|
||||
assert.equal(report.operatingMaximum, 515)
|
||||
assert.equal(report.withinBudget, false)
|
||||
})
|
||||
|
||||
test('excludes fenced cell pools and reads per-cell pool overrides', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget({
|
||||
proposedAsiaCellCount: 1,
|
||||
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
|
||||
sources: {
|
||||
productionTfvars: `
|
||||
relay_max_instances = 1
|
||||
relay_gce_fenced_cells = ["production-gce-c1"]
|
||||
relay_gce_cells = {
|
||||
"production-gce-c1" = { database_pool_max = 99
|
||||
}
|
||||
"production-gce-c2" = { database_pool_max = 4
|
||||
}
|
||||
}
|
||||
`,
|
||||
terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 100,
|
||||
maintenanceAdminAllowance: 1,
|
||||
explicitReserve: 1
|
||||
})
|
||||
|
||||
assert.equal(report.consumers.cells, 14)
|
||||
assert.equal(report.operatingMaximum, 46)
|
||||
assert.equal(report.budgetedTotal, 47)
|
||||
})
|
||||
|
||||
test('requires strict headroom below the physical ceiling', () => {
|
||||
const report = calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal: 20,
|
||||
asiaCellCount: 0,
|
||||
asiaPoolMax: 10,
|
||||
directorInstances: 1,
|
||||
directorPoolMax: 3,
|
||||
authInstances: 1,
|
||||
authPoolMax: 10,
|
||||
apiInstances: 1,
|
||||
apiPoolMax: 5,
|
||||
maxConnections: 50,
|
||||
maintenanceAdminAllowance: 9,
|
||||
explicitReserve: 3
|
||||
})
|
||||
|
||||
assert.equal(report.budgetedTotal, 63)
|
||||
assert.equal(report.withinBudget, false)
|
||||
})
|
||||
|
||||
test('pages Relay channels when Cloud SQL backends consume headroom', () => {
|
||||
const terraform = readFileSync(
|
||||
new URL('../../infra/terraform/relay-observability.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const policy = terraform.match(
|
||||
/resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" \{([\s\S]*?)\n\}/
|
||||
)?.[1]
|
||||
|
||||
assert.ok(policy)
|
||||
assert.match(policy, /notification_channels\s*=\s*var\.relay_alert_notification_channels/)
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
export function relayLoadFailureReason(error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
|
||||
if (tokenExchange) return `token_http_${tokenExchange[1]}`
|
||||
const assignment =
|
||||
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_capacity_exhausted|relay_connection_headroom_exhausted))?$/.exec(
|
||||
message
|
||||
)
|
||||
if (assignment?.[1] === '503' && assignment[2]) return 'assignment_capacity_exhausted'
|
||||
if (assignment) return `assignment_http_${assignment[1]}`
|
||||
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
|
||||
if (closed) return `control_close_${closed[1]}`
|
||||
if (message === 'control open timeout') return 'control_open_timeout'
|
||||
if (message === 'control response timeout') return 'control_response_timeout'
|
||||
if (message === 'relay token exchange timeout') return 'token_timeout'
|
||||
if (message === 'relay assignment timeout') return 'assignment_timeout'
|
||||
if (message === 'WebSocket was closed before the connection was established') {
|
||||
return 'socket_closed_before_open'
|
||||
}
|
||||
if (message === 'relay token exchange omitted token') return 'token_response_invalid'
|
||||
if (message === 'relay assignment response invalid') return 'assignment_response_invalid'
|
||||
if (message === 'expected host challenge') return 'host_challenge_invalid'
|
||||
if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed'
|
||||
if (message === 'expected host hello acknowledgement') return 'host_ack_invalid'
|
||||
const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message)
|
||||
if (socketResponse) return `socket_http_${socketResponse[1]}`
|
||||
if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) {
|
||||
return 'socket_transport'
|
||||
}
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
export function discardFailedLoadSocket(socket) {
|
||||
if (!socket) return
|
||||
socket.on('error', () => undefined)
|
||||
socket.terminate()
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
discardFailedLoadSocket,
|
||||
relayLoadFailureReason
|
||||
} from './relay-load-connection-failure.mjs'
|
||||
|
||||
test('classifies only bounded aggregate connection failure reasons', () => {
|
||||
assert.equal(relayLoadFailureReason(new Error('relay token exchange failed: 503')), 'token_http_503')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay assignment failed: 503')), 'assignment_http_503')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(
|
||||
new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
|
||||
),
|
||||
'assignment_capacity_exhausted'
|
||||
)
|
||||
for (const status of [400, 429, 500]) {
|
||||
assert.equal(
|
||||
relayLoadFailureReason(
|
||||
new Error(`${`relay assignment failed: ${status}`} relay_capacity_exhausted`)
|
||||
),
|
||||
`assignment_http_${status}`
|
||||
)
|
||||
}
|
||||
assert.equal(relayLoadFailureReason(new Error('control closed: 4404 wrong cell')), 'control_close_4404')
|
||||
assert.equal(relayLoadFailureReason(new Error('control open timeout')), 'control_open_timeout')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay token exchange timeout')), 'token_timeout')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay assignment timeout')), 'assignment_timeout')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('relay token exchange omitted token')),
|
||||
'token_response_invalid'
|
||||
)
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('relay assignment response invalid')),
|
||||
'assignment_response_invalid'
|
||||
)
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('Unexpected server response: 503 Service Unavailable')),
|
||||
'socket_http_503'
|
||||
)
|
||||
assert.equal(relayLoadFailureReason(new Error('connect ECONNRESET 127.0.0.1')), 'socket_transport')
|
||||
assert.equal(relayLoadFailureReason(new Error('expected host challenge')), 'host_challenge_invalid')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('host proof challenge did not decrypt')),
|
||||
'host_challenge_decrypt_failed'
|
||||
)
|
||||
assert.equal(relayLoadFailureReason(new Error('expected host hello acknowledgement')), 'host_ack_invalid')
|
||||
assert.equal(relayLoadFailureReason(new Error('host-sensitive detail')), 'unknown')
|
||||
})
|
||||
|
||||
test('absorbs the setup error emitted while discarding a failed socket', () => {
|
||||
const socket = new EventEmitter()
|
||||
socket.terminate = () => socket.emit('error', new Error('closed before open'))
|
||||
|
||||
assert.doesNotThrow(() => discardFailedLoadSocket(socket))
|
||||
})
|
||||
@@ -0,0 +1,891 @@
|
||||
import { createHash, createHmac } from 'node:crypto'
|
||||
import { createRequire } from 'node:module'
|
||||
import { controlPhase } from './relay-load-model.mjs'
|
||||
import { discardFailedLoadSocket } from './relay-load-connection-failure.mjs'
|
||||
|
||||
const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url))
|
||||
const nacl = requireFromRelay('tweetnacl')
|
||||
const WebSocket = requireFromRelay('ws')
|
||||
const { SignJWT } = await import(requireFromRelay.resolve('jose'))
|
||||
const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import(
|
||||
requireFromRelay.resolve('@orca-cloud/relay-contract')
|
||||
)
|
||||
|
||||
const CAPACITY_ASSIGNMENT_ERRORS = [
|
||||
'relay_capacity_exhausted',
|
||||
'relay_connection_headroom_exhausted'
|
||||
]
|
||||
|
||||
function waitForOpen(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs)
|
||||
const finish = (error) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('open', onOpen)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve()
|
||||
}
|
||||
const onOpen = () => finish()
|
||||
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('open', onOpen)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function nextJson(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data) => {
|
||||
try {
|
||||
finish(undefined, JSON.parse(data.toString()))
|
||||
} catch (error) {
|
||||
finish(error)
|
||||
}
|
||||
}
|
||||
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function nextFrame(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data, binary) => finish(undefined, { bytes: Buffer.from(data), binary })
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function receiveBinaryStream(socket, expectedBytes, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let receivedBytes = 0
|
||||
const hash = createHash('sha256')
|
||||
const timer = setTimeout(() => finish(new Error('relay stream timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data, binary) => {
|
||||
if (!binary) return finish(new Error('relay changed stream opcode'))
|
||||
const bytes = Buffer.from(data)
|
||||
receivedBytes += bytes.byteLength
|
||||
hash.update(bytes)
|
||||
if (receivedBytes > expectedBytes) return finish(new Error('relay expanded reader stream'))
|
||||
if (receivedBytes === expectedBytes) {
|
||||
finish(undefined, { bytes: receivedBytes, digest: hash.digest('hex') })
|
||||
}
|
||||
}
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.on('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function closeInfo(socket, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('reader close timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onClose = (code, reason) => finish(undefined, { code, reason: reason.toString() })
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
export function relayLoadWedgedCloseAccepted(closeCodes) {
|
||||
return closeCodes.length === 2 && closeCodes[1] === 4429 &&
|
||||
(closeCodes[0] === 4429 || closeCodes[0] === 1006)
|
||||
}
|
||||
|
||||
function waitForClose(socket, timeoutMs = 10_000) {
|
||||
if (!socket || socket.readyState === socket.CLOSED) return Promise.resolve()
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
socket.off('close', onClose)
|
||||
discardFailedLoadSocket(socket)
|
||||
reject(new Error('control close timeout'))
|
||||
}, timeoutMs)
|
||||
const onClose = () => {
|
||||
clearTimeout(timer)
|
||||
resolve()
|
||||
}
|
||||
socket.once('close', onClose)
|
||||
})
|
||||
}
|
||||
|
||||
async function cancelResponse(response) {
|
||||
try {
|
||||
await response.body?.cancel()
|
||||
} catch {
|
||||
// Preserve the bounded failure classification.
|
||||
}
|
||||
}
|
||||
|
||||
function proofForChallenge(challenge, hostSecretKey) {
|
||||
const plaintext = nacl.box.open(
|
||||
Buffer.from(challenge.ciphertextB64, 'base64'),
|
||||
Buffer.from(challenge.nonceB64, 'base64'),
|
||||
Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'),
|
||||
hostSecretKey
|
||||
)
|
||||
if (!plaintext) throw new Error('host proof challenge did not decrypt')
|
||||
const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`)
|
||||
const transcriptLength = new DataView(
|
||||
plaintext.buffer,
|
||||
plaintext.byteOffset + domain.length,
|
||||
4
|
||||
).getUint32(0, false)
|
||||
const transcriptStart = domain.length + 4
|
||||
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
|
||||
const secret = plaintext.slice(transcriptStart + transcriptLength)
|
||||
return createHmac('sha256', secret).update(buildHostProofMacInput(transcript)).digest('base64')
|
||||
}
|
||||
|
||||
export class RelayLoadControlPeer {
|
||||
constructor(index, options, observe) {
|
||||
if (options.directorOrigin && options.targetOrigin) {
|
||||
throw new Error('provide either directorOrigin or targetOrigin, not both')
|
||||
}
|
||||
this.index = index
|
||||
this.options = options
|
||||
this.observe = observe
|
||||
this.keys = nacl.box.keyPair()
|
||||
this.relayHostId = createHash('sha256')
|
||||
.update(this.keys.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
this.phase = controlPhase(index, options.seed)
|
||||
this.socket = null
|
||||
this.generation = undefined
|
||||
this.controlResumeSecret = undefined
|
||||
this.lastAssignment = undefined
|
||||
this.refreshTimer = null
|
||||
this.stopped = false
|
||||
this.connecting = false
|
||||
this.inFlight = new Set()
|
||||
this.shutdownPromise = null
|
||||
this.abortController = new AbortController()
|
||||
this.drainExpected = false
|
||||
this.controlWaiters = new Set()
|
||||
this.spliceSockets = new Set()
|
||||
this.spliceSequence = 0
|
||||
}
|
||||
|
||||
connect() {
|
||||
if (
|
||||
this.stopped ||
|
||||
this.connecting ||
|
||||
(this.socket !== null && this.socket.readyState === this.socket.OPEN)
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
this.connecting = true
|
||||
const operation = this.connectOnce()
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => {
|
||||
this.connecting = false
|
||||
this.inFlight.delete(operation)
|
||||
}
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
assignedCellUrl() {
|
||||
return this.lastAssignment?.cellUrl
|
||||
}
|
||||
|
||||
async connectOnce() {
|
||||
let socket = null
|
||||
try {
|
||||
const relayToken = await this.relayToken()
|
||||
if (this.stopped) return
|
||||
const assignment = await this.assignment(relayToken)
|
||||
if (this.stopped) return
|
||||
this.lastAssignment = assignment
|
||||
socket = this.createSocket(assignment, relayToken)
|
||||
this.socket = socket
|
||||
this.drainExpected = false
|
||||
await waitForOpen(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
|
||||
appVersion: 'relay-load',
|
||||
...(this.generation === undefined ? {} : { previousGeneration: this.generation }),
|
||||
...(this.controlResumeSecret === undefined
|
||||
? {}
|
||||
: { controlResumeSecret: this.controlResumeSecret })
|
||||
})
|
||||
)
|
||||
const challenge = await nextJson(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-challenge-ack',
|
||||
challengeId: challenge.challengeId,
|
||||
proofB64: proofForChallenge(challenge, this.keys.secretKey)
|
||||
})
|
||||
)
|
||||
const ack = await nextJson(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
if (ack.type !== 'host-hello-ack') throw new Error('expected host hello acknowledgement')
|
||||
this.generation = ack.generation
|
||||
this.controlResumeSecret = ack.controlResumeSecret
|
||||
socket.on('message', (data) => this.onMessage(socket, data))
|
||||
socket.once('close', (code) => this.onClose(socket, code))
|
||||
socket.once('error', (error) => this.observe('socketError', { index: this.index, error }))
|
||||
this.observe('connected', { index: this.index })
|
||||
this.scheduleRefresh(this.phase.refreshOffsetMs)
|
||||
} catch (error) {
|
||||
discardFailedLoadSocket(socket)
|
||||
if (this.socket === socket) this.socket = null
|
||||
if (this.stopped) return
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
createSocket(assignment, relayToken) {
|
||||
return new WebSocket(`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, {
|
||||
headers: { authorization: `Bearer ${relayToken}` },
|
||||
perMessageDeflate: false
|
||||
})
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
if (this.shutdownPromise) return this.shutdownPromise
|
||||
this.stopped = true
|
||||
this.abortController.abort()
|
||||
if (this.refreshTimer) clearTimeout(this.refreshTimer)
|
||||
this.refreshTimer = null
|
||||
this.shutdownPromise = this.shutdownOnce()
|
||||
return this.shutdownPromise
|
||||
}
|
||||
|
||||
async shutdownOnce() {
|
||||
const socket = this.socket
|
||||
const closed = waitForClose(socket)
|
||||
for (const spliceSocket of this.spliceSockets) {
|
||||
if (
|
||||
spliceSocket.readyState !== spliceSocket.CLOSED &&
|
||||
spliceSocket.readyState !== spliceSocket.CLOSING
|
||||
) {
|
||||
spliceSocket.close(1000, 'load complete')
|
||||
}
|
||||
}
|
||||
this.rejectControlWaiters(new Error('control stopped'))
|
||||
if (socket && socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'load complete')
|
||||
}
|
||||
const settled = async () => {
|
||||
while (this.inFlight.size > 0) {
|
||||
await Promise.allSettled([...this.inFlight])
|
||||
}
|
||||
}
|
||||
await Promise.all([closed, settled()])
|
||||
this.observe('shutdown', {
|
||||
index: this.index,
|
||||
activeControls: this.socket?.readyState === this.socket?.OPEN ? 1 : 0,
|
||||
activeSpliceSockets: this.spliceSockets.size,
|
||||
inFlightOperations: this.inFlight.size,
|
||||
refreshTimerActive: this.refreshTimer !== null
|
||||
})
|
||||
}
|
||||
|
||||
openSplice(options = {}) {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
const operation = this.openSpliceOnce(options)
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
openInviteOffer() {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
const operation = this.openInviteOfferOnce()
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
async openInviteOfferOnce() {
|
||||
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
|
||||
throw new Error('active control required for invite offer')
|
||||
}
|
||||
const sequence = this.spliceSequence++
|
||||
const reqId = `load-offer-${this.index}-${sequence}`
|
||||
const response = this.waitForControlMessage(
|
||||
(message) =>
|
||||
message.reqId === reqId &&
|
||||
(message.type === 'invite-created' || message.type === 'control-error')
|
||||
)
|
||||
this.socket.send(JSON.stringify({
|
||||
type: 'invite-create',
|
||||
reqId,
|
||||
relayDeviceId: `load-offer-device-${this.index}-${sequence}`
|
||||
}))
|
||||
const result = await response
|
||||
if (result.type === 'control-error') throw new Error(`invite offer failed: ${result.code}`)
|
||||
if (
|
||||
typeof result.inviteToken !== 'string' ||
|
||||
!Number.isSafeInteger(result.expiresAt) ||
|
||||
result.expiresAt <= Date.now()
|
||||
) throw new Error('relay invite offer response invalid')
|
||||
}
|
||||
|
||||
async openSpliceOnce({
|
||||
payloadBytes = 64,
|
||||
readerMode = 'normal',
|
||||
readerHoldMs = 0,
|
||||
streamBytes = payloadBytes,
|
||||
frameBytes = payloadBytes,
|
||||
observeReaderPressure = async () => undefined,
|
||||
readerDelay = async (ms) => await new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
slowReaderHoldMs = 0,
|
||||
holdMs = 0
|
||||
} = {}) {
|
||||
if (
|
||||
!this.socket ||
|
||||
this.socket.readyState !== this.socket.OPEN ||
|
||||
this.generation === undefined ||
|
||||
this.lastAssignment === undefined
|
||||
) {
|
||||
throw new Error('active control required for splice')
|
||||
}
|
||||
if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 1) {
|
||||
throw new Error('splice payload bytes must be positive')
|
||||
}
|
||||
const sequence = this.spliceSequence++
|
||||
const reqId = `load-invite-${this.index}-${sequence}`
|
||||
const relayDeviceId = `load-device-${this.index}-${sequence}`
|
||||
let phone
|
||||
let data
|
||||
let opened = false
|
||||
try {
|
||||
const invitePromise = this.waitForControlMessage(
|
||||
(message) => message.type === 'invite-created' && message.reqId === reqId
|
||||
)
|
||||
this.socket.send(JSON.stringify({ type: 'invite-create', reqId, relayDeviceId }))
|
||||
const invite = await invitePromise
|
||||
if (typeof invite.inviteToken !== 'string') throw new Error('relay invite response invalid')
|
||||
|
||||
phone = this.createClientSocket(this.lastAssignment)
|
||||
this.trackSpliceSocket(phone)
|
||||
await waitForOpen(phone)
|
||||
const connectionPromise = this.waitForControlMessage(
|
||||
(message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId
|
||||
)
|
||||
phone.send(
|
||||
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
|
||||
)
|
||||
const connection = await connectionPromise
|
||||
if (typeof connection.connId !== 'string' || typeof connection.connTicket !== 'string') {
|
||||
throw new Error('relay connection response invalid')
|
||||
}
|
||||
|
||||
data = this.createHostDataSocket(this.lastAssignment, connection.connId)
|
||||
this.trackSpliceSocket(data)
|
||||
await waitForOpen(data)
|
||||
const phoneHello = nextJson(phone)
|
||||
data.send(
|
||||
JSON.stringify({
|
||||
type: 'host-data-auth',
|
||||
v: 1,
|
||||
connTicket: connection.connTicket,
|
||||
generation: this.generation
|
||||
})
|
||||
)
|
||||
if ((await phoneHello).ok !== true) throw new Error('relay rejected load splice')
|
||||
|
||||
if (slowReaderHoldMs > 0 && readerMode === 'normal') {
|
||||
readerMode = 'slow'
|
||||
readerHoldMs = slowReaderHoldMs
|
||||
streamBytes = payloadBytes
|
||||
frameBytes = payloadBytes
|
||||
}
|
||||
if (!['normal', 'slow', 'wedged'].includes(readerMode)) {
|
||||
throw new Error('reader mode is invalid')
|
||||
}
|
||||
const pausedSocket = readerMode === 'normal' ? undefined : phone._socket
|
||||
if (readerMode !== 'normal' && !pausedSocket) throw new Error('reader transport unavailable')
|
||||
if (readerMode === 'wedged') {
|
||||
const closes = [closeInfo(phone, readerHoldMs + 10_000), closeInfo(data, readerHoldMs + 10_000)]
|
||||
pausedSocket.pause()
|
||||
const readerPausedAt = Date.now()
|
||||
const readerPressure = observeReaderPressure({
|
||||
cellOrigin: this.lastAssignment.cellUrl,
|
||||
readerMode,
|
||||
streamBytes
|
||||
})
|
||||
const [sent] = await Promise.all([
|
||||
this.sendReaderStream(data, sequence, streamBytes, frameBytes, readerDelay),
|
||||
readerPressure
|
||||
])
|
||||
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
|
||||
pausedSocket.resume()
|
||||
const closeEvidence = await Promise.all(closes)
|
||||
const closeCodes = closeEvidence.map(({ code }) => code)
|
||||
if (!relayLoadWedgedCloseAccepted(closeCodes)) {
|
||||
throw new Error(`wedged reader close codes: ${closeCodes.join(',')}`)
|
||||
}
|
||||
opened = true
|
||||
this.observe('spliceOpened', { index: this.index, readerMode })
|
||||
this.observe('spliceWedged', { index: this.index, code: 4429, streamBytes: sent.bytes })
|
||||
return
|
||||
}
|
||||
|
||||
const expectedStreamBytes = readerMode === 'slow' ? streamBytes : payloadBytes
|
||||
const expectedFrameBytes = readerMode === 'slow' ? frameBytes : payloadBytes
|
||||
const phoneStream = receiveBinaryStream(
|
||||
phone,
|
||||
expectedStreamBytes,
|
||||
readerMode === 'slow' ? readerHoldMs + 10_000 : 10_000
|
||||
)
|
||||
const readerPausedAt = pausedSocket ? Date.now() : 0
|
||||
if (pausedSocket) pausedSocket.pause()
|
||||
const readerPressure = readerMode === 'slow'
|
||||
? observeReaderPressure({
|
||||
cellOrigin: this.lastAssignment.cellUrl,
|
||||
readerMode,
|
||||
streamBytes: expectedStreamBytes
|
||||
})
|
||||
: Promise.resolve()
|
||||
const [sent] = await Promise.all([
|
||||
this.sendReaderStream(
|
||||
data,
|
||||
sequence,
|
||||
expectedStreamBytes,
|
||||
expectedFrameBytes,
|
||||
readerDelay
|
||||
),
|
||||
readerPressure
|
||||
])
|
||||
if (readerMode === 'slow') {
|
||||
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
|
||||
pausedSocket.resume()
|
||||
}
|
||||
const receivedByPhone = await phoneStream
|
||||
if (receivedByPhone.bytes !== sent.bytes || receivedByPhone.digest !== sent.digest) {
|
||||
throw new Error('relay changed host-to-client splice payload')
|
||||
}
|
||||
|
||||
const textPayload = `orca-relay-load:${this.index}:${sequence}:${payloadBytes}`
|
||||
const dataFrame = nextFrame(data)
|
||||
phone.send(textPayload)
|
||||
const receivedByHost = await dataFrame
|
||||
if (receivedByHost.binary || receivedByHost.bytes.toString() !== textPayload) {
|
||||
throw new Error('relay changed client-to-host splice payload')
|
||||
}
|
||||
opened = true
|
||||
this.observe('spliceOpened', { index: this.index, readerMode })
|
||||
if (holdMs > 0 && !(await this.waitForSpliceHold(holdMs, [phone, data]))) return
|
||||
this.observe('spliceCompleted', { index: this.index, readerMode })
|
||||
} catch (error) {
|
||||
if (!this.stopped) this.observe('spliceFailed', { index: this.index, error })
|
||||
throw error
|
||||
} finally {
|
||||
await Promise.all([this.closeSpliceSocket(phone), this.closeSpliceSocket(data)])
|
||||
if (opened) this.observe('spliceClosed', { index: this.index })
|
||||
}
|
||||
}
|
||||
|
||||
waitForSpliceHold(holdMs, sockets) {
|
||||
if (this.stopped) return Promise.resolve(false)
|
||||
return new Promise((resolve, reject) => {
|
||||
const finish = (error, completed = false) => {
|
||||
clearTimeout(timer)
|
||||
this.abortController.signal.removeEventListener('abort', onAbort)
|
||||
for (const socket of sockets) {
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
}
|
||||
if (error) reject(error)
|
||||
else resolve(completed)
|
||||
}
|
||||
const onAbort = () => finish(undefined, false)
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
const timer = setTimeout(() => finish(undefined, true), holdMs)
|
||||
this.abortController.signal.addEventListener('abort', onAbort, { once: true })
|
||||
for (const socket of sockets) {
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
createClientSocket(assignment) {
|
||||
return new WebSocket(
|
||||
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/connect/${this.relayHostId}`,
|
||||
{ perMessageDeflate: false }
|
||||
)
|
||||
}
|
||||
|
||||
createHostDataSocket(assignment, connId) {
|
||||
return new WebSocket(
|
||||
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/data/${connId}`,
|
||||
{ perMessageDeflate: false }
|
||||
)
|
||||
}
|
||||
|
||||
splicePayload(sequence, payloadBytes) {
|
||||
const seed = createHash('sha256')
|
||||
.update(`orca-relay-load:${this.index}:${sequence}`)
|
||||
.digest()
|
||||
return Buffer.allocUnsafe(payloadBytes).map((_, index) => seed[index % seed.length])
|
||||
}
|
||||
|
||||
async sendReaderStream(socket, sequence, streamBytes, frameBytes, delay) {
|
||||
const hash = createHash('sha256')
|
||||
let sentBytes = 0
|
||||
let frameIndex = 0
|
||||
while (sentBytes < streamBytes) {
|
||||
const bytes = Math.min(frameBytes, streamBytes - sentBytes)
|
||||
const payload = this.splicePayload(sequence + frameIndex, bytes)
|
||||
await this.sendReaderFrame(socket, payload)
|
||||
hash.update(payload)
|
||||
sentBytes += bytes
|
||||
frameIndex++
|
||||
while (socket.bufferedAmount > frameBytes) await delay(10)
|
||||
}
|
||||
return { bytes: sentBytes, digest: hash.digest('hex') }
|
||||
}
|
||||
|
||||
sendReaderFrame(socket, payload) {
|
||||
if (socket.send.length < 2) {
|
||||
socket.send(payload)
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('reader send timeout')), 10_000)
|
||||
socket.send(payload, (error) => {
|
||||
clearTimeout(timer)
|
||||
if (error) reject(error)
|
||||
else resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
trackSpliceSocket(socket) {
|
||||
this.spliceSockets.add(socket)
|
||||
socket.once('close', () => this.spliceSockets.delete(socket))
|
||||
}
|
||||
|
||||
async closeSpliceSocket(socket) {
|
||||
if (!socket) return
|
||||
const closed = waitForClose(socket).catch(() => undefined)
|
||||
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'splice complete')
|
||||
}
|
||||
await closed
|
||||
this.spliceSockets.delete(socket)
|
||||
}
|
||||
|
||||
async openRebindProbe() {
|
||||
if (
|
||||
!this.socket ||
|
||||
this.socket.readyState !== this.socket.OPEN ||
|
||||
this.generation === undefined ||
|
||||
this.controlResumeSecret === undefined ||
|
||||
this.lastAssignment === undefined
|
||||
) {
|
||||
throw new Error('active control required for rebind probe')
|
||||
}
|
||||
const relayToken = await this.relayToken()
|
||||
const socket = new WebSocket(
|
||||
`${this.lastAssignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`,
|
||||
{
|
||||
headers: { authorization: `Bearer ${relayToken}` },
|
||||
perMessageDeflate: false
|
||||
}
|
||||
)
|
||||
try {
|
||||
await waitForOpen(socket)
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
assignmentEpoch: this.lastAssignment.assignmentEpoch,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
|
||||
appVersion: 'relay-load-rebind-proof',
|
||||
previousGeneration: this.generation,
|
||||
controlResumeSecret: this.controlResumeSecret
|
||||
})
|
||||
)
|
||||
const challenge = await nextJson(socket)
|
||||
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
|
||||
socket.on('error', () => undefined)
|
||||
const closed = new Promise((resolve) => socket.once('close', resolve))
|
||||
return {
|
||||
close: async () => {
|
||||
const closeCompleted = waitForClose(socket)
|
||||
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'rebind boundary proved')
|
||||
}
|
||||
await closeCompleted
|
||||
},
|
||||
closed,
|
||||
isOpen: () => socket.readyState === socket.OPEN
|
||||
}
|
||||
} catch (error) {
|
||||
discardFailedLoadSocket(socket)
|
||||
await waitForClose(socket).catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async relayToken() {
|
||||
const accessToken = this.options.accessTokenProvider
|
||||
? await this.options.accessTokenProvider()
|
||||
: this.options.accessToken
|
||||
if (accessToken) {
|
||||
const body = await this.requestJson(
|
||||
`${this.options.authOrigin}/v1/desktop/auth/relay-token`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${accessToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
relayHostId: this.relayHostId,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64')
|
||||
})
|
||||
},
|
||||
'relay token exchange timeout',
|
||||
(status) => `relay token exchange failed: ${status}`
|
||||
)
|
||||
if (typeof body.relayToken !== 'string') throw new Error('relay token exchange omitted token')
|
||||
if (!this.stopped) this.observe('token', { index: this.index })
|
||||
return body.relayToken
|
||||
}
|
||||
const token = await new SignJWT({
|
||||
prof: `load-profile-${this.index}`,
|
||||
org: 'relay-load',
|
||||
purpose: 'host-control',
|
||||
relayHostId: this.relayHostId
|
||||
})
|
||||
.setProtectedHeader({ alg: 'ES256', kid: this.options.signingKeyId })
|
||||
.setIssuer(this.options.authOrigin)
|
||||
.setAudience('orca-relay')
|
||||
.setSubject(`load-user-${this.index}`)
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('5m')
|
||||
.sign(this.options.signingKey)
|
||||
if (!this.stopped) this.observe('token', { index: this.index })
|
||||
return token
|
||||
}
|
||||
|
||||
async requestAssignment(preferredRegion) {
|
||||
return await this.assignment(await this.relayToken(), preferredRegion)
|
||||
}
|
||||
|
||||
async assignment(relayToken, preferredRegion = this.options.preferredRegion) {
|
||||
if (!this.options.directorOrigin) {
|
||||
if (!this.options.targetOrigin) throw new Error('relay target origin missing')
|
||||
return { cellUrl: this.options.targetOrigin, assignmentEpoch: 1 }
|
||||
}
|
||||
const body = await this.requestJson(
|
||||
`${this.options.directorOrigin}/v1/assign`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${relayToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
...(preferredRegion ? { preferredRegion } : {})
|
||||
})
|
||||
},
|
||||
'relay assignment timeout',
|
||||
(status, errorCode) =>
|
||||
`relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`,
|
||||
CAPACITY_ASSIGNMENT_ERRORS
|
||||
)
|
||||
if (
|
||||
typeof body.cellUrl !== 'string' ||
|
||||
!Number.isSafeInteger(body.assignmentEpoch) ||
|
||||
body.assignmentEpoch < 1
|
||||
) {
|
||||
throw new Error('relay assignment response invalid')
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async requestJson(url, init, timeoutMessage, httpErrorMessage, allowedErrorCodes = []) {
|
||||
const controller = new AbortController()
|
||||
const onShutdown = () => controller.abort()
|
||||
if (this.abortController.signal.aborted) controller.abort()
|
||||
else this.abortController.signal.addEventListener('abort', onShutdown, { once: true })
|
||||
let timedOut = false
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true
|
||||
controller.abort()
|
||||
}, this.options.requestTimeoutMs ?? 10_000)
|
||||
try {
|
||||
const response = await fetch(url, { ...init, signal: controller.signal })
|
||||
if (!response.ok) {
|
||||
let bodyConsumed = false
|
||||
let errorCode
|
||||
if (allowedErrorCodes.length > 0) {
|
||||
try {
|
||||
const body = await response.json()
|
||||
bodyConsumed = true
|
||||
if (allowedErrorCodes.includes(body?.error)) errorCode = body.error
|
||||
} catch {
|
||||
// Preserve the bounded status-only classification.
|
||||
}
|
||||
}
|
||||
if (!bodyConsumed) await cancelResponse(response)
|
||||
throw new Error(httpErrorMessage(response.status, errorCode))
|
||||
}
|
||||
return await response.json()
|
||||
} catch (error) {
|
||||
if (timedOut) throw new Error(timeoutMessage, { cause: error })
|
||||
throw error
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
this.abortController.signal.removeEventListener('abort', onShutdown)
|
||||
}
|
||||
}
|
||||
|
||||
onMessage(socket, data) {
|
||||
let message
|
||||
try {
|
||||
message = JSON.parse(data.toString())
|
||||
} catch {
|
||||
this.observe('protocolError', { index: this.index })
|
||||
return
|
||||
}
|
||||
for (const waiter of this.controlWaiters) {
|
||||
if (waiter.matches(message)) {
|
||||
this.controlWaiters.delete(waiter)
|
||||
clearTimeout(waiter.timer)
|
||||
waiter.resolve(message)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (message.type === 'ping') {
|
||||
socket.send(JSON.stringify({ type: 'pong', t: message.t }))
|
||||
this.observe('ping', { index: this.index })
|
||||
} else if (message.type === 'drain') {
|
||||
this.drainExpected = true
|
||||
this.observe('drain', { index: this.index })
|
||||
}
|
||||
}
|
||||
|
||||
onClose(socket, code) {
|
||||
if (this.socket !== socket) return
|
||||
this.socket = null
|
||||
if (this.refreshTimer) clearTimeout(this.refreshTimer)
|
||||
this.refreshTimer = null
|
||||
this.rejectControlWaiters(new Error(`control closed: ${code}`))
|
||||
this.observe('closed', {
|
||||
index: this.index,
|
||||
code,
|
||||
stopped: this.stopped,
|
||||
expectedDrain: this.drainExpected
|
||||
})
|
||||
this.drainExpected = false
|
||||
}
|
||||
|
||||
waitForControlMessage(matches, timeoutMs = 10_000) {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = {
|
||||
matches,
|
||||
resolve,
|
||||
reject,
|
||||
timer: setTimeout(() => {
|
||||
this.controlWaiters.delete(waiter)
|
||||
reject(new Error('control response timeout'))
|
||||
}, timeoutMs)
|
||||
}
|
||||
this.controlWaiters.add(waiter)
|
||||
})
|
||||
}
|
||||
|
||||
rejectControlWaiters(error) {
|
||||
for (const waiter of this.controlWaiters) {
|
||||
clearTimeout(waiter.timer)
|
||||
waiter.reject(error)
|
||||
}
|
||||
this.controlWaiters.clear()
|
||||
}
|
||||
|
||||
scheduleRefresh(delayMs) {
|
||||
if (this.stopped) return
|
||||
this.refreshTimer = setTimeout(() => {
|
||||
void this.refresh().then(
|
||||
() => this.scheduleRefresh(this.phase.refreshIntervalMs),
|
||||
() => this.scheduleRefresh(this.phase.refreshIntervalMs)
|
||||
)
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
refresh() {
|
||||
if (this.stopped) return Promise.resolve()
|
||||
const operation = this.refreshOnce()
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
async refreshOnce() {
|
||||
const socket = this.socket
|
||||
if (this.stopped || !socket || socket.readyState !== socket.OPEN) return
|
||||
try {
|
||||
const relayJwt = await this.relayToken()
|
||||
if (this.stopped || this.socket !== socket || socket.readyState !== socket.OPEN) return
|
||||
socket.send(JSON.stringify({ type: 'auth-refresh', relayJwt }))
|
||||
this.observe('refresh', { index: this.index })
|
||||
} catch (error) {
|
||||
if (!this.stopped) this.observe('refreshError', { index: this.index, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,903 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { generateKeyPairSync } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { createRequire } from 'node:module'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
RelayLoadControlPeer,
|
||||
relayLoadWedgedCloseAccepted
|
||||
} from './relay-load-control-peer.mjs'
|
||||
|
||||
const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url))
|
||||
const nacl = requireFromRelay('tweetnacl')
|
||||
const { buildHostChallengePlaintext } = await import(
|
||||
requireFromRelay.resolve('@orca-cloud/relay-contract')
|
||||
)
|
||||
|
||||
function deferred() {
|
||||
let resolve
|
||||
let reject
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise
|
||||
reject = rejectPromise
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
function peerOptions(overrides = {}) {
|
||||
const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' })
|
||||
return {
|
||||
authOrigin: 'https://auth.test',
|
||||
directorOrigin: 'https://director.test',
|
||||
reconnectMaxMs: 0,
|
||||
seed: 1,
|
||||
signingKey: privateKey,
|
||||
signingKeyId: 'test-key',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function response(body) {
|
||||
return { ok: true, status: 200, json: async () => body }
|
||||
}
|
||||
|
||||
function fakeOpenSocket() {
|
||||
const socket = new EventEmitter()
|
||||
socket.OPEN = 1
|
||||
socket.CLOSING = 2
|
||||
socket.CLOSED = 3
|
||||
socket.readyState = socket.OPEN
|
||||
socket.sent = []
|
||||
socket.send = (message) => socket.sent.push(message)
|
||||
socket.close = (code = 1000, reason = '') => {
|
||||
socket.readyState = socket.CLOSING
|
||||
queueMicrotask(() => {
|
||||
socket.readyState = socket.CLOSED
|
||||
socket.emit('close', code, Buffer.from(reason))
|
||||
})
|
||||
}
|
||||
socket.terminate = socket.close
|
||||
return socket
|
||||
}
|
||||
|
||||
function fakeHandshakeSocket() {
|
||||
const socket = fakeOpenSocket()
|
||||
socket.CONNECTING = 0
|
||||
socket.readyState = socket.CONNECTING
|
||||
socket.open = () => {
|
||||
socket.readyState = socket.OPEN
|
||||
socket.emit('open')
|
||||
}
|
||||
socket.message = (message) => socket.emit('message', Buffer.from(JSON.stringify(message)))
|
||||
return socket
|
||||
}
|
||||
|
||||
function openOnNextTurn(socket) {
|
||||
queueMicrotask(() => socket.open())
|
||||
return socket
|
||||
}
|
||||
|
||||
function validChallenge(peer) {
|
||||
const relayKeys = nacl.box.keyPair()
|
||||
const nonce = nacl.randomBytes(nacl.box.nonceLength)
|
||||
const plaintext = buildHostChallengePlaintext(
|
||||
new Uint8Array([1, 2, 3]),
|
||||
nacl.randomBytes(32)
|
||||
)
|
||||
const ciphertext = nacl.box(plaintext, nonce, peer.keys.publicKey, relayKeys.secretKey)
|
||||
return {
|
||||
type: 'host-challenge',
|
||||
challengeId: 'test-challenge',
|
||||
ciphertextB64: Buffer.from(ciphertext).toString('base64'),
|
||||
nonceB64: Buffer.from(nonce).toString('base64'),
|
||||
relayEphemeralPublicKeyB64: Buffer.from(relayKeys.publicKey).toString('base64')
|
||||
}
|
||||
}
|
||||
|
||||
test('shutdown waits for pending assignment and prevents a late connection', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
const assignment = deferred()
|
||||
const assignmentStarted = deferred()
|
||||
global.fetch = () => {
|
||||
assignmentStarted.resolve()
|
||||
return assignment.promise
|
||||
}
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(0, peerOptions(), (type) => observations.push(type))
|
||||
|
||||
const connecting = peer.connect()
|
||||
await assignmentStarted.promise
|
||||
let shutdownFinished = false
|
||||
const shutdown = peer.shutdown().then(() => {
|
||||
shutdownFinished = true
|
||||
})
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
assert.equal(shutdownFinished, false)
|
||||
|
||||
assignment.resolve(response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 }))
|
||||
await Promise.all([connecting, shutdown])
|
||||
assert.equal(peer.socket, null)
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('shutdown after socket open prevents a late handshake', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
const socket = fakeHandshakeSocket()
|
||||
const socketCreated = deferred()
|
||||
peer.createSocket = () => {
|
||||
socketCreated.resolve()
|
||||
return socket
|
||||
}
|
||||
|
||||
const connecting = peer.connect()
|
||||
await socketCreated.promise
|
||||
socket.open()
|
||||
await Promise.all([connecting, peer.shutdown()])
|
||||
|
||||
assert.deepEqual(socket.sent, [])
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('shutdown after challenge prevents a late acknowledgement', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
const socket = fakeHandshakeSocket()
|
||||
const socketCreated = deferred()
|
||||
const helloSent = deferred()
|
||||
socket.send = (message) => {
|
||||
socket.sent.push(message)
|
||||
helloSent.resolve()
|
||||
}
|
||||
peer.createSocket = () => {
|
||||
socketCreated.resolve()
|
||||
return socket
|
||||
}
|
||||
|
||||
const connecting = peer.connect()
|
||||
await socketCreated.promise
|
||||
socket.open()
|
||||
await helloSent.promise
|
||||
socket.message({ type: 'host-challenge' })
|
||||
await Promise.all([connecting, peer.shutdown()])
|
||||
|
||||
assert.equal(socket.sent.length, 1)
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('shutdown after host acknowledgement prevents a late connected observation', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
const socket = fakeHandshakeSocket()
|
||||
const socketCreated = deferred()
|
||||
const helloSent = deferred()
|
||||
const proofSent = deferred()
|
||||
socket.send = (message) => {
|
||||
socket.sent.push(message)
|
||||
if (socket.sent.length === 1) helloSent.resolve()
|
||||
else proofSent.resolve()
|
||||
}
|
||||
peer.createSocket = () => {
|
||||
socketCreated.resolve()
|
||||
return socket
|
||||
}
|
||||
|
||||
const connecting = peer.connect()
|
||||
await socketCreated.promise
|
||||
socket.open()
|
||||
await helloSent.promise
|
||||
socket.message(validChallenge(peer))
|
||||
await proofSent.promise
|
||||
socket.message({ type: 'host-hello-ack', generation: 1, controlResumeSecret: 'test-secret' })
|
||||
await Promise.all([connecting, peer.shutdown()])
|
||||
|
||||
assert.equal(socket.sent.length, 2)
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('shutdown prevents a pending refresh from sending or reporting success', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
const token = deferred()
|
||||
const tokenStarted = deferred()
|
||||
global.fetch = () => {
|
||||
tokenStarted.resolve()
|
||||
return token.promise
|
||||
}
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
const socket = fakeOpenSocket()
|
||||
peer.socket = socket
|
||||
|
||||
const refreshing = peer.refresh()
|
||||
await tokenStarted.promise
|
||||
const shutdown = peer.shutdown()
|
||||
token.resolve(response({ relayToken: 'test-relay-token' }))
|
||||
await Promise.all([refreshing, shutdown])
|
||||
|
||||
assert.deepEqual(socket.sent, [])
|
||||
assert.equal(observations.includes('refresh'), false)
|
||||
assert.equal(observations.includes('refreshError'), false)
|
||||
})
|
||||
|
||||
test('shutdown suppresses a late refresh error', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
const token = deferred()
|
||||
const tokenStarted = deferred()
|
||||
global.fetch = () => {
|
||||
tokenStarted.resolve()
|
||||
return token.promise
|
||||
}
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
peer.socket = fakeOpenSocket()
|
||||
|
||||
const refreshing = peer.refresh()
|
||||
await tokenStarted.promise
|
||||
const shutdown = peer.shutdown()
|
||||
token.reject(new Error('late token failure'))
|
||||
await Promise.all([refreshing, shutdown])
|
||||
|
||||
assert.equal(observations.includes('refreshError'), false)
|
||||
})
|
||||
|
||||
test('shutdown aborts an HTTP request that would otherwise remain pending', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
const requestStarted = deferred()
|
||||
let aborted = false
|
||||
global.fetch = (_url, { signal }) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
requestStarted.resolve()
|
||||
signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
aborted = true
|
||||
reject(new Error('request aborted'))
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
|
||||
const connecting = peer.connect()
|
||||
await requestStarted.promise
|
||||
await Promise.all([connecting, peer.shutdown()])
|
||||
|
||||
assert.equal(aborted, true)
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('bounds a pending HTTP request with a classified timeout', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
global.fetch = (_url, { signal }) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
signal.addEventListener('abort', () => reject(new Error('request aborted')), { once: true })
|
||||
})
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({
|
||||
accessToken: 'test-access-token',
|
||||
directorOrigin: undefined,
|
||||
requestTimeoutMs: 1
|
||||
}),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
await assert.rejects(peer.connect(), /relay token exchange timeout/)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('bounds a stalled token response body', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
global.fetch = async (_url, { signal }) => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () =>
|
||||
new Promise((_resolve, reject) => {
|
||||
signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true })
|
||||
})
|
||||
})
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({
|
||||
accessToken: 'test-access-token',
|
||||
directorOrigin: undefined,
|
||||
requestTimeoutMs: 1
|
||||
}),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
await assert.rejects(peer.connect(), /relay token exchange timeout/)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('bounds a stalled assignment response body', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
global.fetch = async (_url, { signal }) => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () =>
|
||||
new Promise((_resolve, reject) => {
|
||||
signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true })
|
||||
})
|
||||
})
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ requestTimeoutMs: 1 }),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
await assert.rejects(peer.connect(), /relay assignment timeout/)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('shutdown aborts a stalled successful response body', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
const bodyStarted = deferred()
|
||||
let bodyAborted = false
|
||||
global.fetch = async (_url, { signal }) => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () =>
|
||||
new Promise((_resolve, reject) => {
|
||||
bodyStarted.resolve()
|
||||
signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
bodyAborted = true
|
||||
reject(new Error('body aborted'))
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
})
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
|
||||
(type) => observations.push(type)
|
||||
)
|
||||
|
||||
const connecting = peer.connect()
|
||||
await bodyStarted.promise
|
||||
await Promise.all([connecting, peer.shutdown()])
|
||||
|
||||
assert.equal(bodyAborted, true)
|
||||
assert.equal(observations.includes('connected'), false)
|
||||
})
|
||||
|
||||
test('cancels a rejected HTTP response body', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
let canceled = false
|
||||
global.fetch = async () => ({
|
||||
ok: false,
|
||||
status: 503,
|
||||
body: {
|
||||
cancel: async () => {
|
||||
canceled = true
|
||||
}
|
||||
}
|
||||
})
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
await assert.rejects(peer.connect(), /relay token exchange failed: 503/)
|
||||
await peer.shutdown()
|
||||
assert.equal(canceled, true)
|
||||
})
|
||||
|
||||
test('preserves only an exact capacity assignment rejection reason', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
global.fetch = async () => ({
|
||||
ok: false,
|
||||
status: 503,
|
||||
json: async () => ({ error: 'relay_connection_headroom_exhausted' })
|
||||
})
|
||||
const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined)
|
||||
|
||||
await assert.rejects(
|
||||
peer.connect(),
|
||||
/relay assignment failed: 503 relay_connection_headroom_exhausted/
|
||||
)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('sends preferred region and preserves the genuine director epoch', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
let assignmentRequest
|
||||
global.fetch = async (_url, init) => {
|
||||
assignmentRequest = JSON.parse(init.body)
|
||||
return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 47 })
|
||||
}
|
||||
const peer = new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ preferredRegion: 'asia-east2' }),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
const assignment = await peer.assignment('relay-token')
|
||||
|
||||
assert.deepEqual(assignmentRequest, {
|
||||
v: 1,
|
||||
relayHostId: peer.relayHostId,
|
||||
preferredRegion: 'asia-east2'
|
||||
})
|
||||
assert.equal(assignment.assignmentEpoch, 47)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('omits preferred region and rejects a fabricated director epoch', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => {
|
||||
global.fetch = originalFetch
|
||||
})
|
||||
let assignmentRequest
|
||||
global.fetch = async (_url, init) => {
|
||||
assignmentRequest = JSON.parse(init.body)
|
||||
return response({ cellUrl: 'https://cell.test', assignmentEpoch: 0 })
|
||||
}
|
||||
const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined)
|
||||
|
||||
await assert.rejects(peer.assignment('relay-token'), /assignment response invalid/)
|
||||
assert.equal('preferredRegion' in assignmentRequest, false)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('rejects ambiguous direct and director assignment modes', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
new RelayLoadControlPeer(
|
||||
0,
|
||||
peerOptions({ targetOrigin: 'https://cell.test' }),
|
||||
() => undefined
|
||||
),
|
||||
/either directorOrigin or targetOrigin/
|
||||
)
|
||||
})
|
||||
|
||||
test('opens a genuine splice and verifies payloads in both directions', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(3, peerOptions(), (type, detail) => {
|
||||
observations.push({ type, detail })
|
||||
})
|
||||
const control = fakeOpenSocket()
|
||||
const phone = fakeHandshakeSocket()
|
||||
const data = fakeHandshakeSocket()
|
||||
let dataPayloadBytes = 0
|
||||
let observationStarted = false
|
||||
let sentBeforeObservation = false
|
||||
let paused = 0
|
||||
let resumed = 0
|
||||
phone._socket = {
|
||||
pause: () => paused++,
|
||||
resume: () => resumed++
|
||||
}
|
||||
control.send = (raw) => {
|
||||
const message = JSON.parse(raw)
|
||||
if (message.type === 'invite-create') {
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'invite-created',
|
||||
reqId: message.reqId,
|
||||
inviteToken: 'invite-token'
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
phone.send = (raw) => {
|
||||
if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') {
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'conn-open',
|
||||
relayDeviceId: 'load-device-3-0',
|
||||
connId: 'connection-1',
|
||||
connTicket: 'connection-ticket'
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
queueMicrotask(() => data.emit('message', Buffer.from(raw), false))
|
||||
}
|
||||
data.send = (raw) => {
|
||||
if (typeof raw === 'string') {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
|
||||
return
|
||||
}
|
||||
const dataPayload = Buffer.from(raw)
|
||||
if (!observationStarted) sentBeforeObservation = true
|
||||
dataPayloadBytes += dataPayload.byteLength
|
||||
queueMicrotask(() => phone.emit('message', dataPayload, true))
|
||||
}
|
||||
peer.socket = control
|
||||
peer.generation = 9
|
||||
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 47 }
|
||||
control.on('message', (raw) => peer.onMessage(control, raw))
|
||||
peer.createClientSocket = () => openOnNextTurn(phone)
|
||||
peer.createHostDataSocket = () => openOnNextTurn(data)
|
||||
|
||||
await peer.openSplice({
|
||||
readerMode: 'slow',
|
||||
readerHoldMs: 1,
|
||||
streamBytes: 300 * 1024,
|
||||
frameBytes: 64 * 1024,
|
||||
observeReaderPressure: async () => { observationStarted = true }
|
||||
})
|
||||
|
||||
assert.equal(dataPayloadBytes, 300 * 1024)
|
||||
assert.equal(sentBeforeObservation, false)
|
||||
assert.equal(paused, 1)
|
||||
assert.equal(resumed, 1)
|
||||
assert.equal(observations.filter(({ type }) => type === 'spliceCompleted').length, 1)
|
||||
assert.equal(observations.some(({ type }) => type === 'spliceFailed'), false)
|
||||
await peer.shutdown()
|
||||
assert.deepEqual(observations.at(-1), {
|
||||
type: 'shutdown',
|
||||
detail: {
|
||||
index: 3,
|
||||
activeControls: 0,
|
||||
activeSpliceSockets: 0,
|
||||
inFlightOperations: 0,
|
||||
refreshTimerActive: false
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('opens invitation leases and preserves exact capacity errors', async () => {
|
||||
const peer = new RelayLoadControlPeer(4, peerOptions(), () => undefined)
|
||||
const control = fakeOpenSocket()
|
||||
peer.socket = control
|
||||
control.on('message', (raw) => peer.onMessage(control, raw))
|
||||
let calls = 0
|
||||
control.send = (raw) => {
|
||||
const request = JSON.parse(raw)
|
||||
calls++
|
||||
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify(
|
||||
calls === 1
|
||||
? {
|
||||
type: 'invite-created', reqId: request.reqId,
|
||||
inviteToken: 'invite-token', expiresAt: Date.now() + 60_000
|
||||
}
|
||||
: { type: 'control-error', reqId: request.reqId, code: 'relay_capacity_exhausted' }
|
||||
))))
|
||||
}
|
||||
|
||||
await peer.openInviteOffer()
|
||||
await assert.rejects(peer.openInviteOffer(), /invite offer failed: relay_capacity_exhausted/)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('can request the same assignment with an explicit replacement preference', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => { global.fetch = originalFetch })
|
||||
const requests = []
|
||||
global.fetch = async (url, init) => {
|
||||
if (String(url).endsWith('/v1/assign')) {
|
||||
requests.push(JSON.parse(init.body))
|
||||
return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 3 })
|
||||
}
|
||||
return response({ relayToken: 'relay-token' })
|
||||
}
|
||||
const peer = new RelayLoadControlPeer(
|
||||
5,
|
||||
peerOptions({ accessToken: 'access-token', preferredRegion: 'asia-east2' }),
|
||||
() => undefined
|
||||
)
|
||||
|
||||
assert.equal((await peer.requestAssignment('us-central1')).cellUrl, 'https://asia-cell.test')
|
||||
assert.equal(requests[0].preferredRegion, 'us-central1')
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('uses a refreshable workflow access-token provider', async (context) => {
|
||||
const originalFetch = global.fetch
|
||||
context.after(() => { global.fetch = originalFetch })
|
||||
let providerCalls = 0
|
||||
let authorization
|
||||
global.fetch = async (url, init) => {
|
||||
if (String(url).endsWith('/v1/desktop/auth/relay-token')) {
|
||||
authorization = init.headers.authorization
|
||||
return response({ relayToken: 'relay-token' })
|
||||
}
|
||||
return response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 })
|
||||
}
|
||||
const peer = new RelayLoadControlPeer(6, peerOptions({
|
||||
accessTokenProvider: async () => {
|
||||
providerCalls++
|
||||
return 'refreshed-access-token'
|
||||
}
|
||||
}), () => undefined)
|
||||
|
||||
await peer.requestAssignment('asia-east2')
|
||||
assert.equal(providerCalls, 1)
|
||||
assert.equal(authorization, 'Bearer refreshed-access-token')
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('accepts a forced close only when the responsive splice leg receives 4429', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(7, peerOptions(), (type, detail) => {
|
||||
observations.push({ type, detail })
|
||||
})
|
||||
const control = fakeOpenSocket()
|
||||
const phone = fakeHandshakeSocket()
|
||||
const data = fakeHandshakeSocket()
|
||||
let streamStarted
|
||||
const streamStartedPromise = new Promise((resolve) => { streamStarted = resolve })
|
||||
phone._socket = { pause: () => undefined, resume: () => undefined }
|
||||
control.send = (raw) => {
|
||||
const message = JSON.parse(raw)
|
||||
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({
|
||||
type: 'invite-created', reqId: message.reqId, inviteToken: 'invite-token'
|
||||
}))))
|
||||
}
|
||||
phone.send = (raw) => {
|
||||
if (typeof raw === 'string' && raw.startsWith('{')) {
|
||||
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({
|
||||
type: 'conn-open', relayDeviceId: 'load-device-7-0',
|
||||
connId: 'connection-7', connTicket: 'connection-ticket'
|
||||
}))))
|
||||
}
|
||||
}
|
||||
data.send = (raw) => {
|
||||
if (typeof raw === 'string') {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
|
||||
} else {
|
||||
streamStarted()
|
||||
}
|
||||
}
|
||||
peer.socket = control
|
||||
peer.generation = 11
|
||||
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 9 }
|
||||
control.on('message', (raw) => peer.onMessage(control, raw))
|
||||
peer.createClientSocket = () => openOnNextTurn(phone)
|
||||
peer.createHostDataSocket = () => openOnNextTurn(data)
|
||||
|
||||
await peer.openSplice({
|
||||
readerMode: 'wedged',
|
||||
readerHoldMs: 10_001,
|
||||
streamBytes: 300 * 1024,
|
||||
frameBytes: 64 * 1024,
|
||||
observeReaderPressure: async () => {
|
||||
await streamStartedPromise
|
||||
phone.close(1006)
|
||||
data.close(4429, 'wedged relay link')
|
||||
},
|
||||
readerDelay: async () => undefined
|
||||
})
|
||||
|
||||
assert.equal(observations.filter(({ type }) => type === 'spliceWedged').length, 1)
|
||||
assert.equal(observations.find(({ type }) => type === 'spliceWedged').detail.code, 4429)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('requires one 4429 and rejects unrelated wedged close codes', () => {
|
||||
assert.equal(relayLoadWedgedCloseAccepted([4429, 4429]), true)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([1006, 4429]), true)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([4429, 1006]), false)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([1006, 1006]), false)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([1000, 4429]), false)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([4429]), false)
|
||||
assert.equal(relayLoadWedgedCloseAccepted([1006, 4429, 4429]), false)
|
||||
})
|
||||
|
||||
test('streams reader frames with bounded send-side backpressure', async () => {
|
||||
const peer = new RelayLoadControlPeer(9, peerOptions(), () => undefined)
|
||||
let inFlight = 0
|
||||
let peakInFlight = 0
|
||||
let sentBytes = 0
|
||||
const socket = {
|
||||
bufferedAmount: 0,
|
||||
send(payload, callback) {
|
||||
inFlight++
|
||||
peakInFlight = Math.max(peakInFlight, inFlight)
|
||||
sentBytes += payload.byteLength
|
||||
this.bufferedAmount = payload.byteLength
|
||||
queueMicrotask(() => {
|
||||
this.bufferedAmount = 0
|
||||
inFlight--
|
||||
callback()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const result = await peer.sendReaderStream(
|
||||
socket,
|
||||
0,
|
||||
1024 * 1024,
|
||||
64 * 1024,
|
||||
async () => undefined
|
||||
)
|
||||
|
||||
assert.equal(result.bytes, 1024 * 1024)
|
||||
assert.equal(sentBytes, 1024 * 1024)
|
||||
assert.equal(peakInFlight, 1)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('reports a bidirectional splice payload mismatch', async () => {
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(2, peerOptions(), (type) => observations.push(type))
|
||||
const control = fakeOpenSocket()
|
||||
const phone = fakeHandshakeSocket()
|
||||
const data = fakeHandshakeSocket()
|
||||
control.send = (raw) => {
|
||||
const message = JSON.parse(raw)
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' })
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
phone.send = (raw) => {
|
||||
if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') {
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'conn-open',
|
||||
relayDeviceId: 'load-device-2-0',
|
||||
connId: 'connection-2',
|
||||
connTicket: 'ticket'
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
data.send = (raw) => {
|
||||
if (typeof raw === 'string') {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
|
||||
} else {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.alloc(Buffer.from(raw).byteLength), true))
|
||||
}
|
||||
}
|
||||
peer.socket = control
|
||||
peer.generation = 4
|
||||
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 2 }
|
||||
control.on('message', (raw) => peer.onMessage(control, raw))
|
||||
peer.createClientSocket = () => openOnNextTurn(phone)
|
||||
peer.createHostDataSocket = () => openOnNextTurn(data)
|
||||
|
||||
await assert.rejects(peer.openSplice(), /changed host-to-client splice payload/)
|
||||
assert.equal(observations.includes('spliceFailed'), true)
|
||||
await peer.shutdown()
|
||||
})
|
||||
|
||||
test('shutdown closes both splice legs and waits for the in-flight splice', async () => {
|
||||
const spliceOpened = deferred()
|
||||
const observations = []
|
||||
const peer = new RelayLoadControlPeer(5, peerOptions(), (type, detail) => {
|
||||
observations.push({ type, detail })
|
||||
if (type === 'spliceOpened') spliceOpened.resolve()
|
||||
})
|
||||
const control = fakeOpenSocket()
|
||||
const phone = fakeHandshakeSocket()
|
||||
const data = fakeHandshakeSocket()
|
||||
control.send = (raw) => {
|
||||
const message = JSON.parse(raw)
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' })
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
phone.send = (raw) => {
|
||||
if (typeof raw === 'string' && raw.startsWith('{')) {
|
||||
queueMicrotask(() =>
|
||||
control.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'conn-open',
|
||||
relayDeviceId: 'load-device-5-0',
|
||||
connId: 'connection-5',
|
||||
connTicket: 'ticket'
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
} else {
|
||||
queueMicrotask(() => data.emit('message', Buffer.from(raw), false))
|
||||
}
|
||||
}
|
||||
data.send = (raw) => {
|
||||
if (typeof raw === 'string') {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
|
||||
} else {
|
||||
queueMicrotask(() => phone.emit('message', Buffer.from(raw), true))
|
||||
}
|
||||
}
|
||||
peer.socket = control
|
||||
peer.generation = 8
|
||||
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 3 }
|
||||
control.on('message', (raw) => peer.onMessage(control, raw))
|
||||
peer.createClientSocket = () => openOnNextTurn(phone)
|
||||
peer.createHostDataSocket = () => openOnNextTurn(data)
|
||||
|
||||
const splice = peer.openSplice({ holdMs: 60_000 })
|
||||
await spliceOpened.promise
|
||||
await Promise.all([splice, peer.shutdown()])
|
||||
|
||||
assert.equal(phone.readyState, phone.CLOSED)
|
||||
assert.equal(data.readyState, data.CLOSED)
|
||||
assert.equal(peer.inFlight.size, 0)
|
||||
assert.equal(observations.at(-1).type, 'shutdown')
|
||||
assert.equal(observations.at(-1).detail.activeSpliceSockets, 0)
|
||||
})
|
||||
@@ -0,0 +1,147 @@
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
function integer(value) {
|
||||
return Number.isSafeInteger(value) && value >= 0 ? value : undefined
|
||||
}
|
||||
|
||||
export function assertRelayLoadDirectorCapacityToken(config, now = Date.now, timeoutMs = 0) {
|
||||
if (!config.adminToken || config.adminToken.length > 8_192) {
|
||||
throw new Error('director capacity identity token is unavailable')
|
||||
}
|
||||
const origin = new URL(config.directorOrigin)
|
||||
if (origin.protocol !== 'https:' || origin.origin !== config.directorOrigin) {
|
||||
throw new Error('director capacity origin must be canonical HTTPS')
|
||||
}
|
||||
let claims
|
||||
try {
|
||||
const parts = config.adminToken.split('.')
|
||||
if (parts.length !== 3) throw new Error('invalid token shape')
|
||||
claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'))
|
||||
} catch {
|
||||
throw new Error('director capacity identity token is invalid')
|
||||
}
|
||||
const expectedAudience = new URL('/v1/admin/drain', origin).toString()
|
||||
const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud]
|
||||
const expiresAt = integer(claims.exp)
|
||||
if (
|
||||
!audiences.includes(expectedAudience) ||
|
||||
typeof claims.email !== 'string' ||
|
||||
claims.email.length === 0 ||
|
||||
claims.email_verified !== true ||
|
||||
expiresAt === undefined ||
|
||||
expiresAt * 1_000 <= now() + timeoutMs
|
||||
) {
|
||||
throw new Error('director capacity identity token is not bound to this proof')
|
||||
}
|
||||
}
|
||||
|
||||
function matchingHeartbeat(status, config) {
|
||||
const capacity = status?.connectionCapacity
|
||||
const runtime = status?.runtime
|
||||
const heartbeatAt = integer(runtime?.lastHeartbeatAt)
|
||||
const matches =
|
||||
status?.cellId === config.cellId &&
|
||||
status?.admissionState === 'general' &&
|
||||
runtime?.ready === true &&
|
||||
runtime?.heartbeatFresh === true &&
|
||||
capacity?.heartbeatFresh === true &&
|
||||
integer(capacity?.hardCap) === config.hardCap &&
|
||||
integer(capacity?.unobservedBound) === config.unobservedBound &&
|
||||
integer(capacity?.normalAdmissionPause) === config.requiredConnections &&
|
||||
integer(capacity?.observedConnections) === config.requiredConnections &&
|
||||
integer(capacity?.enforcedConnectionUnits) === config.requiredConnections &&
|
||||
integer(capacity?.inFlightConnections) === 0 &&
|
||||
integer(capacity?.reservedConnectionUnits) === 0 &&
|
||||
integer(capacity?.pendingControlReservations) === 0 &&
|
||||
heartbeatAt !== undefined
|
||||
return matches ? heartbeatAt : undefined
|
||||
}
|
||||
|
||||
async function cellStatus(fetchImpl, config) {
|
||||
const response = await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.adminToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ v: 1, cellId: config.cellId }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new Error('director capacity identity was rejected')
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.arrayBuffer().catch(() => undefined)
|
||||
return undefined
|
||||
}
|
||||
const result = await response.json().catch(() => undefined)
|
||||
if (!result?.status) throw new Error('director capacity status is invalid')
|
||||
return result.status
|
||||
}
|
||||
|
||||
export async function waitForRelayLoadDirectorCapacity(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? 120_000
|
||||
const pollMs = overrides.pollMs ?? 1_000
|
||||
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
|
||||
const deadline = now() + timeoutMs
|
||||
let baselineHeartbeatAt = config.baselineHeartbeatAt
|
||||
let previousHeartbeatAt
|
||||
let matchingSamples = 0
|
||||
const requiredSamples = config.requiredSamples ?? 2
|
||||
for (;;) {
|
||||
const status = await cellStatus(fetchImpl, config)
|
||||
const currentHeartbeatAt = integer(status?.runtime?.lastHeartbeatAt)
|
||||
if (baselineHeartbeatAt === undefined && currentHeartbeatAt !== undefined) {
|
||||
baselineHeartbeatAt = currentHeartbeatAt
|
||||
}
|
||||
const heartbeatAt = status ? matchingHeartbeat(status, config) : undefined
|
||||
if (heartbeatAt !== undefined && heartbeatAt > baselineHeartbeatAt) {
|
||||
if (previousHeartbeatAt === undefined || heartbeatAt > previousHeartbeatAt) {
|
||||
previousHeartbeatAt = heartbeatAt
|
||||
matchingSamples++
|
||||
if (matchingSamples === requiredSamples) return { heartbeatAt }
|
||||
}
|
||||
} else {
|
||||
previousHeartbeatAt = undefined
|
||||
matchingSamples = 0
|
||||
}
|
||||
if (now() >= deadline) throw new Error('director capacity did not converge after recovery')
|
||||
await delay(pollMs)
|
||||
}
|
||||
}
|
||||
|
||||
function matchingRequestUnits(status, config) {
|
||||
return (
|
||||
status?.cellId === config.cellId &&
|
||||
status?.admissionState === 'general' &&
|
||||
status?.capacityRequests === config.capacityRequests &&
|
||||
status?.reservedRequests === config.expectedRequestUnits &&
|
||||
status?.activityRequestUnits === config.expectedRequestUnits &&
|
||||
status?.activityLeases === config.expectedActivityLeases &&
|
||||
status?.runtime?.observedRequests === config.expectedRequestUnits &&
|
||||
status?.runtime?.ready === true &&
|
||||
status?.runtime?.heartbeatFresh === true
|
||||
)
|
||||
}
|
||||
|
||||
export async function waitForRelayLoadRequestUnits(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs ?? 120_000
|
||||
const pollMs = overrides.pollMs ?? 1_000
|
||||
const requiredSamples = overrides.requiredSamples ?? 2
|
||||
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
|
||||
const deadline = now() + timeoutMs
|
||||
let matches = 0
|
||||
for (;;) {
|
||||
const status = await cellStatus(fetchImpl, config)
|
||||
matches = matchingRequestUnits(status, config) ? matches + 1 : 0
|
||||
if (matches === requiredSamples) return
|
||||
if (now() >= deadline) throw new Error('Relay request-unit accounting did not converge')
|
||||
await delay(pollMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertRelayLoadDirectorCapacityToken,
|
||||
waitForRelayLoadDirectorCapacity,
|
||||
waitForRelayLoadRequestUnits
|
||||
} from './relay-load-director-capacity-gate.mjs'
|
||||
|
||||
function token(claims) {
|
||||
const encode = (value) => Buffer.from(JSON.stringify(value)).toString('base64url')
|
||||
return `${encode({ alg: 'none' })}.${encode(claims)}.signature`
|
||||
}
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay-staging.example.com',
|
||||
adminToken: token({
|
||||
aud: 'https://relay-staging.example.com/v1/admin/drain',
|
||||
email: 'capacity@example.com',
|
||||
email_verified: true,
|
||||
exp: 4_000_000_000
|
||||
}),
|
||||
cellId: 'staging-gce-c3',
|
||||
hardCap: 1_000,
|
||||
unobservedBound: 60,
|
||||
requiredConnections: 840
|
||||
}
|
||||
|
||||
function status(lastHeartbeatAt, overrides = {}) {
|
||||
return {
|
||||
cellId: config.cellId,
|
||||
admissionState: 'general',
|
||||
runtime: { ready: true, heartbeatFresh: true, lastHeartbeatAt, observedRequests: 0 },
|
||||
connectionCapacity: {
|
||||
hardCap: 1_000,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 840,
|
||||
observedConnections: 840,
|
||||
enforcedConnectionUnits: 840,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
pendingControlReservations: 0,
|
||||
heartbeatFresh: true,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function response(value, responseStatus = 200) {
|
||||
return {
|
||||
ok: responseStatus >= 200 && responseStatus < 300,
|
||||
status: responseStatus,
|
||||
json: async () => value,
|
||||
arrayBuffer: async () => new ArrayBuffer(0)
|
||||
}
|
||||
}
|
||||
|
||||
test('requires two advancing exact director capacity heartbeats', async () => {
|
||||
const heartbeats = [101, 116, 131]
|
||||
let calls = 0
|
||||
const result = await waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: status(heartbeats[calls++]) }),
|
||||
delay: async () => undefined
|
||||
})
|
||||
assert.equal(calls, 3)
|
||||
assert.deepEqual(result, { heartbeatAt: 131 })
|
||||
})
|
||||
|
||||
test('resets after a newer heartbeat undercounts recovered controls', async () => {
|
||||
const samples = [
|
||||
status(101),
|
||||
status(116),
|
||||
status(131, { observedConnections: 839 }),
|
||||
status(146),
|
||||
status(161)
|
||||
]
|
||||
let calls = 0
|
||||
await waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: samples[calls++] }),
|
||||
delay: async () => undefined
|
||||
})
|
||||
assert.equal(calls, 5)
|
||||
})
|
||||
|
||||
test('fails closed when exact advancing telemetry never converges', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: status(101) }),
|
||||
delay: async (milliseconds) => {
|
||||
elapsed += milliseconds
|
||||
},
|
||||
now: () => elapsed,
|
||||
timeoutMs: 2_000,
|
||||
pollMs: 1_000
|
||||
}),
|
||||
/did not converge/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects an unauthorized capacity identity without retrying', async () => {
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return response({}, 401)
|
||||
},
|
||||
delay: async () => undefined
|
||||
}),
|
||||
/identity was rejected/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('binds the admin token to the canonical director audience', async () => {
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(
|
||||
{
|
||||
...config,
|
||||
directorOrigin: 'https://relay-staging.example.com/path'
|
||||
},
|
||||
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
|
||||
),
|
||||
/canonical HTTPS/
|
||||
)
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(
|
||||
{
|
||||
...config,
|
||||
adminToken: token({
|
||||
aud: 'https://other.example.com/v1/admin/drain',
|
||||
email: 'capacity@example.com',
|
||||
email_verified: true,
|
||||
exp: 4_000_000_000
|
||||
})
|
||||
},
|
||||
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
|
||||
),
|
||||
/not bound/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects a missing or malformed admin token during startup preflight', () => {
|
||||
assert.throws(
|
||||
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: undefined }, () => 0),
|
||||
/unavailable/
|
||||
)
|
||||
assert.throws(
|
||||
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: 'not-a-jwt' }, () => 0),
|
||||
/invalid/
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
assertRelayLoadDirectorCapacityToken(
|
||||
{
|
||||
...config,
|
||||
adminToken: token({
|
||||
aud: 'https://relay-staging.example.com/v1/admin/drain',
|
||||
exp: 4_000_000_000
|
||||
})
|
||||
},
|
||||
() => 0
|
||||
),
|
||||
/not bound/
|
||||
)
|
||||
})
|
||||
|
||||
test('supports one newer exact post-probe heartbeat', async () => {
|
||||
const heartbeats = [146, 161]
|
||||
let calls = 0
|
||||
const result = await waitForRelayLoadDirectorCapacity(
|
||||
{ ...config, requiredSamples: 1 },
|
||||
{
|
||||
fetch: async () => response({ status: status(heartbeats[calls++]) }),
|
||||
delay: async () => undefined,
|
||||
now: () => 0
|
||||
}
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(result, { heartbeatAt: 161 })
|
||||
})
|
||||
|
||||
test('requires consecutive exact request-unit accounting samples', async () => {
|
||||
const requestConfig = {
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 6_000,
|
||||
expectedActivityLeases: 6_000
|
||||
}
|
||||
const samples = [5_999, 6_000, 6_000]
|
||||
let calls = 0
|
||||
await waitForRelayLoadRequestUnits(requestConfig, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: {
|
||||
...status(100).runtime,
|
||||
observedRequests: samples[calls]
|
||||
},
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: samples[calls],
|
||||
activityRequestUnits: samples[calls],
|
||||
activityLeases: samples[calls++]
|
||||
}
|
||||
}),
|
||||
delay: async () => undefined,
|
||||
now: () => 0
|
||||
})
|
||||
assert.equal(calls, 3)
|
||||
})
|
||||
|
||||
test('requires the cell runtime to observe every request unit', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(waitForRelayLoadRequestUnits({
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 6_000,
|
||||
expectedActivityLeases: 6_000,
|
||||
timeoutMs: 1_000
|
||||
}, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: { ...status(100).runtime, observedRequests: 5_999 },
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: 6_000,
|
||||
activityRequestUnits: 6_000,
|
||||
activityLeases: 6_000
|
||||
}
|
||||
}),
|
||||
delay: async (milliseconds) => { elapsed += milliseconds },
|
||||
now: () => elapsed,
|
||||
pollMs: 1_000
|
||||
}), /did not converge/)
|
||||
})
|
||||
|
||||
test('fails closed when request-unit accounting does not clean up', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(waitForRelayLoadRequestUnits({
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 0,
|
||||
expectedActivityLeases: 0,
|
||||
timeoutMs: 2_000
|
||||
}, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: { ...status(100).runtime, observedRequests: 1 },
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: 1,
|
||||
activityRequestUnits: 1,
|
||||
activityLeases: 1
|
||||
}
|
||||
}),
|
||||
delay: async (milliseconds) => { elapsed += milliseconds },
|
||||
now: () => elapsed,
|
||||
pollMs: 1_000
|
||||
}), /did not converge/)
|
||||
})
|
||||
@@ -0,0 +1,76 @@
|
||||
const HEARTBEAT_INTERVAL_MS = 15_000
|
||||
const REFRESH_MIN_MS = 180_000
|
||||
const REFRESH_MAX_MS = 240_000
|
||||
|
||||
function mix32(value) {
|
||||
let mixed = value >>> 0
|
||||
mixed = Math.imul(mixed ^ (mixed >>> 16), 0x21f0aaad)
|
||||
mixed = Math.imul(mixed ^ (mixed >>> 15), 0x735a2d97)
|
||||
return (mixed ^ (mixed >>> 15)) >>> 0
|
||||
}
|
||||
|
||||
function fraction(seed, index, stream) {
|
||||
return mix32(seed ^ Math.imul(index + 1, 0x9e3779b1) ^ stream) / 0x1_0000_0000
|
||||
}
|
||||
|
||||
export function controlPhase(controlIndex, seed = 0x4f524341) {
|
||||
const refreshIntervalMs = Math.round(
|
||||
REFRESH_MIN_MS + fraction(seed, controlIndex, 2) * (REFRESH_MAX_MS - REFRESH_MIN_MS)
|
||||
)
|
||||
return {
|
||||
heartbeatOffsetMs: Math.floor(fraction(seed, controlIndex, 1) * HEARTBEAT_INTERVAL_MS),
|
||||
refreshIntervalMs,
|
||||
refreshOffsetMs: Math.floor(fraction(seed, controlIndex, 3) * refreshIntervalMs),
|
||||
reconnectJitterMs: Math.floor(fraction(seed, controlIndex, 4) * 30_000)
|
||||
}
|
||||
}
|
||||
|
||||
export function modeledRelayLoad(controlCount, durationMs = 15 * 60_000, seed) {
|
||||
if (!Number.isInteger(controlCount) || controlCount < 1) throw new Error('controlCount must be positive')
|
||||
const bins = Array.from({ length: Math.ceil(durationMs / 1000) }, () => ({ pings: 0, refreshes: 0 }))
|
||||
for (let controlIndex = 0; controlIndex < controlCount; controlIndex++) {
|
||||
const phase = controlPhase(controlIndex, seed)
|
||||
for (let at = phase.heartbeatOffsetMs; at < durationMs; at += HEARTBEAT_INTERVAL_MS) {
|
||||
bins[Math.floor(at / 1000)].pings++
|
||||
}
|
||||
for (let at = phase.refreshOffsetMs; at < durationMs; at += phase.refreshIntervalMs) {
|
||||
bins[Math.floor(at / 1000)].refreshes++
|
||||
}
|
||||
}
|
||||
const totals = bins.reduce(
|
||||
(result, bin) => ({
|
||||
pings: result.pings + bin.pings,
|
||||
refreshes: result.refreshes + bin.refreshes
|
||||
}),
|
||||
{ pings: 0, refreshes: 0 }
|
||||
)
|
||||
const durationSeconds = durationMs / 1000
|
||||
return {
|
||||
controlCount,
|
||||
durationMs,
|
||||
expectedPingRate: controlCount / (HEARTBEAT_INTERVAL_MS / 1000),
|
||||
expectedRefreshRate: controlCount / ((REFRESH_MIN_MS + REFRESH_MAX_MS) / 2 / 1000),
|
||||
observedPingRate: totals.pings / durationSeconds,
|
||||
observedRefreshRate: totals.refreshes / durationSeconds,
|
||||
maxPingBurst: Math.max(...bins.map(({ pings }) => pings)),
|
||||
maxRefreshBurst: Math.max(...bins.map(({ refreshes }) => refreshes))
|
||||
}
|
||||
}
|
||||
|
||||
export function assertSpreadModel(model) {
|
||||
const pingTolerance = model.expectedPingRate * 0.03 + 1
|
||||
const refreshTolerance = model.expectedRefreshRate * 0.08 + 1
|
||||
if (Math.abs(model.observedPingRate - model.expectedPingRate) > pingTolerance) {
|
||||
throw new Error('modeled heartbeat rate diverged from the 15-second contract')
|
||||
}
|
||||
if (Math.abs(model.observedRefreshRate - model.expectedRefreshRate) > refreshTolerance) {
|
||||
throw new Error('modeled token refresh rate diverged from the 180-240 second contract')
|
||||
}
|
||||
if (model.maxPingBurst > model.expectedPingRate * 1.3 + 5) {
|
||||
throw new Error('heartbeat phase spreading produced a reconnect cliff')
|
||||
}
|
||||
// One-second bins have Poisson-sized tails even with uniform phase spreading.
|
||||
if (model.maxRefreshBurst > model.expectedRefreshRate * 1.75 + 5) {
|
||||
throw new Error('refresh phase spreading produced an auth herd')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { assertSpreadModel, controlPhase, modeledRelayLoad } from './relay-load-model.mjs'
|
||||
|
||||
test('phases are deterministic, bounded, and separated by stream', () => {
|
||||
assert.deepEqual(controlPhase(42), controlPhase(42))
|
||||
assert.notDeepEqual(controlPhase(42), controlPhase(43))
|
||||
const phase = controlPhase(42)
|
||||
assert.ok(phase.heartbeatOffsetMs >= 0 && phase.heartbeatOffsetMs < 15_000)
|
||||
assert.ok(phase.refreshIntervalMs >= 180_000 && phase.refreshIntervalMs <= 240_000)
|
||||
assert.ok(phase.refreshOffsetMs >= 0 && phase.refreshOffsetMs < phase.refreshIntervalMs)
|
||||
assert.ok(phase.reconnectJitterMs >= 0 && phase.reconnectJitterMs < 30_000)
|
||||
})
|
||||
|
||||
for (const [controls, expectedPings, expectedRefreshes] of [
|
||||
[4_000, 267, 19],
|
||||
[10_000, 667, 48]
|
||||
]) {
|
||||
test(`${controls} modeled controls spread heartbeat and token refresh load`, () => {
|
||||
const model = modeledRelayLoad(controls)
|
||||
assert.equal(Math.round(model.expectedPingRate), expectedPings)
|
||||
assert.equal(Math.round(model.expectedRefreshRate), expectedRefreshes)
|
||||
assert.doesNotThrow(() => assertSpreadModel(model))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { access, mkdir, open } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
export async function waitForRelayLoadPhaseBarrier(config, overrides = {}) {
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs
|
||||
if (
|
||||
typeof config.directory !== 'string' || config.directory.length === 0 ||
|
||||
!Number.isSafeInteger(config.shardCount) || config.shardCount < 2 ||
|
||||
!Number.isSafeInteger(config.shardIndex) || config.shardIndex < 0 ||
|
||||
config.shardIndex >= config.shardCount ||
|
||||
!Number.isSafeInteger(timeoutMs) || timeoutMs < 1
|
||||
) throw new Error('invalid Relay load phase barrier')
|
||||
|
||||
await mkdir(config.directory, { recursive: true })
|
||||
const marker = join(config.directory, `${config.shardIndex}.ready`)
|
||||
const handle = await open(marker, 'wx')
|
||||
await handle.close()
|
||||
const deadline = now() + timeoutMs
|
||||
for (;;) {
|
||||
const ready = await Promise.all(
|
||||
Array.from({ length: config.shardCount }, async (_, index) => {
|
||||
try {
|
||||
await access(join(config.directory, `${index}.ready`))
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
})
|
||||
)
|
||||
if (ready.every(Boolean)) return
|
||||
if (now() >= deadline) throw new Error('Relay load phase barrier timed out')
|
||||
await delay(100)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
|
||||
|
||||
const loadHarness = await readFile(new URL('./load-relay-controls.mjs', import.meta.url), 'utf8')
|
||||
|
||||
test('releases every shard only after all readiness markers exist', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
|
||||
try {
|
||||
let firstResolved = false
|
||||
const first = waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1_000
|
||||
}).then(() => { firstResolved = true })
|
||||
await new Promise((resolve) => setTimeout(resolve, 20))
|
||||
assert.equal(firstResolved, false)
|
||||
await Promise.all([
|
||||
first,
|
||||
waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 1, timeoutMs: 1_000
|
||||
})
|
||||
])
|
||||
assert.equal(firstResolved, true)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('fails closed on a duplicate shard or incomplete barrier', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
|
||||
try {
|
||||
const nowValues = [0, 2]
|
||||
await assert.rejects(
|
||||
waitForRelayLoadPhaseBarrier(
|
||||
{ directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 },
|
||||
{ now: () => nowValues.shift() ?? 2, delay: async () => undefined }
|
||||
),
|
||||
/timed out/
|
||||
)
|
||||
await assert.rejects(
|
||||
waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1
|
||||
}),
|
||||
/EEXIST/
|
||||
)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('synchronizes splice ramps after every shard finishes reader baselines', () => {
|
||||
assert.match(
|
||||
loadHarness,
|
||||
/createRelayLoadReaderEvidence[\s\S]*?phaseBarrierDir\}-splices[\s\S]*?splicePromises/
|
||||
)
|
||||
})
|
||||
|
||||
test('budgets both shard barriers and the splice ramp in token lifetime', () => {
|
||||
assert.match(
|
||||
loadHarness,
|
||||
/phaseBarrierDir \? 2 \* config\.phaseBarrierTimeoutMs : 0[\s\S]*?config\.spliceRampMs/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,29 @@
|
||||
export async function proveRelayLoadPlacementBoundary({ peer, failureReason }) {
|
||||
let connected = false
|
||||
try {
|
||||
await peer.connect()
|
||||
connected = true
|
||||
} catch (error) {
|
||||
const reason = failureReason(error)
|
||||
if (reason !== 'assignment_capacity_exhausted') {
|
||||
throw new Error(`placement overflow was not rejected: ${reason}`)
|
||||
}
|
||||
return reason
|
||||
} finally {
|
||||
await peer.shutdown()
|
||||
}
|
||||
if (connected) throw new Error('placement overflow unexpectedly connected')
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRegionalFallback({ peer, blockedOrigin }) {
|
||||
try {
|
||||
await peer.connect()
|
||||
const assignedOrigin = peer.assignedCellUrl()
|
||||
if (!assignedOrigin || assignedOrigin === blockedOrigin) {
|
||||
throw new Error('regional fallback did not leave the full preferred cell')
|
||||
}
|
||||
return true
|
||||
} finally {
|
||||
await peer.shutdown()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
proveRelayLoadPlacementBoundary,
|
||||
proveRelayLoadRegionalFallback
|
||||
} from './relay-load-placement-boundary.mjs'
|
||||
|
||||
function peer(connect) {
|
||||
return { connect, shutdown: async () => undefined }
|
||||
}
|
||||
|
||||
test('requires the next fresh placement to receive HTTP 503', async () => {
|
||||
assert.equal(
|
||||
await proveRelayLoadPlacementBoundary({
|
||||
peer: peer(async () => {
|
||||
throw new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
|
||||
}),
|
||||
failureReason: (error) =>
|
||||
error.message === 'relay assignment failed: 503 relay_connection_headroom_exhausted'
|
||||
? 'assignment_capacity_exhausted'
|
||||
: 'unknown'
|
||||
}),
|
||||
'assignment_capacity_exhausted'
|
||||
)
|
||||
await assert.rejects(
|
||||
proveRelayLoadPlacementBoundary({
|
||||
peer: peer(async () => undefined),
|
||||
failureReason: () => 'unknown'
|
||||
}),
|
||||
/placement overflow unexpectedly connected/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires a preferred-region fallback to leave the full cell', async () => {
|
||||
let shutdowns = 0
|
||||
assert.equal(await proveRelayLoadRegionalFallback({
|
||||
peer: {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => 'https://c3.relay-staging.onorca.dev',
|
||||
shutdown: async () => { shutdowns++ }
|
||||
},
|
||||
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
|
||||
}), true)
|
||||
assert.equal(shutdowns, 1)
|
||||
await assert.rejects(proveRelayLoadRegionalFallback({
|
||||
peer: {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => 'https://c4.relay-staging.onorca.dev',
|
||||
shutdown: async () => undefined
|
||||
},
|
||||
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
|
||||
}), /did not leave/)
|
||||
})
|
||||
@@ -0,0 +1,404 @@
|
||||
export const RELAY_CONTROL_LEASE_HORIZON_SECONDS = 105
|
||||
export const RELAY_LOAD_SPLICE_HIGH_WATER_BYTES = 256 * 1024
|
||||
export const RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS = 10_000
|
||||
export const RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES = 16
|
||||
export const RELAY_LOAD_MAX_AGGREGATE_READER_BYTES = 64 * 1024 * 1024
|
||||
|
||||
const DEFAULT_SLOW_READER_STREAM_BYTES = 1024 * 1024
|
||||
const DEFAULT_WEDGED_READER_STREAM_BYTES = 8 * 1024 * 1024
|
||||
const DEFAULT_READER_FRAME_BYTES = 64 * 1024
|
||||
|
||||
export function parseRelayLoadArguments(argv) {
|
||||
const values = new Map()
|
||||
const flags = new Set()
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const argument = argv[index]
|
||||
if (argument === '--') continue
|
||||
if (
|
||||
[
|
||||
'--allow-partial',
|
||||
'--allow-planned-transition-retries',
|
||||
'--skip-rebind-overflow-check'
|
||||
].includes(argument)
|
||||
) {
|
||||
flags.add(argument)
|
||||
continue
|
||||
}
|
||||
if (!argument.startsWith('--') || index + 1 >= argv.length) {
|
||||
throw new Error(`invalid argument: ${argument}`)
|
||||
}
|
||||
values.set(argument, argv[++index])
|
||||
}
|
||||
const integer = (name, fallback) => {
|
||||
const parsed = Number(values.get(name) ?? fallback)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a nonnegative integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
const targetOrigin = values.get('--target-origin')?.replace(/\/$/, '')
|
||||
const directorOrigin = values.get('--director-origin')?.replace(/\/$/, '')
|
||||
const authOrigin = values.get('--auth-origin')?.replace(/\/$/, '')
|
||||
if ((!targetOrigin && !directorOrigin) || (targetOrigin && directorOrigin) || !authOrigin) {
|
||||
throw new Error('provide --auth-origin and exactly one target or director origin')
|
||||
}
|
||||
const controls = integer('--controls', 100)
|
||||
const maxRampConnectionFailures = integer('--max-ramp-connection-failures', 0)
|
||||
const maxUnexpectedCloses = integer('--max-unexpected-closes', 0)
|
||||
const rebindProbes = integer('--rebind-probes', 0)
|
||||
const placementOverflowProbes = integer('--placement-overflow-probes', 0)
|
||||
const regionalFallbackProbes = integer('--regional-fallback-probes', 0)
|
||||
const regionBehaviorProbes = integer('--region-behavior-probes', 0)
|
||||
const requestUnitInvites = integer('--request-unit-invites', 0)
|
||||
const requestUnitInvitesPerSecond = integer('--request-unit-invites-per-second', 0)
|
||||
const requestUnitPrincipalCount = integer('--request-unit-principals', 0)
|
||||
const relayAsiaLoadPrincipalCount = integer('--relay-asia-load-principals', 0)
|
||||
const requestUnitOverflowProbes = integer('--request-unit-overflow-probes', 0)
|
||||
const requestUnitCapacity = values.has('--request-unit-capacity')
|
||||
? integer('--request-unit-capacity', 0)
|
||||
: undefined
|
||||
const requestUnitCleanupTimeoutMs = integer(
|
||||
'--request-unit-cleanup-timeout-seconds',
|
||||
0
|
||||
) * 1000
|
||||
const rebindHoldMs = integer('--rebind-hold-ms', 4_000)
|
||||
const rebindDelayMs = integer('--rebind-delay-seconds', 0) * 1000
|
||||
const capacityCellId = values.get('--capacity-cell-id')
|
||||
const capacityCellOrigin = values.get('--capacity-cell-origin')?.replace(/\/$/, '')
|
||||
const capacityHardCap = values.has('--capacity-hard-cap')
|
||||
? integer('--capacity-hard-cap', 0)
|
||||
: undefined
|
||||
const capacityUnobservedBound = values.has('--capacity-unobserved-bound')
|
||||
? integer('--capacity-unobserved-bound', 0)
|
||||
: undefined
|
||||
const shardCount = integer('--shard-count', 1)
|
||||
const shardIndex = integer('--shard-index', 0)
|
||||
const durationMs = integer('--duration-seconds', 900) * 1000
|
||||
const spliceHoldMs = integer(
|
||||
'--splice-hold-seconds',
|
||||
values.get('--duration-seconds') ?? 900
|
||||
) * 1000
|
||||
const splices = integer('--splices', 0)
|
||||
const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000
|
||||
const slowReaderSplices = integer('--slow-reader-splices', 0)
|
||||
const wedgedReaderSplices = integer('--wedged-reader-splices', 0)
|
||||
const splicePayloadBytes = integer('--splice-payload-bytes', 1_024)
|
||||
const slowReaderStreamBytes = integer(
|
||||
'--slow-reader-stream-bytes',
|
||||
DEFAULT_SLOW_READER_STREAM_BYTES
|
||||
)
|
||||
const wedgedReaderStreamBytes = integer(
|
||||
'--wedged-reader-stream-bytes',
|
||||
DEFAULT_WEDGED_READER_STREAM_BYTES
|
||||
)
|
||||
const readerFrameBytes = integer('--reader-frame-bytes', DEFAULT_READER_FRAME_BYTES)
|
||||
const slowReaderHoldMs = integer('--slow-reader-hold-ms', 2_000)
|
||||
const wedgedReaderHoldMs = integer('--wedged-reader-hold-ms', 12_000)
|
||||
const maxGeneratorRssGrowthMiB = integer('--max-generator-rss-growth-mib', 512)
|
||||
const requiredLeaseHorizons = integer('--required-lease-horizons', 0)
|
||||
const aggregateControls = values.has('--aggregate-controls')
|
||||
? integer('--aggregate-controls', 0)
|
||||
: controls
|
||||
const aggregateSplices = values.has('--aggregate-splices')
|
||||
? integer('--aggregate-splices', 0)
|
||||
: splices
|
||||
const aggregateRequestUnitInvites = values.has('--aggregate-request-unit-invites')
|
||||
? integer('--aggregate-request-unit-invites', 0)
|
||||
: requestUnitInvites
|
||||
const phaseBarrierDir = values.get('--phase-barrier-dir')
|
||||
const phaseBarrierTimeoutMs = integer('--phase-barrier-timeout-seconds', 180) * 1000
|
||||
if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000')
|
||||
if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls')
|
||||
if (splices > controls) throw new Error('--splices cannot exceed --controls')
|
||||
if (shardCount > 1 && capacityHardCap !== undefined) {
|
||||
if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) {
|
||||
throw new Error('capacity-bound sharding requires explicit aggregate controls and splices')
|
||||
}
|
||||
if (aggregateControls !== controls * shardCount || aggregateSplices !== splices * shardCount) {
|
||||
throw new Error('aggregate controls and splices must match every equal-sized shard')
|
||||
}
|
||||
} else if (aggregateControls !== controls || aggregateSplices !== splices) {
|
||||
throw new Error('aggregate controls and splices require matching sharded local counts')
|
||||
}
|
||||
if (
|
||||
capacityHardCap !== undefined &&
|
||||
aggregateControls + 2 * aggregateSplices > capacityHardCap - 100
|
||||
) {
|
||||
throw new Error('controls plus splice connection units exceed ordinary cell admission')
|
||||
}
|
||||
if (slowReaderSplices + wedgedReaderSplices > splices) {
|
||||
throw new Error('reader splice counts cannot exceed --splices')
|
||||
}
|
||||
if (splices > 0 && (spliceHoldMs < 1_000 || spliceHoldMs > durationMs)) {
|
||||
throw new Error('--splice-hold-seconds must be between 1 and the steady duration')
|
||||
}
|
||||
if (slowReaderSplices + wedgedReaderSplices > 0 && !directorOrigin) {
|
||||
throw new Error('reader evidence requires --director-origin')
|
||||
}
|
||||
if (splicePayloadBytes < 1 || splicePayloadBytes > 1_048_576) {
|
||||
throw new Error('--splice-payload-bytes must be between 1 and 1048576')
|
||||
}
|
||||
if (
|
||||
slowReaderSplices > 0 &&
|
||||
slowReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
|
||||
) {
|
||||
throw new Error('--slow-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
|
||||
}
|
||||
if (
|
||||
wedgedReaderSplices > 0 &&
|
||||
wedgedReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
|
||||
) {
|
||||
throw new Error('--wedged-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
|
||||
}
|
||||
const localReaderSplices = slowReaderSplices + wedgedReaderSplices
|
||||
const localReaderBytes = slowReaderSplices * slowReaderStreamBytes +
|
||||
wedgedReaderSplices * wedgedReaderStreamBytes
|
||||
const aggregateReaderSplices = values.has('--aggregate-reader-splices')
|
||||
? integer('--aggregate-reader-splices', 0)
|
||||
: localReaderSplices * shardCount
|
||||
const aggregateReaderBytes = values.has('--aggregate-reader-bytes')
|
||||
? integer('--aggregate-reader-bytes', 0)
|
||||
: localReaderBytes * shardCount
|
||||
if (
|
||||
aggregateReaderSplices < localReaderSplices ||
|
||||
aggregateReaderBytes < localReaderBytes ||
|
||||
(shardCount === 1 &&
|
||||
(aggregateReaderSplices !== localReaderSplices || aggregateReaderBytes !== localReaderBytes))
|
||||
) {
|
||||
throw new Error('aggregate reader bounds do not cover the local shard')
|
||||
}
|
||||
if (aggregateReaderSplices > RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES) {
|
||||
throw new Error('aggregate reader splice count exceeds the reviewed bound')
|
||||
}
|
||||
if (aggregateReaderBytes > RELAY_LOAD_MAX_AGGREGATE_READER_BYTES) {
|
||||
throw new Error('aggregate reader stream bytes exceed the reviewed bound')
|
||||
}
|
||||
if (readerFrameBytes < 1 || readerFrameBytes > 1_048_576) {
|
||||
throw new Error('--reader-frame-bytes must be between 1 and 1048576')
|
||||
}
|
||||
if (slowReaderSplices > 0 && slowReaderHoldMs >= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
|
||||
throw new Error('--slow-reader-hold-ms must stay below the wedged timeout')
|
||||
}
|
||||
if (wedgedReaderSplices > 0 && wedgedReaderHoldMs <= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
|
||||
throw new Error('--wedged-reader-hold-ms must exceed the wedged timeout')
|
||||
}
|
||||
if (wedgedReaderHoldMs > 30_000) {
|
||||
throw new Error('--wedged-reader-hold-ms cannot exceed 30000')
|
||||
}
|
||||
if (maxGeneratorRssGrowthMiB < 1) {
|
||||
throw new Error('--max-generator-rss-growth-mib must be positive')
|
||||
}
|
||||
if (placementOverflowProbes > 1) {
|
||||
throw new Error('--placement-overflow-probes must be zero or one')
|
||||
}
|
||||
if (regionalFallbackProbes > 1) {
|
||||
throw new Error('--regional-fallback-probes must be zero or one')
|
||||
}
|
||||
if (regionBehaviorProbes > 1 || requestUnitOverflowProbes > 1) {
|
||||
throw new Error('regional behavior and request-unit overflow probes must be zero or one')
|
||||
}
|
||||
if (placementOverflowProbes > 0 && shardCount > 1) {
|
||||
throw new Error('placement overflow proof requires one coordinated generator')
|
||||
}
|
||||
if (
|
||||
placementOverflowProbes > 0 &&
|
||||
(!directorOrigin ||
|
||||
!capacityCellId ||
|
||||
capacityHardCap === undefined ||
|
||||
capacityUnobservedBound === undefined)
|
||||
) {
|
||||
throw new Error('placement overflow requires exact capacity cell, hard cap, and bound')
|
||||
}
|
||||
if (regionalFallbackProbes > 0 && (
|
||||
!directorOrigin || !capacityCellId || !capacityCellOrigin ||
|
||||
capacityHardCap === undefined || capacityUnobservedBound === undefined ||
|
||||
(shardCount > 1 && shardIndex !== 0)
|
||||
)) throw new Error('regional fallback requires the coordinating capacity shard')
|
||||
if (regionBehaviorProbes > 0 && (
|
||||
!directorOrigin || !capacityCellOrigin || preferredRegionValue(values) !== 'asia-east2' ||
|
||||
(shardCount > 1 && shardIndex !== 0)
|
||||
)) throw new Error('regional behavior proof requires the coordinating Asia shard')
|
||||
if (phaseBarrierDir && shardCount < 2) {
|
||||
throw new Error('phase barrier requires multiple shards')
|
||||
}
|
||||
if (phaseBarrierTimeoutMs < 1_000) {
|
||||
throw new Error('phase barrier timeout must be at least one second')
|
||||
}
|
||||
if (requestUnitInvites > 0) {
|
||||
if (
|
||||
!directorOrigin || !capacityCellId || requestUnitCapacity === undefined ||
|
||||
requestUnitCapacity < 1 || requestUnitInvitesPerSecond < 1 ||
|
||||
requestUnitInvitesPerSecond > 20 || requestUnitPrincipalCount < 1 ||
|
||||
requestUnitPrincipalCount > 32 || requestUnitInvites > requestUnitPrincipalCount * 30 ||
|
||||
aggregateSplices !== 0 || !phaseBarrierDir ||
|
||||
aggregateRequestUnitInvites !== requestUnitInvites * shardCount ||
|
||||
aggregateControls + aggregateRequestUnitInvites !== requestUnitCapacity
|
||||
) throw new Error('request-unit proof does not reach the exact reviewed capacity')
|
||||
} else if (
|
||||
requestUnitCapacity !== undefined || requestUnitInvitesPerSecond !== 0 ||
|
||||
requestUnitPrincipalCount !== 0 ||
|
||||
requestUnitOverflowProbes > 0 ||
|
||||
requestUnitCleanupTimeoutMs > 0 || aggregateRequestUnitInvites !== 0
|
||||
) throw new Error('request-unit proof options require invite offers')
|
||||
if (
|
||||
requestUnitOverflowProbes > 0 &&
|
||||
(shardIndex !== 0 || requestUnitCleanupTimeoutMs < 600_000)
|
||||
) throw new Error('request-unit overflow requires the cleanup-owning coordinator')
|
||||
if (requestUnitCleanupTimeoutMs > 0 && requestUnitOverflowProbes !== 1) {
|
||||
throw new Error('request-unit cleanup requires the overflow proof')
|
||||
}
|
||||
if (
|
||||
relayAsiaLoadPrincipalCount > 32 ||
|
||||
(relayAsiaLoadPrincipalCount > 0 &&
|
||||
(!directorOrigin || preferredRegionValue(values) !== 'asia-east2'))
|
||||
) throw new Error('Relay Asia load principals require a regional director proof')
|
||||
if (capacityCellOrigin) {
|
||||
const origin = new URL(capacityCellOrigin)
|
||||
if (origin.protocol !== 'https:' || origin.origin !== capacityCellOrigin) {
|
||||
throw new Error('--capacity-cell-origin must be canonical HTTPS')
|
||||
}
|
||||
}
|
||||
if (shardCount < 1 || shardIndex >= shardCount) throw new Error('invalid shard index/count')
|
||||
const minimumDurationMs = requiredLeaseHorizons * RELAY_CONTROL_LEASE_HORIZON_SECONDS * 1000
|
||||
if (durationMs < minimumDurationMs) {
|
||||
throw new Error(`--duration-seconds must cover ${requiredLeaseHorizons} lease horizons`)
|
||||
}
|
||||
return {
|
||||
targetOrigin,
|
||||
directorOrigin,
|
||||
authOrigin,
|
||||
preferredRegion: preferredRegionValue(values),
|
||||
controls,
|
||||
maxRampConnectionFailures,
|
||||
maxUnexpectedCloses,
|
||||
rebindProbes,
|
||||
placementOverflowProbes,
|
||||
regionalFallbackProbes,
|
||||
regionBehaviorProbes,
|
||||
requestUnitInvites,
|
||||
requestUnitInvitesPerSecond,
|
||||
requestUnitPrincipalCount,
|
||||
relayAsiaLoadPrincipalCount,
|
||||
requestUnitOverflowProbes,
|
||||
requestUnitCapacity,
|
||||
requestUnitCleanupTimeoutMs,
|
||||
rebindHoldMs,
|
||||
rebindDelayMs,
|
||||
capacityCellId,
|
||||
capacityCellOrigin,
|
||||
capacityHardCap,
|
||||
capacityUnobservedBound,
|
||||
durationMs,
|
||||
spliceHoldMs,
|
||||
rampMs: integer('--ramp-seconds', 60) * 1000,
|
||||
rampStartDelayMs: integer('--ramp-start-delay-ms', 0),
|
||||
reconnectMaxMs: integer('--reconnect-max-seconds', 30) * 1000,
|
||||
shardCount,
|
||||
shardIndex,
|
||||
aggregateControls,
|
||||
aggregateSplices,
|
||||
aggregateRequestUnitInvites,
|
||||
phaseBarrierDir,
|
||||
phaseBarrierTimeoutMs,
|
||||
aggregateReaderSplices,
|
||||
aggregateReaderBytes,
|
||||
signingKeyFile: values.get('--signing-key-file'),
|
||||
splices,
|
||||
spliceRampMs,
|
||||
splicePayloadBytes,
|
||||
slowReaderSplices,
|
||||
wedgedReaderSplices,
|
||||
slowReaderStreamBytes,
|
||||
wedgedReaderStreamBytes,
|
||||
readerFrameBytes,
|
||||
slowReaderHoldMs,
|
||||
wedgedReaderHoldMs,
|
||||
maxGeneratorRssGrowthMiB,
|
||||
requiredLeaseHorizons,
|
||||
allowPartial: flags.has('--allow-partial'),
|
||||
allowPlannedTransitionRetries: flags.has('--allow-planned-transition-retries'),
|
||||
requireRebindOverflow: !flags.has('--skip-rebind-overflow-check')
|
||||
}
|
||||
}
|
||||
|
||||
function preferredRegionValue(values) {
|
||||
return values.get('--preferred-region')
|
||||
}
|
||||
|
||||
export function relayLoadSpliceIndexes(config) {
|
||||
return Array.from(
|
||||
{ length: config.splices },
|
||||
(_, localIndex) => localIndex * config.shardCount + config.shardIndex
|
||||
)
|
||||
}
|
||||
|
||||
export function relayLoadSpliceStartDelayMs(config, localIndex) {
|
||||
if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) {
|
||||
throw new Error('invalid local splice index')
|
||||
}
|
||||
const totalSplices = config.splices * config.shardCount
|
||||
if (totalSplices <= 1) return 0
|
||||
const globalOrdinal = localIndex * config.shardCount + config.shardIndex
|
||||
return Math.floor(globalOrdinal * config.spliceRampMs / (totalSplices - 1))
|
||||
}
|
||||
|
||||
export function relayLoadPrincipalIndex(peerIndex, shardCount, principalCount) {
|
||||
if (
|
||||
!Number.isSafeInteger(peerIndex) || peerIndex < 0 ||
|
||||
!Number.isSafeInteger(shardCount) || shardCount < 1 ||
|
||||
!Number.isSafeInteger(principalCount) || principalCount < 1
|
||||
) throw new Error('invalid Relay load principal mapping')
|
||||
return Math.floor(peerIndex / shardCount) % principalCount
|
||||
}
|
||||
|
||||
export function relayLoadSpliceProfile(config, spliceIndex) {
|
||||
if (spliceIndex < config.wedgedReaderSplices) {
|
||||
return {
|
||||
readerMode: 'wedged',
|
||||
readerHoldMs: config.wedgedReaderHoldMs,
|
||||
streamBytes: config.wedgedReaderStreamBytes,
|
||||
frameBytes: config.readerFrameBytes
|
||||
}
|
||||
}
|
||||
if (spliceIndex < config.wedgedReaderSplices + config.slowReaderSplices) {
|
||||
return {
|
||||
readerMode: 'slow',
|
||||
readerHoldMs: config.slowReaderHoldMs,
|
||||
streamBytes: config.slowReaderStreamBytes,
|
||||
frameBytes: config.readerFrameBytes
|
||||
}
|
||||
}
|
||||
return {
|
||||
readerMode: 'normal',
|
||||
readerHoldMs: 0,
|
||||
streamBytes: config.splicePayloadBytes,
|
||||
frameBytes: config.splicePayloadBytes
|
||||
}
|
||||
}
|
||||
|
||||
export function relayLoadReaderEvidenceError(result, config) {
|
||||
const readerSplices = config.slowReaderSplices + config.wedgedReaderSplices
|
||||
if (readerSplices > 0 && result.generatorRssGrowthMiB > config.maxGeneratorRssGrowthMiB) {
|
||||
return 'load generator exceeded its RSS growth budget'
|
||||
}
|
||||
if (result.slowReaderSplicesCompleted !== config.slowReaderSplices) {
|
||||
return 'slow-reader streams did not all complete'
|
||||
}
|
||||
if (result.wedgedReaderSplicesClosed !== config.wedgedReaderSplices) {
|
||||
return 'wedged-reader streams did not all close at the relay limit'
|
||||
}
|
||||
if (
|
||||
readerSplices > 0 &&
|
||||
(result.readerQueueEvidence.length === 0 ||
|
||||
result.readerQueueEvidence.some(({ increaseBytes }) => increaseBytes < 1))
|
||||
) {
|
||||
return 'reader streams produced no causal Relay queued-byte evidence'
|
||||
}
|
||||
if (
|
||||
config.wedgedReaderSplices > 0 &&
|
||||
result.readerClosesByCode['4429'] !== config.wedgedReaderSplices
|
||||
) {
|
||||
return 'wedged-reader streams did not close with 4429'
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
parseRelayLoadArguments,
|
||||
relayLoadPrincipalIndex,
|
||||
relayLoadReaderEvidenceError,
|
||||
relayLoadSpliceIndexes,
|
||||
relayLoadSpliceProfile,
|
||||
relayLoadSpliceStartDelayMs
|
||||
} from './relay-load-profile.mjs'
|
||||
|
||||
const required = ['--auth-origin', 'https://auth.test', '--director-origin', 'https://relay.test']
|
||||
|
||||
test('accepts the 2840-control two-lease-horizon Asia proof profile', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls',
|
||||
'2840',
|
||||
'--duration-seconds',
|
||||
'210',
|
||||
'--required-lease-horizons',
|
||||
'2',
|
||||
'--preferred-region',
|
||||
'asia-east2',
|
||||
'--relay-asia-load-principals',
|
||||
'32',
|
||||
'--capacity-hard-cap',
|
||||
'3000'
|
||||
])
|
||||
|
||||
assert.equal(config.controls, 2840)
|
||||
assert.equal(config.durationMs, 210_000)
|
||||
assert.equal(config.requiredLeaseHorizons, 2)
|
||||
assert.equal(config.preferredRegion, 'asia-east2')
|
||||
assert.equal(config.relayAsiaLoadPrincipalCount, 32)
|
||||
assert.equal(config.splices, 0)
|
||||
})
|
||||
|
||||
test('binds synthetic Relay principals to a bounded Asia director proof', () => {
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--relay-asia-load-principals', '1'
|
||||
]), /require a regional director proof/)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'asia-east2',
|
||||
'--relay-asia-load-principals', '33'
|
||||
]), /require a regional director proof/)
|
||||
})
|
||||
|
||||
test('rejects a mixed profile beyond the ordinary 2900-unit boundary', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '2840',
|
||||
'--splices', '31',
|
||||
'--capacity-hard-cap', '3000'
|
||||
]),
|
||||
/exceed ordinary cell admission/
|
||||
)
|
||||
expectMixedProfile(parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '2600',
|
||||
'--splices', '120',
|
||||
'--capacity-hard-cap', '3000'
|
||||
]))
|
||||
})
|
||||
|
||||
test('requires reviewed aggregate totals for capacity-bound shards', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0'
|
||||
]),
|
||||
/requires explicit aggregate controls and splices/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0'
|
||||
]),
|
||||
/must match every equal-sized shard/
|
||||
)
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0'
|
||||
])
|
||||
assert.equal(config.aggregateControls, 2840)
|
||||
assert.equal(config.aggregateSplices, 0)
|
||||
})
|
||||
|
||||
test('allows one coordinating shard to prove regional fallback', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--capacity-unobserved-bound', '60', '--rebind-probes', '160'
|
||||
])
|
||||
assert.equal(config.regionalFallbackProbes, 1)
|
||||
assert.equal(config.capacityCellOrigin, 'https://c4.relay-staging.onorca.dev')
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '1',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--capacity-unobserved-bound', '60'
|
||||
]), /coordinating capacity shard/)
|
||||
})
|
||||
|
||||
test('accepts the exact sharded request-unit and region behavior proof', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'asia-east2',
|
||||
'--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--request-unit-invites', '790', '--request-unit-invites-per-second', '2',
|
||||
'--request-unit-principals', '32',
|
||||
'--aggregate-request-unit-invites', '3160',
|
||||
'--request-unit-capacity', '6000', '--request-unit-overflow-probes', '1',
|
||||
'--request-unit-cleanup-timeout-seconds', '630',
|
||||
'--region-behavior-probes', '1', '--phase-barrier-dir', '/tmp/load-barrier'
|
||||
])
|
||||
assert.equal(config.aggregateRequestUnitInvites, 3_160)
|
||||
assert.equal(config.requestUnitCapacity, 6_000)
|
||||
assert.equal(config.requestUnitPrincipalCount, 32)
|
||||
assert.equal(config.requestUnitCleanupTimeoutMs, 630_000)
|
||||
assert.equal(config.regionBehaviorProbes, 1)
|
||||
assert.equal(config.phaseBarrierDir, '/tmp/load-barrier')
|
||||
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '789',
|
||||
'--request-unit-invites-per-second', '2',
|
||||
'--request-unit-principals', '32',
|
||||
'--aggregate-request-unit-invites', '3156', '--request-unit-capacity', '6000',
|
||||
'--phase-barrier-dir', '/tmp/load-barrier'
|
||||
]), /does not reach the exact reviewed capacity/)
|
||||
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '790',
|
||||
'--request-unit-invites-per-second', '2', '--request-unit-principals', '26',
|
||||
'--aggregate-request-unit-invites', '3160', '--request-unit-capacity', '6000',
|
||||
'--phase-barrier-dir', '/tmp/load-barrier'
|
||||
]), /does not reach the exact reviewed capacity/)
|
||||
})
|
||||
|
||||
function expectMixedProfile(config) {
|
||||
assert.equal(config.controls + 2 * config.splices, 2840)
|
||||
}
|
||||
|
||||
test('rejects a run shorter than its required lease horizons', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--duration-seconds',
|
||||
'209',
|
||||
'--required-lease-horizons',
|
||||
'2'
|
||||
]),
|
||||
/must cover 2 lease horizons/
|
||||
)
|
||||
})
|
||||
|
||||
test('bounds an explicit splice hold within the steady window', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '1',
|
||||
'--splices', '1',
|
||||
'--duration-seconds', '300',
|
||||
'--splice-hold-seconds', '60'
|
||||
])
|
||||
assert.equal(config.durationMs, 300_000)
|
||||
assert.equal(config.spliceHoldMs, 60_000)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '1',
|
||||
'--splices', '1',
|
||||
'--duration-seconds', '300',
|
||||
'--splice-hold-seconds', '301'
|
||||
]), /between 1 and the steady duration/)
|
||||
})
|
||||
|
||||
test('maps splice ownership deterministically within a shard', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls',
|
||||
'4',
|
||||
'--splices',
|
||||
'3',
|
||||
'--shard-count',
|
||||
'4',
|
||||
'--shard-index',
|
||||
'2'
|
||||
])
|
||||
|
||||
assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10])
|
||||
})
|
||||
|
||||
test('staggered shards form one deterministic splice ramp', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '4',
|
||||
'--splices', '3',
|
||||
'--splice-ramp-seconds', '11',
|
||||
'--shard-count', '4',
|
||||
'--shard-index', '2'
|
||||
])
|
||||
|
||||
assert.equal(config.spliceRampMs, 11_000)
|
||||
assert.deepEqual(
|
||||
[0, 1, 2].map((index) => relayLoadSpliceStartDelayMs(config, index)),
|
||||
[2_000, 6_000, 10_000]
|
||||
)
|
||||
assert.throws(() => relayLoadSpliceStartDelayMs(config, 3), /invalid local splice index/)
|
||||
})
|
||||
|
||||
test('distributes each shard invite wave below the account rate limit', () => {
|
||||
const identities = Array.from(
|
||||
{ length: 710 },
|
||||
(_, localIndex) => relayLoadPrincipalIndex(localIndex * 4 + 2, 4, 32)
|
||||
)
|
||||
const offers = Array.from({ length: 790 }, (_, index) => identities[index % identities.length])
|
||||
const counts = new Map()
|
||||
for (const principal of offers) counts.set(principal, (counts.get(principal) ?? 0) + 1)
|
||||
assert.equal(counts.size, 32)
|
||||
assert.equal(Math.max(...counts.values()), 26)
|
||||
})
|
||||
|
||||
test('requires exactly one assignment mode', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--target-origin',
|
||||
'https://cell.test'
|
||||
]),
|
||||
/exactly one target or director origin/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires separate recoverable and wedged reader profiles', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '4',
|
||||
'--splices', '3',
|
||||
'--slow-reader-splices', '1',
|
||||
'--wedged-reader-splices', '1',
|
||||
'--slow-reader-stream-bytes', '524288',
|
||||
'--wedged-reader-stream-bytes', '1048576',
|
||||
'--slow-reader-hold-ms', '9000',
|
||||
'--wedged-reader-hold-ms', '11000'
|
||||
])
|
||||
|
||||
assert.deepEqual(relayLoadSpliceProfile(config, 0), {
|
||||
readerMode: 'wedged', readerHoldMs: 11_000, streamBytes: 1_048_576, frameBytes: 65_536
|
||||
})
|
||||
assert.deepEqual(relayLoadSpliceProfile(config, 1), {
|
||||
readerMode: 'slow', readerHoldMs: 9_000, streamBytes: 524_288, frameBytes: 65_536
|
||||
})
|
||||
assert.equal(relayLoadSpliceProfile(config, 2).readerMode, 'normal')
|
||||
})
|
||||
|
||||
test('bounds reader stream, timeout, and splice load per shard', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '3']),
|
||||
/splices cannot exceed/
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--splices',
|
||||
'1',
|
||||
'--slow-reader-splices',
|
||||
'2'
|
||||
]),
|
||||
/reader splice counts cannot exceed/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--slow-reader-splices', '1',
|
||||
'--slow-reader-stream-bytes', '262144'
|
||||
]),
|
||||
/must exceed the 256 KiB/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--wedged-reader-splices', '1',
|
||||
'--wedged-reader-stream-bytes', '262144'
|
||||
]),
|
||||
/must exceed the 256 KiB/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--slow-reader-splices', '1',
|
||||
'--slow-reader-hold-ms', '10000'
|
||||
]),
|
||||
/must stay below the wedged timeout/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--wedged-reader-splices', '1',
|
||||
'--wedged-reader-hold-ms', '10000'
|
||||
]),
|
||||
/must exceed the wedged timeout/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '17', '--splices', '17', '--slow-reader-splices', '17'
|
||||
]),
|
||||
/reader splice count exceeds/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '9', '--splices', '9', '--wedged-reader-splices', '9'
|
||||
]),
|
||||
/reader stream bytes exceed/
|
||||
)
|
||||
})
|
||||
|
||||
test('supports one bounded reader-owning shard without multiplying its pressure', () => {
|
||||
const owner = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2600', '--aggregate-splices', '120',
|
||||
'--slow-reader-splices', '10', '--wedged-reader-splices', '1',
|
||||
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
|
||||
])
|
||||
const peer = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '1',
|
||||
'--aggregate-controls', '2600', '--aggregate-splices', '120',
|
||||
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
|
||||
])
|
||||
|
||||
assert.equal(owner.aggregateReaderSplices, 11)
|
||||
assert.equal(owner.aggregateReaderBytes, 18 * 1024 * 1024)
|
||||
assert.equal(peer.slowReaderSplices + peer.wedgedReaderSplices, 0)
|
||||
assert.equal(peer.aggregateReaderSplices, 11)
|
||||
})
|
||||
|
||||
test('requires queue, memory, and expected close evidence', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--splices', '2', '--slow-reader-splices', '1',
|
||||
'--wedged-reader-splices', '1', '--max-generator-rss-growth-mib', '100'
|
||||
])
|
||||
const passing = {
|
||||
generatorRssGrowthMiB: 50,
|
||||
slowReaderSplicesCompleted: 1,
|
||||
wedgedReaderSplicesClosed: 1,
|
||||
readerQueueEvidence: [
|
||||
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 65_544, increaseBytes: 65_536 }
|
||||
],
|
||||
readerClosesByCode: { '4429': 1 }
|
||||
}
|
||||
|
||||
assert.equal(relayLoadReaderEvidenceError(passing, config), undefined)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({
|
||||
...passing,
|
||||
readerQueueEvidence: [
|
||||
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 8, increaseBytes: 0 }
|
||||
]
|
||||
}, config),
|
||||
/no causal Relay queued-byte evidence/
|
||||
)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({ ...passing, generatorRssGrowthMiB: 101 }, config),
|
||||
/RSS growth budget/
|
||||
)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({ ...passing, readerClosesByCode: { '4429': 0 } }, config),
|
||||
/did not close with 4429/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
const DEFAULT_TIMEOUT_MS = 8_000
|
||||
const DEFAULT_POLL_MS = 100
|
||||
|
||||
export async function createRelayLoadReaderEvidence(origins, dependencies) {
|
||||
const distinctOrigins = [...new Set(origins)].sort()
|
||||
const baselines = new Map(await Promise.all(distinctOrigins.map(async (origin) => [
|
||||
origin,
|
||||
await dependencies.readQueuedBytes(origin)
|
||||
])))
|
||||
const peaks = new Map(baselines)
|
||||
const pending = new Map()
|
||||
const now = dependencies.now ?? Date.now
|
||||
const delay = dependencies.delay
|
||||
const timeoutMs = dependencies.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
||||
const pollMs = dependencies.pollMs ?? DEFAULT_POLL_MS
|
||||
|
||||
const observe = async ({ cellOrigin }) => {
|
||||
if (!baselines.has(cellOrigin)) throw new Error('reader origin lacks a run baseline')
|
||||
if (peaks.get(cellOrigin) > baselines.get(cellOrigin)) return
|
||||
const current = pending.get(cellOrigin)
|
||||
if (current) return await current
|
||||
const proof = (async () => {
|
||||
const baseline = baselines.get(cellOrigin)
|
||||
const deadline = now() + timeoutMs
|
||||
for (;;) {
|
||||
const queuedBytes = await dependencies.readQueuedBytes(cellOrigin)
|
||||
peaks.set(cellOrigin, Math.max(peaks.get(cellOrigin), queuedBytes))
|
||||
if (queuedBytes > baseline) return
|
||||
if (now() >= deadline) {
|
||||
throw new Error('reader stream produced no causal Relay queued-byte increase')
|
||||
}
|
||||
await delay(pollMs)
|
||||
}
|
||||
})()
|
||||
pending.set(cellOrigin, proof)
|
||||
try {
|
||||
await proof
|
||||
} finally {
|
||||
pending.delete(cellOrigin)
|
||||
}
|
||||
}
|
||||
|
||||
const snapshot = () => distinctOrigins.map((origin) => ({
|
||||
origin,
|
||||
baselineBytes: baselines.get(origin),
|
||||
peakBytes: peaks.get(origin),
|
||||
increaseBytes: peaks.get(origin) - baselines.get(origin)
|
||||
}))
|
||||
|
||||
return { observe, snapshot }
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
|
||||
|
||||
test('requires a queue increase above the pre-injection baseline for every origin', async () => {
|
||||
const samples = new Map([
|
||||
['https://a.test', [7, 7, 11]],
|
||||
['https://b.test', [0, 3]]
|
||||
])
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence([...samples.keys()], {
|
||||
readQueuedBytes: async (origin) => samples.get(origin).shift(),
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await Promise.all([
|
||||
evidence.observe({ cellOrigin: 'https://a.test' }),
|
||||
evidence.observe({ cellOrigin: 'https://b.test' })
|
||||
])
|
||||
|
||||
assert.deepEqual(evidence.snapshot(), [
|
||||
{ origin: 'https://a.test', baselineBytes: 7, peakBytes: 11, increaseBytes: 4 },
|
||||
{ origin: 'https://b.test', baselineBytes: 0, peakBytes: 3, increaseBytes: 3 }
|
||||
])
|
||||
})
|
||||
|
||||
test('shares one causal proof across concurrent readers on the same cell', async () => {
|
||||
const samples = [4, 4, 9]
|
||||
let reads = 0
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => { reads++; return samples.shift() },
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await Promise.all([
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' }),
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
])
|
||||
|
||||
assert.equal(reads, 3)
|
||||
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
|
||||
})
|
||||
|
||||
test('reuses a completed causal proof for later readers on the same cell', async () => {
|
||||
const samples = [4, 9]
|
||||
let reads = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => { reads++; return samples.shift() },
|
||||
delay: async () => undefined
|
||||
})
|
||||
|
||||
await evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
await evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
|
||||
assert.equal(reads, 2)
|
||||
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
|
||||
})
|
||||
|
||||
test('rejects a pre-existing nonzero queue that never increases', async () => {
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => 9,
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now,
|
||||
timeoutMs: 200,
|
||||
pollMs: 100
|
||||
})
|
||||
|
||||
await assert.rejects(
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' }),
|
||||
/no causal Relay queued-byte increase/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
export async function waitForRelayLoadRebindGate({
|
||||
delay,
|
||||
delayMs,
|
||||
activeCount,
|
||||
requiredCount
|
||||
}) {
|
||||
await delay(delayMs)
|
||||
const active = activeCount()
|
||||
if (active !== requiredCount) {
|
||||
throw new Error(`rebind boundary requires ${requiredCount} active controls, found ${active}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount,
|
||||
holdMs,
|
||||
delay,
|
||||
failureReason,
|
||||
requireOverflow = true
|
||||
}) {
|
||||
if (probeCount === 0) return { opened: 0, overflowReason: null }
|
||||
if (peers.length < probeCount) throw new Error('insufficient active controls for rebind proof')
|
||||
|
||||
const probes = []
|
||||
try {
|
||||
const opened = await Promise.allSettled(
|
||||
peers.slice(0, probeCount).map((peer) => peer.openRebindProbe())
|
||||
)
|
||||
for (const result of opened) {
|
||||
if (result.status === 'fulfilled') probes.push(result.value)
|
||||
}
|
||||
const rejected = opened.find((result) => result.status === 'rejected')
|
||||
if (rejected) throw rejected.reason
|
||||
|
||||
let overflowReason = null
|
||||
if (requireOverflow) {
|
||||
try {
|
||||
const overflow = await peers[0].openRebindProbe()
|
||||
await overflow.close()
|
||||
} catch (error) {
|
||||
overflowReason = failureReason(error)
|
||||
}
|
||||
if (overflowReason !== 'socket_http_503') {
|
||||
throw new Error(`rebind overflow was not rejected at the hard cap: ${overflowReason}`)
|
||||
}
|
||||
}
|
||||
const closedIndex = await Promise.race([
|
||||
delay(holdMs).then(() => -1),
|
||||
...probes.map((probe, index) => probe.closed.then(() => index))
|
||||
])
|
||||
if (closedIndex >= 0 || probes.some((probe) => !probe.isOpen())) {
|
||||
throw new Error('rebind probe closed before the hold completed')
|
||||
}
|
||||
return { opened: probes.length, overflowReason }
|
||||
} finally {
|
||||
await Promise.all(probes.map((probe) => probe.close()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
proveRelayLoadRebindBoundary,
|
||||
waitForRelayLoadRebindGate
|
||||
} from './relay-load-rebind-boundary.mjs'
|
||||
|
||||
function peer(open) {
|
||||
return { openRebindProbe: open }
|
||||
}
|
||||
|
||||
function probe(onClose = () => undefined) {
|
||||
let open = true
|
||||
let resolveClosed
|
||||
const closed = new Promise((resolve) => {
|
||||
resolveClosed = resolve
|
||||
})
|
||||
return {
|
||||
close: () => {
|
||||
if (!open) return
|
||||
open = false
|
||||
onClose()
|
||||
resolveClosed()
|
||||
},
|
||||
closed,
|
||||
isOpen: () => open
|
||||
}
|
||||
}
|
||||
|
||||
test('holds the requested rebind overlap and requires a hard-cap rejection', async () => {
|
||||
let openCalls = 0
|
||||
let closes = 0
|
||||
let heldFor = null
|
||||
const peers = [
|
||||
peer(async () => {
|
||||
openCalls++
|
||||
if (openCalls === 3) throw new Error('Unexpected server response: 503')
|
||||
return probe(() => closes++)
|
||||
}),
|
||||
peer(async () => {
|
||||
openCalls++
|
||||
return probe(() => closes++)
|
||||
})
|
||||
]
|
||||
const result = await proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount: 2,
|
||||
holdMs: 4_000,
|
||||
delay: async (milliseconds) => {
|
||||
heldFor = milliseconds
|
||||
},
|
||||
failureReason: (error) =>
|
||||
error.message.includes('503') ? 'socket_http_503' : 'unknown'
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { opened: 2, overflowReason: 'socket_http_503' })
|
||||
assert.equal(heldFor, 4_000)
|
||||
assert.equal(closes, 2)
|
||||
})
|
||||
|
||||
test('closes successful probes when a boundary probe fails', async () => {
|
||||
let closes = 0
|
||||
const peers = [
|
||||
peer(async () => probe(() => closes++)),
|
||||
peer(async () => {
|
||||
throw new Error('probe failed')
|
||||
})
|
||||
]
|
||||
|
||||
await assert.rejects(
|
||||
proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount: 2,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown'
|
||||
}),
|
||||
/probe failed/
|
||||
)
|
||||
assert.equal(closes, 1)
|
||||
})
|
||||
|
||||
test('waits for every replacement socket to finish closing', async () => {
|
||||
let finishClose
|
||||
const closeFinished = new Promise((resolve) => {
|
||||
finishClose = resolve
|
||||
})
|
||||
const closingProbe = probe()
|
||||
closingProbe.close = () => closeFinished
|
||||
let completed = false
|
||||
const boundary = proveRelayLoadRebindBoundary({
|
||||
peers: [peer(async () => closingProbe)],
|
||||
probeCount: 1,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
}).then(() => {
|
||||
completed = true
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
assert.equal(completed, false)
|
||||
finishClose()
|
||||
await boundary
|
||||
assert.equal(completed, true)
|
||||
})
|
||||
|
||||
test('can prove reserved replacement headroom below the physical cap', async () => {
|
||||
let closes = 0
|
||||
const result = await proveRelayLoadRebindBoundary({
|
||||
peers: [peer(async () => probe(() => closes++))],
|
||||
probeCount: 1,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
})
|
||||
assert.deepEqual(result, { opened: 1, overflowReason: null })
|
||||
assert.equal(closes, 1)
|
||||
})
|
||||
|
||||
test('fails when a replacement closes before the hold completes', async () => {
|
||||
let heldProbe
|
||||
await assert.rejects(
|
||||
proveRelayLoadRebindBoundary({
|
||||
peers: [
|
||||
peer(async () => {
|
||||
heldProbe = probe()
|
||||
return heldProbe
|
||||
})
|
||||
],
|
||||
probeCount: 1,
|
||||
holdMs: 4_000,
|
||||
delay: async () => {
|
||||
heldProbe.close()
|
||||
},
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
}),
|
||||
/closed before the hold completed/
|
||||
)
|
||||
})
|
||||
|
||||
test('delays the boundary until every ordinary control has recovered', async () => {
|
||||
let active = 899
|
||||
await assert.rejects(
|
||||
waitForRelayLoadRebindGate({
|
||||
delay: async () => undefined,
|
||||
delayMs: 0,
|
||||
activeCount: () => active,
|
||||
requiredCount: 900
|
||||
}),
|
||||
/requires 900 active controls/
|
||||
)
|
||||
await waitForRelayLoadRebindGate({
|
||||
delay: async () => {
|
||||
active = 900
|
||||
},
|
||||
delayMs: 1,
|
||||
activeCount: () => active,
|
||||
requiredCount: 900
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,35 @@
|
||||
const ASSIGNMENT_RETRY_DELAY_MS = 5_100
|
||||
|
||||
const waitPastAssignmentRateLimit = (schedule) =>
|
||||
new Promise((resolve) => schedule(resolve, ASSIGNMENT_RETRY_DELAY_MS))
|
||||
|
||||
export async function proveRelayLoadRegionBehavior({
|
||||
oldClientPeer,
|
||||
stickyPeer,
|
||||
asiaOrigin,
|
||||
scheduleAssignmentRetry = setTimeout
|
||||
}) {
|
||||
try {
|
||||
await oldClientPeer.connect()
|
||||
if (!oldClientPeer.assignedCellUrl() || oldClientPeer.assignedCellUrl() === asiaOrigin) {
|
||||
throw new Error('unhinted client did not use the US-first path')
|
||||
}
|
||||
} finally {
|
||||
await oldClientPeer.shutdown()
|
||||
}
|
||||
|
||||
try {
|
||||
await stickyPeer.connect()
|
||||
if (stickyPeer.assignedCellUrl() !== asiaOrigin) {
|
||||
throw new Error('preferred Asia client did not reach the Asia cell')
|
||||
}
|
||||
await waitPastAssignmentRateLimit(scheduleAssignmentRetry)
|
||||
const reassigned = await stickyPeer.requestAssignment('us-central1')
|
||||
if (reassigned.cellUrl !== asiaOrigin) {
|
||||
throw new Error('valid sticky assignment moved after preference changed')
|
||||
}
|
||||
} finally {
|
||||
await stickyPeer.shutdown()
|
||||
}
|
||||
return { oldClientUsFirst: true, stickyAssignmentPreserved: true }
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
|
||||
|
||||
function peer(origin, reassigned = origin) {
|
||||
let shutdowns = 0
|
||||
return {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => origin,
|
||||
requestAssignment: async () => ({ cellUrl: reassigned }),
|
||||
shutdown: async () => { shutdowns++ },
|
||||
shutdowns: () => shutdowns
|
||||
}
|
||||
}
|
||||
|
||||
test('proves unhinted US-first placement and sticky Asia preservation', async () => {
|
||||
const oldClientPeer = peer('https://c3.relay-staging.onorca.dev')
|
||||
const stickyPeer = peer('https://c4.relay-staging.onorca.dev')
|
||||
let retryDelayMs = 0
|
||||
assert.deepEqual(await proveRelayLoadRegionBehavior({
|
||||
oldClientPeer,
|
||||
stickyPeer,
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve, delayMs) => {
|
||||
retryDelayMs = delayMs
|
||||
resolve()
|
||||
}
|
||||
}), { oldClientUsFirst: true, stickyAssignmentPreserved: true })
|
||||
assert.equal(retryDelayMs, 5_100)
|
||||
assert.equal(oldClientPeer.shutdowns(), 1)
|
||||
assert.equal(stickyPeer.shutdowns(), 1)
|
||||
})
|
||||
|
||||
test('rejects Asia placement for an unhinted client or a moved sticky assignment', async () => {
|
||||
await assert.rejects(proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: peer('https://c4.relay-staging.onorca.dev'),
|
||||
stickyPeer: peer('https://c4.relay-staging.onorca.dev'),
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve) => resolve()
|
||||
}), /US-first/)
|
||||
await assert.rejects(proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: peer('https://c3.relay-staging.onorca.dev'),
|
||||
stickyPeer: peer('https://c4.relay-staging.onorca.dev', 'https://c3.relay-staging.onorca.dev'),
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve) => resolve()
|
||||
}), /sticky assignment moved/)
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
export async function openRelayLoadInviteOffers({
|
||||
peers,
|
||||
count,
|
||||
ratePerSecond,
|
||||
concurrency = 8,
|
||||
delay = delayDefault,
|
||||
now = Date.now
|
||||
}) {
|
||||
if (
|
||||
!Array.isArray(peers) || peers.length === 0 ||
|
||||
!Number.isSafeInteger(count) || count < 0 ||
|
||||
!Number.isSafeInteger(ratePerSecond) || ratePerSecond < 1 || ratePerSecond > 20 ||
|
||||
!Number.isSafeInteger(concurrency) || concurrency < 1
|
||||
) throw new Error('invalid Relay invite-offer load')
|
||||
let next = 0
|
||||
let nextStartAt = now()
|
||||
const workers = Array.from({ length: Math.min(concurrency, count) }, async () => {
|
||||
for (;;) {
|
||||
const index = next++
|
||||
if (index >= count) return
|
||||
const scheduledAt = Math.max(nextStartAt, now())
|
||||
nextStartAt = scheduledAt + 1_000 / ratePerSecond
|
||||
await delay(Math.max(0, scheduledAt - now()))
|
||||
await peers[index % peers.length].openInviteOffer()
|
||||
}
|
||||
})
|
||||
await Promise.all(workers)
|
||||
return count
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRequestUnitBoundary(peer) {
|
||||
try {
|
||||
await peer.openInviteOffer()
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'invite offer failed: relay_capacity_exhausted') {
|
||||
return 'relay_capacity_exhausted'
|
||||
}
|
||||
throw new Error('request-unit overflow was not rejected safely', { cause: error })
|
||||
}
|
||||
throw new Error('request-unit overflow unexpectedly succeeded')
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
openRelayLoadInviteOffers,
|
||||
proveRelayLoadRequestUnitBoundary
|
||||
} from './relay-load-request-unit-boundary.mjs'
|
||||
|
||||
test('distributes the exact invite count with bounded concurrency', async () => {
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const delays = []
|
||||
const calls = [0, 0, 0]
|
||||
const peers = calls.map((_, index) => ({
|
||||
async openInviteOffer() {
|
||||
calls[index]++
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await Promise.resolve()
|
||||
active--
|
||||
}
|
||||
}))
|
||||
assert.equal(await openRelayLoadInviteOffers({
|
||||
peers,
|
||||
count: 8,
|
||||
ratePerSecond: 2,
|
||||
concurrency: 2,
|
||||
delay: async (milliseconds) => { delays.push(milliseconds) },
|
||||
now: () => 0
|
||||
}), 8)
|
||||
assert.deepEqual(calls, [3, 3, 2])
|
||||
assert.ok(peak <= 2)
|
||||
assert.equal(delays.length, 8)
|
||||
assert.ok(Math.max(...delays) >= 3_500)
|
||||
})
|
||||
|
||||
test('accepts only the exact request-unit exhaustion error', async () => {
|
||||
assert.equal(await proveRelayLoadRequestUnitBoundary({
|
||||
openInviteOffer: async () => {
|
||||
throw new Error('invite offer failed: relay_capacity_exhausted')
|
||||
}
|
||||
}), 'relay_capacity_exhausted')
|
||||
await assert.rejects(
|
||||
proveRelayLoadRequestUnitBoundary({
|
||||
openInviteOffer: async () => { throw new Error('control response timeout') }
|
||||
}),
|
||||
/not rejected safely/
|
||||
)
|
||||
await assert.rejects(
|
||||
proveRelayLoadRequestUnitBoundary({ openInviteOffer: async () => undefined }),
|
||||
/unexpectedly succeeded/
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,25 @@
|
||||
export function assertRelayLoadRampAccepted(rampConnectionFailures, maximum) {
|
||||
if (rampConnectionFailures > maximum) {
|
||||
throw new Error('relay load ramp exceeded the allowed connection failures')
|
||||
}
|
||||
}
|
||||
|
||||
export async function runRelayLoadWithShutdown(operation, shutdown) {
|
||||
try {
|
||||
return await operation()
|
||||
} finally {
|
||||
await shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
export function relayLoadRunHasDisallowedFailures(result, config) {
|
||||
return (
|
||||
result.rampConnectionFailures > config.maxRampConnectionFailures ||
|
||||
(!config.allowPlannedTransitionRetries && result.transitionConnectionFailures > 0) ||
|
||||
result.steadyConnectionFailures > 0 ||
|
||||
result.unexpectedCloses > config.maxUnexpectedCloses ||
|
||||
result.protocolErrors > 0 ||
|
||||
result.refreshErrors > 0 ||
|
||||
result.socketErrors > 0
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
assertRelayLoadRampAccepted,
|
||||
relayLoadRunHasDisallowedFailures,
|
||||
runRelayLoadWithShutdown
|
||||
} from './relay-load-run-lifecycle.mjs'
|
||||
|
||||
test('always shuts down peers when a load phase fails', async () => {
|
||||
const events = []
|
||||
await assert.rejects(
|
||||
runRelayLoadWithShutdown(
|
||||
async () => {
|
||||
events.push('run')
|
||||
throw new Error('boundary failed')
|
||||
},
|
||||
async () => events.push('shutdown')
|
||||
),
|
||||
/boundary failed/
|
||||
)
|
||||
assert.deepEqual(events, ['run', 'shutdown'])
|
||||
})
|
||||
|
||||
test('fails immediately when the strict ramp budget is exceeded', () => {
|
||||
assert.doesNotThrow(() => assertRelayLoadRampAccepted(0, 0))
|
||||
assert.throws(() => assertRelayLoadRampAccepted(1, 0), /ramp exceeded/)
|
||||
})
|
||||
|
||||
test('rejects connection failures during the transition window', () => {
|
||||
const result = {
|
||||
rampConnectionFailures: 0,
|
||||
transitionConnectionFailures: 1,
|
||||
steadyConnectionFailures: 0,
|
||||
unexpectedCloses: 0,
|
||||
protocolErrors: 0,
|
||||
refreshErrors: 0,
|
||||
socketErrors: 0
|
||||
}
|
||||
assert.equal(
|
||||
relayLoadRunHasDisallowedFailures(result, {
|
||||
maxRampConnectionFailures: 0,
|
||||
maxUnexpectedCloses: 0
|
||||
}),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('allows only explicitly planned transition retries', () => {
|
||||
const result = {
|
||||
rampConnectionFailures: 0,
|
||||
transitionConnectionFailures: 1,
|
||||
steadyConnectionFailures: 0,
|
||||
unexpectedCloses: 0,
|
||||
protocolErrors: 0,
|
||||
refreshErrors: 0,
|
||||
socketErrors: 0
|
||||
}
|
||||
const config = {
|
||||
allowPlannedTransitionRetries: true,
|
||||
maxRampConnectionFailures: 0,
|
||||
maxUnexpectedCloses: 0
|
||||
}
|
||||
assert.equal(relayLoadRunHasDisallowedFailures(result, config), false)
|
||||
assert.equal(
|
||||
relayLoadRunHasDisallowedFailures({ ...result, steadyConnectionFailures: 1 }, config),
|
||||
true
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,355 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
|
||||
import { basename, join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
|
||||
const SHA = /^[a-f0-9]{40}$/
|
||||
const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
|
||||
const EVIDENCE_MAX_AGE_MS = 5 * 60_000
|
||||
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
|
||||
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
|
||||
const WAVE_INDEX = /^[0-3]$/
|
||||
const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000
|
||||
const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000
|
||||
const MIGRATION_POLICIES = new Set([
|
||||
'strict',
|
||||
'recover-forward',
|
||||
'capacity-transition'
|
||||
])
|
||||
const MUTATION_MODES = new Set([
|
||||
'capacity-transition',
|
||||
'continue-evacuation',
|
||||
'disable-cell',
|
||||
'enable-empty-cell',
|
||||
'execute',
|
||||
'fence-source',
|
||||
'recover-forward',
|
||||
'reset-empty-candidate'
|
||||
])
|
||||
|
||||
function argumentsByName(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const name = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!name?.startsWith('--') || !value || value.startsWith('--')) {
|
||||
throw new Error('relay monitor evidence arguments are invalid')
|
||||
}
|
||||
values[name.slice(2)] = value
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
async function sha256(path) {
|
||||
return createHash('sha256').update(await readFile(path)).digest('hex')
|
||||
}
|
||||
|
||||
async function regularFile(path) {
|
||||
try {
|
||||
return (await stat(path)).isFile()
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function provenance(values) {
|
||||
const runAttempt = Number(values['run-attempt'])
|
||||
if (
|
||||
!SAFE_ID.test(values['incident-id'] ?? '') ||
|
||||
!SAFE_ID.test(values['run-id'] ?? '') ||
|
||||
!Number.isSafeInteger(runAttempt) ||
|
||||
runAttempt < 1 ||
|
||||
!SHA.test(values['commit-sha'] ?? '') ||
|
||||
!['dry-run', 'monitor'].includes(values.mode)
|
||||
) {
|
||||
throw new Error('relay monitor evidence provenance is invalid')
|
||||
}
|
||||
return {
|
||||
incidentId: values['incident-id'],
|
||||
runId: values['run-id'],
|
||||
runAttempt,
|
||||
commitSha: values['commit-sha'],
|
||||
mode: values.mode
|
||||
}
|
||||
}
|
||||
|
||||
export async function createEvidenceManifest(argv) {
|
||||
const values = argumentsByName(argv)
|
||||
const directory = resolve(values.directory ?? '')
|
||||
const expected = provenance(values)
|
||||
const candidates = [
|
||||
`${expected.incidentId}.state.json`,
|
||||
`${expected.incidentId}.summaries.jsonl`,
|
||||
`${expected.incidentId}.summary.md`
|
||||
]
|
||||
const files = {}
|
||||
for (const name of candidates) {
|
||||
const path = join(directory, name)
|
||||
if (await regularFile(path)) files[name] = await sha256(path)
|
||||
}
|
||||
if (!files[`${expected.incidentId}.state.json`]) {
|
||||
throw new Error('relay monitor durable state is missing')
|
||||
}
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
...expected,
|
||||
files
|
||||
}
|
||||
const path = join(directory, 'evidence-manifest.json')
|
||||
await writeFile(path, `${JSON.stringify(manifest)}\n`, { mode: 0o600 })
|
||||
await chmod(path, 0o600)
|
||||
return manifest
|
||||
}
|
||||
|
||||
async function readAndVerifyManifest(directory, expected) {
|
||||
const manifest = JSON.parse(
|
||||
await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
|
||||
)
|
||||
if (
|
||||
manifest.schemaVersion !== 1 ||
|
||||
manifest.incidentId !== expected.incidentId ||
|
||||
manifest.runId !== expected.runId ||
|
||||
manifest.runAttempt !== expected.runAttempt ||
|
||||
manifest.commitSha !== expected.commitSha ||
|
||||
manifest.mode !== expected.mode
|
||||
) {
|
||||
throw new Error('relay monitor evidence provenance does not match')
|
||||
}
|
||||
const names = Object.keys(manifest.files ?? {})
|
||||
if (!names.includes(`${expected.incidentId}.state.json`)) {
|
||||
throw new Error('relay monitor evidence has no durable state')
|
||||
}
|
||||
for (const name of names) {
|
||||
if (basename(name) !== name || !/^[A-Za-z0-9._-]+$/.test(name)) {
|
||||
throw new Error('relay monitor evidence file name is invalid')
|
||||
}
|
||||
if (await sha256(join(directory, name)) !== manifest.files[name]) {
|
||||
throw new Error('relay monitor evidence hash does not match')
|
||||
}
|
||||
}
|
||||
const allowed = new Set([...names, 'evidence-manifest.json'])
|
||||
const unexpected = (await readdir(directory)).filter((name) => !allowed.has(name))
|
||||
if (unexpected.length > 0) throw new Error('relay monitor evidence has unexpected files')
|
||||
return manifest
|
||||
}
|
||||
|
||||
function validMigrationPolicyState(state) {
|
||||
return (
|
||||
(
|
||||
state.migrationPolicy === 'strict' &&
|
||||
state.recoverySourceCellId === null &&
|
||||
state.capacityCellId === null
|
||||
) ||
|
||||
(
|
||||
state.migrationPolicy === 'recover-forward' &&
|
||||
state.capacityCellId === null &&
|
||||
typeof state.recoverySourceCellId === 'string' &&
|
||||
state.expectedSelector?.membership?.existingOnly?.includes(
|
||||
state.recoverySourceCellId
|
||||
)
|
||||
) ||
|
||||
(
|
||||
state.migrationPolicy === 'capacity-transition' &&
|
||||
state.recoverySourceCellId === null &&
|
||||
typeof state.capacityCellId === 'string' &&
|
||||
state.expectedSelector?.membership?.general?.includes(state.capacityCellId)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
export async function verifyRestoredEvidence(argv) {
|
||||
const values = argumentsByName(argv)
|
||||
const directory = resolve(values.directory ?? '')
|
||||
const expected = provenance(values)
|
||||
await readAndVerifyManifest(directory, expected)
|
||||
const state = JSON.parse(
|
||||
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
|
||||
)
|
||||
if (
|
||||
state.schemaVersion !== 4 ||
|
||||
state.incidentId !== expected.incidentId ||
|
||||
state.environment !== 'production' ||
|
||||
state.preDrainDryRun !== (expected.mode === 'dry-run') ||
|
||||
!MIGRATION_POLICIES.has(state.migrationPolicy) ||
|
||||
!validMigrationPolicyState(state)
|
||||
) {
|
||||
throw new Error('relay monitor restored state does not match provenance')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
|
||||
const completedAt = Date.parse(state.completedAt)
|
||||
const startedAt = Date.parse(state.startedAt)
|
||||
const windowStartedAt = Date.parse(state.windowStartedAt)
|
||||
const lastSampleAt = Date.parse(state.lastSampleAt)
|
||||
const age = nowMs - completedAt
|
||||
return (
|
||||
state.schemaVersion === 4 &&
|
||||
state.incidentId === expected.incidentId &&
|
||||
state.environment === 'production' &&
|
||||
state.preDrainDryRun === true &&
|
||||
validMigrationPolicyState(state) &&
|
||||
state.durationMinutes === 15 &&
|
||||
state.intervalMs === EVIDENCE_SAMPLE_INTERVAL_MS &&
|
||||
state.sampleCount >= 16 &&
|
||||
state.frozenAt === null &&
|
||||
Number.isFinite(startedAt) &&
|
||||
completedAt - startedAt >= 0 &&
|
||||
completedAt - startedAt <= EVIDENCE_MAX_LINEAGE_MS &&
|
||||
Number.isFinite(windowStartedAt) &&
|
||||
completedAt - windowStartedAt >= 15 * 60_000 &&
|
||||
Number.isFinite(lastSampleAt) &&
|
||||
lastSampleAt <= completedAt &&
|
||||
completedAt - lastSampleAt <= state.intervalMs &&
|
||||
Number.isFinite(completedAt) &&
|
||||
age >= 0 &&
|
||||
age <= maxAgeMs
|
||||
)
|
||||
}
|
||||
|
||||
export async function verifyDryRunAuthority(argv, now = Date.now) {
|
||||
const values = argumentsByName(argv)
|
||||
const directory = resolve(values.directory ?? '')
|
||||
const expected = provenance(values)
|
||||
if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
|
||||
const manifest = await readAndVerifyManifest(directory, expected)
|
||||
const state = JSON.parse(
|
||||
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
|
||||
)
|
||||
const requiredMigrationPolicy = values['required-migration-policy']
|
||||
// Later same-cap waves start after sequential predecessor cell rolls, so the
|
||||
// freshness bound grows by one cell-job timeout per predecessor; single-use
|
||||
// consumption, needs-chaining, and each wave's live preflight recheck keep
|
||||
// holding the mutation to current health.
|
||||
const waveIndex = values['wave-index'] ?? '0'
|
||||
if (!WAVE_INDEX.test(waveIndex)) {
|
||||
throw new Error('relay monitor wave index is invalid')
|
||||
}
|
||||
const maxAgeMs =
|
||||
EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
|
||||
if (
|
||||
!MIGRATION_POLICIES.has(requiredMigrationPolicy) ||
|
||||
state.migrationPolicy !== requiredMigrationPolicy ||
|
||||
!validCompletedDryRunState(state, expected, now(), maxAgeMs)
|
||||
) {
|
||||
throw new Error('relay monitor dry-run authority is incomplete or stale')
|
||||
}
|
||||
return { manifest, state }
|
||||
}
|
||||
|
||||
function exactSelector(actual, expected) {
|
||||
const membership = (selector) => {
|
||||
if (
|
||||
!selector?.membership ||
|
||||
!['existingOnly', 'migrationOnly', 'general'].every((key) =>
|
||||
Array.isArray(selector.membership[key])
|
||||
)
|
||||
) return null
|
||||
const normalized = Object.fromEntries(
|
||||
['existingOnly', 'migrationOnly', 'general'].map((key) => [
|
||||
key,
|
||||
[...selector.membership[key]].sort()
|
||||
])
|
||||
)
|
||||
const all = Object.values(normalized).flat()
|
||||
return new Set(all).size === all.length ? normalized : null
|
||||
}
|
||||
const actualMembership = membership(actual)
|
||||
const expectedMembership = membership(expected)
|
||||
return Boolean(
|
||||
actualMembership &&
|
||||
expectedMembership &&
|
||||
actual?.generation === expected?.generation &&
|
||||
JSON.stringify(actualMembership) === JSON.stringify(expectedMembership)
|
||||
)
|
||||
}
|
||||
|
||||
export async function verifyMutationEvidence(
|
||||
argv,
|
||||
environment = process.env,
|
||||
fetchImpl = fetch,
|
||||
now = Date.now
|
||||
) {
|
||||
const values = argumentsByName(argv)
|
||||
const directory = resolve(values.directory ?? '')
|
||||
const expected = provenance(values)
|
||||
if (expected.mode !== 'dry-run') throw new Error('mutation requires dry-run evidence')
|
||||
const manifest = await readAndVerifyManifest(directory, expected)
|
||||
const state = JSON.parse(
|
||||
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
|
||||
)
|
||||
const mutationMode = values['mutation-mode']
|
||||
if (!MUTATION_MODES.has(mutationMode)) {
|
||||
throw new Error('relay monitor mutation mode is invalid')
|
||||
}
|
||||
const scopedRecoverySourceCellId =
|
||||
values['scoped-recovery-source-cell-id']
|
||||
const recoveryMutation = ['fence-source', 'recover-forward'].includes(mutationMode)
|
||||
const scopedRecoveryMutation =
|
||||
['execute', 'recover-forward'].includes(mutationMode) &&
|
||||
Boolean(scopedRecoverySourceCellId)
|
||||
if (scopedRecoverySourceCellId && !scopedRecoveryMutation) {
|
||||
throw new Error('relay monitor scoped recovery evidence is invalid')
|
||||
}
|
||||
const requiredMigrationPolicy = mutationMode === 'capacity-transition'
|
||||
? 'capacity-transition'
|
||||
: recoveryMutation || scopedRecoveryMutation ? 'recover-forward' : 'strict'
|
||||
if (state.migrationPolicy !== requiredMigrationPolicy) {
|
||||
throw new Error('relay monitor migration policy does not match mutation')
|
||||
}
|
||||
const expectedRecoverySourceCellId = scopedRecoveryMutation
|
||||
? scopedRecoverySourceCellId
|
||||
: values['source-cell-id']
|
||||
if (
|
||||
(recoveryMutation || scopedRecoveryMutation) &&
|
||||
state.recoverySourceCellId !== expectedRecoverySourceCellId
|
||||
) {
|
||||
throw new Error('relay monitor recovery source does not match mutation')
|
||||
}
|
||||
if (
|
||||
mutationMode === 'capacity-transition' &&
|
||||
state.capacityCellId !== values['source-cell-id']
|
||||
) {
|
||||
throw new Error('relay monitor capacity cell does not match mutation')
|
||||
}
|
||||
if (
|
||||
!validCompletedDryRunState(state, expected, now(), EVIDENCE_MAX_AGE_MS)
|
||||
) {
|
||||
throw new Error('relay monitor dry-run evidence is incomplete or stale')
|
||||
}
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
const origin = values['director-origin']
|
||||
if (!token || !JWT.test(token) || !origin?.startsWith('https://')) {
|
||||
throw new Error('relay monitor live selector verification is unavailable')
|
||||
}
|
||||
const response = await fetchImpl(`${origin}/v1/admin/admission-selector/status`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error('relay monitor live selector verification failed')
|
||||
const current = (await response.json()).selector
|
||||
if (!exactSelector(current, state.expectedSelector)) {
|
||||
throw new Error('relay admission selector changed after the dry run')
|
||||
}
|
||||
return { manifest, state }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [command, ...argv] = process.argv.slice(2)
|
||||
if (command === 'create') await createEvidenceManifest(argv)
|
||||
else if (command === 'verify-restore') await verifyRestoredEvidence(argv)
|
||||
else if (command === 'verify-authority') await verifyDryRunAuthority(argv)
|
||||
else if (command === 'verify-mutation') await verifyMutationEvidence(argv)
|
||||
else throw new Error('relay monitor evidence command is invalid')
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : 'relay monitor evidence failed')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs'
|
||||
import {
|
||||
createEvidenceManifest,
|
||||
verifyDryRunAuthority,
|
||||
verifyMutationEvidence,
|
||||
verifyRestoredEvidence
|
||||
} from './relay-monitor-evidence.mjs'
|
||||
|
||||
const now = Date.parse('2026-07-28T12:00:00.000Z')
|
||||
const provenance = [
|
||||
'--incident-id',
|
||||
'relay-123',
|
||||
'--run-id',
|
||||
'123',
|
||||
'--run-attempt',
|
||||
'1',
|
||||
'--commit-sha',
|
||||
'a'.repeat(40),
|
||||
'--mode',
|
||||
'dry-run'
|
||||
]
|
||||
const selector = {
|
||||
generation: 2,
|
||||
membership: {
|
||||
existingOnly: ['c1'],
|
||||
migrationOnly: ['c2'],
|
||||
general: ['c3']
|
||||
}
|
||||
}
|
||||
|
||||
async function evidenceDirectory(migrationPolicy = 'strict') {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-monitor-evidence-'))
|
||||
const state = {
|
||||
schemaVersion: 4,
|
||||
incidentId: 'relay-123',
|
||||
environment: 'production',
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy,
|
||||
recoverySourceCellId: migrationPolicy === 'recover-forward' ? 'c1' : null,
|
||||
capacityCellId: migrationPolicy === 'capacity-transition' ? 'c3' : null,
|
||||
startedAt: new Date(now - 17 * 60_000).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000,
|
||||
sampleCount: 16,
|
||||
windowStartedAt: new Date(now - 16 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(now - 60_007).toISOString(),
|
||||
completedAt: new Date(now - 60_000).toISOString(),
|
||||
frozenAt: null,
|
||||
expectedSelector: selector
|
||||
}
|
||||
await writeFile(
|
||||
join(directory, 'relay-123.state.json'),
|
||||
`${JSON.stringify(state)}\n`
|
||||
)
|
||||
return directory
|
||||
}
|
||||
|
||||
test('creates and verifies exact restart provenance and hashes', async () => {
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
await assert.doesNotReject(
|
||||
verifyRestoredEvidence(['--directory', directory, ...provenance])
|
||||
)
|
||||
await writeFile(join(directory, 'relay-123.state.json'), '{}\n')
|
||||
await assert.rejects(
|
||||
verifyRestoredEvidence(['--directory', directory, ...provenance]),
|
||||
/hash does not match/
|
||||
)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('requires fresh green evidence and rechecks the live selector', async () => {
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
const verifyAuthority = () => verifyDryRunAuthority(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--required-migration-policy',
|
||||
'strict'
|
||||
],
|
||||
() => now
|
||||
)
|
||||
await assert.doesNotReject(verifyAuthority())
|
||||
const fetchImpl = async (_input, init) => {
|
||||
assert.equal(
|
||||
new Headers(init.headers).get('authorization'),
|
||||
'Bearer aaa.bbb.ccc'
|
||||
)
|
||||
return Response.json({ selector })
|
||||
}
|
||||
await assert.doesNotReject(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
)
|
||||
)
|
||||
const statePath = join(directory, 'relay-123.state.json')
|
||||
const state = JSON.parse(await readFile(statePath, 'utf8'))
|
||||
state.completedAt = new Date(now - 300_001).toISOString()
|
||||
await writeFile(statePath, `${JSON.stringify(state)}\n`)
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
await assert.rejects(
|
||||
verifyAuthority(),
|
||||
/authority is incomplete or stale/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/incomplete or stale/
|
||||
)
|
||||
state.completedAt = new Date(now - 60_000).toISOString()
|
||||
state.lastSampleAt = new Date(now - 120_001).toISOString()
|
||||
await writeFile(statePath, `${JSON.stringify(state)}\n`)
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/incomplete or stale/
|
||||
)
|
||||
state.lastSampleAt = new Date(now - 60_007).toISOString()
|
||||
state.startedAt = new Date(now - 26 * 60_000 - 1).toISOString()
|
||||
await writeFile(statePath, `${JSON.stringify(state)}\n`)
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
await assert.rejects(
|
||||
verifyAuthority(),
|
||||
/authority is incomplete or stale/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/incomplete or stale/
|
||||
)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('later same-cap waves accept evidence aged by predecessor cell rolls', async () => {
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
const statePath = join(directory, 'relay-123.state.json')
|
||||
const state = JSON.parse(await readFile(statePath, 'utf8'))
|
||||
const authorityAt = (...waveArgs) => verifyDryRunAuthority(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--required-migration-policy',
|
||||
'strict',
|
||||
...waveArgs.flatMap((waveIndex) => ['--wave-index', waveIndex])
|
||||
],
|
||||
() => now
|
||||
)
|
||||
const ageState = async (ageMs) => {
|
||||
state.completedAt = new Date(now - ageMs).toISOString()
|
||||
state.lastSampleAt = new Date(now - ageMs - 7).toISOString()
|
||||
state.startedAt = new Date(now - ageMs - 17 * 60_000).toISOString()
|
||||
state.windowStartedAt = new Date(now - ageMs - 16 * 60_000).toISOString()
|
||||
await writeFile(statePath, `${JSON.stringify(state)}\n`)
|
||||
await createEvidenceManifest(['--directory', directory, ...provenance])
|
||||
}
|
||||
// The wave-0 bound in isolation: exactly 5 minutes, flag or no flag.
|
||||
await ageState(5 * 60_000)
|
||||
await assert.doesNotReject(authorityAt())
|
||||
await assert.doesNotReject(authorityAt('0'))
|
||||
await ageState(5 * 60_000 + 1)
|
||||
await assert.rejects(authorityAt(), /authority is incomplete or stale/)
|
||||
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
|
||||
// One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1.
|
||||
await ageState(17 * 60_000)
|
||||
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
|
||||
await assert.doesNotReject(authorityAt('1'))
|
||||
await assert.rejects(authorityAt('4'), /wave index is invalid/)
|
||||
await assert.rejects(authorityAt('x'), /wave index is invalid/)
|
||||
// Both edges of one predecessor job timeout: 5min + 75min exactly.
|
||||
await ageState(80 * 60_000)
|
||||
await assert.doesNotReject(authorityAt('1'))
|
||||
await ageState(80 * 60_000 + 1)
|
||||
await assert.rejects(authorityAt('1'), /authority is incomplete or stale/)
|
||||
await assert.doesNotReject(authorityAt('2'))
|
||||
// Wave 2 and wave 3 edges: 5min + 2 * 75min and 5min + 3 * 75min exactly.
|
||||
await ageState(155 * 60_000)
|
||||
await assert.doesNotReject(authorityAt('2'))
|
||||
await ageState(155 * 60_000 + 1)
|
||||
await assert.rejects(authorityAt('2'), /authority is incomplete or stale/)
|
||||
await ageState(230 * 60_000)
|
||||
await assert.doesNotReject(authorityAt('3'))
|
||||
await ageState(230 * 60_000 + 1)
|
||||
await assert.rejects(authorityAt('3'), /authority is incomplete or stale/)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('binds migration policies to their exact mutations', async () => {
|
||||
const strictDirectory = await evidenceDirectory()
|
||||
const recoveryDirectory = await evidenceDirectory('recover-forward')
|
||||
const capacityDirectory = await evidenceDirectory('capacity-transition')
|
||||
const fetchImpl = async () => Response.json({ selector })
|
||||
try {
|
||||
await createEvidenceManifest(['--directory', strictDirectory, ...provenance])
|
||||
await createEvidenceManifest(['--directory', recoveryDirectory, ...provenance])
|
||||
await createEvidenceManifest(['--directory', capacityDirectory, ...provenance])
|
||||
await assert.rejects(
|
||||
verifyDryRunAuthority(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--required-migration-policy',
|
||||
'strict'
|
||||
],
|
||||
() => now
|
||||
),
|
||||
/authority is incomplete or stale/
|
||||
)
|
||||
const verify = (directory, mutationMode, sourceCellId = 'c1') => verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
mutationMode,
|
||||
'--source-cell-id',
|
||||
sourceCellId,
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
)
|
||||
await assert.doesNotReject(verify(strictDirectory, 'execute'))
|
||||
await assert.doesNotReject(
|
||||
verify(capacityDirectory, 'capacity-transition', 'c3')
|
||||
)
|
||||
await assert.doesNotReject(verify(recoveryDirectory, 'recover-forward'))
|
||||
await assert.doesNotReject(verify(recoveryDirectory, 'fence-source'))
|
||||
await assert.doesNotReject(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c12',
|
||||
'--scoped-recovery-source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
)
|
||||
)
|
||||
await assert.doesNotReject(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'recover-forward',
|
||||
'--source-cell-id',
|
||||
'c12',
|
||||
'--scoped-recovery-source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
)
|
||||
)
|
||||
await assert.rejects(
|
||||
verify(recoveryDirectory, 'execute'),
|
||||
/migration policy does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verify(strictDirectory, 'recover-forward'),
|
||||
/migration policy does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verify(strictDirectory, 'capacity-transition', 'c3'),
|
||||
/migration policy does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verify(capacityDirectory, 'capacity-transition', 'c1'),
|
||||
/capacity cell does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'recover-forward',
|
||||
'--source-cell-id',
|
||||
'c9',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/recovery source does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'fence-source',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--scoped-recovery-source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/scoped recovery evidence is invalid/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
recoveryDirectory,
|
||||
...provenance,
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c12',
|
||||
'--scoped-recovery-source-cell-id',
|
||||
'c9',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
fetchImpl,
|
||||
() => now
|
||||
),
|
||||
/recovery source does not match/
|
||||
)
|
||||
} finally {
|
||||
await rm(strictDirectory, { recursive: true, force: true })
|
||||
await rm(recoveryDirectory, { recursive: true, force: true })
|
||||
await rm(capacityDirectory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('workflow reruns restore the prior attempt into one stable incident', async () => {
|
||||
const workflow = await readFile(
|
||||
relayWorkflowUrl('monitor-relay-production-job.yml'),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(workflow, /INCIDENT_ID: relay-\$\{\{ github\.run_id \}\}-\$\{\{ inputs\.mode \}\}/)
|
||||
assert.doesNotMatch(workflow, /INCIDENT_ID:.*run_attempt/)
|
||||
assert.match(workflow, /actions\/download-artifact@v4/)
|
||||
assert.match(workflow, /verify-restore/)
|
||||
assert.match(workflow, /RESTART_FLAG=--restart/)
|
||||
assert.equal(workflow.match(/--capacity-cell-id/g)?.length, 3)
|
||||
const dispatchWorkflow = await readFile(
|
||||
relayWorkflowUrl('monitor-relay-production.yml'),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(dispatchWorkflow, /- capacity-transition/)
|
||||
assert.match(dispatchWorkflow, /capacity-cell-id: \$\{\{ inputs\.capacity-cell-id \}\}/)
|
||||
})
|
||||
|
||||
test('same-cap and rehome mutations require complete strict dry-run authority', async () => {
|
||||
for (const name of [
|
||||
'deploy-relay-production-same-cap-job.yml',
|
||||
'operate-relay-production-rehome-job.yml'
|
||||
]) {
|
||||
const workflow = await readFile(
|
||||
relayWorkflowUrl(name),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(workflow, /relay-monitor-evidence\.mjs verify-authority/)
|
||||
assert.match(workflow, /--required-migration-policy strict/)
|
||||
assert.doesNotMatch(workflow, /relay-monitor-evidence\.mjs verify-restore/)
|
||||
}
|
||||
})
|
||||
|
||||
test('production mutation workflows consume and live-recheck dry-run evidence', async () => {
|
||||
for (const name of [
|
||||
'deploy-relay-production.yml',
|
||||
'deploy-relay-production-multi-target.yml'
|
||||
]) {
|
||||
const workflow = await readFile(
|
||||
relayWorkflowUrl(name),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(workflow, /actions\/download-artifact@v4/)
|
||||
assert.match(workflow, /verify-mutation/)
|
||||
assert.match(workflow, /--mutation-mode "\$\{DEPLOY_MODE\}"/)
|
||||
assert.match(workflow, /--source-cell-id "\$\{SOURCE_CELL_ID\}"/)
|
||||
assert.match(workflow, /incident:relay-preflight/)
|
||||
assert.match(workflow, /Reject previously consumed dry-run evidence/)
|
||||
assert.match(workflow, /actions\/upload-artifact@v4/)
|
||||
assert.match(workflow, /relay-monitor-consumed-/)
|
||||
assert.match(workflow, /ORCA_RELAY_ADMIN_ID_TOKEN/)
|
||||
assert.match(workflow, /github\.ref == 'refs\/heads\/main'/)
|
||||
assert.ok(
|
||||
workflow.indexOf('pnpm install --frozen-lockfile') <
|
||||
workflow.indexOf('id: google-auth')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('monitor and mutation workflows share the production Cloud SQL rollout lock', async () => {
|
||||
for (const name of [
|
||||
'monitor-relay-production.yml',
|
||||
'deploy-relay-production.yml',
|
||||
'deploy-relay-production-multi-target.yml',
|
||||
'deploy-relay-production-capacity.yml'
|
||||
]) {
|
||||
const workflow = await readFile(
|
||||
relayWorkflowUrl(name),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(workflow, /group: production-cloud-sql-rollout/)
|
||||
}
|
||||
})
|
||||
|
||||
test('monitor uses a reusable job so exact job_workflow_ref is present', async () => {
|
||||
const wrapper = await readFile(
|
||||
relayWorkflowUrl('monitor-relay-production.yml'),
|
||||
'utf8'
|
||||
)
|
||||
assert.ok(wrapper.includes(`uses: ./${relayWorkflowPath('monitor-relay-production-job.yml')}`))
|
||||
const job = await readFile(
|
||||
relayWorkflowUrl('monitor-relay-production-job.yml'),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(job, /workflow_call:/)
|
||||
assert.match(job, /environment: production/)
|
||||
})
|
||||
@@ -0,0 +1,172 @@
|
||||
import { readFile, writeFile } from 'node:fs/promises'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
|
||||
|
||||
const APPROVED_CELLS = new Set(PRODUCTION_CAPACITY_CELL_IDS)
|
||||
|
||||
function argumentsByName(argv, allowed) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const argument = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!argument?.startsWith('--') || !value || value.startsWith('--')) {
|
||||
throw new Error('capacity wave arguments are invalid')
|
||||
}
|
||||
const name = argument.slice(2)
|
||||
if (!allowed.has(name) || name in values) {
|
||||
throw new Error(`capacity wave argument --${name} is invalid`)
|
||||
}
|
||||
values[name] = value
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
export function parseCapacityWave(waveCellIds, confirmation) {
|
||||
const cells = waveCellIds.split(',')
|
||||
if (
|
||||
cells.length < 2 ||
|
||||
cells.length > 4 ||
|
||||
cells.some((cell) => !APPROVED_CELLS.has(cell)) ||
|
||||
new Set(cells).size !== cells.length ||
|
||||
cells.join(',') !== waveCellIds
|
||||
) {
|
||||
throw new Error('capacity wave must contain two to four unique approved cells')
|
||||
}
|
||||
if (confirmation !== `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`) {
|
||||
throw new Error('capacity wave confirmation does not match the selected cells')
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function exactMembership(membership) {
|
||||
const keys = ['existingOnly', 'migrationOnly', 'general']
|
||||
if (!keys.every((key) => Array.isArray(membership?.[key]))) return false
|
||||
const cells = keys.flatMap((key) => membership[key])
|
||||
return cells.every((cell) => typeof cell === 'string') && new Set(cells).size === cells.length
|
||||
}
|
||||
|
||||
export function capacityWavePreflightState(state, waveCellIds, waveIndex, targetCellId) {
|
||||
const cells = parseCapacityWave(
|
||||
waveCellIds,
|
||||
`RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`
|
||||
)
|
||||
const index = Number(waveIndex)
|
||||
const membership = state.expectedSelector?.membership
|
||||
if (
|
||||
!Number.isSafeInteger(index) ||
|
||||
index < 0 ||
|
||||
index >= cells.length ||
|
||||
targetCellId !== cells[index] ||
|
||||
state.schemaVersion !== 4 ||
|
||||
state.environment !== 'production' ||
|
||||
state.preDrainDryRun !== true ||
|
||||
state.migrationPolicy !== 'capacity-transition' ||
|
||||
state.recoverySourceCellId !== null ||
|
||||
state.capacityCellId !== cells[0] ||
|
||||
!Number.isSafeInteger(state.expectedSelector?.generation) ||
|
||||
!exactMembership(membership) ||
|
||||
!cells.every((cell) => membership.general.includes(cell)) ||
|
||||
state.sampleCount < 16 ||
|
||||
state.frozenAt !== null ||
|
||||
typeof state.completedAt !== 'string'
|
||||
) {
|
||||
throw new Error('capacity wave evidence does not match this step')
|
||||
}
|
||||
return {
|
||||
schemaVersion: 4,
|
||||
environment: 'production',
|
||||
expectedSelector: {
|
||||
generation: state.expectedSelector.generation + index * 2,
|
||||
membership
|
||||
},
|
||||
migrationPolicy: 'capacity-transition',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: targetCellId
|
||||
}
|
||||
}
|
||||
|
||||
export function capacityWaveResumePreflightState(state, waveCellIds, targetCellId) {
|
||||
const cells = parseCapacityWave(
|
||||
waveCellIds,
|
||||
`RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`
|
||||
)
|
||||
const index = cells.indexOf(targetCellId)
|
||||
const base = capacityWavePreflightState(
|
||||
state,
|
||||
waveCellIds,
|
||||
String(index),
|
||||
targetCellId
|
||||
)
|
||||
const membership = base.expectedSelector.membership
|
||||
const general = membership.general.filter((cell) => cell !== targetCellId)
|
||||
const capacityCellId = general.includes(state.capacityCellId)
|
||||
? state.capacityCellId
|
||||
: cells.find((cell) => general.includes(cell))
|
||||
if (!capacityCellId) throw new Error('capacity wave resume has no general evidence cell')
|
||||
return {
|
||||
...base,
|
||||
expectedSelector: {
|
||||
generation: base.expectedSelector.generation + 1,
|
||||
membership: {
|
||||
existingOnly: membership.existingOnly,
|
||||
migrationOnly: [...membership.migrationOnly, targetCellId].sort(),
|
||||
general
|
||||
}
|
||||
},
|
||||
capacityCellId
|
||||
}
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const [command, ...arguments_] = argv
|
||||
if (command === 'validate') {
|
||||
const values = argumentsByName(
|
||||
arguments_,
|
||||
new Set(['wave-cell-ids', 'confirmation'])
|
||||
)
|
||||
process.stdout.write(`${JSON.stringify(parseCapacityWave(
|
||||
values['wave-cell-ids'] ?? '',
|
||||
values.confirmation ?? ''
|
||||
))}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'build-preflight' || command === 'build-resume-preflight') {
|
||||
const values = argumentsByName(
|
||||
arguments_,
|
||||
new Set([
|
||||
'state-file',
|
||||
'wave-cell-ids',
|
||||
...(command === 'build-preflight' ? ['wave-index'] : []),
|
||||
'target-cell-id',
|
||||
'output-file'
|
||||
])
|
||||
)
|
||||
const state = JSON.parse(await readFile(values['state-file'] ?? '', 'utf8'))
|
||||
const preflight = command === 'build-preflight'
|
||||
? capacityWavePreflightState(
|
||||
state,
|
||||
values['wave-cell-ids'] ?? '',
|
||||
values['wave-index'] ?? '',
|
||||
values['target-cell-id'] ?? ''
|
||||
)
|
||||
: capacityWaveResumePreflightState(
|
||||
state,
|
||||
values['wave-cell-ids'] ?? '',
|
||||
values['target-cell-id'] ?? ''
|
||||
)
|
||||
await writeFile(
|
||||
values['output-file'] ?? '',
|
||||
`${JSON.stringify(preflight)}\n`,
|
||||
{ mode: 0o600 }
|
||||
)
|
||||
return
|
||||
}
|
||||
throw new Error('capacity wave command is invalid')
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main(process.argv.slice(2)).catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : 'capacity wave failed')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
capacityWavePreflightState,
|
||||
capacityWaveResumePreflightState,
|
||||
parseCapacityWave
|
||||
} from './relay-production-capacity-wave.mjs'
|
||||
|
||||
const wave = [
|
||||
'production-gce-c22',
|
||||
'production-gce-c21',
|
||||
'production-gce-c20',
|
||||
'production-gce-c19'
|
||||
]
|
||||
|
||||
function evidence(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 4,
|
||||
environment: 'production',
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'capacity-transition',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: wave[0],
|
||||
expectedSelector: {
|
||||
generation: 39,
|
||||
membership: {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c18'],
|
||||
general: [...wave, 'production-gce-c23']
|
||||
}
|
||||
},
|
||||
sampleCount: 16,
|
||||
frozenAt: null,
|
||||
completedAt: '2026-08-11T21:00:00.000Z',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('accepts only exact confirmed waves of two to four approved cells', () => {
|
||||
for (const cells of [wave.slice(0, 2), wave]) {
|
||||
const value = cells.join(',')
|
||||
assert.deepEqual(
|
||||
parseCapacityWave(value, `RAISE_SELECTED_WAVE_TO_1000 ${value}`),
|
||||
cells
|
||||
)
|
||||
}
|
||||
for (const [cells, confirmation] of [
|
||||
[[wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}`],
|
||||
[[...wave, 'production-gce-c16'], `RAISE_SELECTED_WAVE_TO_1000 ${wave.join(',')}`],
|
||||
[[wave[0], wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},${wave[0]}`],
|
||||
[[wave[0], 'production-gce-c17'], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},production-gce-c17`],
|
||||
[[wave[0], ` ${wave[1]}`], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}, ${wave[1]}`],
|
||||
[wave, 'RAISE_SELECTED_WAVE_TO_1000 production-gce-c22']
|
||||
]) {
|
||||
assert.throws(() => parseCapacityWave(cells.join(','), confirmation))
|
||||
}
|
||||
})
|
||||
|
||||
test('derives each continuation preflight from the sealed selector generation', () => {
|
||||
for (const [index, cell] of wave.entries()) {
|
||||
const state = capacityWavePreflightState(
|
||||
evidence(),
|
||||
wave.join(','),
|
||||
String(index),
|
||||
cell
|
||||
)
|
||||
assert.equal(state.expectedSelector.generation, 39 + index * 2)
|
||||
assert.equal(state.capacityCellId, cell)
|
||||
assert.deepEqual(state.expectedSelector.membership, evidence().expectedSelector.membership)
|
||||
}
|
||||
})
|
||||
|
||||
test('derives an exact isolated-cell resume state from sealed wave evidence', () => {
|
||||
const state = capacityWaveResumePreflightState(
|
||||
evidence(),
|
||||
wave.join(','),
|
||||
wave[3]
|
||||
)
|
||||
assert.equal(state.expectedSelector.generation, 46)
|
||||
assert.equal(state.capacityCellId, wave[0])
|
||||
assert.deepEqual(state.expectedSelector.membership, {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c18', wave[3]].sort(),
|
||||
general: [wave[0], wave[1], wave[2], 'production-gce-c23']
|
||||
})
|
||||
})
|
||||
|
||||
test('resume rejects a target outside the exact sealed wave', () => {
|
||||
assert.throws(
|
||||
() => capacityWaveResumePreflightState(
|
||||
evidence(),
|
||||
wave.join(','),
|
||||
'production-gce-c16'
|
||||
),
|
||||
/does not match/
|
||||
)
|
||||
})
|
||||
|
||||
test('resume rebinds first-cell evidence to another general wave cell', () => {
|
||||
const state = capacityWaveResumePreflightState(
|
||||
evidence(),
|
||||
wave.join(','),
|
||||
wave[0]
|
||||
)
|
||||
assert.equal(state.capacityCellId, wave[1])
|
||||
assert.equal(state.expectedSelector.generation, 40)
|
||||
assert.ok(state.expectedSelector.membership.migrationOnly.includes(wave[0]))
|
||||
assert.ok(state.expectedSelector.membership.general.includes(wave[1]))
|
||||
})
|
||||
|
||||
test('rejects reordered, incomplete, frozen, or mismatched wave evidence', () => {
|
||||
const calls = [
|
||||
() => capacityWavePreflightState(evidence(), wave.join(','), '1', wave[0]),
|
||||
() => capacityWavePreflightState(evidence({ capacityCellId: wave[1] }), wave.join(','), '0', wave[0]),
|
||||
() => capacityWavePreflightState(evidence({ sampleCount: 15 }), wave.join(','), '0', wave[0]),
|
||||
() => capacityWavePreflightState(evidence({ frozenAt: '2026-08-11T20:59:00.000Z' }), wave.join(','), '0', wave[0]),
|
||||
() => capacityWavePreflightState(evidence({ completedAt: null }), wave.join(','), '0', wave[0]),
|
||||
() => capacityWavePreflightState(evidence({
|
||||
expectedSelector: {
|
||||
...evidence().expectedSelector,
|
||||
membership: {
|
||||
...evidence().expectedSelector.membership,
|
||||
general: wave.slice(1)
|
||||
}
|
||||
}
|
||||
}), wave.join(','), '0', wave[0])
|
||||
]
|
||||
for (const call of calls) assert.throws(call, /does not match/)
|
||||
})
|
||||
@@ -0,0 +1,440 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { readRelayWorkflow, relayWorkflowPath } from './relay-repository.mjs'
|
||||
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
|
||||
|
||||
const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml')
|
||||
const workflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
|
||||
const terraform = source('infra/terraform/relay-github-actions.tf')
|
||||
const production = source('infra/terraform/environments/production.tfvars')
|
||||
const capacityCells = PRODUCTION_CAPACITY_CELL_IDS
|
||||
|
||||
function source(path) {
|
||||
return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
function resource(type, name) {
|
||||
const start = terraform.indexOf(`resource "${type}" "${name}"`)
|
||||
assert.notEqual(start, -1, `${type}.${name} is missing`)
|
||||
const next = terraform.indexOf('\nresource "', start + 1)
|
||||
return terraform.slice(start, next === -1 ? undefined : next)
|
||||
}
|
||||
|
||||
function ordered(...markers) {
|
||||
let previous = -1
|
||||
for (const marker of markers) {
|
||||
const current = workflow.indexOf(marker)
|
||||
assert.ok(current > previous, `${marker} is missing or out of order`)
|
||||
previous = current
|
||||
}
|
||||
}
|
||||
|
||||
function mutationConfirmation(mode, targetCellId, confirmation) {
|
||||
const stepStart = workflow.indexOf(' - name: Require exact mutation confirmation')
|
||||
const runMarker = ' run: |\n'
|
||||
const runStart = workflow.indexOf(runMarker, stepStart) + runMarker.length
|
||||
const runEnd = workflow.indexOf('\n - name:', runStart)
|
||||
const script = workflow.slice(runStart, runEnd).replace(/^ {10}/gm, '')
|
||||
return spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
|
||||
env: {
|
||||
...process.env,
|
||||
DEPLOY_MODE: mode,
|
||||
TARGET_CELL_ID: targetCellId,
|
||||
CONFIRMATION: confirmation
|
||||
}
|
||||
}).status
|
||||
}
|
||||
|
||||
function imageCompatibility(activeImage, desiredImage) {
|
||||
const start = workflow.indexOf(' ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"')
|
||||
const end = workflow.indexOf('\n CURRENT_CELLS_JSON=', start)
|
||||
const script = workflow.slice(start, end).replace(/^ {10}/gm, '')
|
||||
return spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
|
||||
env: {
|
||||
...process.env,
|
||||
ACTIVE_IMAGE: activeImage,
|
||||
DESIRED_IMAGE: desiredImage,
|
||||
DESIRED_IMAGE_DIGEST: desiredImage.split('@').at(-1),
|
||||
COMPATIBLE_DIRECTOR_IMAGE_DIGEST: 'sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73',
|
||||
COMPATIBLE_CELL_IMAGE_DIGEST: 'sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f'
|
||||
}
|
||||
}).status
|
||||
}
|
||||
|
||||
test('production capacity mutation is restricted to the exact serving rollout set', () => {
|
||||
assert.match(workflow, /TARGET_CELL_ID: \$\{\{ inputs\.target-cell-id \}\}/)
|
||||
const targetInput = dispatchWorkflow.slice(
|
||||
dispatchWorkflow.indexOf(' target-cell-id:'),
|
||||
dispatchWorkflow.indexOf(' wave-cell-ids:')
|
||||
)
|
||||
assert.deepEqual(
|
||||
[...targetInput.matchAll(/^\s+- (production-gce-c\d+)$/gm)].map((match) => match[1]),
|
||||
capacityCells
|
||||
)
|
||||
assert.match(workflow, new RegExp(`CAPACITY_CELL_IDS: ${capacityCells.join(',')}`))
|
||||
for (const cellId of capacityCells) {
|
||||
assert.match(dispatchWorkflow, new RegExp(`^\\s+- ${cellId}$`, 'm'))
|
||||
}
|
||||
assert.match(workflow, /CELL_ORIGIN="https:\/\/\$\{TARGET_HOSTNAME\}\.relay\.onorca\.dev"/)
|
||||
assert.match(workflow, /echo "TARGET_HOSTNAME=\$\{TARGET_HOSTNAME\}"/)
|
||||
assert.match(workflow, /\} >> "\$\{GITHUB_ENV\}"/)
|
||||
assert.match(workflow, /RAISE_SELECTED_CELL_TO_1000/)
|
||||
assert.match(workflow, /ROLL_BACK_SELECTED_CELL_TO_600/)
|
||||
assert.match(workflow, /TARGET_HARD_CAP=600[\s\S]*?else[\s\S]*?TARGET_HARD_CAP=1000/)
|
||||
})
|
||||
|
||||
test('rollback confirmation is bound to the exact selected cell', () => {
|
||||
assert.equal(
|
||||
mutationConfirmation('apply', 'production-gce-c25', 'RAISE_SELECTED_CELL_TO_1000'),
|
||||
0
|
||||
)
|
||||
assert.equal(
|
||||
mutationConfirmation(
|
||||
'rollback',
|
||||
'production-gce-c25',
|
||||
'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c25'
|
||||
),
|
||||
0
|
||||
)
|
||||
assert.notEqual(
|
||||
mutationConfirmation('rollback', 'production-gce-c25', 'ROLL_BACK_SELECTED_CELL_TO_600'),
|
||||
0
|
||||
)
|
||||
assert.notEqual(
|
||||
mutationConfirmation(
|
||||
'rollback',
|
||||
'production-gce-c25',
|
||||
'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c26'
|
||||
),
|
||||
0
|
||||
)
|
||||
})
|
||||
|
||||
test('production configuration selects only serving cells for 1,000 and the compatible image', () => {
|
||||
const cell = (cellId) => production.slice(
|
||||
production.indexOf(`"${cellId}"`),
|
||||
production.indexOf('\n }', production.indexOf(`"${cellId}"`))
|
||||
)
|
||||
for (const cellId of capacityCells) {
|
||||
assert.match(cell(cellId), /connection_hard_cap\s+= 1000/)
|
||||
assert.match(
|
||||
cell(cellId),
|
||||
/sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563/
|
||||
)
|
||||
}
|
||||
for (const cellId of ['production-gce-c17', 'production-gce-c18']) {
|
||||
assert.match(cell(cellId), /connection_hard_cap\s+= 600/)
|
||||
assert.doesNotMatch(cell(cellId), /connection_hard_cap\s+= 1000/)
|
||||
assert.match(cell(cellId), /sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d/)
|
||||
}
|
||||
assert.equal(production.match(/connection_hard_cap\s+= 1000/g)?.length, capacityCells.length)
|
||||
// Asia cells legitimately share this digest, so scope the uniqueness check to the capacity set.
|
||||
assert.equal(
|
||||
new Set(capacityCells.map((cellId) => cell(cellId).match(/sha256:[0-9a-f]{64}/)[0])).size,
|
||||
1
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{ACTIVE_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_DIRECTOR_IMAGE_DIGEST\}"/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{DESIRED_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/
|
||||
)
|
||||
assert.match(workflow, /\.\[\$cell\]\.connection_hard_cap = \$cap/)
|
||||
assert.match(workflow, /baseCells:\$baseCells/)
|
||||
assert.match(workflow, /capacityCellIds:\(\$capacityCellIds \| split\(","\)\)/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/Verify current selected-cell capacity[\s\S]*?TOPOLOGY_PHASE.*predecessor[\s\S]*?CURRENT_CAP=600[\s\S]*?DESIRED_IMAGE_DIGEST.*PREDECESSOR_IMAGE_DIGEST/
|
||||
)
|
||||
})
|
||||
|
||||
test('director and cell image compatibility is an exact reviewed pair', () => {
|
||||
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@'
|
||||
const director = `${repository}sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73`
|
||||
const cell = `${repository}sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f`
|
||||
const other = `${repository}sha256:${'a'.repeat(64)}`
|
||||
assert.equal(imageCompatibility(cell, cell), 0)
|
||||
assert.equal(imageCompatibility(director, cell), 0)
|
||||
assert.notEqual(imageCompatibility(director, other), 0)
|
||||
assert.notEqual(imageCompatibility(other, cell), 0)
|
||||
})
|
||||
|
||||
// The matching check on google_project_service.required lives with the foundation root, which
|
||||
// stays in the private repository.
|
||||
|
||||
test('apply consumes fresh evidence before arming mutation cleanup', () => {
|
||||
for (const marker of [
|
||||
'Require fresh dry-run evidence reference',
|
||||
'Verify dry-run artifact before cloud authentication',
|
||||
'Reject previously consumed dry-run evidence',
|
||||
'Verify fresh dry-run evidence against the live selector',
|
||||
'Recheck every live safety signal',
|
||||
'Publish the consumed-evidence marker'
|
||||
]) {
|
||||
assert.match(workflow, new RegExp(marker))
|
||||
}
|
||||
assert.match(workflow, /--mutation-mode capacity-transition/)
|
||||
assert.match(workflow, /--source-cell-id "\$\{TARGET_CELL_ID\}"/)
|
||||
ordered(
|
||||
'Publish the consumed-evidence marker',
|
||||
'Arm fail-closed mutation cleanup',
|
||||
'Reversibly isolate only the selected cell',
|
||||
'Deploy only the reviewed director topology',
|
||||
'Plan and apply only the empty selected cell',
|
||||
'Restore only the selected cell to general admission',
|
||||
'Verify the live general selected cell'
|
||||
)
|
||||
})
|
||||
|
||||
test('wave apply consumes one proof and runs fail-closed cells sequentially', () => {
|
||||
assert.match(dispatchWorkflow, /- wave-apply/)
|
||||
assert.match(dispatchWorkflow, /group: production-cloud-sql-rollout/)
|
||||
assert.match(dispatchWorkflow, /Validate the exact wave request/)
|
||||
assert.match(dispatchWorkflow, /Verify wave evidence against the live selector/)
|
||||
assert.match(dispatchWorkflow, /Require exact 600\/60 predecessor wave cells/)
|
||||
assert.match(
|
||||
dispatchWorkflow,
|
||||
/COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f/
|
||||
)
|
||||
assert.match(
|
||||
dispatchWorkflow,
|
||||
/Require exact 600\/60 predecessor wave cells[\s\S]*?--expected-image-digests \\\n\s+"\$\{PREDECESSOR_IMAGE_DIGEST\},\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/
|
||||
)
|
||||
assert.match(dispatchWorkflow, /Publish the consumed-evidence marker/)
|
||||
assert.match(
|
||||
dispatchWorkflow,
|
||||
/OUTPUT_DIRECTORY: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/
|
||||
)
|
||||
assert.match(
|
||||
dispatchWorkflow,
|
||||
/path: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/
|
||||
)
|
||||
assert.doesNotMatch(dispatchWorkflow, /strategy:/)
|
||||
for (const [index, dependency] of [
|
||||
[1, 'wave_gate'],
|
||||
[2, 'wave_cell_1'],
|
||||
[3, 'wave_cell_2'],
|
||||
[4, 'wave_cell_3']
|
||||
]) {
|
||||
const start = dispatchWorkflow.indexOf(` wave_cell_${index}:`)
|
||||
const end = dispatchWorkflow.indexOf(`\n wave_cell_${index + 1}:`, start)
|
||||
const job = dispatchWorkflow.slice(start, end === -1 ? undefined : end)
|
||||
assert.match(job, new RegExp(`needs: (?:\\[wave_gate, )?${dependency}`))
|
||||
assert.match(job, /evidence-mode: continuation/)
|
||||
assert.match(job, new RegExp(`wave-index: '${index - 1}'`))
|
||||
}
|
||||
assert.match(workflow, /Download this workflow's wave authority/)
|
||||
assert.match(workflow, /run-id: \$\{\{ github\.run_id \}\}/)
|
||||
assert.match(workflow, /relay-production-capacity-wave\.mjs build-preflight/)
|
||||
assert.match(workflow, /Require the exact wave predecessor topology/)
|
||||
assert.match(workflow, /test "\$\{TOPOLOGY_PHASE\}" = predecessor/)
|
||||
assert.match(workflow, /Recheck exact wave state and every live safety signal/)
|
||||
assert.match(workflow, /if test "\$\{WAVE_INDEX\}" != 0; then RETRY_ARGS=\(--retry-freshness\); fi/)
|
||||
assert.equal(workflow.match(/--retry-freshness/g)?.length, 2)
|
||||
ordered(
|
||||
'Recheck exact wave state and every live safety signal',
|
||||
'Arm fail-closed mutation cleanup',
|
||||
'Reversibly isolate only the selected cell',
|
||||
'Verify the live general selected cell'
|
||||
)
|
||||
})
|
||||
|
||||
test('Terraform mutation targets only the selected cell and has fail-closed recovery', () => {
|
||||
assert.match(
|
||||
workflow,
|
||||
/google_compute_instance_template\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/google_compute_instance_group_manager\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/
|
||||
)
|
||||
assert.doesNotMatch(workflow, /relay_gce_cell\["production-gce-c26"\]/)
|
||||
assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/)
|
||||
assert.match(workflow, /validate-relay-capacity-plan\.mjs/)
|
||||
assert.match(workflow, /--mode bootstrap-cell/)
|
||||
assert.match(workflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/)
|
||||
assert.match(workflow, /failure\(\) && inputs\.mode != 'verify'/)
|
||||
assert.match(workflow, /test "\$\{MUTATION_STARTED:-false\}" = true \|\| exit 0/)
|
||||
assert.match(workflow, /--mode isolate/)
|
||||
assert.doesNotMatch(workflow, /rolling-action restart/)
|
||||
assert.equal(workflow.match(/manage_artifact_dns=false/g)?.length, 5)
|
||||
assert.match(workflow, /OFFLINE_ROLLBACK=true/)
|
||||
assert.match(workflow, /--runtime unavailable/)
|
||||
assert.equal(workflow.match(/--expected-image-digests/g)?.length, 7)
|
||||
assert.match(workflow, /PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b/)
|
||||
assert.match(workflow, /classify-relay-production-capacity-director\.mjs/)
|
||||
assert.match(workflow, /CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON/)
|
||||
assert.match(workflow, /if test "\$\{DIRECTOR_READY\}" = true; then exit 0; fi/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/Keep the selected cell isolated after a failed mutation[\s\S]*?--mode isolate[\s\S]*?--mode drain/
|
||||
)
|
||||
const cleanup = workflow.slice(workflow.indexOf('id: cleanup-auth'))
|
||||
assert.match(cleanup, /Keep the selected cell isolated after a failed mutation/)
|
||||
assert.match(cleanup, /steps\.cleanup-auth\.outputs\.id_token/)
|
||||
assert.doesNotMatch(cleanup, /steps\.deploy-auth\.outputs\.id_token/)
|
||||
ordered(
|
||||
'Reversibly isolate only the selected cell',
|
||||
'Drain the selected cell or prove an offline rollback',
|
||||
'id: restart-auth-one',
|
||||
'Require restart-safe selected-cell activity',
|
||||
'id: restart-auth-two',
|
||||
'Require extended restart-safe selected-cell activity',
|
||||
'Deploy only the reviewed director topology',
|
||||
'id: director-transition-auth',
|
||||
'Require fail-closed director transition',
|
||||
'id: capacity-auth',
|
||||
'Plan and apply only the empty selected cell',
|
||||
'id: capacity-transition-auth',
|
||||
'Verify fresh exact selected-cell heartbeat before admission'
|
||||
)
|
||||
const restartGate = workflow.slice(
|
||||
workflow.indexOf('id: restart-auth-one'),
|
||||
workflow.indexOf('Deploy only the reviewed director topology')
|
||||
)
|
||||
assert.equal(restartGate.match(/--timeout-ms 450000/g)?.length, 2)
|
||||
assert.match(restartGate, /steps\.restart-auth-one\.outputs\.id_token/)
|
||||
assert.match(restartGate, /steps\.restart-auth-two\.outputs\.id_token/)
|
||||
assert.match(restartGate, /capacity transition verification timed out:/)
|
||||
assert.match(workflow, /timeout-minutes: 75/)
|
||||
})
|
||||
|
||||
test('wave resume is bound to the failed run and exact isolated selector state', () => {
|
||||
assert.match(dispatchWorkflow, /- wave-resume/)
|
||||
assert.match(dispatchWorkflow, /evidence-mode: resume/)
|
||||
assert.match(dispatchWorkflow, /source-wave-run-id: \$\{\{ inputs\.source-wave-run-id \}\}/)
|
||||
assert.match(workflow, /Download the failed wave authority for resume/)
|
||||
assert.match(workflow, /test "\$\{SOURCE_SHA\}" = "\$\{MONITOR_SHA\}"/)
|
||||
assert.match(workflow, /test "\$\{SOURCE_ATTEMPT\}" = "\$\{EXPECTED_SOURCE_ATTEMPT\}"/)
|
||||
for (const boundary of [
|
||||
'31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13',
|
||||
'31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10',
|
||||
'31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8',
|
||||
'a917e8e1fc1a2654e8cb81ba39b57733ec56be9c',
|
||||
'6082e9ca89a918ca51f0c87db003f5e8805b64b7',
|
||||
'e59958130c9d9b7a6cd805df2678d08997842c7c',
|
||||
'EXPECTED_SOURCE_ATTEMPT=2'
|
||||
]) {
|
||||
assert.match(workflow, new RegExp(boundary))
|
||||
}
|
||||
assert.match(workflow, /\*\) exit 1 ;;/)
|
||||
assert.match(workflow, /test "\$\{MONITOR_SHA\}" = "\$\{EXPECTED_SHA\}"/)
|
||||
assert.match(workflow, /\.head_branch == "main"/)
|
||||
assert.match(workflow, /\.head_repository\.full_name == env\.GITHUB_REPOSITORY/)
|
||||
assert.ok(workflow.includes(`.path == "${relayWorkflowPath('monitor-relay-production.yml')}"`))
|
||||
assert.ok(workflow.includes(`.path == "${relayWorkflowPath('deploy-relay-production-capacity.yml')}"`))
|
||||
assert.match(workflow, /build-resume-preflight/)
|
||||
assert.match(workflow, /RESUME_SELECTED_CELL_TO_1000 \$\{TARGET_CELL_ID\}/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/Recheck exact isolated resume state[\s\S]*?--hard-cap 600[\s\S]*?--admission migration-only[\s\S]*?--draining required/
|
||||
)
|
||||
})
|
||||
|
||||
test('GCE capacity identity is exact-workflow and narrowly permissioned', () => {
|
||||
const provider = resource(
|
||||
'google_iam_workload_identity_pool_provider',
|
||||
'github_production_relay_capacity'
|
||||
)
|
||||
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/)
|
||||
for (const boundary of [
|
||||
"assertion.ref == 'refs/heads/main'",
|
||||
"assertion.environment == 'production'",
|
||||
'local.relay_github_workflow_conditions["github_production_relay_capacity"]'
|
||||
]) {
|
||||
assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
|
||||
}
|
||||
// The workflow pair itself is pinned in the clause the provider renders, once per accepted
|
||||
// repository, and each repository supplies its own workflow-ref head.
|
||||
for (const boundary of [
|
||||
"assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main'",
|
||||
"assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main'"
|
||||
]) {
|
||||
assert.match(terraform, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
|
||||
}
|
||||
const role = resource(
|
||||
'google_project_iam_custom_role',
|
||||
'github_production_relay_capacity_mutation'
|
||||
)
|
||||
assert.match(role, /compute\.instanceGroupManagers\.update/)
|
||||
assert.match(role, /compute\.instanceTemplates\.create/)
|
||||
assert.doesNotMatch(
|
||||
role,
|
||||
/compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/
|
||||
)
|
||||
const state = resource(
|
||||
'google_storage_bucket_iam_member',
|
||||
'github_production_relay_capacity_state'
|
||||
)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
|
||||
})
|
||||
|
||||
test('deploy and capacity identities are used in their intended phases', () => {
|
||||
const jobStart = workflow.indexOf(' capacity:')
|
||||
const stepsStart = workflow.indexOf(' steps:', jobStart)
|
||||
const jobHeader = workflow.slice(jobStart, stepsStart)
|
||||
assert.deepEqual(jobHeader.match(/^\s+if:.*$/gm), [
|
||||
" if: ${{ github.ref == 'refs/heads/main' }}"
|
||||
])
|
||||
const configurationStart = workflow.indexOf('Require production workflow configuration')
|
||||
const configurationEnd = workflow.indexOf('- uses: actions/checkout@v4', configurationStart)
|
||||
assert.ok(configurationStart >= 0)
|
||||
assert.ok(configurationEnd > configurationStart)
|
||||
const configurationStep = workflow.slice(configurationStart, configurationEnd)
|
||||
for (const name of [
|
||||
'GCP_REGION',
|
||||
'DEPLOY_WORKLOAD_IDENTITY_PROVIDER',
|
||||
'DEPLOY_SERVICE_ACCOUNT',
|
||||
'CAPACITY_WORKLOAD_IDENTITY_PROVIDER',
|
||||
'CAPACITY_SERVICE_ACCOUNT'
|
||||
]) {
|
||||
assert.match(configurationStep, new RegExp(`test -n "\\$\\{${name}\\}"`))
|
||||
}
|
||||
ordered('Require production workflow configuration', 'id: deploy-auth')
|
||||
ordered('id: deploy-auth', 'Reversibly isolate only the selected cell', 'id: capacity-auth')
|
||||
assert.match(workflow, /steps\.deploy-auth\.outputs\.id_token/)
|
||||
assert.doesNotMatch(workflow, /steps\.capacity-auth\.outputs\.id_token/)
|
||||
assert.equal(
|
||||
workflow.match(/steps\.capacity-transition-auth\.outputs\.id_token/g)?.length,
|
||||
3
|
||||
)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
assert.match(workflow, /read-relay-production-capacity-identity\.mjs/)
|
||||
assert.match(workflow, /\(\.directorReady \| type\) == "boolean"/)
|
||||
assert.match(workflow, /\(\.directorReady \| tostring\)/)
|
||||
})
|
||||
|
||||
test('director readiness extraction preserves only JSON booleans', {
|
||||
skip: spawnSync('jq', ['--version']).status !== 0
|
||||
}, () => {
|
||||
const filter = `if (.directorReady | type) == "boolean" then
|
||||
(.directorReady | tostring)
|
||||
else error("invalid directorReady classification") end`
|
||||
const extract = (input) => spawnSync('jq', ['-er', filter], {
|
||||
encoding: 'utf8',
|
||||
input: JSON.stringify(input)
|
||||
})
|
||||
for (const value of [true, false]) {
|
||||
const result = extract({ directorReady: value })
|
||||
assert.equal(result.status, 0)
|
||||
assert.equal(result.stdout.trim(), String(value))
|
||||
}
|
||||
for (const input of [
|
||||
{ directorReady: 'true' },
|
||||
{ directorReady: 'false' },
|
||||
{ directorReady: null },
|
||||
{}
|
||||
]) {
|
||||
assert.notEqual(extract(input).status, 0)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,169 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import test from 'node:test'
|
||||
import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'
|
||||
import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs'
|
||||
|
||||
async function source(path) {
|
||||
return await readFile(new URL(`../../${path}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
// Why: the shared deploy identity is relay-owned in production and moves with the relay
|
||||
// extraction, so it needs a name the public repo can carry without touching the app pair. The
|
||||
// generic names are retired; a workflow that still reads them would silently resolve to nothing.
|
||||
test('no workflow names the retired generic production deploy identity', async () => {
|
||||
const files = workflowFiles()
|
||||
assert.ok(files.length > 20)
|
||||
const relayReaders = []
|
||||
for (const file of files) {
|
||||
const workflow = readWorkflow(file)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER\b/, file)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT\b/, file)
|
||||
if (/PRODUCTION_GCP_RELAY_DEPLOY_/.test(workflow)) relayReaders.push(file)
|
||||
}
|
||||
assert.deepEqual(relayReaders.sort(), [
|
||||
'deploy-relay-fence-broker.yml',
|
||||
'deploy-relay-production-capacity-job.yml',
|
||||
'deploy-relay-production-capacity.yml',
|
||||
'deploy-relay-production-director.yml',
|
||||
'deploy-relay-production-multi-target.yml',
|
||||
'deploy-relay-production-same-cap-job.yml',
|
||||
'deploy-relay-production-same-cap.yml',
|
||||
'deploy-relay-production.yml',
|
||||
'operate-relay-asia-admission.yml',
|
||||
'operate-relay-production-rehome-job.yml',
|
||||
'publish-relay-production.yml'
|
||||
].map((name) => relayWorkflowFile(name)).sort())
|
||||
})
|
||||
|
||||
test('monitor workflow has no shared deploy identity fallback', async () => {
|
||||
const workflow = readRelayWorkflow('monitor-relay-production-job.yml')
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
})
|
||||
|
||||
test('relay fencing uses the dedicated requester and private broker', async () => {
|
||||
const workflow = readRelayWorkflow('deploy-relay-production-multi-target.yml')
|
||||
assert.match(workflow, /Reject direct-runner Terraform fence aborts/)
|
||||
assert.match(workflow, /inputs\.mode == 'fence-source'/)
|
||||
assert.match(workflow, /inputs\.mode == 'abort-fence-source'/)
|
||||
assert.match(workflow, /inputs\.mode == 'supersede-target'/)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT/)
|
||||
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_BROKER_URI/)
|
||||
assert.match(workflow, /Invoke private target-supersession broker/)
|
||||
assert.match(workflow, /Invoke private source-fence broker/)
|
||||
assert.match(workflow, /Require exact broker cell contract/)
|
||||
assert.match(workflow, /Require exact source-fence broker contract/)
|
||||
assert.match(workflow, /Require private fence-broker environment/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/DEPLOY_MODE\}" = "execute" \|\|\s+"\$\{DEPLOY_MODE\}" = "recover-forward"\) &&\s+"\$\{SOURCE_CELL_ID\}" = "production-gce-c12"/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/--scoped-recovery-source-cell-id\s+production-gce-c3/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{FAILED_TARGET_CELL_ID\}" = "production-gce-c12"/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{REPLACEMENT_TARGET_CELL_ID\}" = "production-gce-c13"/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{TARGET_CELL_IDS\}" = "production-gce-c12,production-gce-c13"/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{TARGET_CELL_IDS\}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"/
|
||||
)
|
||||
const jobGate = workflow.slice(
|
||||
workflow.indexOf('jobs:'),
|
||||
workflow.indexOf('runs-on:')
|
||||
)
|
||||
assert.doesNotMatch(jobGate, /PRODUCTION_GCP_RELAY_FENCE_/)
|
||||
const brokerStep = workflow.slice(
|
||||
workflow.indexOf('- name: Invoke private target-supersession broker'),
|
||||
workflow.indexOf('- name: Preflight or run multi-target evacuation')
|
||||
)
|
||||
assert.match(brokerStep, /steps\.google-fence-broker-auth\.outputs\.id_token/)
|
||||
assert.doesNotMatch(brokerStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
const sourceFenceStep = workflow.slice(
|
||||
workflow.indexOf('- name: Invoke private source-fence broker'),
|
||||
workflow.indexOf('- name: Preflight or run multi-target evacuation')
|
||||
)
|
||||
assert.match(sourceFenceStep, /steps\.google-fence-broker-auth\.outputs\.id_token/)
|
||||
assert.match(sourceFenceStep, /\/v1\/fence-source/)
|
||||
assert.doesNotMatch(sourceFenceStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
})
|
||||
|
||||
test('Terraform binds dedicated identities to exact OIDC and resource boundaries', async () => {
|
||||
const terraform = await source('infra/terraform/relay-github-actions.tf')
|
||||
for (const claim of ['job_workflow_ref', 'workflow_ref', 'ref', 'environment']) {
|
||||
assert.match(terraform, new RegExp(`assertion\\.${claim}`))
|
||||
}
|
||||
assert.match(terraform, /github_monitor_workflow_file/)
|
||||
assert.match(terraform, /github_fence_workflow_file/)
|
||||
assert.match(terraform, /github_production_relay_capacity_job_workflow_file/)
|
||||
assert.match(terraform, /google_service_account" "github_monitor"/)
|
||||
assert.match(terraform, /google_service_account" "github_fence"/)
|
||||
assert.match(terraform, /google_service_account\.github_monitor\[0\]\.member/)
|
||||
assert.match(terraform, /service_account_id = google_service_account\.github_fence\[0\]\.name/)
|
||||
assert.match(terraform, /attribute\.relay_ops_identity\/monitor/)
|
||||
assert.match(terraform, /attribute\.relay_ops_identity\/fence/)
|
||||
assert.doesNotMatch(terraform, /github_relay_fence_operator/)
|
||||
assert.doesNotMatch(terraform, /github_terraform_fence_state_writer/)
|
||||
const broker = await source('infra/terraform/relay-fence-broker.tf')
|
||||
assert.match(broker, /max_instance_request_concurrency = 1/)
|
||||
assert.match(broker, /max_instance_count = 1/)
|
||||
assert.match(broker, /roles\/run\.invoker/)
|
||||
assert.match(broker, /google_service_account\.github_fence\[0\]\.member/)
|
||||
assert.doesNotMatch(broker, /allUsers/)
|
||||
const brokerDeploy = readRelayWorkflow('deploy-relay-fence-broker.yml')
|
||||
assert.match(brokerDeploy, /sha-\$\{GITHUB_SHA\}/)
|
||||
assert.match(brokerDeploy, /gcloud run services update/)
|
||||
assert.match(brokerDeploy, /\.status\.traffic/)
|
||||
assert.doesNotMatch(brokerDeploy, /latestReadyRevisionName/)
|
||||
assert.doesNotMatch(brokerDeploy, /--set-env-vars/)
|
||||
})
|
||||
|
||||
test('Terraform exposes the audited production environment values', async () => {
|
||||
const outputs = await source('infra/terraform/outputs.tf')
|
||||
for (const output of [
|
||||
'github_relay_monitor_workload_identity_provider',
|
||||
'github_relay_monitor_service_account',
|
||||
'github_relay_fence_workload_identity_provider',
|
||||
'github_relay_fence_service_account'
|
||||
]) {
|
||||
assert.match(outputs, new RegExp(`output "${output}"`))
|
||||
}
|
||||
})
|
||||
|
||||
test('production mutations pass the minted admin token to live preflight', async () => {
|
||||
const workflow = readRelayWorkflow('deploy-relay-production.yml')
|
||||
const recheck = workflow.slice(
|
||||
workflow.indexOf('- name: Recheck all live safety signals'),
|
||||
workflow.indexOf('- name: Create single-use dry-run marker')
|
||||
)
|
||||
assert.match(
|
||||
recheck,
|
||||
/ORCA_RELAY_ADMIN_ID_TOKEN: \$\{\{ steps\.google-auth\.outputs\.id_token \}\}/
|
||||
)
|
||||
const multiTarget = readRelayWorkflow('deploy-relay-production-multi-target.yml')
|
||||
const multiTargetRecheck = multiTarget.slice(
|
||||
multiTarget.indexOf('- name: Recheck all live safety signals'),
|
||||
multiTarget.indexOf('- name: Create single-use dry-run marker')
|
||||
)
|
||||
assert.match(multiTargetRecheck, /steps\.google-auth\.outputs\.id_token/)
|
||||
assert.match(multiTargetRecheck, /inputs\.mode != 'supersede-target'/)
|
||||
})
|
||||
|
||||
test('fence broker pins the production-proven Terraform planner', async () => {
|
||||
const dockerfile = await source('apps/relay-fence-broker/Dockerfile')
|
||||
assert.match(dockerfile, /FROM hashicorp\/terraform:1\.15\.8 AS terraform/)
|
||||
})
|
||||
@@ -0,0 +1,161 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
export const SAME_CAP_CELLS = [
|
||||
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
|
||||
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
|
||||
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
|
||||
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
|
||||
]
|
||||
|
||||
function digest(value, name) {
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
|
||||
return value
|
||||
}
|
||||
|
||||
function cells(value) {
|
||||
const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
|
||||
if (
|
||||
parsed.length < 1 ||
|
||||
parsed.length > 4 ||
|
||||
new Set(parsed).size !== parsed.length ||
|
||||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
|
||||
) throw new Error('same-cap wave cells are invalid')
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function validateSameCapWave(input) {
|
||||
if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) {
|
||||
throw new Error('same-cap wave mode is invalid')
|
||||
}
|
||||
const selected = cells(input.cellIds)
|
||||
const targetDigest = digest(input.targetDigest, 'target digest')
|
||||
const rollbackDigest = digest(input.rollbackDigest, 'rollback digest')
|
||||
if (targetDigest === rollbackDigest) throw new Error('target and rollback digests must differ')
|
||||
if (input.mode === 'canary-apply' && selected.length !== 1) {
|
||||
throw new Error('canary mode requires exactly one cell')
|
||||
}
|
||||
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
|
||||
throw new Error('batch mode requires two to four cells')
|
||||
}
|
||||
// Later waves expect the selector to advance by exactly 2 per predecessor,
|
||||
// which a resumed rollback cell (isolate skipped, +1) violates.
|
||||
if (input.mode === 'rollback' && selected.length !== 1) {
|
||||
throw new Error('rollback mode requires exactly one cell')
|
||||
}
|
||||
const mutation = input.mode !== 'verify'
|
||||
const expectedConfirmation = input.mode === 'rollback'
|
||||
? `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${selected.join(',')}`
|
||||
: `ROLL_RELAY_SAME_CAP ${targetDigest} ${selected.join(',')}`
|
||||
if (mutation && input.confirmation !== expectedConfirmation) {
|
||||
throw new Error('same-cap confirmation does not match the exact digest and cells')
|
||||
}
|
||||
if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation')
|
||||
if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) {
|
||||
throw new Error('batch mode requires a canary run ID')
|
||||
}
|
||||
if (input.mode !== 'batch-apply' && input.canaryRunId) {
|
||||
throw new Error('only batch mode accepts a canary run ID')
|
||||
}
|
||||
return { cells: selected, targetDigest, rollbackDigest }
|
||||
}
|
||||
|
||||
export function canaryAuthority(input) {
|
||||
const wave = validateSameCapWave({ ...input, mode: 'canary-apply', canaryRunId: '' })
|
||||
if (!/^[0-9a-f]{40}$/.test(input.commitSha ?? '')) throw new Error('commit SHA is invalid')
|
||||
if (!/^[1-9][0-9]*$/.test(input.runId ?? '')) throw new Error('run ID is invalid')
|
||||
const selectorGeneration = Number(input.selectorGeneration)
|
||||
const rehomeGeneration = Number(input.rehomeGeneration)
|
||||
if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) {
|
||||
throw new Error('selector generation is invalid')
|
||||
}
|
||||
if (!Number.isSafeInteger(rehomeGeneration) || rehomeGeneration < 0) {
|
||||
throw new Error('rehome generation is invalid')
|
||||
}
|
||||
return {
|
||||
v: 1,
|
||||
commitSha: input.commitSha,
|
||||
runId: input.runId,
|
||||
cellId: wave.cells[0],
|
||||
targetDigest: wave.targetDigest,
|
||||
rollbackDigest: wave.rollbackDigest,
|
||||
selectorGeneration: selectorGeneration + 2,
|
||||
rehomeGeneration
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyCanaryAuthority(authority, expected) {
|
||||
if (
|
||||
authority?.v !== 1 ||
|
||||
authority.commitSha !== expected.commitSha ||
|
||||
authority.runId !== expected.runId ||
|
||||
authority.targetDigest !== expected.targetDigest ||
|
||||
authority.rollbackDigest !== expected.rollbackDigest ||
|
||||
authority.selectorGeneration !== Number(expected.selectorGeneration) ||
|
||||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
|
||||
!SAME_CAP_CELLS.includes(authority.cellId)
|
||||
) throw new Error('canary authority does not match this batch')
|
||||
return authority
|
||||
}
|
||||
|
||||
function values(argv) {
|
||||
const result = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
|
||||
throw new Error('invalid arguments')
|
||||
}
|
||||
result[argv[index].slice(2)] = argv[index + 1]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
const command = argv.shift()
|
||||
const input = values(argv)
|
||||
if (command === 'validate') {
|
||||
const wave = validateSameCapWave({
|
||||
mode: input.mode,
|
||||
cellIds: input['cell-ids'],
|
||||
targetDigest: input['target-digest'],
|
||||
rollbackDigest: input['rollback-digest'],
|
||||
confirmation: input.confirmation,
|
||||
canaryRunId: input['canary-run-id']
|
||||
})
|
||||
process.stdout.write(`${JSON.stringify(wave.cells)}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'create-canary') {
|
||||
process.stdout.write(`${JSON.stringify(canaryAuthority({
|
||||
mode: 'canary-apply',
|
||||
cellIds: input['cell-id'],
|
||||
targetDigest: input['target-digest'],
|
||||
rollbackDigest: input['rollback-digest'],
|
||||
confirmation: input.confirmation,
|
||||
commitSha: input['commit-sha'],
|
||||
runId: input['run-id'],
|
||||
selectorGeneration: input['selector-generation'],
|
||||
rehomeGeneration: input['rehome-generation']
|
||||
}))}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'verify-canary') {
|
||||
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
|
||||
commitSha: input['commit-sha'],
|
||||
runId: input['run-id'],
|
||||
targetDigest: input['target-digest'],
|
||||
rollbackDigest: input['rollback-digest'],
|
||||
selectorGeneration: input['selector-generation'],
|
||||
rehomeGeneration: input['rehome-generation']
|
||||
})
|
||||
return
|
||||
}
|
||||
throw new Error('unknown same-cap wave command')
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try { main() } catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
canaryAuthority,
|
||||
validateSameCapWave,
|
||||
verifyCanaryAuthority
|
||||
} from './relay-production-same-cap-wave.mjs'
|
||||
|
||||
const targetDigest = `sha256:${'a'.repeat(64)}`
|
||||
const rollbackDigest = `sha256:${'b'.repeat(64)}`
|
||||
|
||||
test('requires one canary or a bounded reviewed batch', () => {
|
||||
assert.deepEqual(validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c7',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
|
||||
}).cells, ['production-gce-c7'])
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c7,production-gce-c8',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: 'wrong'
|
||||
}), /canary/)
|
||||
assert.deepEqual(validateSameCapWave({
|
||||
mode: 'batch-apply',
|
||||
cellIds: 'production-gce-c8,production-gce-c9',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c8,production-gce-c9`,
|
||||
canaryRunId: '42'
|
||||
}).cells, ['production-gce-c8', 'production-gce-c9'])
|
||||
assert.deepEqual(validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c28',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c28`
|
||||
}).cells, ['production-gce-c28'])
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c30',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30`
|
||||
}), /cells/)
|
||||
})
|
||||
|
||||
test('binds rollback confirmation to the exact digest and ordered cells', () => {
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'rollback',
|
||||
cellIds: 'production-gce-c7',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
|
||||
}), /confirmation/)
|
||||
})
|
||||
|
||||
test('rollback rolls exactly one cell so later waves stay unreachable', () => {
|
||||
const cellIds = 'production-gce-c7,production-gce-c8'
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'rollback',
|
||||
cellIds,
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${cellIds}`
|
||||
}), /rollback mode requires exactly one cell/)
|
||||
assert.deepEqual(validateSameCapWave({
|
||||
mode: 'rollback',
|
||||
cellIds: 'production-gce-c7',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7`
|
||||
}).cells, ['production-gce-c7'])
|
||||
})
|
||||
|
||||
test('seals and verifies canary authority for later batches', () => {
|
||||
const authority = canaryAuthority({
|
||||
cellIds: 'production-gce-c7',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
|
||||
commitSha: 'c'.repeat(40),
|
||||
runId: '42',
|
||||
selectorGeneration: '11',
|
||||
rehomeGeneration: '4'
|
||||
})
|
||||
assert.equal(verifyCanaryAuthority(authority, {
|
||||
commitSha: 'c'.repeat(40),
|
||||
runId: '42',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
selectorGeneration: '13',
|
||||
rehomeGeneration: '4'
|
||||
}).cellId, 'production-gce-c7')
|
||||
assert.throws(() => verifyCanaryAuthority(authority, {
|
||||
commitSha: 'd'.repeat(40),
|
||||
runId: '42',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
selectorGeneration: '11',
|
||||
rehomeGeneration: '4'
|
||||
}), /does not match/)
|
||||
})
|
||||
@@ -0,0 +1,82 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
isEntrypoint,
|
||||
jobIf,
|
||||
jobNeeds,
|
||||
jobs,
|
||||
readWorkflow,
|
||||
workflowFiles
|
||||
} from './cloud-sql-rollout-lock-census.mjs'
|
||||
import { relayWorkflowFile } from './relay-repository.mjs'
|
||||
|
||||
// Why: this repository publishes the relay's operate surface next to the desktop app. Three
|
||||
// invariants make that safe, and each of them is one careless edit away from being lost.
|
||||
const OPERATIONS_GATE = "vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"
|
||||
|
||||
// Cloud Verify is the only cloud workflow that must run on every pull request.
|
||||
const UNGATED = relayWorkflowFile('verify.yml')
|
||||
|
||||
const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
|
||||
|
||||
test('the copy carries every relay workflow', () => {
|
||||
assert.equal(relayWorkflows().length, 24)
|
||||
})
|
||||
|
||||
// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming
|
||||
// one of these silently breaks the recovery chain with no failing run to notice.
|
||||
test('the recovery chain keeps the display names it is matched by', () => {
|
||||
const names = Object.fromEntries(
|
||||
['prove-relay-staging-capacity.yml', 'recover-relay-staging-c4-image.yml', 'requeue-relay-staging-c4-recovery.yml'].map(
|
||||
(name) => [name, /^name: (.+)$/m.exec(readWorkflow(relayWorkflowFile(name)))?.[1]]
|
||||
)
|
||||
)
|
||||
assert.deepEqual(names, {
|
||||
'prove-relay-staging-capacity.yml': 'Prove Relay Staging Capacity',
|
||||
'recover-relay-staging-c4-image.yml': 'Recover Relay Staging C4 Image',
|
||||
'requeue-relay-staging-c4-recovery.yml': 'Requeue Relay Staging C4 Recovery'
|
||||
})
|
||||
const recover = readWorkflow(relayWorkflowFile('recover-relay-staging-c4-image.yml'))
|
||||
const requeue = readWorkflow(relayWorkflowFile('requeue-relay-staging-c4-recovery.yml'))
|
||||
assert.ok(recover.includes(`workflows: [${names['prove-relay-staging-capacity.yml']}]`))
|
||||
assert.ok(requeue.includes(`workflows: [${names['recover-relay-staging-c4-image.yml']}]`))
|
||||
})
|
||||
|
||||
// Why: this repository holds none of the GCP credentials these workflows would need. Every one
|
||||
// authenticates through Workload Identity read from a variable, so any repository secret other
|
||||
// than the automatic token would be a credential the owner has to store here.
|
||||
test('no cloud workflow reads a repository secret', () => {
|
||||
for (const file of workflowFiles()) {
|
||||
for (const [, name] of readWorkflow(file).matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
|
||||
assert.equal(name, 'GITHUB_TOKEN', `${file} reads secrets.${name}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Why: the operations gate is what makes the whole surface inert until the owner enables it. A
|
||||
// job that can start without a gated dependency would run the moment someone dispatches it.
|
||||
test('every job that can start on its own is gated on the operations variable', () => {
|
||||
const reachable = []
|
||||
for (const file of relayWorkflows()) {
|
||||
const text = readWorkflow(file)
|
||||
if (!isEntrypoint(text)) continue
|
||||
for (const job of jobs(text)) {
|
||||
if (jobNeeds(job.text).length > 0) continue
|
||||
reachable.push(`${file}:${job.id}`)
|
||||
assert.ok(jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} is not gated`)
|
||||
}
|
||||
}
|
||||
assert.ok(reachable.length >= 20, `only ${reachable.length} root jobs were checked`)
|
||||
})
|
||||
|
||||
// Why: reusable jobs inherit the caller's gate. Gating them again would be dead configuration
|
||||
// that reads as protection, and every caller is already checked above.
|
||||
test('reusable workflows carry no gate of their own', () => {
|
||||
for (const file of relayWorkflows()) {
|
||||
const text = readWorkflow(file)
|
||||
if (isEntrypoint(text)) continue
|
||||
for (const job of jobs(text)) {
|
||||
assert.ok(!jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} regates a reusable job`)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,517 @@
|
||||
const EXPECTED_KEYS = {
|
||||
report: ['schemaVersion', 'environment', 'load', 'outcomes'],
|
||||
environment: [
|
||||
'projectId',
|
||||
'directorOrigin',
|
||||
'databaseVcpu',
|
||||
'databasePoolMax',
|
||||
'publicConcurrentMax',
|
||||
'resolvePrioritySlots',
|
||||
'directorMinInstances',
|
||||
'directorMaxInstances',
|
||||
'cloudRunConcurrency',
|
||||
'rolloutOldPublicConcurrentMax',
|
||||
'rolloutOldResolvePrioritySlots',
|
||||
'rolloutNewPublicConcurrentMax',
|
||||
'rolloutNewResolvePrioritySlots'
|
||||
],
|
||||
load: [
|
||||
'drainingDesktops',
|
||||
'backgroundRequestsPerMinute',
|
||||
'backgroundAssignmentRequestsPerMinute',
|
||||
'backgroundAssignment503PerMinute',
|
||||
'targetConnectionCap',
|
||||
'targetCells'
|
||||
],
|
||||
targetCell: ['cellId', 'peakConnections', 'recoveredControls'],
|
||||
outcomes: [
|
||||
'migrationExpirations',
|
||||
'migrationAborts',
|
||||
'transactionRetryExhaustions',
|
||||
'keyProvenTargetRegistrations',
|
||||
'oldestMigrationLeaseRemainingAtDrainMs',
|
||||
'targetRegistrationDurationMs',
|
||||
'assignmentSuccessesPerMinuteBaseline',
|
||||
'assignmentSuccessesPerMinuteMinimum',
|
||||
'eligibleResolveRequests',
|
||||
'resolve2xx',
|
||||
'resolveOverload',
|
||||
'readinessChecks',
|
||||
'readinessFailures',
|
||||
'maintenanceOperations',
|
||||
'maintenanceFailures',
|
||||
'directorPeakInstances',
|
||||
'rolloutOverlapPeakInstances',
|
||||
'rolloutOverlapPeakPublicOperations',
|
||||
'rolloutOverlapEligibleResolveRequests',
|
||||
'rolloutOverlapResolve2xx',
|
||||
'rolloutOverlapResolveOverload',
|
||||
'rolloutOverlapReadinessFailures',
|
||||
'rolloutOverlapPoolWaitP95Ms',
|
||||
'rolloutOverlapDatabaseCpuPercentMax',
|
||||
'poolWaitP95Ms',
|
||||
'poolWaitMaxMs',
|
||||
'databaseCpuPercentP95',
|
||||
'databaseCpuPercentMax',
|
||||
'recoveryDurationMs'
|
||||
]
|
||||
}
|
||||
|
||||
const LIMITS = {
|
||||
drainingDesktopsMin: 760,
|
||||
drainingDesktopsMax: 840,
|
||||
backgroundRequestsPerMinuteMin: 10_450,
|
||||
backgroundRequestsPerMinuteMax: 11_550,
|
||||
backgroundAssignment503PerMinuteMin: 8_500,
|
||||
backgroundAssignment503PerMinuteMax: 10_500,
|
||||
targetCellCount: 2,
|
||||
targetConnectionCap: 600,
|
||||
databaseVcpu: 2,
|
||||
databasePoolMax: 3,
|
||||
publicConcurrentMax: 2,
|
||||
resolvePrioritySlots: 1,
|
||||
directorMinInstances: 1,
|
||||
directorMaxInstances: 2,
|
||||
directorPeakInstances: 2,
|
||||
rolloutOverlapPeakInstances: 4,
|
||||
rolloutOverlapPeakPublicOperations: 8,
|
||||
cloudRunConcurrency: 80,
|
||||
rolloutOldPublicConcurrentMax: 2,
|
||||
rolloutOldResolvePrioritySlots: 0,
|
||||
rolloutNewPublicConcurrentMax: 2,
|
||||
rolloutNewResolvePrioritySlots: 1,
|
||||
assignmentThroughputRetentionMin: 0.9,
|
||||
resolveSuccessRateMin: 0.95,
|
||||
resolveOverloadRateMaxExclusive: 0.01,
|
||||
poolWaitP95MsMaxExclusive: 500,
|
||||
poolWaitMaxMsMaxExclusive: 5_000,
|
||||
databaseCpuPercentP95MaxExclusive: 70,
|
||||
databaseCpuPercentMaxMaxExclusive: 85,
|
||||
oldestMigrationLeaseRemainingAtDrainMsMin: 10 * 60_000,
|
||||
targetRegistrationDurationMsMax: 5 * 60_000,
|
||||
recoveryDurationMsMax: 14 * 60_000
|
||||
}
|
||||
|
||||
export function evaluateRecoveryWaveReport(input) {
|
||||
const report = parseReport(input)
|
||||
const recoveredControls = report.load.targetCells.reduce(
|
||||
(total, cell) => total + cell.recoveredControls,
|
||||
0
|
||||
)
|
||||
const peakTargetConnections = Math.max(
|
||||
...report.load.targetCells.map((cell) => cell.peakConnections)
|
||||
)
|
||||
const assignmentThroughputRetention = ratio(
|
||||
report.outcomes.assignmentSuccessesPerMinuteMinimum,
|
||||
report.outcomes.assignmentSuccessesPerMinuteBaseline
|
||||
)
|
||||
const resolveSuccessRate = ratio(
|
||||
report.outcomes.resolve2xx,
|
||||
report.outcomes.eligibleResolveRequests
|
||||
)
|
||||
const resolveOverloadRate = ratio(
|
||||
report.outcomes.resolveOverload,
|
||||
report.outcomes.eligibleResolveRequests
|
||||
)
|
||||
const rolloutOverlapResolveSuccessRate = ratio(
|
||||
report.outcomes.rolloutOverlapResolve2xx,
|
||||
report.outcomes.rolloutOverlapEligibleResolveRequests
|
||||
)
|
||||
const rolloutOverlapResolveOverloadRate = ratio(
|
||||
report.outcomes.rolloutOverlapResolveOverload,
|
||||
report.outcomes.rolloutOverlapEligibleResolveRequests
|
||||
)
|
||||
const thresholds = [
|
||||
equal('database_vcpu', report.environment.databaseVcpu, LIMITS.databaseVcpu),
|
||||
equal('database_pool_max', report.environment.databasePoolMax, LIMITS.databasePoolMax),
|
||||
equal(
|
||||
'public_concurrent_max',
|
||||
report.environment.publicConcurrentMax,
|
||||
LIMITS.publicConcurrentMax
|
||||
),
|
||||
equal(
|
||||
'resolve_priority_slots',
|
||||
report.environment.resolvePrioritySlots,
|
||||
LIMITS.resolvePrioritySlots
|
||||
),
|
||||
equal(
|
||||
'director_min_instances',
|
||||
report.environment.directorMinInstances,
|
||||
LIMITS.directorMinInstances
|
||||
),
|
||||
equal(
|
||||
'director_max_instances',
|
||||
report.environment.directorMaxInstances,
|
||||
LIMITS.directorMaxInstances
|
||||
),
|
||||
equal(
|
||||
'cloud_run_concurrency',
|
||||
report.environment.cloudRunConcurrency,
|
||||
LIMITS.cloudRunConcurrency
|
||||
),
|
||||
equal(
|
||||
'rollout_old_public_concurrent_max',
|
||||
report.environment.rolloutOldPublicConcurrentMax,
|
||||
LIMITS.rolloutOldPublicConcurrentMax
|
||||
),
|
||||
equal(
|
||||
'rollout_old_resolve_priority_slots',
|
||||
report.environment.rolloutOldResolvePrioritySlots,
|
||||
LIMITS.rolloutOldResolvePrioritySlots
|
||||
),
|
||||
equal(
|
||||
'rollout_new_public_concurrent_max',
|
||||
report.environment.rolloutNewPublicConcurrentMax,
|
||||
LIMITS.rolloutNewPublicConcurrentMax
|
||||
),
|
||||
equal(
|
||||
'rollout_new_resolve_priority_slots',
|
||||
report.environment.rolloutNewResolvePrioritySlots,
|
||||
LIMITS.rolloutNewResolvePrioritySlots
|
||||
),
|
||||
between(
|
||||
'draining_desktops',
|
||||
report.load.drainingDesktops,
|
||||
LIMITS.drainingDesktopsMin,
|
||||
LIMITS.drainingDesktopsMax
|
||||
),
|
||||
between(
|
||||
'background_requests_per_minute',
|
||||
report.load.backgroundRequestsPerMinute,
|
||||
LIMITS.backgroundRequestsPerMinuteMin,
|
||||
LIMITS.backgroundRequestsPerMinuteMax
|
||||
),
|
||||
between(
|
||||
'background_assignment_requests_per_minute',
|
||||
report.load.backgroundAssignmentRequestsPerMinute,
|
||||
LIMITS.backgroundRequestsPerMinuteMin,
|
||||
LIMITS.backgroundRequestsPerMinuteMax
|
||||
),
|
||||
between(
|
||||
'background_assignment_503_per_minute',
|
||||
report.load.backgroundAssignment503PerMinute,
|
||||
LIMITS.backgroundAssignment503PerMinuteMin,
|
||||
LIMITS.backgroundAssignment503PerMinuteMax
|
||||
),
|
||||
atMost(
|
||||
'background_assignment_requests_within_total',
|
||||
report.load.backgroundAssignmentRequestsPerMinute,
|
||||
report.load.backgroundRequestsPerMinute
|
||||
),
|
||||
atMost(
|
||||
'background_assignment_503_within_assignments',
|
||||
report.load.backgroundAssignment503PerMinute,
|
||||
report.load.backgroundAssignmentRequestsPerMinute
|
||||
),
|
||||
equal('target_cell_count', report.load.targetCells.length, LIMITS.targetCellCount),
|
||||
equal(
|
||||
'target_connection_cap',
|
||||
report.load.targetConnectionCap,
|
||||
LIMITS.targetConnectionCap
|
||||
),
|
||||
atMost(
|
||||
'peak_target_connections',
|
||||
peakTargetConnections,
|
||||
report.load.targetConnectionCap
|
||||
),
|
||||
equal('recovered_controls', recoveredControls, report.load.drainingDesktops),
|
||||
equal('migration_expirations', report.outcomes.migrationExpirations, 0),
|
||||
equal('migration_aborts', report.outcomes.migrationAborts, 0),
|
||||
equal('transaction_retry_exhaustions', report.outcomes.transactionRetryExhaustions, 0),
|
||||
equal(
|
||||
'key_proven_target_registrations',
|
||||
report.outcomes.keyProvenTargetRegistrations,
|
||||
report.load.drainingDesktops
|
||||
),
|
||||
atLeast(
|
||||
'oldest_migration_lease_remaining_at_drain_ms',
|
||||
report.outcomes.oldestMigrationLeaseRemainingAtDrainMs,
|
||||
LIMITS.oldestMigrationLeaseRemainingAtDrainMsMin
|
||||
),
|
||||
atMost(
|
||||
'target_registration_duration_ms',
|
||||
report.outcomes.targetRegistrationDurationMs,
|
||||
LIMITS.targetRegistrationDurationMsMax
|
||||
),
|
||||
atLeast(
|
||||
'assignment_throughput_retention',
|
||||
assignmentThroughputRetention,
|
||||
LIMITS.assignmentThroughputRetentionMin
|
||||
),
|
||||
atLeast('resolve_success_rate', resolveSuccessRate, LIMITS.resolveSuccessRateMin),
|
||||
lessThan(
|
||||
'resolve_overload_rate',
|
||||
resolveOverloadRate,
|
||||
LIMITS.resolveOverloadRateMaxExclusive
|
||||
),
|
||||
atLeast('eligible_resolve_requests', report.outcomes.eligibleResolveRequests, 100),
|
||||
equal('readiness_failures', report.outcomes.readinessFailures, 0),
|
||||
atLeast('readiness_checks', report.outcomes.readinessChecks, 1),
|
||||
equal('maintenance_failures', report.outcomes.maintenanceFailures, 0),
|
||||
atLeast('maintenance_operations', report.outcomes.maintenanceOperations, 1),
|
||||
equal(
|
||||
'director_peak_instances',
|
||||
report.outcomes.directorPeakInstances,
|
||||
LIMITS.directorPeakInstances
|
||||
),
|
||||
equal(
|
||||
'rollout_overlap_peak_instances',
|
||||
report.outcomes.rolloutOverlapPeakInstances,
|
||||
LIMITS.rolloutOverlapPeakInstances
|
||||
),
|
||||
equal(
|
||||
'rollout_overlap_peak_public_operations',
|
||||
report.outcomes.rolloutOverlapPeakPublicOperations,
|
||||
LIMITS.rolloutOverlapPeakPublicOperations
|
||||
),
|
||||
atLeast(
|
||||
'rollout_overlap_eligible_resolve_requests',
|
||||
report.outcomes.rolloutOverlapEligibleResolveRequests,
|
||||
100
|
||||
),
|
||||
atLeast(
|
||||
'rollout_overlap_resolve_success_rate',
|
||||
rolloutOverlapResolveSuccessRate,
|
||||
LIMITS.resolveSuccessRateMin
|
||||
),
|
||||
lessThan(
|
||||
'rollout_overlap_resolve_overload_rate',
|
||||
rolloutOverlapResolveOverloadRate,
|
||||
LIMITS.resolveOverloadRateMaxExclusive
|
||||
),
|
||||
equal(
|
||||
'rollout_overlap_readiness_failures',
|
||||
report.outcomes.rolloutOverlapReadinessFailures,
|
||||
0
|
||||
),
|
||||
lessThan(
|
||||
'rollout_overlap_pool_wait_p95_ms',
|
||||
report.outcomes.rolloutOverlapPoolWaitP95Ms,
|
||||
LIMITS.poolWaitP95MsMaxExclusive
|
||||
),
|
||||
lessThan(
|
||||
'rollout_overlap_database_cpu_percent_max',
|
||||
report.outcomes.rolloutOverlapDatabaseCpuPercentMax,
|
||||
LIMITS.databaseCpuPercentMaxMaxExclusive
|
||||
),
|
||||
lessThan(
|
||||
'pool_wait_p95_ms',
|
||||
report.outcomes.poolWaitP95Ms,
|
||||
LIMITS.poolWaitP95MsMaxExclusive
|
||||
),
|
||||
lessThan(
|
||||
'pool_wait_max_ms',
|
||||
report.outcomes.poolWaitMaxMs,
|
||||
LIMITS.poolWaitMaxMsMaxExclusive
|
||||
),
|
||||
lessThan(
|
||||
'database_cpu_percent_p95',
|
||||
report.outcomes.databaseCpuPercentP95,
|
||||
LIMITS.databaseCpuPercentP95MaxExclusive
|
||||
),
|
||||
lessThan(
|
||||
'database_cpu_percent_max',
|
||||
report.outcomes.databaseCpuPercentMax,
|
||||
LIMITS.databaseCpuPercentMaxMaxExclusive
|
||||
),
|
||||
atMost(
|
||||
'recovery_duration_ms',
|
||||
report.outcomes.recoveryDurationMs,
|
||||
LIMITS.recoveryDurationMsMax
|
||||
)
|
||||
]
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
status: thresholds.every(({ pass }) => pass) ? 'PASS' : 'FAIL',
|
||||
environment: {
|
||||
projectId: report.environment.projectId,
|
||||
directorOrigin: report.environment.directorOrigin
|
||||
},
|
||||
metrics: {
|
||||
recoveredControls,
|
||||
peakTargetConnections,
|
||||
assignmentThroughputRetention,
|
||||
resolveSuccessRate,
|
||||
resolveOverloadRate,
|
||||
rolloutOverlapResolveSuccessRate,
|
||||
rolloutOverlapResolveOverloadRate
|
||||
},
|
||||
thresholds
|
||||
}
|
||||
}
|
||||
|
||||
function parseReport(input) {
|
||||
const report = strictObject(input, EXPECTED_KEYS.report, 'report')
|
||||
if (report.schemaVersion !== 1) throw new Error('unsupported report schemaVersion')
|
||||
const environment = strictObject(
|
||||
report.environment,
|
||||
EXPECTED_KEYS.environment,
|
||||
'environment'
|
||||
)
|
||||
assertSafeEnvironment(environment)
|
||||
const load = strictObject(report.load, EXPECTED_KEYS.load, 'load')
|
||||
if (!Array.isArray(load.targetCells)) throw new Error('load.targetCells must be an array')
|
||||
const targetCells = load.targetCells.map((value, index) => {
|
||||
const cell = strictObject(value, EXPECTED_KEYS.targetCell, `load.targetCells[${index}]`)
|
||||
if (!/^[a-z0-9-]{1,128}$/.test(cell.cellId)) throw new Error('target cellId is invalid')
|
||||
return {
|
||||
cellId: cell.cellId,
|
||||
peakConnections: nonnegativeNumber(cell.peakConnections, 'peakConnections'),
|
||||
recoveredControls: nonnegativeNumber(cell.recoveredControls, 'recoveredControls')
|
||||
}
|
||||
})
|
||||
if (new Set(targetCells.map(({ cellId }) => cellId)).size !== targetCells.length) {
|
||||
throw new Error('target cell IDs must be unique')
|
||||
}
|
||||
const outcomes = strictObject(report.outcomes, EXPECTED_KEYS.outcomes, 'outcomes')
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
environment: {
|
||||
projectId: environment.projectId,
|
||||
directorOrigin: environment.directorOrigin,
|
||||
databaseVcpu: positiveNumber(environment.databaseVcpu, 'databaseVcpu'),
|
||||
databasePoolMax: positiveNumber(environment.databasePoolMax, 'databasePoolMax'),
|
||||
publicConcurrentMax: positiveNumber(
|
||||
environment.publicConcurrentMax,
|
||||
'publicConcurrentMax'
|
||||
),
|
||||
resolvePrioritySlots: positiveNumber(
|
||||
environment.resolvePrioritySlots,
|
||||
'resolvePrioritySlots'
|
||||
),
|
||||
directorMinInstances: positiveNumber(
|
||||
environment.directorMinInstances,
|
||||
'directorMinInstances'
|
||||
),
|
||||
directorMaxInstances: positiveNumber(
|
||||
environment.directorMaxInstances,
|
||||
'directorMaxInstances'
|
||||
),
|
||||
cloudRunConcurrency: positiveNumber(
|
||||
environment.cloudRunConcurrency,
|
||||
'cloudRunConcurrency'
|
||||
),
|
||||
rolloutOldPublicConcurrentMax: positiveNumber(
|
||||
environment.rolloutOldPublicConcurrentMax,
|
||||
'rolloutOldPublicConcurrentMax'
|
||||
),
|
||||
rolloutOldResolvePrioritySlots: nonnegativeNumber(
|
||||
environment.rolloutOldResolvePrioritySlots,
|
||||
'rolloutOldResolvePrioritySlots'
|
||||
),
|
||||
rolloutNewPublicConcurrentMax: positiveNumber(
|
||||
environment.rolloutNewPublicConcurrentMax,
|
||||
'rolloutNewPublicConcurrentMax'
|
||||
),
|
||||
rolloutNewResolvePrioritySlots: positiveNumber(
|
||||
environment.rolloutNewResolvePrioritySlots,
|
||||
'rolloutNewResolvePrioritySlots'
|
||||
)
|
||||
},
|
||||
load: {
|
||||
drainingDesktops: positiveNumber(load.drainingDesktops, 'drainingDesktops'),
|
||||
backgroundRequestsPerMinute: positiveNumber(
|
||||
load.backgroundRequestsPerMinute,
|
||||
'backgroundRequestsPerMinute'
|
||||
),
|
||||
backgroundAssignmentRequestsPerMinute: positiveNumber(
|
||||
load.backgroundAssignmentRequestsPerMinute,
|
||||
'backgroundAssignmentRequestsPerMinute'
|
||||
),
|
||||
backgroundAssignment503PerMinute: nonnegativeNumber(
|
||||
load.backgroundAssignment503PerMinute,
|
||||
'backgroundAssignment503PerMinute'
|
||||
),
|
||||
targetConnectionCap: positiveNumber(load.targetConnectionCap, 'targetConnectionCap'),
|
||||
targetCells
|
||||
},
|
||||
outcomes: Object.fromEntries(
|
||||
EXPECTED_KEYS.outcomes.map((key) => [key, nonnegativeNumber(outcomes[key], `outcomes.${key}`)])
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeEnvironment(environment) {
|
||||
if (typeof environment.projectId !== 'string') throw new Error('projectId must be a string')
|
||||
if (
|
||||
environment.projectId !== 'local' &&
|
||||
!environment.projectId.endsWith('-staging') &&
|
||||
!environment.projectId.endsWith('-test')
|
||||
) {
|
||||
throw new Error('recovery-wave reports must come from an isolated non-production project')
|
||||
}
|
||||
if (typeof environment.directorOrigin !== 'string') {
|
||||
throw new Error('directorOrigin must be a string')
|
||||
}
|
||||
const origin = new URL(environment.directorOrigin)
|
||||
const loopback = ['localhost', '127.0.0.1', '::1', '[::1]'].includes(origin.hostname)
|
||||
const isolatedHost =
|
||||
loopback || origin.hostname.endsWith('.test') || origin.hostname.includes('staging')
|
||||
if (
|
||||
origin.origin !== environment.directorOrigin ||
|
||||
origin.pathname !== '/' ||
|
||||
(!loopback && origin.protocol !== 'https:') ||
|
||||
!isolatedHost
|
||||
) {
|
||||
throw new Error('directorOrigin must identify a canonical isolated non-production origin')
|
||||
}
|
||||
}
|
||||
|
||||
function strictObject(value, keys, name) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${name} must be an object`)
|
||||
}
|
||||
const actual = Object.keys(value).sort()
|
||||
const expected = [...keys].sort()
|
||||
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
|
||||
throw new Error(`${name} has unexpected or missing fields`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveNumber(value, name) {
|
||||
const number = nonnegativeNumber(value, name)
|
||||
if (number <= 0) throw new Error(`${name} must be positive`)
|
||||
return number
|
||||
}
|
||||
|
||||
function nonnegativeNumber(value, name) {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`${name} must be a finite nonnegative number`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function ratio(numerator, denominator) {
|
||||
return denominator === 0 ? 0 : numerator / denominator
|
||||
}
|
||||
|
||||
function equal(name, observed, limit) {
|
||||
return threshold(name, observed, '==', limit, observed === limit)
|
||||
}
|
||||
|
||||
function atLeast(name, observed, limit) {
|
||||
return threshold(name, observed, '>=', limit, observed >= limit)
|
||||
}
|
||||
|
||||
function atMost(name, observed, limit) {
|
||||
return threshold(name, observed, '<=', limit, observed <= limit)
|
||||
}
|
||||
|
||||
function lessThan(name, observed, limit) {
|
||||
return threshold(name, observed, '<', limit, observed < limit)
|
||||
}
|
||||
|
||||
function between(name, observed, minimum, maximum) {
|
||||
return threshold(
|
||||
name,
|
||||
observed,
|
||||
'between_inclusive',
|
||||
[minimum, maximum],
|
||||
observed >= minimum && observed <= maximum
|
||||
)
|
||||
}
|
||||
|
||||
function threshold(name, observed, operator, limit, pass) {
|
||||
return { name, observed, operator, limit, pass }
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs'
|
||||
|
||||
test('passes a complete isolated production-shaped recovery report', () => {
|
||||
const result = evaluateRecoveryWaveReport(passingReport())
|
||||
|
||||
assert.equal(result.status, 'PASS')
|
||||
assert.equal(result.metrics.recoveredControls, 800)
|
||||
assert.equal(result.metrics.peakTargetConnections, 425)
|
||||
assert.equal(result.metrics.resolveSuccessRate, 0.99)
|
||||
assert.equal(result.thresholds.every(({ pass }) => pass), true)
|
||||
assert.equal(
|
||||
result.thresholds.find(({ name }) => name === 'recovery_duration_ms')?.limit,
|
||||
840_000
|
||||
)
|
||||
})
|
||||
|
||||
for (const [name, mutate, failedThreshold] of [
|
||||
[
|
||||
'target connection ceiling',
|
||||
(report) => {
|
||||
report.load.targetCells[0].peakConnections = 601
|
||||
},
|
||||
'peak_target_connections'
|
||||
],
|
||||
[
|
||||
'migration expiration',
|
||||
(report) => {
|
||||
report.outcomes.migrationExpirations = 1
|
||||
},
|
||||
'migration_expirations'
|
||||
],
|
||||
[
|
||||
'migration abort',
|
||||
(report) => {
|
||||
report.outcomes.migrationAborts = 1
|
||||
},
|
||||
'migration_aborts'
|
||||
],
|
||||
[
|
||||
'full-wave registration deadline',
|
||||
(report) => {
|
||||
report.outcomes.targetRegistrationDurationMs = 300_001
|
||||
},
|
||||
'target_registration_duration_ms'
|
||||
],
|
||||
[
|
||||
'legacy assignment background shape',
|
||||
(report) => {
|
||||
report.load.backgroundAssignment503PerMinute = 100
|
||||
},
|
||||
'background_assignment_503_per_minute'
|
||||
],
|
||||
[
|
||||
'production director instance topology',
|
||||
(report) => {
|
||||
report.outcomes.directorPeakInstances = 1
|
||||
},
|
||||
'director_peak_instances'
|
||||
],
|
||||
[
|
||||
'old/new rollout overlap topology',
|
||||
(report) => {
|
||||
report.outcomes.rolloutOverlapPeakInstances = 2
|
||||
},
|
||||
'rollout_overlap_peak_instances'
|
||||
],
|
||||
[
|
||||
'old revision shared admission mode',
|
||||
(report) => {
|
||||
report.environment.rolloutOldResolvePrioritySlots = 1
|
||||
},
|
||||
'rollout_old_resolve_priority_slots'
|
||||
],
|
||||
[
|
||||
'old/new rollout overlap resolve availability',
|
||||
(report) => {
|
||||
report.outcomes.rolloutOverlapResolveOverload = 2
|
||||
},
|
||||
'rollout_overlap_resolve_overload_rate'
|
||||
],
|
||||
[
|
||||
'resolve availability',
|
||||
(report) => {
|
||||
report.outcomes.resolve2xx = 940
|
||||
report.outcomes.resolveOverload = 20
|
||||
},
|
||||
'resolve_success_rate'
|
||||
],
|
||||
[
|
||||
'pool wait',
|
||||
(report) => {
|
||||
report.outcomes.poolWaitP95Ms = 500
|
||||
},
|
||||
'pool_wait_p95_ms'
|
||||
],
|
||||
[
|
||||
'database CPU',
|
||||
(report) => {
|
||||
report.outcomes.databaseCpuPercentMax = 85
|
||||
},
|
||||
'database_cpu_percent_max'
|
||||
],
|
||||
[
|
||||
'recovery deadline',
|
||||
(report) => {
|
||||
report.outcomes.recoveryDurationMs = 840_001
|
||||
},
|
||||
'recovery_duration_ms'
|
||||
],
|
||||
[
|
||||
'non-public database maintenance',
|
||||
(report) => {
|
||||
report.outcomes.maintenanceFailures = 1
|
||||
},
|
||||
'maintenance_failures'
|
||||
]
|
||||
]) {
|
||||
test(`fails closed on ${name}`, () => {
|
||||
const report = passingReport()
|
||||
mutate(report)
|
||||
const result = evaluateRecoveryWaveReport(report)
|
||||
|
||||
assert.equal(result.status, 'FAIL')
|
||||
assert.equal(
|
||||
result.thresholds.find(({ name: thresholdName }) => thresholdName === failedThreshold)?.pass,
|
||||
false
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
test('rejects production provenance and unexpected report fields', () => {
|
||||
const productionProject = passingReport()
|
||||
productionProject.environment.projectId = 'onorca-cloud'
|
||||
assert.throws(
|
||||
() => evaluateRecoveryWaveReport(productionProject),
|
||||
/isolated non-production project/
|
||||
)
|
||||
|
||||
const productionOrigin = passingReport()
|
||||
productionOrigin.environment.directorOrigin = 'https://relay.onorca.dev'
|
||||
assert.throws(
|
||||
() => evaluateRecoveryWaveReport(productionOrigin),
|
||||
/isolated non-production origin/
|
||||
)
|
||||
|
||||
const extraField = passingReport()
|
||||
extraField.environment.accessToken = 'must-not-be-accepted'
|
||||
assert.throws(() => evaluateRecoveryWaveReport(extraField), /unexpected or missing fields/)
|
||||
})
|
||||
|
||||
function passingReport() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
environment: {
|
||||
projectId: 'onorca-cloud-staging',
|
||||
directorOrigin: 'https://relay-staging.onorca.dev',
|
||||
databaseVcpu: 2,
|
||||
databasePoolMax: 3,
|
||||
publicConcurrentMax: 2,
|
||||
resolvePrioritySlots: 1,
|
||||
directorMinInstances: 1,
|
||||
directorMaxInstances: 2,
|
||||
cloudRunConcurrency: 80,
|
||||
rolloutOldPublicConcurrentMax: 2,
|
||||
rolloutOldResolvePrioritySlots: 0,
|
||||
rolloutNewPublicConcurrentMax: 2,
|
||||
rolloutNewResolvePrioritySlots: 1
|
||||
},
|
||||
load: {
|
||||
drainingDesktops: 800,
|
||||
backgroundRequestsPerMinute: 11_000,
|
||||
backgroundAssignmentRequestsPerMinute: 10_950,
|
||||
backgroundAssignment503PerMinute: 9_500,
|
||||
targetConnectionCap: 600,
|
||||
targetCells: [
|
||||
{ cellId: 'target-a', peakConnections: 425, recoveredControls: 400 },
|
||||
{ cellId: 'target-b', peakConnections: 419, recoveredControls: 400 }
|
||||
]
|
||||
},
|
||||
outcomes: {
|
||||
migrationExpirations: 0,
|
||||
migrationAborts: 0,
|
||||
transactionRetryExhaustions: 0,
|
||||
keyProvenTargetRegistrations: 800,
|
||||
oldestMigrationLeaseRemainingAtDrainMs: 660_000,
|
||||
targetRegistrationDurationMs: 240_000,
|
||||
assignmentSuccessesPerMinuteBaseline: 1_400,
|
||||
assignmentSuccessesPerMinuteMinimum: 1_330,
|
||||
eligibleResolveRequests: 1_000,
|
||||
resolve2xx: 990,
|
||||
resolveOverload: 5,
|
||||
readinessChecks: 180,
|
||||
readinessFailures: 0,
|
||||
maintenanceOperations: 30,
|
||||
maintenanceFailures: 0,
|
||||
directorPeakInstances: 2,
|
||||
rolloutOverlapPeakInstances: 4,
|
||||
rolloutOverlapPeakPublicOperations: 8,
|
||||
rolloutOverlapEligibleResolveRequests: 100,
|
||||
rolloutOverlapResolve2xx: 99,
|
||||
rolloutOverlapResolveOverload: 0,
|
||||
rolloutOverlapReadinessFailures: 0,
|
||||
rolloutOverlapPoolWaitP95Ms: 180,
|
||||
rolloutOverlapDatabaseCpuPercentMax: 78,
|
||||
poolWaitP95Ms: 120,
|
||||
poolWaitMaxMs: 900,
|
||||
databaseCpuPercentP95: 55,
|
||||
databaseCpuPercentMax: 72,
|
||||
recoveryDurationMs: 360_000
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const WINDOW_MS = 24 * 60 * 60_000
|
||||
const BUCKET_MS = 60 * 60_000
|
||||
const METRICS = [
|
||||
'requestedRegionsDelta',
|
||||
'selectedRegionsDelta',
|
||||
'regionFallbacksDelta',
|
||||
'unavailableRegionsDelta'
|
||||
]
|
||||
const REGION_KEYS = new Set(['asia-east2', 'us-central1', 'unhinted'])
|
||||
|
||||
function integer(value, name) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`)
|
||||
return parsed
|
||||
}
|
||||
|
||||
function digest(value, name) {
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
|
||||
return value
|
||||
}
|
||||
|
||||
function metric(value, name) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${name} is invalid`)
|
||||
}
|
||||
return Object.fromEntries(Object.entries(value).map(([key, count]) => {
|
||||
if (!REGION_KEYS.has(key)) throw new Error(`${name} has an unknown aggregate key`)
|
||||
return [key, integer(count, `${name}.${key}`)]
|
||||
}))
|
||||
}
|
||||
|
||||
function sumMetric(total, value) {
|
||||
for (const [key, count] of Object.entries(value)) total[key] = (total[key] ?? 0) + count
|
||||
}
|
||||
|
||||
function evidenceSha256(evidence) {
|
||||
return createHash('sha256').update(JSON.stringify(evidence)).digest('hex')
|
||||
}
|
||||
|
||||
export function createRegionObservationEvidence(entries, bindings, now = Date.now()) {
|
||||
if (!Array.isArray(entries)) throw new Error('runtime metrics response must be an array')
|
||||
if (!/^[0-9a-f]{40}$/.test(bindings.commitSha ?? '')) throw new Error('commit SHA is invalid')
|
||||
const directorImageDigest = digest(bindings.directorImageDigest, 'director digest')
|
||||
const selectorGeneration = integer(bindings.selectorGeneration, 'selector generation')
|
||||
const controlGeneration = integer(bindings.controlGeneration, 'control generation')
|
||||
const start = now - WINDOW_MS
|
||||
const buckets = Array.from({ length: 24 }, () => 0)
|
||||
const totals = Object.fromEntries(METRICS.map((name) => [name, {}]))
|
||||
let samples = 0
|
||||
for (const entry of entries) {
|
||||
const timestamp = Date.parse(entry?.timestamp ?? '')
|
||||
const payload = entry?.jsonPayload
|
||||
if (
|
||||
!Number.isFinite(timestamp) ||
|
||||
timestamp < start ||
|
||||
timestamp > now + 60_000 ||
|
||||
payload?.event !== 'orca_relay_runtime_metrics' ||
|
||||
payload.role !== 'director'
|
||||
) continue
|
||||
const bucket = Math.min(23, Math.floor((timestamp - start) / BUCKET_MS))
|
||||
buckets[bucket] += 1
|
||||
samples += 1
|
||||
for (const name of METRICS) sumMetric(totals[name], metric(payload[name], name))
|
||||
}
|
||||
if (buckets.some((count) => count === 0)) {
|
||||
throw new Error('24-hour region evidence has a missing hourly bucket')
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(totals.requestedRegionsDelta['asia-east2']) ||
|
||||
totals.requestedRegionsDelta['asia-east2'] < 1 ||
|
||||
!Number.isSafeInteger(totals.selectedRegionsDelta['asia-east2']) ||
|
||||
totals.selectedRegionsDelta['asia-east2'] < 1
|
||||
) throw new Error('24-hour region evidence has no Asia request and selection activity')
|
||||
const evidence = {
|
||||
v: 1,
|
||||
commitSha: bindings.commitSha,
|
||||
directorImageDigest,
|
||||
selectorGeneration,
|
||||
controlGeneration,
|
||||
windowStartedAt: start,
|
||||
windowEndedAt: now,
|
||||
hourlySampleCounts: buckets,
|
||||
samples,
|
||||
totals
|
||||
}
|
||||
return { evidence, sha256: evidenceSha256(evidence) }
|
||||
}
|
||||
|
||||
export function verifyRegionObservationEvidence(sealed, bindings) {
|
||||
if (
|
||||
sealed?.sha256 !== evidenceSha256(sealed?.evidence) ||
|
||||
sealed.evidence?.commitSha !== bindings.commitSha ||
|
||||
sealed.evidence?.directorImageDigest !== bindings.directorImageDigest ||
|
||||
sealed.evidence?.selectorGeneration !== Number(bindings.selectorGeneration) ||
|
||||
sealed.evidence?.controlGeneration !== Number(bindings.controlGeneration) ||
|
||||
!Array.isArray(sealed.evidence?.hourlySampleCounts) ||
|
||||
sealed.evidence.hourlySampleCounts.length !== 24 ||
|
||||
sealed.evidence.hourlySampleCounts.some((count) => integer(count, 'bucket') < 1) ||
|
||||
integer(sealed.evidence?.totals?.requestedRegionsDelta?.['asia-east2'], 'Asia requests') < 1 ||
|
||||
integer(sealed.evidence?.totals?.selectedRegionsDelta?.['asia-east2'], 'Asia selections') < 1
|
||||
) throw new Error('sealed 24-hour region evidence does not match enable authority')
|
||||
return sealed.evidence
|
||||
}
|
||||
|
||||
function values(argv) {
|
||||
const result = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
|
||||
throw new Error('invalid arguments')
|
||||
}
|
||||
result[argv[index].slice(2)] = argv[index + 1]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
async function stdinJson(input) {
|
||||
const chunks = []
|
||||
for await (const chunk of input) chunks.push(chunk)
|
||||
return JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), input = process.stdin) {
|
||||
const command = argv.shift()
|
||||
const args = values(argv)
|
||||
const bindings = {
|
||||
commitSha: args['commit-sha'],
|
||||
directorImageDigest: args['director-image-digest'],
|
||||
selectorGeneration: args['selector-generation'],
|
||||
controlGeneration: args['control-generation']
|
||||
}
|
||||
if (command === 'create') {
|
||||
const sealed = createRegionObservationEvidence(await stdinJson(input), bindings)
|
||||
process.stdout.write(`${JSON.stringify(sealed)}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'verify') {
|
||||
verifyRegionObservationEvidence(JSON.parse(readFileSync(args.file, 'utf8')), bindings)
|
||||
return
|
||||
}
|
||||
throw new Error('unknown region observation evidence command')
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
createRegionObservationEvidence,
|
||||
verifyRegionObservationEvidence
|
||||
} from './relay-region-observation-evidence.mjs'
|
||||
|
||||
const now = Date.parse('2026-08-14T12:00:00Z')
|
||||
const bindings = {
|
||||
commitSha: 'a'.repeat(40),
|
||||
directorImageDigest: `sha256:${'b'.repeat(64)}`,
|
||||
selectorGeneration: 11,
|
||||
controlGeneration: 4
|
||||
}
|
||||
|
||||
function entries() {
|
||||
return Array.from({ length: 24 }, (_, index) => ({
|
||||
timestamp: new Date(now - (index * 60 + 30) * 60_000).toISOString(),
|
||||
jsonPayload: {
|
||||
event: 'orca_relay_runtime_metrics',
|
||||
role: 'director',
|
||||
requestedRegionsDelta: { 'asia-east2': index === 0 ? 2 : 0 },
|
||||
selectedRegionsDelta: { 'asia-east2': index === 0 ? 1 : 0 },
|
||||
regionFallbacksDelta: { 'asia-east2': index === 0 ? 1 : 0 },
|
||||
unavailableRegionsDelta: {}
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
test('seals all 24 hourly aggregate region buckets', () => {
|
||||
const sealed = createRegionObservationEvidence(entries(), bindings, now)
|
||||
assert.equal(sealed.evidence.hourlySampleCounts.length, 24)
|
||||
assert.equal(sealed.evidence.totals.requestedRegionsDelta['asia-east2'], 2)
|
||||
assert.equal(verifyRegionObservationEvidence(sealed, bindings).samples, 24)
|
||||
})
|
||||
|
||||
test('rejects missing coverage, missing Asia activity, and changed bindings', () => {
|
||||
assert.throws(() => createRegionObservationEvidence(entries().slice(1), bindings, now), /missing hourly/)
|
||||
const noAsia = entries().map((entry) => ({
|
||||
...entry,
|
||||
jsonPayload: {
|
||||
...entry.jsonPayload,
|
||||
requestedRegionsDelta: {},
|
||||
selectedRegionsDelta: {}
|
||||
}
|
||||
}))
|
||||
assert.throws(() => createRegionObservationEvidence(noAsia, bindings, now), /no Asia/)
|
||||
const sealed = createRegionObservationEvidence(entries(), bindings, now)
|
||||
assert.throws(() => verifyRegionObservationEvidence(sealed, {
|
||||
...bindings,
|
||||
commitSha: 'c'.repeat(40)
|
||||
}), /does not match/)
|
||||
})
|
||||
@@ -0,0 +1,193 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
function workflow(name) {
|
||||
return readFileSync(
|
||||
fileURLToPath(relayWorkflowUrl(name)),
|
||||
'utf8'
|
||||
)
|
||||
}
|
||||
|
||||
test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
|
||||
const wrapper = workflow('deploy-relay-production-same-cap.yml')
|
||||
const job = workflow('deploy-relay-production-same-cap-job.yml')
|
||||
assert.match(wrapper, /options: \[verify, canary-apply, batch-apply, rollback\]/)
|
||||
assert.match(wrapper, /relay-same-cap-canary-\$\{\{ inputs\.canary-run-id \}\}/)
|
||||
assert.match(wrapper, /needs: \[gate, cell_1\]/)
|
||||
assert.match(wrapper, /needs: \[gate, cell_2\]/)
|
||||
assert.match(wrapper, /needs: \[gate, cell_3\]/)
|
||||
assert.match(job, /on:\n workflow_call:/)
|
||||
assert.match(job, /c27\|c28\|c29/)
|
||||
assert.match(job, /EXPECTED_HARD_CAP=3000/)
|
||||
assert.match(job, /EXPECTED_REGION=asia-east2/)
|
||||
assert.match(job, /--hard-cap "\$\{EXPECTED_HARD_CAP\}"/)
|
||||
assert.match(job, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
|
||||
assert.match(job, /--argjson protocol "\$\{PREDECESSOR_REHOME_PROTOCOL\}"/)
|
||||
assert.match(job, /runtime predecessor mismatch fields=/)
|
||||
// A rollback interrupted between apply and restore must be resumable.
|
||||
assert.match(job, /ROLLBACK_RESUME=true/)
|
||||
assert.match(job, /test "\$\{LIVE_IMAGE_DIGEST\}" = "\$\{DESIRED_IMAGE_DIGEST\}"/)
|
||||
// Resume must skip BOTH the drain (no restart will clear the flag) and the
|
||||
// apply (state already converged), and prove convergence instead.
|
||||
assert.match(
|
||||
job,
|
||||
/Reversibly isolate and drain only the selected cell\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/Apply only the selected same-cap template and MIG\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/Require converged Terraform state and a stable MIG on resume\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME == 'true' \}\}/
|
||||
)
|
||||
assert.match(job, /resume found unconverged resources/)
|
||||
// A canary or batch cell that failed before its template apply also
|
||||
// resumes here with template drift from repo changes since its last roll;
|
||||
// only a plan the reviewed validator approves for the image the cell
|
||||
// already serves may pass, and resume still applies nothing.
|
||||
assert.match(job, /requiring reviewed rollback-image drift/)
|
||||
assert.match(
|
||||
job,
|
||||
/--image "\$\{DESIRED_IMAGE\}" \\\n {16}--rollback-image "\$\{DESIRED_IMAGE\}"/
|
||||
)
|
||||
// The relaxation is only safe if the reviewed validator actually runs on
|
||||
// the NON-converged branch, in same-cap-cell mode, with the trust config
|
||||
// the validator requires, restricted to the template-and-MIG change pair.
|
||||
assert.match(
|
||||
job,
|
||||
/if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n then\n/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/requiring reviewed rollback-image drift'\n[\s\S]{0,400}?\n {16}--mode same-cap-cell --cell-id "\$\{TARGET_CELL_ID\}" \\\n/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
|
||||
)
|
||||
assert.match(
|
||||
job,
|
||||
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
|
||||
)
|
||||
assert.match(job, /host-drain \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/)
|
||||
assert.match(job, /resume requires the isolated migration-only cell/)
|
||||
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
|
||||
assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
|
||||
assert.match(job, /\(\.draining == false or \$drainingOk\)/)
|
||||
// Selector expectations must follow the mutations' returned generations,
|
||||
// not fixed offsets: isolate is a no-op on a cell a failed canary already
|
||||
// isolated, and the restore inspect must expect post-restore membership.
|
||||
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
|
||||
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
|
||||
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
|
||||
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/)
|
||||
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
|
||||
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
|
||||
assert.match(job, /FAILSAFE_GENERATION/)
|
||||
// Later batch waves start after ~16-min predecessor rolls, so BOTH evidence
|
||||
// age checks must scale by wave or cell_2+ can never pass; the bound's
|
||||
// per-wave step is the cell job timeout, so the two must move together.
|
||||
assert.match(job, /--required-migration-policy strict \\\n --wave-index "\$\{WAVE_INDEX\}"/)
|
||||
assert.match(job, /dry-run\.state\.json" \\\n --wave-index "\$\{WAVE_INDEX\}" "\$\{RETRY_ARGS\[@\]\}"/)
|
||||
assert.match(job, /timeout-minutes: 75/)
|
||||
// Both age gates step by the cell job timeout above; the constant is
|
||||
// duplicated across the two languages, so pin each copy to it.
|
||||
for (const source of [
|
||||
'../../dev/scripts/relay-monitor-evidence.mjs',
|
||||
'../../apps/relay-ops/src/incident-live-preflight-cli.ts'
|
||||
]) {
|
||||
const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
|
||||
assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
|
||||
assert.match(body, /\^\[0-3\]\$/)
|
||||
}
|
||||
// Aged-evidence replay via job re-runs is fenced: mutations are
|
||||
// single-dispatch, so a failed cell needs a fresh gate and monitor run.
|
||||
assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
|
||||
for (const index of [0, 1, 2, 3]) {
|
||||
assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
|
||||
}
|
||||
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
|
||||
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 2\)/)
|
||||
assert.match(job, /\$region == "us-central1" and \$protocol == 0 and [.]region == null/)
|
||||
assert.match(job, /[.]regionalRehomeProtocol \/\/ 0/)
|
||||
assert.match(job, /runtime predecessor normalized legacy fields=/)
|
||||
assert.match(job, /probe-relay-rehome-trust[.]mjs/)
|
||||
assert.doesNotMatch(job, /service_account: \$\{\{ vars\.PRODUCTION_GCP_RELAY_(?:DIRECTOR_)?RUNTIME_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(job, /roles\/iam\.serviceAccountTokenCreator/)
|
||||
})
|
||||
|
||||
// Why: the same-cap caller defines release_lease itself, and a caller-defined job presents the
|
||||
// caller as job_workflow_ref, so the pair must admit the caller alongside its reusable job.
|
||||
test('shared deploy WIF admits the exact same-cap reusable workflow pair and the caller itself', () => {
|
||||
const terraform = readFileSync(
|
||||
fileURLToPath(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url)),
|
||||
'utf8'
|
||||
)
|
||||
const providerStart = terraform.indexOf(
|
||||
'resource "google_iam_workload_identity_pool_provider" "github"'
|
||||
)
|
||||
const providerEnd = terraform.indexOf('\nresource "', providerStart + 1)
|
||||
const sharedProvider = terraform.slice(providerStart, providerEnd)
|
||||
assert.ok(providerStart >= 0 && providerEnd > providerStart)
|
||||
assert.match(sharedProvider, /local\.relay_github_workflow_conditions\["github"\]/)
|
||||
// The pairing itself now lives in the clause the provider renders, once per accepted repository.
|
||||
assert.match(
|
||||
terraform,
|
||||
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main' && \(assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_job_workflow_file\}@refs\/heads\/main' \|\| assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main'\)/
|
||||
)
|
||||
})
|
||||
|
||||
test('pause and disable precede optional installation and cloud diagnostics', () => {
|
||||
const job = workflow('operate-relay-production-rehome-job.yml')
|
||||
const emergency = job.indexOf('Apply emergency durable pause or disable before diagnostics')
|
||||
const install = job.indexOf('pnpm install --frozen-lockfile')
|
||||
const revision = job.indexOf('Verify exact serving and rollback director identities')
|
||||
assert.ok(emergency > 0)
|
||||
assert.ok(emergency < install)
|
||||
assert.ok(emergency < revision)
|
||||
assert.match(job, /inputs\.mode == 'pause' \|\| inputs\.mode == 'disable'/)
|
||||
assert.match(job, /Seal 24-hour aggregate region observation evidence/)
|
||||
assert.match(job, /--freshness=25h --limit=30000/)
|
||||
assert.match(job, /relay-region-observation-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
|
||||
assert.match(job, /test "\$\{RATE_PER_MINUTE\}" = 10/)
|
||||
})
|
||||
|
||||
test('a failed enable independently restores and verifies durable disabled state', () => {
|
||||
const job = workflow('operate-relay-production-rehome-job.yml')
|
||||
const enable = job.indexOf('Apply exact durable regional rehome enable')
|
||||
const evidence = job.indexOf('Read fresh aggregate completion and abort evidence')
|
||||
const summary = job.indexOf('Publish aggregate control evidence')
|
||||
const recovery = job.indexOf('Fail closed after an unsuccessful enable run')
|
||||
assert.ok(enable > 0 && enable < evidence && evidence < summary && summary < recovery)
|
||||
const recoveryStep = job.slice(recovery)
|
||||
assert.match(
|
||||
recoveryStep,
|
||||
/failure\(\) && inputs\.mode == 'enable' && steps\.google-auth\.outcome == 'success'/
|
||||
)
|
||||
assert.match(recoveryStep, /--mode recover-enable/)
|
||||
assert.match(recoveryStep, /--expected-control-generation "\$\{EXPECTED_CONTROL_GENERATION\}"/)
|
||||
assert.match(recoveryStep, /RECOVER_FAILED_REGIONAL_REHOME_ENABLE/)
|
||||
assert.match(recoveryStep, /\.control\.enabled == false/)
|
||||
assert.doesNotMatch(recoveryStep, /gcloud|pnpm/)
|
||||
})
|
||||
|
||||
test('director rollout has a strict one-time identity bootstrap', () => {
|
||||
const workflowBody = workflow('deploy-relay-production-director.yml')
|
||||
const script = readFileSync(
|
||||
fileURLToPath(new URL('./deploy-relay-blue-green.mjs', import.meta.url)),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(workflowBody, /BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY/)
|
||||
assert.match(workflowBody, /--predecessor-runtime-service-account/)
|
||||
assert.match(workflowBody, /--expected-rehome-generation/)
|
||||
assert.match(script, /args\.push\('--service-account', config\['runtime-service-account'\]\)/)
|
||||
assert.match(script, /director predecessor runtime service account does not match/)
|
||||
const candidateProof = script.indexOf('await verifyRehomeDisabled(candidate.origin)')
|
||||
const trafficMove = script.indexOf('operations.updateTraffic(config, [`--to-tags=')
|
||||
assert.ok(candidateProof > 0 && candidateProof < trafficMove)
|
||||
assert.equal(script.indexOf('verifyRehomeDisabled', trafficMove), -1)
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const INVENTORY = /^\[orca-relay\] regional rehome inventory active=(\d+) awaitingReceipt=(\d+) targetRegistered=(\d+) completedLast24Hours=(\d+) abortedLast24Hours=(\d+) oldestActiveAgeMs=(none|\d+)$/
|
||||
|
||||
function count(value, name) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`)
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseRegionalRehomeInventory(entries, options = {}) {
|
||||
if (!Array.isArray(entries)) throw new Error('logging response must be an array')
|
||||
const parsed = entries.flatMap((entry) => {
|
||||
const match = INVENTORY.exec(entry?.textPayload ?? '')
|
||||
const timestamp = Date.parse(entry?.timestamp ?? '')
|
||||
if (!match || !Number.isFinite(timestamp)) return []
|
||||
return [{
|
||||
timestamp,
|
||||
active: count(match[1], 'active'),
|
||||
awaitingReceipt: count(match[2], 'awaiting receipt'),
|
||||
targetRegistered: count(match[3], 'target registered'),
|
||||
completedLast24Hours: count(match[4], 'completed'),
|
||||
abortedLast24Hours: count(match[5], 'aborted'),
|
||||
oldestActiveAgeMs: match[6] === 'none' ? null : count(match[6], 'oldest active age')
|
||||
}]
|
||||
}).sort((left, right) => right.timestamp - left.timestamp)
|
||||
if (parsed.length === 0) throw new Error('no aggregate regional rehome inventory evidence')
|
||||
const latest = parsed[0]
|
||||
const now = options.now ?? Date.now()
|
||||
const maxAgeMs = options.maxAgeMs ?? 15 * 60_000
|
||||
if (latest.timestamp > now + 60_000 || latest.timestamp < now - maxAgeMs) {
|
||||
throw new Error('aggregate regional rehome inventory evidence is stale')
|
||||
}
|
||||
return latest
|
||||
}
|
||||
|
||||
function argumentsMap(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
|
||||
throw new Error('invalid arguments')
|
||||
}
|
||||
values[argv[index].slice(2)] = argv[index + 1]
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), input = process.stdin) {
|
||||
const values = argumentsMap(argv)
|
||||
const maxAgeMs = count(values['max-age-ms'] ?? 900_000, '--max-age-ms')
|
||||
const chunks = []
|
||||
for await (const chunk of input) chunks.push(chunk)
|
||||
const evidence = parseRegionalRehomeInventory(
|
||||
JSON.parse(Buffer.concat(chunks).toString('utf8')),
|
||||
{ maxAgeMs }
|
||||
)
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_aggregate_evidence', ...evidence })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { parseRegionalRehomeInventory } from './relay-rehome-aggregate-evidence.mjs'
|
||||
|
||||
const now = Date.parse('2026-08-14T12:00:00Z')
|
||||
|
||||
test('selects the newest fresh aggregate-only regional rehome inventory', () => {
|
||||
const result = parseRegionalRehomeInventory([
|
||||
{
|
||||
timestamp: '2026-08-14T11:58:00Z',
|
||||
textPayload: '[orca-relay] regional rehome inventory active=2 awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=0 oldestActiveAgeMs=30000'
|
||||
},
|
||||
{
|
||||
timestamp: '2026-08-14T11:50:00Z',
|
||||
textPayload: '[orca-relay] regional rehome inventory active=1 awaitingReceipt=0 targetRegistered=1 completedLast24Hours=8 abortedLast24Hours=0 oldestActiveAgeMs=none'
|
||||
}
|
||||
], { now, maxAgeMs: 5 * 60_000 })
|
||||
assert.deepEqual(result, {
|
||||
timestamp: Date.parse('2026-08-14T11:58:00Z'),
|
||||
active: 2,
|
||||
awaitingReceipt: 1,
|
||||
targetRegistered: 1,
|
||||
completedLast24Hours: 9,
|
||||
abortedLast24Hours: 0,
|
||||
oldestActiveAgeMs: 30_000
|
||||
})
|
||||
})
|
||||
|
||||
test('rejects stale, malformed, and identity-bearing lookalikes', () => {
|
||||
assert.throws(() => parseRegionalRehomeInventory([{
|
||||
timestamp: '2026-08-14T11:00:00Z',
|
||||
textPayload: '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0 completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none'
|
||||
}], { now, maxAgeMs: 5 * 60_000 }), /stale/)
|
||||
assert.throws(() => parseRegionalRehomeInventory([{
|
||||
timestamp: '2026-08-14T11:59:00Z',
|
||||
textPayload: '[orca-relay] regional rehome inventory active=0 hostId=secret'
|
||||
}], { now }), /no aggregate/)
|
||||
})
|
||||
@@ -0,0 +1,29 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
|
||||
// Single place naming the repository the Relay workflows live in and where their files sit. The
|
||||
// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the
|
||||
// owning repository, so only this module changes: nothing else may restate any of the three.
|
||||
export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca'
|
||||
|
||||
export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-'
|
||||
|
||||
// Where .github/workflows sits relative to this file. Workflows stay at the repository root while
|
||||
// this tree moves under cloud/, so the depth changes at the copy even though the layout does not.
|
||||
export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url)
|
||||
|
||||
export function relayWorkflowFile(name) {
|
||||
return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
|
||||
}
|
||||
|
||||
// Repository-relative path, the shape GitHub reports in workflow_ref and evidence payloads.
|
||||
export function relayWorkflowPath(name) {
|
||||
return `.github/workflows/${relayWorkflowFile(name)}`
|
||||
}
|
||||
|
||||
export function relayWorkflowUrl(name) {
|
||||
return new URL(relayWorkflowFile(name), RELAY_WORKFLOW_DIRECTORY)
|
||||
}
|
||||
|
||||
export function readRelayWorkflow(name) {
|
||||
return readFileSync(relayWorkflowUrl(name), 'utf8')
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
RELAY_GITHUB_REPOSITORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
readRelayWorkflow,
|
||||
relayWorkflowFile,
|
||||
relayWorkflowPath,
|
||||
relayWorkflowUrl
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
const directory = fileURLToPath(new URL('.', import.meta.url))
|
||||
// The Relay copy takes the scripts named for it. Everything else stays with the applications.
|
||||
const relayScripts = readdirSync(directory)
|
||||
.filter((name) => name.includes('relay') && name.endsWith('.mjs'))
|
||||
.filter((name) => !name.startsWith('relay-repository.'))
|
||||
|
||||
test('workflow identity is derived, never restated', () => {
|
||||
assert.equal(relayWorkflowFile('deploy-relay-staging.yml'), `${RELAY_WORKFLOW_FILE_PREFIX}deploy-relay-staging.yml`)
|
||||
assert.equal(relayWorkflowPath('deploy-relay-staging.yml'), `.github/workflows/${relayWorkflowFile('deploy-relay-staging.yml')}`)
|
||||
assert.ok(relayWorkflowUrl('deploy-relay-staging.yml').pathname.endsWith(relayWorkflowPath('deploy-relay-staging.yml')))
|
||||
assert.match(readRelayWorkflow('deploy-relay-staging.yml'), /^name:/m)
|
||||
assert.match(RELAY_GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/)
|
||||
})
|
||||
|
||||
// Why: the public-repo copy changes the owning repository, the workflow filenames, and the depth
|
||||
// this tree sits at. Each has to be one edit here, so no Relay script may restate any of them.
|
||||
test('no Relay script restates the repository or the workflow directory', () => {
|
||||
for (const name of relayScripts) {
|
||||
const text = readFileSync(`${directory}${name}`, 'utf8')
|
||||
assert.doesNotMatch(text, /stablyai\//, `${name} restates the GitHub repository`)
|
||||
assert.doesNotMatch(text, /\.github\/workflows/, `${name} restates the workflow directory`)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,159 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowFile, relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const recoveryWorkflow = readFileSync(
|
||||
relayWorkflowUrl('recover-relay-staging-c4-image.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const requeueWorkflow = readFileSync(
|
||||
relayWorkflowUrl('requeue-relay-staging-c4-recovery.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const githubActions = readFileSync(
|
||||
new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const cells = readFileSync(
|
||||
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8')
|
||||
const stagingTfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const productionTfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563'
|
||||
|
||||
// Scoped to the Asia cells by name: the production capacity cells now serve this digest too,
|
||||
// so a file-wide count no longer isolates Asia.
|
||||
const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
|
||||
|
||||
function productionCell(cellId) {
|
||||
const start = productionTfvars.indexOf(`"${cellId}"`)
|
||||
assert.notEqual(start, -1, `${cellId} is missing`)
|
||||
return productionTfvars.slice(start, productionTfvars.indexOf('\n }', start))
|
||||
}
|
||||
|
||||
test('pins staging C4 and all production Asia cells to the same launch image', () => {
|
||||
assert.equal(stagingTfvars.match(new RegExp(launchDigest, 'g'))?.length, 1)
|
||||
for (const cellId of asiaCells) {
|
||||
assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId)
|
||||
}
|
||||
assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`))
|
||||
})
|
||||
|
||||
test('refreshes only empty staging C4 through the trusted capacity identity', () => {
|
||||
const refresh = workflow.slice(workflow.indexOf(' refresh-asia-c4-image:'))
|
||||
assert.match(refresh, /terraform_version: 1\.15\.8/)
|
||||
assert.doesNotMatch(refresh, /terraform_version: 1\.5\.7/)
|
||||
assert.match(refresh, /github\.ref == 'refs\/heads\/main'/)
|
||||
assert.match(refresh, /REFRESH_STAGING_ASIA_C4_IMAGE/)
|
||||
assert.match(refresh, /APPROVED_PREDECESSOR_IMAGE_DIGEST/)
|
||||
assert.match(refresh, /--activity quiescent/)
|
||||
assert.match(refresh, /--admission migration-only/)
|
||||
assert.match(refresh, /fence_digests="\$\{PREDECESSOR_IMAGE_DIGEST\}"/)
|
||||
assert.match(refresh, /--expected-image-digests "\$\{TARGET_IMAGE_DIGEST\}"/)
|
||||
assert.match(refresh, /--mode same-cap-image/)
|
||||
assert.match(refresh, /test "\$\(jq -r '\.changes'/)
|
||||
assert.match(refresh, /REFRESH_PHASE=\$\{refresh_phase\}/)
|
||||
assert.match(refresh, /MUTATION_STARTED=false/)
|
||||
assert.match(refresh, /MIG_STABLE_AT_MS=/)
|
||||
assert.match(refresh, /PLAN_CHANGES=/)
|
||||
assert.match(refresh, /obsolete-template-delete/)
|
||||
assert.match(
|
||||
refresh,
|
||||
/\*:manager-convergence\|\*:replacement-with-obsolete-template\) refresh_phase=converging/
|
||||
)
|
||||
assert.match(refresh, /--mode isolate/)
|
||||
assert.match(refresh, /--mode verify/)
|
||||
assert.match(refresh, /\.status\.runtime\.ready/)
|
||||
assert.match(refresh, /\.status\.runtime\.startedAt/)
|
||||
assert.match(refresh, /\.status\.runtime\.lastHeartbeatAt/)
|
||||
assert.match(refresh, /--runtime unavailable/)
|
||||
const plan = refresh.indexOf('Save, validate, and classify the exact C4 plan')
|
||||
const currentState = refresh.indexOf('Verify the exact selector and current C4 state')
|
||||
const isolate = refresh.indexOf('--mode isolate')
|
||||
const apply = refresh.indexOf('terraform -chdir=infra/terraform apply -auto-approve')
|
||||
assert.ok(plan < currentState && currentState < isolate && isolate < apply)
|
||||
assert.match(refresh, /Require an empty targeted Terraform readback/)
|
||||
})
|
||||
|
||||
test('recovers a failed or cancelled C4 refresh from an independent workflow', () => {
|
||||
assert.match(recoveryWorkflow, /terraform_version: 1\.15\.8/)
|
||||
assert.doesNotMatch(recoveryWorkflow, /terraform_version: 1\.5\.7/)
|
||||
assert.match(recoveryWorkflow, /workflow_run:/)
|
||||
assert.match(recoveryWorkflow, /workflows: \[Prove Relay Staging Capacity\]/)
|
||||
assert.match(recoveryWorkflow, /RECOVER_STAGING_ASIA_C4_IMAGE/)
|
||||
assert.match(recoveryWorkflow, /outputs:\n\s+recover: \$\{\{ steps\.trigger\.outputs\.recover \}\}/)
|
||||
assert.match(recoveryWorkflow, /if test "\$\{count\}" = 0; then\n\s+echo "recover=false"/)
|
||||
assert.match(recoveryWorkflow, /needs: gate\n\s+if: \$\{\{ needs\.gate\.outputs\.recover == 'true' \}\}/)
|
||||
assert.match(recoveryWorkflow, /concurrency:\n\s+group: relay-staging-mutation/)
|
||||
assert.match(recoveryWorkflow, /group: relay-staging-mutation/)
|
||||
assert.match(recoveryWorkflow, /\.name == "refresh-asia-c4-image"/)
|
||||
assert.match(recoveryWorkflow, /PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13/)
|
||||
assert.match(recoveryWorkflow, /TARGET_IMAGE_DIGEST: sha256:5aedbca5/)
|
||||
assert.match(recoveryWorkflow, /id: preflight-auth/)
|
||||
assert.match(recoveryWorkflow, /id: verify-auth/)
|
||||
assert.match(recoveryWorkflow, /\.status\.runtime\.ready/)
|
||||
assert.match(recoveryWorkflow, /--runtime unavailable/)
|
||||
assert.match(recoveryWorkflow, /current_digest.*\^sha256:\[a-f0-9\]\{64\}\$/)
|
||||
assert.match(recoveryWorkflow, /expected_digests="\$\{expected_digests\},\$\{current_digest\}"/)
|
||||
assert.match(recoveryWorkflow, /--timeout-ms 240000/)
|
||||
assert.doesNotMatch(recoveryWorkflow, /--timeout-ms 900000/)
|
||||
assert.match(recoveryWorkflow, /-var manage_artifact_dns=false -lock-timeout=5m/)
|
||||
assert.match(recoveryWorkflow, /--mode same-cap-image/)
|
||||
assert.match(recoveryWorkflow, /test "\$\(jq -r '\.changes'/)
|
||||
assert.equal(recoveryWorkflow.match(/\*:replacement-with-obsolete-template/g)?.length, 2)
|
||||
assert.match(
|
||||
recoveryWorkflow,
|
||||
/\^\(replacement\|replacement-with-obsolete-template\|manager-convergence\)\$/
|
||||
)
|
||||
const preflightAuth = recoveryWorkflow.indexOf('id: preflight-auth')
|
||||
const preflight = recoveryWorkflow.indexOf('Inspect the exact C4 recovery state')
|
||||
const recoveryPlan = recoveryWorkflow.indexOf('Classify both exact recovery end states')
|
||||
const apply = recoveryWorkflow.indexOf('Apply and stabilize the saved predecessor plan')
|
||||
const restore = recoveryWorkflow.indexOf('Restart only when the plan did not replace C4')
|
||||
const verifyAuth = recoveryWorkflow.indexOf('id: verify-auth')
|
||||
const verify = recoveryWorkflow.indexOf('Verify the recovered image and unchanged isolation')
|
||||
assert.ok(preflightAuth < preflight && preflight < recoveryPlan && recoveryPlan < apply)
|
||||
assert.ok(apply < restore)
|
||||
assert.ok(restore < verifyAuth && verifyAuth < verify)
|
||||
assert.match(githubActions, /"recover-relay-staging-c4-image\.yml"/)
|
||||
})
|
||||
|
||||
test('requeues a protected C4 recovery cancelled while pending', () => {
|
||||
assert.match(requeueWorkflow, /workflows: \[Recover Relay Staging C4 Image\]/)
|
||||
assert.match(requeueWorkflow, /conclusion == 'cancelled'/)
|
||||
assert.match(requeueWorkflow, /permissions:\n\s+actions: write\n\s+contents: read/)
|
||||
assert.match(requeueWorkflow, /group: relay-staging-c4-recovery-requeue/)
|
||||
assert.match(requeueWorkflow, /\.name == "recover" and\n\s+\.conclusion == "cancelled"/)
|
||||
assert.match(requeueWorkflow, /\.started_at == null/)
|
||||
assert.match(requeueWorkflow, /\.name == "gate" and \.conclusion == "success"/)
|
||||
assert.match(requeueWorkflow, /\.name == "recover" and \.status != "completed"/)
|
||||
assert.match(requeueWorkflow, /actions\/runs\/\$\{run_id\}\/jobs\?filter=latest/)
|
||||
assert.match(requeueWorkflow, /if test "\$\{active\}" != 0; then exit 0; fi/)
|
||||
assert.ok(requeueWorkflow.includes(`gh workflow run ${relayWorkflowFile('recover-relay-staging-c4-image.yml')}`))
|
||||
assert.match(requeueWorkflow, /-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE/)
|
||||
assert.doesNotMatch(requeueWorkflow, /id-token: write/)
|
||||
assert.doesNotMatch(requeueWorkflow, /relay-staging-mutation/)
|
||||
})
|
||||
|
||||
test('keeps cell-only plans independent from service-account description drift', () => {
|
||||
assert.match(cells, /runtime_service_account\s+= local\.relay_runtime_service_account_email/)
|
||||
assert.match(
|
||||
cells,
|
||||
/rehome_director_service_account\s+= local\.relay_director_runtime_service_account_email/
|
||||
)
|
||||
assert.match(relay, /var\.environment == "staging" \? "Orca Relay"/)
|
||||
})
|
||||
@@ -0,0 +1,269 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const terraform = readFileSync(
|
||||
new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8')
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const deployWorkflow = readFileSync(
|
||||
relayWorkflowUrl('deploy-relay-staging.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const publishWorkflow = readFileSync(
|
||||
relayWorkflowUrl('publish-relay-production.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const bootstrapWorkflow = readFileSync(
|
||||
relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
function resource(type, name) {
|
||||
const start = terraform.indexOf(`resource "${type}" "${name}"`)
|
||||
assert.notEqual(start, -1, `${type}.${name} is missing`)
|
||||
const next = terraform.indexOf('\nresource "', start + 1)
|
||||
return terraform.slice(start, next === -1 ? undefined : next)
|
||||
}
|
||||
|
||||
function terraformStringList(block, attribute) {
|
||||
const match = block.match(new RegExp(`${attribute}\\s*=\\s*\\[([\\s\\S]*?)\\]`))
|
||||
assert.ok(match, `${attribute} is missing`)
|
||||
return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1])
|
||||
}
|
||||
|
||||
test('capacity workflow uses only its exact staging identity', () => {
|
||||
assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(workflow, /vars\.STAGING_GCP_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.doesNotMatch(workflow, /vars\.STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
|
||||
const provider = resource(
|
||||
'google_iam_workload_identity_pool_provider',
|
||||
'github_staging_relay_capacity'
|
||||
)
|
||||
// The three repository claims are pinned once in relay-shared.tf; every provider concatenates
|
||||
// that list rather than restating the repository on its own.
|
||||
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/)
|
||||
for (const boundary of [
|
||||
"assertion.ref == 'refs/heads/main'",
|
||||
"assertion.environment == 'staging'"
|
||||
]) {
|
||||
assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
|
||||
}
|
||||
assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_capacity"\]/)
|
||||
assert.deepEqual(
|
||||
terraformStringList(terraform, 'github_staging_relay_capacity_workflow_files'),
|
||||
[
|
||||
'bootstrap-relay-staging-capacity.yml',
|
||||
'prove-relay-staging-capacity.yml',
|
||||
'recover-relay-staging-c4-image.yml'
|
||||
]
|
||||
)
|
||||
assert.match(
|
||||
terraform,
|
||||
/for workflow_file in local\.github_staging_relay_capacity_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/
|
||||
)
|
||||
assert.doesNotMatch(terraform, /github_staging_relay_capacity_workflow_file\s*=/)
|
||||
})
|
||||
|
||||
test('job gates do not read environment variables before the environment is attached', () => {
|
||||
for (const source of [workflow, deployWorkflow, bootstrapWorkflow]) {
|
||||
const jobGate = source.match(/^\s{4}if:.*$/m)?.[0] ?? ''
|
||||
assert.doesNotMatch(jobGate, /STAGING_GCP_RELAY_CAPACITY_/)
|
||||
}
|
||||
})
|
||||
|
||||
test('capacity identity has bounded mutation and state permissions', () => {
|
||||
const role = resource(
|
||||
'google_project_iam_custom_role',
|
||||
'github_staging_relay_capacity_mutation'
|
||||
)
|
||||
assert.deepEqual(terraformStringList(role, 'permissions'), [
|
||||
'compute.disks.create',
|
||||
'compute.healthChecks.use',
|
||||
'compute.images.useReadOnly',
|
||||
'compute.instanceGroupManagers.get',
|
||||
'compute.instanceGroupManagers.update',
|
||||
'compute.instances.create',
|
||||
'compute.instances.setLabels',
|
||||
'compute.instances.setMetadata',
|
||||
'compute.instances.setTags',
|
||||
'compute.instanceTemplates.create',
|
||||
'compute.instanceTemplates.delete',
|
||||
'compute.instanceTemplates.get',
|
||||
'compute.instanceTemplates.useReadOnly',
|
||||
'compute.networks.use',
|
||||
'compute.subnetworks.use',
|
||||
'compute.zoneOperations.get'
|
||||
])
|
||||
assert.doesNotMatch(
|
||||
role,
|
||||
/compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/
|
||||
)
|
||||
|
||||
for (const source of [workflow, bootstrapWorkflow]) {
|
||||
assert.match(source, /instance-groups managed recreate-instances/)
|
||||
assert.match(source, /--instances/)
|
||||
assert.doesNotMatch(source, /rolling-action restart/)
|
||||
}
|
||||
|
||||
const state = resource(
|
||||
'google_storage_bucket_iam_member',
|
||||
'github_staging_relay_capacity_state'
|
||||
)
|
||||
assert.match(state, /roles\/storage\.objectAdmin/)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
|
||||
assert.doesNotMatch(state, /resource\.name\.startsWith/)
|
||||
|
||||
const runtime = resource(
|
||||
'google_service_account_iam_member',
|
||||
'github_staging_relay_capacity_runtime_user'
|
||||
)
|
||||
assert.match(runtime, /google_service_account\.relay_runtime\.name/)
|
||||
assert.match(runtime, /roles\/iam\.serviceAccountUser/)
|
||||
|
||||
const cloudRun = resource(
|
||||
'google_cloud_run_v2_service_iam_member',
|
||||
'github_staging_relay_capacity_developer'
|
||||
)
|
||||
assert.match(cloudRun, /name\s*=\s*var\.relay_cloud_run_service_name/)
|
||||
assert.doesNotMatch(cloudRun, /google_cloud_run_v2_service\.relay/)
|
||||
|
||||
const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8')
|
||||
const startup = readFileSync(
|
||||
new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
assert.match(relay, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
assert.match(startup, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
})
|
||||
|
||||
test('capacity identity exposes only its provider and service account', () => {
|
||||
assert.match(outputs, /output "github_staging_relay_capacity_workload_identity_provider"/)
|
||||
assert.match(outputs, /output "github_staging_relay_capacity_service_account"/)
|
||||
})
|
||||
|
||||
test('director capacity configuration stays on the audited blue-green path', () => {
|
||||
assert.match(deployWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
assert.match(deployWorkflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/)
|
||||
assert.match(deployWorkflow, /expected-image-digest/)
|
||||
assert.match(deployWorkflow, /var\.relay_gce_cells\["staging-gce-c4"\]\.image/)
|
||||
assert.match(deployWorkflow, /init -reconfigure \\\n\s+-backend-config=backend\/staging\.hcl/)
|
||||
assert.ok(
|
||||
deployWorkflow.indexOf('id: google-auth') <
|
||||
deployWorkflow.indexOf('Bind the request to the checked-in staging C4 image')
|
||||
)
|
||||
assert.match(deployWorkflow, /artifacts docker images describe "\$\{IMAGE\}"/)
|
||||
assert.doesNotMatch(deployWorkflow, /docker (?:build|push)/)
|
||||
assert.match(workflow, /--director-cells-json "\$\{DESIRED_CELLS_JSON\}"/)
|
||||
assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/)
|
||||
assert.doesNotMatch(workflow, /--mode director/)
|
||||
})
|
||||
|
||||
test('mirrors the exact production manifest through the production deploy identity', () => {
|
||||
assert.match(publishWorkflow, /options: \[publish, mirror-staging\]/)
|
||||
assert.match(publishWorkflow, /MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING/)
|
||||
assert.match(publishWorkflow, /docker pull "\$\{source_image\}"/)
|
||||
assert.match(publishWorkflow, /docker tag "\$\{source_image\}" "\$\{target_tag\}"/)
|
||||
assert.match(publishWorkflow, /test "\$\{source_digest\}" = "\$\{MIRROR_DIGEST\}"/)
|
||||
assert.match(publishWorkflow, /test "\$\{target_digest\}" = "\$\{MIRROR_DIGEST\}"/)
|
||||
const mirrorWriter = resource(
|
||||
'google_artifact_registry_repository_iam_member',
|
||||
'github_production_relay_staging_mirror_writer'
|
||||
)
|
||||
assert.match(mirrorWriter, /var\.environment == "staging"/)
|
||||
assert.match(mirrorWriter, /roles\/artifactregistry\.writer/)
|
||||
assert.match(
|
||||
mirrorWriter,
|
||||
/serviceAccount:orca-cloud-gha-deploy@onorca-cloud\.iam\.gserviceaccount\.com/
|
||||
)
|
||||
})
|
||||
|
||||
test('cells bootstrap one at a time with bounded deploy and capacity identities', () => {
|
||||
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
|
||||
assert.match(bootstrapWorkflow, /--mode bootstrap-cell/)
|
||||
assert.match(bootstrapWorkflow, /--cell-id "\$\{fallback_cell_id\}"/)
|
||||
assert.match(bootstrapWorkflow, /google_compute_instance_template\.relay_gce_cell/)
|
||||
assert.match(bootstrapWorkflow, /google_compute_instance_group_manager\.relay_gce_cell/)
|
||||
assert.match(bootstrapWorkflow, /--mode restore-fallback/)
|
||||
assert.doesNotMatch(bootstrapWorkflow, /target=google_cloud_run_v2_service\.relay/)
|
||||
assert.ok(
|
||||
bootstrapWorkflow.indexOf('id: deploy-auth') < bootstrapWorkflow.indexOf('id: capacity-auth')
|
||||
)
|
||||
assert.match(
|
||||
bootstrapWorkflow,
|
||||
/restore_fallback\(\) \{[\s\S]*?verify_fallback[\s\S]*?--mode restore-fallback/
|
||||
)
|
||||
const rollCell = bootstrapWorkflow.slice(bootstrapWorkflow.indexOf('roll_cell()'))
|
||||
assert.ok(
|
||||
rollCell.indexOf('verify_fallback\n trap restore_fallback EXIT') <
|
||||
rollCell.indexOf('--mode isolate')
|
||||
)
|
||||
assert.match(
|
||||
bootstrapWorkflow,
|
||||
/staging-gce-c2 general[\s\S]*?trap restore_legacy_c3_fallback EXIT[\s\S]*?--mode isolate/
|
||||
)
|
||||
const normalize = bootstrapWorkflow.slice(
|
||||
bootstrapWorkflow.indexOf('normalize_legacy_c3() {'),
|
||||
bootstrapWorkflow.indexOf('\n roll_cell()', bootstrapWorkflow.indexOf('normalize_legacy_c3() {'))
|
||||
)
|
||||
const trapInstalled = normalize.indexOf('trap restore_legacy_c3_fallback EXIT')
|
||||
const isolated = normalize.indexOf('--mode isolate', trapInstalled)
|
||||
const recreated = normalize.indexOf('recreate-instances', isolated)
|
||||
const restored = normalize.indexOf('--mode restore', recreated)
|
||||
const c3Verified = normalize.indexOf('staging-gce-c3 general', restored)
|
||||
const restoreDisabled = normalize.indexOf('legacy_c3_isolated=false', c3Verified)
|
||||
const trapCleared = normalize.indexOf('trap - EXIT', restoreDisabled)
|
||||
assert.ok(
|
||||
trapInstalled < isolated &&
|
||||
isolated < recreated &&
|
||||
recreated < restored &&
|
||||
restored < c3Verified &&
|
||||
c3Verified < restoreDisabled &&
|
||||
restoreDisabled < trapCleared
|
||||
)
|
||||
assert.equal(normalize.indexOf('trap - EXIT', trapInstalled), trapCleared)
|
||||
assert.match(bootstrapWorkflow, /--heartbeat either/)
|
||||
assert.doesNotMatch(bootstrapWorkflow, /heartbeat=stale/)
|
||||
assert.match(
|
||||
rollCell,
|
||||
/"\$\{desired_cap\}" "\$\{desired_bound\}" absent-or-stale[\s\S]*?deploy-relay-blue-green\.mjs[\s\S]*?"\$\{desired_cap\}" "\$\{desired_bound\}"/
|
||||
)
|
||||
})
|
||||
|
||||
test('workflows read desired topology from configuration and gate exact predecessors', () => {
|
||||
assert.match(workflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/)
|
||||
assert.match(bootstrapWorkflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/)
|
||||
assert.match(workflow, /1000\/0\)[\s\S]*?PREDECESSOR_C3_CAP=600/)
|
||||
assert.match(workflow, /1000\/60\)[\s\S]*?PREDECESSOR_C3_BOUND=0/)
|
||||
assert.match(workflow, /600\/60\)[\s\S]*?PREDECESSOR_C3_CAP=1000/)
|
||||
assert.match(workflow, /Unsupported staging capacity transition/)
|
||||
})
|
||||
|
||||
test('capacity apply resumes after director or cell success and preserves the no-op restart proof', () => {
|
||||
for (const phase of ['predecessor', 'director-ready', 'cell-ready', 'cell-active']) {
|
||||
assert.match(workflow, new RegExp(`TRANSITION_PHASE=${phase}`))
|
||||
}
|
||||
assert.match(workflow, /--argjson expected "\$\{PREDECESSOR_CELLS_JSON\}"/)
|
||||
assert.match(workflow, /--argjson expected "\$\{DESIRED_CELLS_JSON\}"/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{TRANSITION_PHASE\}" = cell-ready; then[\s\S]*?test "\$\{CELL_PLAN_CHANGES\}" = 0/
|
||||
)
|
||||
assert.match(
|
||||
workflow,
|
||||
/test "\$\{TRANSITION_PHASE\}" = cell-active; then[\s\S]*?recreate_fixed_one_instance/
|
||||
)
|
||||
assert.match(workflow, /--admission migration-only/)
|
||||
})
|
||||
@@ -0,0 +1,228 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs'
|
||||
import {
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayWorkflowFile,
|
||||
relayWorkflowPath
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
const identity = readFileSync(
|
||||
new URL('../../infra/terraform/relay-staging-deploy-iam.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const shared = readFileSync(
|
||||
new URL('../../infra/terraform/relay-shared.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const variables = readFileSync(
|
||||
new URL('../../infra/terraform/variables.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8')
|
||||
const stagingTfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
// The exact five staging Relay workflows the relay-owned deploy identity serves.
|
||||
const DEPLOY_WORKFLOWS = [
|
||||
'bootstrap-relay-staging-capacity.yml',
|
||||
'deploy-relay-staging-gce-candidate.yml',
|
||||
'deploy-relay-staging.yml',
|
||||
'operate-relay-asia-admission.yml',
|
||||
'power-relay-staging.yml'
|
||||
]
|
||||
|
||||
const GENERIC_STAGING_PAIR =
|
||||
/vars\.STAGING_GCP_(?:WORKLOAD_IDENTITY_PROVIDER|DEPLOY_SERVICE_ACCOUNT)\b/
|
||||
|
||||
const workflowNames = workflowFiles
|
||||
// DEPLOY_WORKFLOWS holds the names Terraform pins; the files on disk carry the copy's prefix.
|
||||
const workflow = (name) => readWorkflow(relayWorkflowFile(name))
|
||||
|
||||
function block(type, name) {
|
||||
const start = identity.indexOf(`resource "${type}" "${name}"`)
|
||||
assert.notEqual(start, -1, `${type}.${name} is missing`)
|
||||
const next = identity.indexOf('\nresource "', start + 1)
|
||||
return identity.slice(start, next === -1 ? undefined : next)
|
||||
}
|
||||
|
||||
function declaredFamilies() {
|
||||
return [...identity.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)]
|
||||
.map((match) => `${match[1]}.${match[2]}`)
|
||||
.sort()
|
||||
}
|
||||
|
||||
function providerWorkflowFiles() {
|
||||
const match = identity.match(
|
||||
/github_staging_relay_deploy_workflow_files\s*=\s*\[([\s\S]*?)\n {2}\]/
|
||||
)
|
||||
assert.ok(match, 'github_staging_relay_deploy_workflow_files is missing')
|
||||
return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1])
|
||||
}
|
||||
|
||||
function variableDefault(name) {
|
||||
const match = variables.match(
|
||||
new RegExp(`variable "${name}" \\{[\\s\\S]*?default\\s*=\\s*"([^"]*)"`)
|
||||
)
|
||||
assert.ok(match, `variable ${name} has no default`)
|
||||
return match[1]
|
||||
}
|
||||
|
||||
test('no Relay workflow authenticates as the shared staging deploy identity', () => {
|
||||
for (const name of workflowNames()) {
|
||||
if (!name.includes('relay')) continue
|
||||
assert.doesNotMatch(readWorkflow(name), GENERIC_STAGING_PAIR, name)
|
||||
}
|
||||
})
|
||||
|
||||
test('the five staging Relay workflows name the relay deploy pair', () => {
|
||||
for (const name of DEPLOY_WORKFLOWS) {
|
||||
const source = workflow(name)
|
||||
assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/, name)
|
||||
assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/, name)
|
||||
}
|
||||
})
|
||||
|
||||
// Why: the Asia workflow serves both environments from one job. Repointing its staging arm must
|
||||
// not move production off the relay-owned shared account.
|
||||
test('the Asia admission production arm keeps the production deploy pair', () => {
|
||||
const source = workflow('operate-relay-asia-admission.yml')
|
||||
assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/)
|
||||
assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/)
|
||||
})
|
||||
|
||||
test('the provider allowlists exactly those five workflow refs', () => {
|
||||
const files = providerWorkflowFiles()
|
||||
assert.deepEqual([...files].sort(), [...DEPLOY_WORKFLOWS].sort())
|
||||
for (const file of files) {
|
||||
assert.match(file, /^[a-z0-9-]+\.yml$/)
|
||||
}
|
||||
// Each accepted repository turns that file list into its own exact refs.
|
||||
assert.match(
|
||||
identity,
|
||||
/for workflow_file in local\.github_staging_relay_deploy_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/
|
||||
)
|
||||
|
||||
const provider = block(
|
||||
'google_iam_workload_identity_pool_provider',
|
||||
'github_staging_relay_deploy'
|
||||
)
|
||||
assert.match(provider, /workload_identity_pool_provider_id\s*=\s*"github-relay-deploy"/)
|
||||
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims/)
|
||||
assert.match(provider, /assertion\.ref == 'refs\/heads\/main'/)
|
||||
assert.match(provider, /assertion\.environment == 'staging'/)
|
||||
assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_deploy"\]/)
|
||||
// A prefix match would turn the allowlist into a namespace grant with no Terraform diff.
|
||||
assert.doesNotMatch(provider, /startsWith|endsWith/)
|
||||
})
|
||||
|
||||
// Why: the documented attribute_condition limit is 4096 characters and the expression grows with
|
||||
// every workflow added. Render it the way Terraform does and keep the headroom visible.
|
||||
test('the rendered attribute condition stays inside the provider limit', () => {
|
||||
const repository = `${variableDefault('github_owner')}/${variableDefault('github_repo')}`
|
||||
const claims = [
|
||||
`assertion.repository == '${repository}'`,
|
||||
`assertion.repository_id == '${variableDefault('github_repo_id')}'`,
|
||||
`assertion.repository_owner_id == '${variableDefault('github_owner_id')}'`
|
||||
]
|
||||
const workflowRefs = providerWorkflowFiles().map(
|
||||
(file) => `${repository}/${relayWorkflowPath(file)}@refs/heads/main`
|
||||
)
|
||||
const rendered = [
|
||||
...claims,
|
||||
"assertion.ref == 'refs/heads/main'",
|
||||
"assertion.environment == 'staging'",
|
||||
`(${workflowRefs.map((ref) => `assertion.workflow_ref == '${ref}'`).join(' || ')})`
|
||||
].join(' && ')
|
||||
assert.ok(rendered.length < 4096, `rendered condition is ${rendered.length} characters`)
|
||||
// 797 is the private repository's rendered length. This copy prefixes every workflow filename,
|
||||
// which is the only difference, so the pin still moves the moment a workflow is added or dropped.
|
||||
assert.equal(rendered.length, 797 + workflowRefs.length * RELAY_WORKFLOW_FILE_PREFIX.length)
|
||||
})
|
||||
|
||||
// Why: the census is the point. A binding added here without a workflow step behind it, or one
|
||||
// silently dropped, changes what the staging Relay credential can reach.
|
||||
test('the staging deploy identity declares exactly its enumerated grants', () => {
|
||||
assert.deepEqual(declaredFamilies(), [
|
||||
'google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader',
|
||||
'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer',
|
||||
'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer',
|
||||
'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
|
||||
'google_project_iam_member.github_staging_relay_deploy_compute_viewer',
|
||||
'google_service_account.github_staging_relay_deploy',
|
||||
'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user',
|
||||
'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user',
|
||||
'google_storage_bucket_iam_member.github_staging_relay_deploy_state',
|
||||
'google_storage_bucket_iam_member.github_staging_relay_deploy_state_list'
|
||||
])
|
||||
// Each grant carries a comment naming the workflow step that needs it; the account, its
|
||||
// provider, and the pool binding are the identity itself and are covered by the file header.
|
||||
const identityFamilies = new Set([
|
||||
'google_service_account.github_staging_relay_deploy',
|
||||
'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
|
||||
'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user'
|
||||
])
|
||||
for (const family of declaredFamilies()) {
|
||||
if (identityFamilies.has(family)) continue
|
||||
const [type, name] = family.split('.')
|
||||
const preceding = identity.slice(0, identity.indexOf(`resource "${type}" "${name}"`)).trimEnd()
|
||||
assert.match(preceding.slice(preceding.lastIndexOf('\n') + 1), /^#/, `${family} has no justifying comment`)
|
||||
}
|
||||
assert.match(identity, /var\.environment == "staging"/)
|
||||
|
||||
const state = block('google_storage_bucket_iam_member', 'github_staging_relay_deploy_state')
|
||||
assert.match(state, /roles\/storage\.objectViewer/)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
|
||||
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
|
||||
assert.doesNotMatch(state, /resource\.name\.startsWith/)
|
||||
assert.doesNotMatch(state, /objectAdmin/)
|
||||
|
||||
const director = block(
|
||||
'google_cloud_run_v2_service_iam_member',
|
||||
'github_staging_relay_deploy_director_developer'
|
||||
)
|
||||
assert.match(director, /name\s*=\s*var\.relay_cloud_run_service_name/)
|
||||
|
||||
// Project-wide run.developer or artifactregistry.writer would let the staging Relay credential
|
||||
// deploy the API service or push images; the shared account holds both today.
|
||||
const projectRoles = declaredFamilies()
|
||||
.filter((family) => family.startsWith('google_project_iam_member.'))
|
||||
.map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1])
|
||||
assert.deepEqual(projectRoles, ['roles/compute.viewer'])
|
||||
assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/)
|
||||
assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/)
|
||||
})
|
||||
|
||||
// Why: the auth-plane grants are guarded on a variable, so an unset tfvars entry would drop them
|
||||
// silently and Power Relay Staging would fail only on the sleep path.
|
||||
test('staging pins the shared auth service the power workflow scales', () => {
|
||||
assert.match(stagingTfvars, /relay_staging_power_auth_service_name\s*=\s*"orca-cloud-auth-staging"/)
|
||||
assert.match(variables, /variable "relay_staging_power_auth_service_name"/)
|
||||
for (const name of [
|
||||
'github_staging_relay_deploy_auth_developer',
|
||||
'github_staging_relay_deploy_auth_runtime_user'
|
||||
]) {
|
||||
const type = name.endsWith('runtime_user')
|
||||
? 'google_service_account_iam_member'
|
||||
: 'google_cloud_run_v2_service_iam_member'
|
||||
assert.match(block(type, name), /var\.relay_staging_power_auth_service_name != ""/)
|
||||
}
|
||||
})
|
||||
|
||||
// Why: flipping this local is what moves the staging cells' startup metadata and the director's
|
||||
// ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT onto the new account. Production must keep the shared one.
|
||||
test('the deploy account email is environment-conditional', () => {
|
||||
assert.match(
|
||||
shared,
|
||||
/relay_github_deploy_service_account_email = \(\s*var\.environment == "production"\s*\? "\$\{var\.name_prefix\}-gha-deploy@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*: "\$\{var\.name_prefix\}-gha-relay@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*\)/
|
||||
)
|
||||
assert.match(identity, /account_id\s*=\s*"\$\{var\.name_prefix\}-gha-relay"/)
|
||||
})
|
||||
|
||||
test('the identity is exposed through its own outputs', () => {
|
||||
assert.match(outputs, /output "github_staging_relay_deploy_workload_identity_provider"/)
|
||||
assert.match(outputs, /output "github_staging_relay_deploy_service_account"/)
|
||||
})
|
||||
@@ -0,0 +1,535 @@
|
||||
// Renders every Workload Identity provider `attribute_condition` exactly as
|
||||
// Terraform would, so contract tests can pin the resulting strings without a
|
||||
// plan. Understands only the HCL subset those expressions use.
|
||||
//
|
||||
// Each root is loaded on its own: the relay and apps roots both declare a provider named
|
||||
// `github` while the staging copy waits on its state surgery, and only separate scopes can
|
||||
// show that the two render the same string.
|
||||
//
|
||||
// Only the relay root ships in this repository. The apps root is still declared so this stays a
|
||||
// straight copy of the private original, and is skipped when its directory is absent.
|
||||
import { existsSync } from 'node:fs'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
|
||||
const TERRAFORM_ROOTS = {
|
||||
relay: {
|
||||
directory: 'infra/terraform',
|
||||
sources: [
|
||||
'infra/terraform/relay-shared.tf',
|
||||
'infra/terraform/relay-github-workflow-trust.tf',
|
||||
'infra/terraform/relay-github-actions.tf',
|
||||
'infra/terraform/relay-staging-deploy-iam.tf',
|
||||
'infra/terraform/relay-asia-topology-iam.tf',
|
||||
'infra/terraform/relay-asia-proof-iam.tf'
|
||||
]
|
||||
},
|
||||
apps: {
|
||||
directory: 'infra/terraform-apps',
|
||||
sources: ['infra/terraform-apps/github-actions.tf']
|
||||
}
|
||||
}
|
||||
|
||||
export function hasTerraformRoot(root) {
|
||||
const directory = TERRAFORM_ROOTS[root]?.directory
|
||||
return directory !== undefined && existsSync(repoFile(directory))
|
||||
}
|
||||
|
||||
export const TERRAFORM_ROOT_NAMES = Object.keys(TERRAFORM_ROOTS).filter(hasTerraformRoot)
|
||||
|
||||
const PROVIDER_RESOURCE = 'google_iam_workload_identity_pool_provider'
|
||||
|
||||
function repoFile(path) {
|
||||
return new URL(`../../${path}`, import.meta.url)
|
||||
}
|
||||
|
||||
function skipTrivia(src, index) {
|
||||
let i = index
|
||||
for (;;) {
|
||||
while (i < src.length && /\s/.test(src[i])) i += 1
|
||||
if (src[i] === '#') {
|
||||
while (i < src.length && src[i] !== '\n') i += 1
|
||||
continue
|
||||
}
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the index just past the closing quote of the string starting at `i`.
|
||||
function endOfString(src, i) {
|
||||
let cursor = i + 1
|
||||
while (src[cursor] !== '"') {
|
||||
if (src[cursor] === '\\') {
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
if (src[cursor] === '$' && src[cursor + 1] === '{') {
|
||||
cursor = endOfInterpolation(src, cursor + 2).next
|
||||
continue
|
||||
}
|
||||
cursor += 1
|
||||
}
|
||||
return cursor + 1
|
||||
}
|
||||
|
||||
function endOfInterpolation(src, i) {
|
||||
let depth = 1
|
||||
let cursor = i
|
||||
while (depth > 0) {
|
||||
const char = src[cursor]
|
||||
if (char === undefined) throw new Error('unterminated interpolation')
|
||||
if (char === '"') {
|
||||
cursor = endOfString(src, cursor)
|
||||
continue
|
||||
}
|
||||
if (char === '{') depth += 1
|
||||
else if (char === '}') {
|
||||
depth -= 1
|
||||
if (depth === 0) break
|
||||
}
|
||||
cursor += 1
|
||||
}
|
||||
return { text: src.slice(i, cursor), next: cursor + 1 }
|
||||
}
|
||||
|
||||
// Stands in for a loop variable when the collection is empty: the body still has to be parsed
|
||||
// once to find where it ends, and any attribute of the probe is another probe.
|
||||
const PROBE = new Proxy(
|
||||
{},
|
||||
{
|
||||
get: (target, key) => (key === Symbol.toPrimitive ? () => '' : PROBE)
|
||||
}
|
||||
)
|
||||
|
||||
function readMember(value, key) {
|
||||
if (value === PROBE) return PROBE
|
||||
if (value === null || value === undefined) throw new Error(`cannot read ${String(key)} of ${value}`)
|
||||
if (Array.isArray(value)) {
|
||||
if (typeof key !== 'number') throw new Error(`list index must be a number, got ${String(key)}`)
|
||||
if (!Number.isInteger(key) || key < 0 || key >= value.length) {
|
||||
throw new Error(`list index ${key} is out of range`)
|
||||
}
|
||||
return value[key]
|
||||
}
|
||||
if (typeof value !== 'object') throw new Error(`cannot index ${typeof value}`)
|
||||
if (!Object.hasOwn(value, key)) throw new Error(`unknown attribute ${String(key)}`)
|
||||
return value[key]
|
||||
}
|
||||
|
||||
// [key, value] pairs the way HCL iterates: list index and element, or object key and value.
|
||||
function collectionEntries(collection) {
|
||||
if (Array.isArray(collection)) return collection.map((item, index) => [index, item])
|
||||
if (collection && typeof collection === 'object') return Object.entries(collection)
|
||||
throw new Error(`cannot iterate ${typeof collection}`)
|
||||
}
|
||||
|
||||
class ExpressionParser {
|
||||
constructor(source, scope) {
|
||||
this.source = source
|
||||
this.scope = scope
|
||||
this.index = 0
|
||||
}
|
||||
|
||||
parse() {
|
||||
const value = this.parseTernary()
|
||||
this.index = skipTrivia(this.source, this.index)
|
||||
if (this.index !== this.source.length) {
|
||||
throw new Error(`trailing expression text: ${this.source.slice(this.index)}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
peek(token) {
|
||||
this.index = skipTrivia(this.source, this.index)
|
||||
return this.source.startsWith(token, this.index)
|
||||
}
|
||||
|
||||
eat(token) {
|
||||
if (!this.peek(token)) return false
|
||||
this.index += token.length
|
||||
return true
|
||||
}
|
||||
|
||||
expect(token) {
|
||||
if (!this.eat(token)) {
|
||||
throw new Error(`expected ${token} at ${this.source.slice(this.index, this.index + 40)}`)
|
||||
}
|
||||
}
|
||||
|
||||
parseTernary() {
|
||||
const condition = this.parseOr()
|
||||
if (!this.eat('?')) return condition
|
||||
const consequent = this.parseTernary()
|
||||
this.expect(':')
|
||||
const alternate = this.parseTernary()
|
||||
return condition ? consequent : alternate
|
||||
}
|
||||
|
||||
parseOr() {
|
||||
let left = this.parseAnd()
|
||||
while (this.eat('||')) left = Boolean(this.parseAnd()) || Boolean(left)
|
||||
return left
|
||||
}
|
||||
|
||||
parseAnd() {
|
||||
let left = this.parseEquality()
|
||||
while (this.eat('&&')) left = Boolean(this.parseEquality()) && Boolean(left)
|
||||
return left
|
||||
}
|
||||
|
||||
parseEquality() {
|
||||
let left = this.parseUnary()
|
||||
for (;;) {
|
||||
if (this.eat('==')) left = left === this.parseUnary()
|
||||
else if (this.eat('!=')) left = left !== this.parseUnary()
|
||||
else return left
|
||||
}
|
||||
}
|
||||
|
||||
parseUnary() {
|
||||
return this.parsePostfix(this.parsePrimary())
|
||||
}
|
||||
|
||||
parsePrimary() {
|
||||
if (this.eat('(')) {
|
||||
const value = this.parseTernary()
|
||||
this.expect(')')
|
||||
return value
|
||||
}
|
||||
if (this.peek('"')) return this.parseString()
|
||||
if (this.peek('[')) return this.parseList()
|
||||
if (this.peek('{')) return this.parseObject()
|
||||
const number = /^[0-9]+/.exec(this.source.slice(this.index))
|
||||
if (number) {
|
||||
this.index += number[0].length
|
||||
return Number(number[0])
|
||||
}
|
||||
return this.parseIdentifier()
|
||||
}
|
||||
|
||||
parsePostfix(value) {
|
||||
let current = value
|
||||
for (;;) {
|
||||
if (this.eat('.')) {
|
||||
current = readMember(current, this.readWord())
|
||||
continue
|
||||
}
|
||||
if (this.peek('[')) {
|
||||
this.index += 1
|
||||
const key = this.parseTernary()
|
||||
this.expect(']')
|
||||
current = readMember(current, key)
|
||||
continue
|
||||
}
|
||||
return current
|
||||
}
|
||||
}
|
||||
|
||||
// `for a in x : body` / `for a, b in x : body`, shared by list and object comprehensions.
|
||||
parseComprehension(readBody) {
|
||||
const names = [this.readWord()]
|
||||
if (this.eat(',')) names.push(this.readWord())
|
||||
this.expect('in')
|
||||
const collection = this.parseUnary()
|
||||
this.expect(':')
|
||||
const bodyStart = skipTrivia(this.source, this.index)
|
||||
const entries = collectionEntries(collection)
|
||||
const bodyParser = ([key, item]) => {
|
||||
const bindings = { ...this.scope.bindings }
|
||||
if (names.length === 1) bindings[names[0]] = Array.isArray(collection) ? item : key
|
||||
else {
|
||||
bindings[names[0]] = key
|
||||
bindings[names[1]] = item
|
||||
}
|
||||
const parser = new ExpressionParser(this.source, { ...this.scope, bindings })
|
||||
parser.index = bodyStart
|
||||
return parser
|
||||
}
|
||||
// Parse once with a probe binding to find where the body ends, because an empty
|
||||
// collection would never parse it.
|
||||
const probe = bodyParser(entries[0] ?? [PROBE, PROBE])
|
||||
readBody(probe)
|
||||
this.index = probe.index
|
||||
return entries.map((entry) => readBody(bodyParser(entry)))
|
||||
}
|
||||
|
||||
parseObject() {
|
||||
this.expect('{')
|
||||
if (this.eat('for')) {
|
||||
const pairs = this.parseComprehension((parser) => {
|
||||
const key = parser.parseTernary()
|
||||
parser.expect('=>')
|
||||
return [key, parser.parseTernary()]
|
||||
})
|
||||
this.expect('}')
|
||||
return Object.fromEntries(pairs)
|
||||
}
|
||||
const object = {}
|
||||
if (this.eat('}')) return object
|
||||
for (;;) {
|
||||
const key = this.peek('"') ? this.parseString() : this.readWord()
|
||||
this.expect('=')
|
||||
object[key] = this.parseTernary()
|
||||
this.eat(',')
|
||||
if (this.eat('}')) return object
|
||||
}
|
||||
}
|
||||
|
||||
parseString() {
|
||||
this.index = skipTrivia(this.source, this.index)
|
||||
const src = this.source
|
||||
let cursor = this.index + 1
|
||||
let rendered = ''
|
||||
while (src[cursor] !== '"') {
|
||||
if (src[cursor] === '\\') {
|
||||
rendered += src[cursor + 1]
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
if (src[cursor] === '$' && src[cursor + 1] === '{') {
|
||||
const { text, next } = endOfInterpolation(src, cursor + 2)
|
||||
rendered += String(evaluate(text, this.scope))
|
||||
cursor = next
|
||||
continue
|
||||
}
|
||||
rendered += src[cursor]
|
||||
cursor += 1
|
||||
}
|
||||
this.index = cursor + 1
|
||||
return rendered
|
||||
}
|
||||
|
||||
parseList() {
|
||||
this.expect('[')
|
||||
if (this.eat('for')) {
|
||||
const items = this.parseComprehension((parser) => parser.parseTernary())
|
||||
this.expect(']')
|
||||
return items
|
||||
}
|
||||
const items = []
|
||||
if (this.eat(']')) return items
|
||||
for (;;) {
|
||||
items.push(this.parseTernary())
|
||||
if (this.eat(',')) {
|
||||
if (this.eat(']')) return items
|
||||
continue
|
||||
}
|
||||
this.expect(']')
|
||||
return items
|
||||
}
|
||||
}
|
||||
|
||||
readWord() {
|
||||
this.index = skipTrivia(this.source, this.index)
|
||||
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(this.source.slice(this.index))
|
||||
if (!match) throw new Error(`expected identifier at ${this.source.slice(this.index, this.index + 40)}`)
|
||||
this.index += match[0].length
|
||||
return match[0]
|
||||
}
|
||||
|
||||
parseIdentifier() {
|
||||
const word = this.readWord()
|
||||
if (word === 'join') {
|
||||
this.expect('(')
|
||||
const separator = this.parseTernary()
|
||||
this.expect(',')
|
||||
const parts = this.parseTernary()
|
||||
this.eat(',')
|
||||
this.expect(')')
|
||||
return parts.join(separator)
|
||||
}
|
||||
if (word === 'concat') {
|
||||
this.expect('(')
|
||||
const lists = []
|
||||
for (;;) {
|
||||
lists.push(this.parseTernary())
|
||||
if (this.eat(',')) {
|
||||
if (this.eat(')')) break
|
||||
continue
|
||||
}
|
||||
this.expect(')')
|
||||
break
|
||||
}
|
||||
return lists.flat()
|
||||
}
|
||||
if (word === 'length') {
|
||||
this.expect('(')
|
||||
const value = this.parseTernary()
|
||||
this.eat(',')
|
||||
this.expect(')')
|
||||
return collectionEntries(value).length
|
||||
}
|
||||
if (word === 'local') {
|
||||
this.expect('.')
|
||||
return resolveLocal(this.readWord(), this.scope)
|
||||
}
|
||||
if (word === 'var') {
|
||||
this.expect('.')
|
||||
const name = this.readWord()
|
||||
if (!(name in this.scope.variables)) throw new Error(`unknown variable ${name}`)
|
||||
return this.scope.variables[name]
|
||||
}
|
||||
if (word in this.scope.bindings) return this.scope.bindings[word]
|
||||
if (word === 'true') return true
|
||||
if (word === 'false') return false
|
||||
throw new Error(`unsupported identifier ${word}`)
|
||||
}
|
||||
}
|
||||
|
||||
function evaluate(source, scope) {
|
||||
return new ExpressionParser(source, scope).parse()
|
||||
}
|
||||
|
||||
function resolveLocal(name, scope) {
|
||||
if (scope.resolved.has(name)) return scope.resolved.get(name)
|
||||
if (!scope.locals.has(name)) throw new Error(`unknown local ${name}`)
|
||||
if (scope.resolving.has(name)) throw new Error(`local cycle at ${name}`)
|
||||
scope.resolving.add(name)
|
||||
const value = evaluate(scope.locals.get(name), { ...scope, bindings: {} })
|
||||
scope.resolving.delete(name)
|
||||
scope.resolved.set(name, value)
|
||||
return value
|
||||
}
|
||||
|
||||
function collectLocals(source, locals) {
|
||||
const blockPattern = /^locals \{$/gm
|
||||
let match
|
||||
while ((match = blockPattern.exec(source)) !== null) {
|
||||
const end = source.indexOf('\n}\n', match.index)
|
||||
const body = source.slice(match.index + match[0].length, end)
|
||||
let name = null
|
||||
let buffer = []
|
||||
const flush = () => {
|
||||
if (name) locals.set(name, buffer.join('\n'))
|
||||
}
|
||||
for (const line of body.split('\n')) {
|
||||
const assignment = /^ {2}([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line)
|
||||
if (assignment) {
|
||||
flush()
|
||||
name = assignment[1]
|
||||
buffer = [assignment[2]]
|
||||
continue
|
||||
}
|
||||
if (name && line.trim() !== '' && !line.trim().startsWith('#')) buffer.push(line)
|
||||
}
|
||||
flush()
|
||||
}
|
||||
}
|
||||
|
||||
function collectProviderFields(source, fields) {
|
||||
const pattern = new RegExp(`resource "${PROVIDER_RESOURCE}" "([A-Za-z_0-9]+)" \\{`, 'g')
|
||||
let match
|
||||
while ((match = pattern.exec(source)) !== null) {
|
||||
const end = source.indexOf('\n}\n', match.index)
|
||||
const body = source.slice(match.index, end)
|
||||
const conditionStart = body.indexOf(' attribute_condition = ')
|
||||
const conditionEnd = body.indexOf('\n\n oidc {', conditionStart)
|
||||
const countStart = body.indexOf(' count = ')
|
||||
fields.set(match[1], {
|
||||
count: body.slice(countStart + ' count = '.length, body.indexOf('\n', countStart)),
|
||||
condition: body.slice(conditionStart + ' attribute_condition = '.length, conditionEnd)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Values that are not a plain quoted string (a list of objects, say) are read with the
|
||||
// expression parser; anything it cannot evaluate is left undefined, exactly as before.
|
||||
function parseValueAt(source, index) {
|
||||
const parser = new ExpressionParser(source, {
|
||||
locals: new Map(),
|
||||
variables: {},
|
||||
bindings: {},
|
||||
resolved: new Map(),
|
||||
resolving: new Set()
|
||||
})
|
||||
parser.index = index
|
||||
return parser.parseTernary()
|
||||
}
|
||||
|
||||
function collectVariableDefaults(source, variables) {
|
||||
const pattern = /variable "([A-Za-z_0-9]+)" \{([\s\S]*?)\n\}/g
|
||||
let match
|
||||
while ((match = pattern.exec(source)) !== null) {
|
||||
const body = match[2]
|
||||
const fallback = /\n\s*default\s*=\s*"([^"]*)"/.exec(body)
|
||||
if (fallback) {
|
||||
variables[match[1]] = fallback[1]
|
||||
continue
|
||||
}
|
||||
const assignment = /\n\s*default\s*=\s*/.exec(body)
|
||||
if (!assignment) continue
|
||||
const start = match.index + match[0].indexOf(body) + assignment.index + assignment[0].length
|
||||
try {
|
||||
variables[match[1]] = parseValueAt(source, start)
|
||||
} catch {
|
||||
// A default this evaluator does not understand is not one any condition reads.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function collectTfvars(source, variables) {
|
||||
let offset = 0
|
||||
for (const line of source.split('\n')) {
|
||||
const quoted = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"([^"]*)"\s*$/.exec(line)
|
||||
if (quoted) {
|
||||
variables[quoted[1]] = quoted[2]
|
||||
offset += line.length + 1
|
||||
continue
|
||||
}
|
||||
const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line)
|
||||
if (structured) {
|
||||
try {
|
||||
variables[structured[1]] = parseValueAt(source, offset + structured[0].length)
|
||||
} catch {
|
||||
// Same as above: unreadable here means unread by every condition.
|
||||
}
|
||||
}
|
||||
offset += line.length + 1
|
||||
}
|
||||
}
|
||||
|
||||
async function loadScope(root, environment) {
|
||||
const { directory, sources } = TERRAFORM_ROOTS[root] ?? {}
|
||||
if (!sources) throw new Error(`unknown terraform root ${root}`)
|
||||
const locals = new Map()
|
||||
const providers = new Map()
|
||||
for (const path of sources) {
|
||||
const source = await readFile(repoFile(path), 'utf8')
|
||||
collectLocals(source, locals)
|
||||
collectProviderFields(source, providers)
|
||||
}
|
||||
const variables = {}
|
||||
collectVariableDefaults(await readFile(repoFile(`${directory}/variables.tf`), 'utf8'), variables)
|
||||
collectTfvars(
|
||||
await readFile(repoFile(`${directory}/environments/${environment}.tfvars`), 'utf8'),
|
||||
variables
|
||||
)
|
||||
return {
|
||||
providers,
|
||||
scope: { locals, variables, bindings: {}, resolved: new Map(), resolving: new Set() }
|
||||
}
|
||||
}
|
||||
|
||||
// Rendered `attribute_condition` per provider that the given root creates in the environment.
|
||||
export async function renderRootAttributeConditions(root, environment) {
|
||||
const { providers, scope } = await loadScope(root, environment)
|
||||
const rendered = {}
|
||||
for (const [name, fields] of providers) {
|
||||
if (evaluate(fields.count, scope) === 0) continue
|
||||
rendered[name] = evaluate(fields.condition, scope)
|
||||
}
|
||||
return rendered
|
||||
}
|
||||
|
||||
// Every root's rendered conditions, keyed by root and then by provider.
|
||||
export async function renderAttributeConditions(environment) {
|
||||
const rendered = {}
|
||||
for (const root of TERRAFORM_ROOT_NAMES) {
|
||||
rendered[root] = await renderRootAttributeConditions(root, environment)
|
||||
}
|
||||
return rendered
|
||||
}
|
||||
|
||||
export async function readTerraformLocal(name, environment, root = 'relay') {
|
||||
const { scope } = await loadScope(root, environment)
|
||||
return resolveLocal(name, scope)
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { assertSpreadModel, modeledRelayLoad } from './relay-load-model.mjs'
|
||||
|
||||
const counts = process.argv.slice(2).length > 0 ? process.argv.slice(2).map(Number) : [4_000, 10_000]
|
||||
for (const count of counts) {
|
||||
const model = modeledRelayLoad(count)
|
||||
assertSpreadModel(model)
|
||||
console.log(JSON.stringify(model))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user