mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
Move Relay deployment configuration into private cloud operations (#31792)
* refactor(cloud): move deployment configuration into private operations repo * docs(cloud): clarify private operator ownership * fix(ci): install pnpm through npm-compatible bootstrap * fix(ci): avoid native downloader metadata timeout * test(push): move deployment preflight contract into private repo * fix(ci): bootstrap Node before installing pnpm in minimal containers * fix(ci): pass Windows boundary test paths through the Node runner * fix(ci): run Windows boundary Vitest with installed Node
This commit is contained in:
Neil
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design),
|
||||
// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses
|
||||
// must be identical before and after a state move. This captures that set deterministically.
|
||||
// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded
|
||||
// because the baseline has none.
|
||||
|
||||
const usage =
|
||||
'usage: capture-terraform-plan-baseline.mjs --root <dir> --env <staging|production> --out <dir> [--tag <name>]'
|
||||
|
||||
export function normalizePlan(planJson) {
|
||||
const changes = (planJson.resource_changes ?? [])
|
||||
.filter((entry) => !entry.change.actions.includes('forget'))
|
||||
.map((entry) => ({
|
||||
address: entry.address,
|
||||
actions: entry.change.actions,
|
||||
before: entry.change.before ?? null,
|
||||
after: entry.change.after ?? null,
|
||||
after_unknown: entry.change.after_unknown ?? null
|
||||
}))
|
||||
.sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))
|
||||
return changes
|
||||
}
|
||||
|
||||
export function summarize(changes) {
|
||||
const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 }
|
||||
for (const change of changes) {
|
||||
const key = change.actions.join('-')
|
||||
if (key === 'create') counts.create += 1
|
||||
else if (key === 'update') counts.update += 1
|
||||
else if (key === 'delete') counts.delete += 1
|
||||
else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1
|
||||
else if (key === 'read') counts.read += 1
|
||||
else counts['no-op'] += 1
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
function argument(flag) {
|
||||
const index = process.argv.indexOf(flag)
|
||||
return index >= 0 ? process.argv[index + 1] : undefined
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) {
|
||||
const root = argument('--root')
|
||||
const environment = argument('--env')
|
||||
const out = argument('--out')
|
||||
const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}`
|
||||
if (!root || !['staging', 'production'].includes(environment) || !out) {
|
||||
process.stderr.write(`${usage}\n`)
|
||||
process.exit(2)
|
||||
}
|
||||
// The Cloudflare override only applies to the root that still declares the records; the relay
|
||||
// root dropped them in the carve and errors on a -var for an undeclared variable.
|
||||
const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes(
|
||||
'variable "manage_artifact_dns"'
|
||||
)
|
||||
mkdirSync(out, { recursive: true })
|
||||
const planFile = join(out, `${tag}.tfplan`)
|
||||
execFileSync(
|
||||
'terraform',
|
||||
[
|
||||
`-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []),
|
||||
`-out=${planFile}`
|
||||
],
|
||||
{ stdio: ['ignore', 'inherit', 'inherit'] }
|
||||
)
|
||||
const json = JSON.parse(
|
||||
execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 256 * 1024 * 1024
|
||||
})
|
||||
)
|
||||
const normalized = normalizePlan(json)
|
||||
writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`)
|
||||
process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs'
|
||||
|
||||
test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => {
|
||||
const changes = normalizePlan({
|
||||
resource_changes: [
|
||||
{ address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } },
|
||||
{ address: 'a.one', change: { actions: ['forget'], before: {}, after: null } },
|
||||
{ address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } }
|
||||
]
|
||||
})
|
||||
assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three'])
|
||||
assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 })
|
||||
})
|
||||
@@ -1,167 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
|
||||
|
||||
// Drained hosts that find no free slot keep redialling and pin the drained cell (c28,
|
||||
// 2026-10-01). Hosts are counted as controls, but each moved host also brings its splices;
|
||||
// the 20% margin is for those.
|
||||
export const MAX_HEADROOM_FRACTION = 0.8
|
||||
|
||||
function count(value, name) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} is invalid`)
|
||||
return value
|
||||
}
|
||||
|
||||
export function parseHeadroomArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-origin', 'cell-id', 'general-cells']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-origin']) {
|
||||
const url = new URL(values[key])
|
||||
if (url.protocol !== 'https:' || url.origin !== values[key]) {
|
||||
throw new Error('origins must be canonical HTTPS origins')
|
||||
}
|
||||
}
|
||||
const generalCells = values['general-cells'] === 'none'
|
||||
? []
|
||||
: values['general-cells'].split(',')
|
||||
if (
|
||||
new Set(generalCells).size !== generalCells.length ||
|
||||
generalCells.some((cellId) => !/^[a-z0-9-]+$/.test(cellId))
|
||||
) {
|
||||
throw new Error('--general-cells is invalid')
|
||||
}
|
||||
return {
|
||||
directorOrigin: values['director-origin'],
|
||||
cellOrigin: values['cell-origin'],
|
||||
cellId: values['cell-id'],
|
||||
generalCells
|
||||
}
|
||||
}
|
||||
|
||||
async function adminJson(fetchImpl, url, token, body, wait) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
},
|
||||
{ wait }
|
||||
)
|
||||
const json = await response.json().catch(() => ({}))
|
||||
// null: the director refused the read because the cell is reserve.
|
||||
if (isReserveCellRefusal(response.status, json)) return null
|
||||
if (!response.ok) throw new Error(`${new URL(url).pathname} returned ${response.status}`)
|
||||
return json
|
||||
}
|
||||
|
||||
// A cell whose capacity view is stale, absent, or not general offers no slot we can count.
|
||||
// Placement admits while enforced units plus outstanding reservations stay under the pause.
|
||||
function freeSlots(status) {
|
||||
const capacity = status.connectionCapacity
|
||||
if (
|
||||
status.enabled !== true ||
|
||||
status.admissionState !== 'general' ||
|
||||
capacity === null ||
|
||||
capacity === undefined ||
|
||||
capacity.heartbeatFresh !== true
|
||||
) {
|
||||
return 0
|
||||
}
|
||||
const pause = count(capacity.normalAdmissionPause, 'normal admission pause')
|
||||
const used = Math.max(
|
||||
count(capacity.observedConnections, 'observed connections'),
|
||||
count(capacity.enforcedConnectionUnits, 'enforced connection units')
|
||||
)
|
||||
const reserved = count(capacity.pendingControlReservations, 'pending control reservations')
|
||||
return Math.max(0, pause - used - reserved)
|
||||
}
|
||||
|
||||
export async function checkSameCapHeadroom(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const cellStatus = async (cellId) => {
|
||||
const answer = await adminJson(
|
||||
fetchImpl,
|
||||
`${config.directorOrigin}/v1/admin/cell-status`,
|
||||
token,
|
||||
{ v: 1, cellId },
|
||||
wait
|
||||
)
|
||||
if (answer === null) return null
|
||||
if (answer.status?.cellId !== cellId) {
|
||||
throw new Error('director status does not match the cell')
|
||||
}
|
||||
return answer.status
|
||||
}
|
||||
const runtime = await adminJson(
|
||||
fetchImpl,
|
||||
`${config.cellOrigin}/v1/admin/runtime-status`,
|
||||
token,
|
||||
{ v: 1 },
|
||||
wait
|
||||
)
|
||||
if (runtime?.role !== 'cell' || runtime.cellId !== config.cellId) {
|
||||
throw new Error('runtime status does not match the cell')
|
||||
}
|
||||
const targetHosts = count(runtime.runtime?.controls, 'target host controls')
|
||||
const targetStatus = await cellStatus(config.cellId)
|
||||
if (targetStatus === null) throw reserveCellRollRefusal(config.cellId)
|
||||
const targetRegion = targetStatus.region ?? null
|
||||
const cells = []
|
||||
for (const cellId of config.generalCells) {
|
||||
if (cellId === config.cellId) continue
|
||||
const status = await cellStatus(cellId)
|
||||
// A reserve cell's free seats are not in the database: counted as none, never guessed.
|
||||
cells.push(
|
||||
status === null
|
||||
? { cellId, region: null, freeSlots: 0, reserve: true }
|
||||
: { cellId, region: status.region ?? null, freeSlots: freeSlots(status) }
|
||||
)
|
||||
}
|
||||
const total = (filter) =>
|
||||
cells.filter(filter).reduce((sum, cell) => sum + cell.freeSlots, 0)
|
||||
const sameRegion = (cell) => targetRegion !== null && cell.region === targetRegion
|
||||
const free = total(() => true)
|
||||
const allowedHosts = Math.floor(free * MAX_HEADROOM_FRACTION)
|
||||
return {
|
||||
cellId: config.cellId,
|
||||
region: targetRegion,
|
||||
targetHosts,
|
||||
freeSlots: free,
|
||||
sameRegionFreeSlots: total(sameRegion),
|
||||
otherRegionFreeSlots: total((cell) => !sameRegion(cell)),
|
||||
allowedHosts,
|
||||
sufficient: targetHosts <= allowedHosts,
|
||||
cells
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await checkSameCapHeadroom(parseHeadroomArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_same_cap_headroom', ...result })}\n`)
|
||||
if (!result.sufficient) {
|
||||
throw new Error(
|
||||
`${result.targetHosts} hosts on ${result.cellId} exceed ` +
|
||||
`${MAX_HEADROOM_FRACTION * 100}% of ${result.freeSlots} free general-cell slots`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
checkSameCapHeadroom,
|
||||
main,
|
||||
parseHeadroomArguments
|
||||
} from './check-relay-same-cap-headroom.mjs'
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
cellOrigin: 'https://c28.relay.example.com',
|
||||
cellId: 'production-gce-c28',
|
||||
generalCells: ['production-gce-c28', 'production-gce-c29', 'production-gce-asia-c1']
|
||||
}
|
||||
|
||||
function cell({
|
||||
region = 'us-central1',
|
||||
pause = 840,
|
||||
observed = 0,
|
||||
enforced = observed,
|
||||
pending = 0,
|
||||
...rest
|
||||
} = {}) {
|
||||
return {
|
||||
enabled: true,
|
||||
admissionState: 'general',
|
||||
region,
|
||||
connectionCapacity: {
|
||||
normalAdmissionPause: pause,
|
||||
observedConnections: observed,
|
||||
enforcedConnectionUnits: enforced,
|
||||
pendingControlReservations: pending,
|
||||
heartbeatFresh: true
|
||||
},
|
||||
...rest
|
||||
}
|
||||
}
|
||||
|
||||
function harness({ controls = 500, cells = {} } = {}) {
|
||||
const statuses = {
|
||||
'production-gce-c28': cell({ observed: 800 }),
|
||||
'production-gce-c29': cell({ observed: 340 }),
|
||||
'production-gce-asia-c1': cell({ region: 'asia-east2', pause: 2_840, observed: 2_640 }),
|
||||
...cells
|
||||
}
|
||||
return async (url, options) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') {
|
||||
return Response.json({ role: 'cell', cellId: config.cellId, runtime: { controls } })
|
||||
}
|
||||
const { cellId } = JSON.parse(options.body)
|
||||
return Response.json({ v: 1, status: { cellId, ...statuses[cellId] } })
|
||||
}
|
||||
}
|
||||
|
||||
const run = async (fetch) =>
|
||||
await checkSameCapHeadroom(config, { fetch, token: 'masked-token', wait: async () => {} })
|
||||
|
||||
test('counts free general-cell slots by region and excludes the target', async () => {
|
||||
const result = await run(harness({ controls: 560 }))
|
||||
assert.deepEqual(
|
||||
{ ...result, cells: undefined },
|
||||
{
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
targetHosts: 560,
|
||||
freeSlots: 700,
|
||||
sameRegionFreeSlots: 500,
|
||||
otherRegionFreeSlots: 200,
|
||||
allowedHosts: 560,
|
||||
sufficient: true,
|
||||
cells: undefined
|
||||
}
|
||||
)
|
||||
assert.deepEqual(result.cells.map((entry) => entry.cellId), [
|
||||
'production-gce-c29',
|
||||
'production-gce-asia-c1'
|
||||
])
|
||||
})
|
||||
|
||||
test('free slots subtract the larger of observed and enforced use plus reservations', async () => {
|
||||
for (const [state, free] of [
|
||||
[{ observed: 340, enforced: 400 }, 640],
|
||||
[{ observed: 400, enforced: 340 }, 640],
|
||||
[{ observed: 340, pending: 60 }, 640],
|
||||
[{ observed: 800, pending: 60 }, 200]
|
||||
]) {
|
||||
const result = await run(harness({ cells: { 'production-gce-c29': cell(state) } }))
|
||||
assert.equal(result.freeSlots, free)
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects malformed capacity counts', async () => {
|
||||
await assert.rejects(
|
||||
run(harness({
|
||||
cells: { 'production-gce-c29': cell({ pending: null }) }
|
||||
})),
|
||||
/pending control reservations is invalid/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses a cell whose hosts exceed 80% of the free slots', async () => {
|
||||
const result = await run(harness({ controls: 561 }))
|
||||
assert.equal(result.sufficient, false)
|
||||
})
|
||||
|
||||
test('a stale, full, or non-general cell offers no slots', async () => {
|
||||
for (const blocked of [
|
||||
cell({
|
||||
connectionCapacity: {
|
||||
normalAdmissionPause: 840,
|
||||
observedConnections: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
pendingControlReservations: 0,
|
||||
heartbeatFresh: false
|
||||
}
|
||||
}),
|
||||
cell({ connectionCapacity: null }),
|
||||
cell({ admissionState: 'migration-only' }),
|
||||
cell({ enabled: false }),
|
||||
cell({ observed: 900 })
|
||||
]) {
|
||||
const result = await run(harness({ cells: { 'production-gce-c29': blocked } }))
|
||||
assert.equal(result.freeSlots, 200)
|
||||
assert.equal(result.sufficient, false)
|
||||
}
|
||||
})
|
||||
|
||||
test('no general cells means no headroom', async () => {
|
||||
const result = await checkSameCapHeadroom(
|
||||
{ ...config, generalCells: [] },
|
||||
{ fetch: harness({ controls: 1 }), token: 'masked-token', wait: async () => {} }
|
||||
)
|
||||
assert.equal(result.freeSlots, 0)
|
||||
assert.equal(result.sufficient, false)
|
||||
})
|
||||
|
||||
const reserveRefusal = () => Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 })
|
||||
|
||||
test('a reserve cell that is not being rolled offers no slots and is named', async () => {
|
||||
const base = harness({ controls: 160 })
|
||||
const result = await run(async (url, options) =>
|
||||
JSON.parse(options.body).cellId === 'production-gce-c29'
|
||||
? reserveRefusal()
|
||||
: await base(url, options))
|
||||
assert.equal(result.freeSlots, 200)
|
||||
assert.equal(result.sufficient, true)
|
||||
assert.deepEqual(result.cells.find((entry) => entry.cellId === 'production-gce-c29'), {
|
||||
cellId: 'production-gce-c29',
|
||||
region: null,
|
||||
freeSlots: 0,
|
||||
reserve: true
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to roll a reserve cell, naming the flip back to db', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
JSON.parse(options.body ?? '{}').cellId === config.cellId
|
||||
? reserveRefusal()
|
||||
: await base(url, options)),
|
||||
/production-gce-c28 is in reserve mode .*flip it to admitMode=db/
|
||||
)
|
||||
})
|
||||
|
||||
test('any other 404 from cell-status still fails', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
JSON.parse(options.body).cellId === 'production-gce-c29'
|
||||
? Response.json({ error: 'cell_not_found' }, { status: 404 })
|
||||
: await base(url, options)),
|
||||
/cell-status returned 404/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects a runtime or director answer for another cell', async () => {
|
||||
const base = harness()
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
new URL(url).pathname === '/v1/admin/runtime-status'
|
||||
? Response.json({ role: 'cell', cellId: 'production-gce-c1', runtime: { controls: 1 } })
|
||||
: await base(url, options)),
|
||||
/runtime status does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
run(async (url, options) =>
|
||||
new URL(url).pathname === '/v1/admin/cell-status'
|
||||
? Response.json({ v: 1, status: { cellId: 'production-gce-c1' } })
|
||||
: await base(url, options)),
|
||||
/director status does not match/
|
||||
)
|
||||
})
|
||||
|
||||
test('parses the general cell list, including none', () => {
|
||||
const base = [
|
||||
'--director-origin', 'https://relay.example.com',
|
||||
'--cell-origin', 'https://c28.relay.example.com',
|
||||
'--cell-id', 'production-gce-c28'
|
||||
]
|
||||
assert.deepEqual(
|
||||
parseHeadroomArguments([...base, '--general-cells', 'a-c1,b-c2']).generalCells,
|
||||
['a-c1', 'b-c2']
|
||||
)
|
||||
assert.deepEqual(parseHeadroomArguments([...base, '--general-cells', 'none']).generalCells, [])
|
||||
assert.throws(() => parseHeadroomArguments([...base, '--general-cells', 'a,a']), /invalid/)
|
||||
assert.throws(() => parseHeadroomArguments(base), /missing --general-cells/)
|
||||
})
|
||||
|
||||
test('main prints the numbers and fails on insufficient headroom', async (t) => {
|
||||
const writes = []
|
||||
t.mock.method(process.stdout, 'write', (text) => writes.push(text))
|
||||
t.mock.method(globalThis, 'fetch', harness({ controls: 700 }))
|
||||
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'masked-token'
|
||||
t.after(() => {
|
||||
delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
})
|
||||
await assert.rejects(
|
||||
main([
|
||||
'--director-origin', config.directorOrigin,
|
||||
'--cell-origin', config.cellOrigin,
|
||||
'--cell-id', config.cellId,
|
||||
'--general-cells', config.generalCells.join(',')
|
||||
]),
|
||||
/700 hosts on production-gce-c28 exceed 80% of 700 free general-cell slots/
|
||||
)
|
||||
const line = JSON.parse(writes.join(''))
|
||||
assert.equal(line.event, 'relay_same_cap_headroom')
|
||||
assert.equal(line.allowedHosts, 560)
|
||||
})
|
||||
@@ -1,109 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
|
||||
function parseArguments(argv) {
|
||||
if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) {
|
||||
throw new Error('missing --capacity-service-account')
|
||||
}
|
||||
return argv[1]
|
||||
}
|
||||
|
||||
export function classifyProductionCapacityDirector(state, capacityServiceAccount) {
|
||||
const currentIdentity = state.currentCapacityServiceAccount
|
||||
if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) {
|
||||
throw new Error('director has an unexpected capacity identity')
|
||||
}
|
||||
const {
|
||||
baseCells,
|
||||
currentCells,
|
||||
capacityCellIds,
|
||||
targetCellId,
|
||||
targetHardCap
|
||||
} = state
|
||||
if (
|
||||
!Array.isArray(baseCells) ||
|
||||
!Array.isArray(currentCells) ||
|
||||
!Array.isArray(capacityCellIds) ||
|
||||
![600, 1000].includes(targetHardCap) ||
|
||||
new Set(capacityCellIds).size !== capacityCellIds.length ||
|
||||
!capacityCellIds.includes(targetCellId) ||
|
||||
baseCells.length !== currentCells.length ||
|
||||
new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length ||
|
||||
new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length
|
||||
) {
|
||||
throw new Error('director topology transition input is invalid')
|
||||
}
|
||||
const capacityCells = new Set(capacityCellIds)
|
||||
const normalizedCurrent = currentCells.map((current, index) => {
|
||||
const base = baseCells[index]
|
||||
if (
|
||||
typeof current?.id !== 'string' ||
|
||||
current.id !== base?.id
|
||||
) {
|
||||
throw new Error('director topology cell identity is invalid')
|
||||
}
|
||||
if (!capacityCells.has(current.id)) {
|
||||
if (!isDeepStrictEqual(current, base)) {
|
||||
throw new Error('director topology changed outside the capacity rollout')
|
||||
}
|
||||
return current
|
||||
}
|
||||
if (
|
||||
base.connectionHardCap !== 1000 ||
|
||||
base.connectionUnobservedBound !== 60 ||
|
||||
![600, 1000].includes(current.connectionHardCap) ||
|
||||
current.connectionUnobservedBound !== 60
|
||||
) {
|
||||
throw new Error('director capacity rollout state is invalid')
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
connectionHardCap: base.connectionHardCap,
|
||||
connectionUnobservedBound: base.connectionUnobservedBound
|
||||
}
|
||||
})
|
||||
if (
|
||||
!isDeepStrictEqual(normalizedCurrent, baseCells) ||
|
||||
capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId))
|
||||
) {
|
||||
throw new Error('director topology is outside the reviewed capacity envelope')
|
||||
}
|
||||
const withTargetCap = (hardCap) => currentCells.map((cell) =>
|
||||
cell.id === targetCellId
|
||||
? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 }
|
||||
: cell
|
||||
)
|
||||
const desiredCells = withTargetCap(targetHardCap)
|
||||
const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600)
|
||||
const topologyPhase = isDeepStrictEqual(currentCells, desiredCells)
|
||||
? 'desired'
|
||||
: isDeepStrictEqual(currentCells, predecessorCells)
|
||||
? 'predecessor'
|
||||
: null
|
||||
if (!topologyPhase) throw new Error('director topology is not a reviewed transition state')
|
||||
return {
|
||||
topologyPhase,
|
||||
directorReady:
|
||||
topologyPhase === 'desired' && currentIdentity === capacityServiceAccount,
|
||||
desiredCells
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
const capacityServiceAccount = parseArguments(argv)
|
||||
const state = JSON.parse(readFileSync(0, 'utf8'))
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
event: 'relay_production_capacity_director_classified',
|
||||
...classifyProductionCapacityDirector(state, capacityServiceAccount)
|
||||
})}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
classifyProductionCapacityDirector
|
||||
} from './classify-relay-production-capacity-director.mjs'
|
||||
|
||||
const capacityServiceAccount =
|
||||
'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com'
|
||||
const capacityCellIds = ['production-gce-c25', 'production-gce-c26']
|
||||
const baseCells = [
|
||||
{ id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 },
|
||||
{ id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 }
|
||||
]
|
||||
const mixedCells = [
|
||||
baseCells[0],
|
||||
{ ...baseCells[1], connectionHardCap: 600 },
|
||||
baseCells[2]
|
||||
]
|
||||
|
||||
function state(overrides = {}) {
|
||||
return {
|
||||
baseCells,
|
||||
currentCells: mixedCells,
|
||||
capacityCellIds,
|
||||
targetCellId: 'production-gce-c25',
|
||||
targetHardCap: 1000,
|
||||
currentCapacityServiceAccount: capacityServiceAccount,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('classifies one target while preserving completed rollout cells', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state(), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: baseCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('skips deployment only for exact topology and identity', () => {
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount),
|
||||
{ topologyPhase: 'desired', directorReady: true, desiredCells: baseCells }
|
||||
)
|
||||
assert.deepEqual(
|
||||
classifyProductionCapacityDirector(state({
|
||||
currentCells: baseCells,
|
||||
targetHardCap: 600
|
||||
}), capacityServiceAccount),
|
||||
{
|
||||
topologyPhase: 'predecessor',
|
||||
directorReady: false,
|
||||
desiredCells: mixedCells
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects an unknown topology or capacity identity', () => {
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/rollout state is invalid/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com'
|
||||
}), capacityServiceAccount),
|
||||
/unexpected capacity identity/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the capacity rollout/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]]
|
||||
}), capacityServiceAccount),
|
||||
/outside the reviewed capacity envelope/
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyProductionCapacityDirector(state({
|
||||
targetCellId: 'production-gce-c17'
|
||||
}), capacityServiceAccount),
|
||||
/transition input is invalid/
|
||||
)
|
||||
})
|
||||
@@ -1,47 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) {
|
||||
for (const kind of [c2Kind, c3Kind]) {
|
||||
if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid')
|
||||
}
|
||||
for (const admission of [c2Admission, c3Admission]) {
|
||||
if (!['general', 'migration-only'].includes(admission)) {
|
||||
throw new Error('bootstrap admission is not recoverable')
|
||||
}
|
||||
}
|
||||
if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete'
|
||||
if (c2Kind === 'modern') return 'roll-c3'
|
||||
if (c3Kind === 'modern') return 'roll-c2'
|
||||
if (c2Admission === 'general') return 'normalize-and-roll-both'
|
||||
if (c3Admission === 'general') return 'resume-c2-then-c3'
|
||||
throw new Error('bootstrap has no general fallback')
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
return {
|
||||
c2Kind: values['c2-kind'],
|
||||
c2Admission: values['c2-admission'],
|
||||
c3Kind: values['c3-kind'],
|
||||
c3Admission: values['c3-admission']
|
||||
}
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs'
|
||||
|
||||
const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({
|
||||
c2Kind,
|
||||
c2Admission,
|
||||
c3Kind,
|
||||
c3Admission
|
||||
})
|
||||
|
||||
test('classifies the fresh legacy bootstrap', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('retries normalization after a failed C3 restart', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')),
|
||||
'normalize-and-roll-both'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 isolation', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')),
|
||||
'resume-c2-then-c3'
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes after C2 apply or a partial C3 transition', () => {
|
||||
for (const c2Admission of ['general', 'migration-only']) {
|
||||
for (const c3Admission of ['general', 'migration-only']) {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)),
|
||||
'roll-c3'
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('repairs an unexpected modern C3 before rolling legacy C2', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')),
|
||||
'roll-c2'
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts already complete modern cells and rejects no-fallback legacy state', () => {
|
||||
assert.equal(
|
||||
classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')),
|
||||
'complete'
|
||||
)
|
||||
assert.throws(
|
||||
() => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')),
|
||||
/no general fallback/
|
||||
)
|
||||
})
|
||||
@@ -1,314 +0,0 @@
|
||||
// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL
|
||||
// instance. Hand lists go stale silently; everything here is read back off disk.
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import {
|
||||
RELAY_WORKFLOW_DIRECTORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayWorkflowFile
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY
|
||||
export const SCRIPT_ROOT = new URL('./', import.meta.url)
|
||||
|
||||
export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease'
|
||||
export const PRODUCTION_LEASE = {
|
||||
bucket: 'onorca-cloud-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/production.lock'
|
||||
}
|
||||
export const STAGING_LEASE = {
|
||||
bucket: 'onorca-cloud-staging-terraform-state',
|
||||
object: 'terraform/state/cloud-sql-rollout/staging.lock'
|
||||
}
|
||||
|
||||
export const PRODUCTION_GROUP = 'production-cloud-sql-rollout'
|
||||
export const STAGING_GROUP = 'relay-staging-mutation'
|
||||
export const SELECTABLE_GROUP =
|
||||
"${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}"
|
||||
// Prove Relay Asia Staging's load mode drives traffic only and must not hold the staging lock.
|
||||
export const STAGING_OR_LOAD_GROUP =
|
||||
"${{ inputs.mode == 'load' && 'relay-staging-load' || 'relay-staging-mutation' }}"
|
||||
|
||||
const selectable = (production, staging) =>
|
||||
`\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}`
|
||||
|
||||
export const SELECTABLE_LEASE = {
|
||||
bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket),
|
||||
object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object)
|
||||
}
|
||||
|
||||
export const LOCK_GROUPS = new Set([
|
||||
PRODUCTION_GROUP,
|
||||
STAGING_GROUP,
|
||||
SELECTABLE_GROUP,
|
||||
STAGING_OR_LOAD_GROUP
|
||||
])
|
||||
|
||||
export function readWorkflow(file) {
|
||||
return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8')
|
||||
}
|
||||
|
||||
export function workflowFiles() {
|
||||
return readdirSync(WORKFLOW_ROOT)
|
||||
.filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX))
|
||||
.sort()
|
||||
}
|
||||
|
||||
// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) ---
|
||||
|
||||
function indentOf(line) {
|
||||
return line.length - line.trimStart().length
|
||||
}
|
||||
|
||||
function blockAfter(lines, index) {
|
||||
const base = indentOf(lines[index])
|
||||
const body = []
|
||||
for (let i = index + 1; i < lines.length; i += 1) {
|
||||
if (lines[i].trim() === '') {
|
||||
body.push(lines[i])
|
||||
continue
|
||||
}
|
||||
if (indentOf(lines[i]) <= base) break
|
||||
body.push(lines[i])
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
export function concurrencyBlocks(text) {
|
||||
const lines = text.split('\n')
|
||||
const blocks = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== 'concurrency:') return
|
||||
const body = blockAfter(lines, index)
|
||||
blocks.push({
|
||||
group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(),
|
||||
cancelInProgress: body
|
||||
.find((l) => l.trim().startsWith('cancel-in-progress:'))
|
||||
?.trim()
|
||||
.slice('cancel-in-progress:'.length)
|
||||
.trim()
|
||||
})
|
||||
})
|
||||
return blocks
|
||||
}
|
||||
|
||||
export function jobs(text) {
|
||||
const lines = text.split('\n')
|
||||
const start = lines.findIndex((line) => line === 'jobs:')
|
||||
if (start === -1) return []
|
||||
const found = []
|
||||
for (let i = start + 1; i < lines.length; i += 1) {
|
||||
const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i])
|
||||
if (!match) continue
|
||||
found.push({ id: match[1], start: i, body: blockAfter(lines, i) })
|
||||
}
|
||||
return found.map((job) => ({ ...job, text: job.body.join('\n') }))
|
||||
}
|
||||
|
||||
function scalarField(jobText, key) {
|
||||
const lines = jobText.split('\n')
|
||||
const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`))
|
||||
if (index === -1) return undefined
|
||||
const inline = lines[index].trim().slice(`${key}:`.length).trim()
|
||||
if (inline !== '' && inline !== '>-' && inline !== '|') return inline
|
||||
return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
export function jobNeeds(jobText) {
|
||||
const raw = scalarField(jobText, 'needs')
|
||||
if (!raw) return []
|
||||
return raw
|
||||
.replace(/^\[|\]$/g, '')
|
||||
.split(/[,\n]|\s+-\s+/)
|
||||
.map((entry) => entry.replace(/^-/, '').trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function jobIf(jobText) {
|
||||
return scalarField(jobText, 'if') ?? ''
|
||||
}
|
||||
|
||||
export function leaseSteps(text) {
|
||||
const lines = text.split('\n')
|
||||
const steps = []
|
||||
lines.forEach((line, index) => {
|
||||
if (line.trim() !== `- uses: ${LEASE_ACTION}`) return
|
||||
const body = blockAfter(lines, index)
|
||||
const read = (key) =>
|
||||
body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim()
|
||||
steps.push({
|
||||
line: index + 1,
|
||||
bucket: read('bucket'),
|
||||
object: read('object'),
|
||||
release: read('release')
|
||||
})
|
||||
})
|
||||
return steps
|
||||
}
|
||||
|
||||
export function leaseStepsByJob(file) {
|
||||
const text = readWorkflow(file)
|
||||
const steps = leaseSteps(text)
|
||||
return jobs(text).map((job) => ({
|
||||
id: job.id,
|
||||
steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length)
|
||||
}))
|
||||
}
|
||||
|
||||
// --- trigger and reusable-call graph ---
|
||||
|
||||
export function triggers(text) {
|
||||
const lines = text.split('\n')
|
||||
const index = lines.findIndex((line) => line === 'on:')
|
||||
if (index === -1) return []
|
||||
return blockAfter(lines, index)
|
||||
.map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1])
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function isEntrypoint(text) {
|
||||
return triggers(text).some((trigger) => trigger !== 'workflow_call')
|
||||
}
|
||||
|
||||
export function reusableCalls(text) {
|
||||
const counts = new Map()
|
||||
for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) {
|
||||
counts.set(match[1], (counts.get(match[1]) ?? 0) + 1)
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
export function entrypointsFor(file, seen = new Set()) {
|
||||
if (seen.has(file)) return new Set()
|
||||
seen.add(file)
|
||||
if (isEntrypoint(readWorkflow(file))) return new Set([file])
|
||||
const reached = new Set()
|
||||
for (const candidate of workflowFiles()) {
|
||||
if (candidate === file) continue
|
||||
if (!reusableCalls(readWorkflow(candidate)).has(file)) continue
|
||||
for (const entry of entrypointsFor(candidate, seen)) reached.add(entry)
|
||||
}
|
||||
return reached
|
||||
}
|
||||
|
||||
// --- what counts as a Cloud SQL connection-budget rollout ---
|
||||
|
||||
const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/
|
||||
|
||||
function commandLines(text) {
|
||||
return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, ''))
|
||||
}
|
||||
|
||||
export function appliesTerraform(text) {
|
||||
return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line))
|
||||
}
|
||||
|
||||
export function runsCloudRunMutation(text) {
|
||||
return commandLines(text).some((line) =>
|
||||
/^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line)
|
||||
)
|
||||
}
|
||||
|
||||
// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them.
|
||||
export function revisionMintingScripts() {
|
||||
const self = new URL(import.meta.url).pathname.split('/').pop()
|
||||
const names = readdirSync(SCRIPT_ROOT).filter(
|
||||
(name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self
|
||||
)
|
||||
const source = new Map(
|
||||
names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')])
|
||||
)
|
||||
const minting = new Set(
|
||||
names.filter((name) => {
|
||||
const text = source.get(name)
|
||||
return (
|
||||
text.includes("'--no-traffic'") ||
|
||||
/'run',\s*'services',\s*'update'/.test(text) ||
|
||||
/'run',\s*'deploy'/.test(text)
|
||||
)
|
||||
})
|
||||
)
|
||||
for (let changed = true; changed; ) {
|
||||
changed = false
|
||||
for (const name of names) {
|
||||
if (minting.has(name)) continue
|
||||
const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map(
|
||||
(match) => match[1]
|
||||
)
|
||||
if (!imports.some((imported) => minting.has(imported))) continue
|
||||
minting.add(name)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return minting
|
||||
}
|
||||
|
||||
export function mutatesSharedInstance(text, minters = revisionMintingScripts()) {
|
||||
if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates'
|
||||
if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision'
|
||||
for (const script of minters) {
|
||||
if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision`
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// --- the declared contract ---
|
||||
|
||||
const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra })
|
||||
const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra })
|
||||
const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP })
|
||||
|
||||
// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place.
|
||||
const named = (entries) =>
|
||||
Object.fromEntries(
|
||||
entries.map(([file, entry]) => [
|
||||
relayWorkflowFile(file),
|
||||
entry.leaseFiles
|
||||
? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) }
|
||||
: entry
|
||||
])
|
||||
)
|
||||
|
||||
export const LEASED_WORKFLOWS = named([
|
||||
['deploy-relay-fence-broker.yml', production()],
|
||||
['deploy-relay-production.yml', production()],
|
||||
['deploy-relay-production-director.yml', production()],
|
||||
['deploy-relay-production-multi-target.yml', production()],
|
||||
[
|
||||
'deploy-relay-production-capacity.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-capacity-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'deploy-relay-production-same-cap.yml',
|
||||
production({
|
||||
leaseFiles: ['deploy-relay-production-same-cap-job.yml'],
|
||||
reentrant: true
|
||||
})
|
||||
],
|
||||
[
|
||||
'operate-relay-production-rehome.yml',
|
||||
production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
|
||||
],
|
||||
['deploy-relay-asia-topology.yml', eitherEnvironment()],
|
||||
['operate-relay-asia-admission.yml', eitherEnvironment()],
|
||||
['deploy-relay-staging.yml', staging()],
|
||||
['deploy-relay-staging-gce-candidate.yml', staging()],
|
||||
['bootstrap-relay-staging-capacity.yml', staging()],
|
||||
['power-relay-staging.yml', staging()],
|
||||
['prove-relay-asia-staging.yml', staging({ group: STAGING_OR_LOAD_GROUP })],
|
||||
[
|
||||
'prove-relay-staging-capacity.yml',
|
||||
staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" })
|
||||
],
|
||||
['recover-relay-staging-c4-image.yml', staging()]
|
||||
])
|
||||
|
||||
export const NOT_A_CLOUD_SQL_CANDIDATE = named([
|
||||
['push-deploy.yml', 'Push uses dedicated SQL and its own production-push-rollout group and durable push-rollout lease.'],
|
||||
[
|
||||
'monitor-relay-production.yml',
|
||||
'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.'
|
||||
]
|
||||
])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,876 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
inspectAdmissionSelector,
|
||||
selectorCellState,
|
||||
transitionAdmissionSelector
|
||||
} from './relay-admission-selector.mjs'
|
||||
import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
|
||||
|
||||
const DEFAULT_POLL_INTERVAL_MS = 5_000
|
||||
const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000
|
||||
const ADMIN_RETRY_ATTEMPTS = 3
|
||||
const ADMIN_RETRY_BASE_MS = 250
|
||||
const CONNECTION_CONTROL_REBIND_RESERVE = 100
|
||||
const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000])
|
||||
const RETRYABLE_ADMIN_PATHS = new Set([
|
||||
'/v1/admin/runtime-status',
|
||||
'/v1/admin/cell-status',
|
||||
'/v1/admin/evacuation-capacity',
|
||||
'/v1/admin/evacuation-status'
|
||||
])
|
||||
|
||||
function canonicalOrigin(value, name) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') {
|
||||
throw new Error(`${name} must be a canonical HTTPS origin`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function adminAudience(value) {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'https:' ||
|
||||
url.pathname !== '/v1/admin/drain' ||
|
||||
url.search ||
|
||||
url.hash ||
|
||||
url.toString() !== value
|
||||
) {
|
||||
throw new Error('--admin-audience must be the canonical HTTPS director drain URL')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) {
|
||||
throw new Error(`${name} must be a positive integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function nonnegativeInteger(value, name) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a nonnegative integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of [
|
||||
'project',
|
||||
'director-origin',
|
||||
'admin-audience',
|
||||
'topology-file',
|
||||
'source-cell-id',
|
||||
'target-cell-id',
|
||||
'runtime-service-account',
|
||||
'mode'
|
||||
]) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (
|
||||
![
|
||||
'audit',
|
||||
'preflight',
|
||||
'recover-forward',
|
||||
'continue-evacuation',
|
||||
'disable-cell',
|
||||
'execute',
|
||||
'reset-empty-candidate',
|
||||
'enable-empty-cell'
|
||||
].includes(values.mode)
|
||||
) {
|
||||
throw new Error(
|
||||
'--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell'
|
||||
)
|
||||
}
|
||||
return {
|
||||
project: values.project,
|
||||
directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'),
|
||||
adminAudience: adminAudience(values['admin-audience']),
|
||||
topologyFile: values['topology-file'],
|
||||
sourceCellId: values['source-cell-id'],
|
||||
targetCellId: values['target-cell-id'],
|
||||
runtimeServiceAccount: values['runtime-service-account'],
|
||||
mode: values.mode,
|
||||
batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100),
|
||||
drainGraceMs: positiveInteger(
|
||||
values['drain-grace-ms'] ?? 120_000,
|
||||
'--drain-grace-ms',
|
||||
60 * 60 * 1_000
|
||||
),
|
||||
pollIntervalMs: positiveInteger(
|
||||
values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS,
|
||||
'--poll-interval-ms',
|
||||
60_000
|
||||
),
|
||||
timeoutMs: positiveInteger(
|
||||
values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS,
|
||||
'--timeout-ms',
|
||||
60 * 60 * 1_000
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function deployment(value, cellId) {
|
||||
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
|
||||
const expected = {
|
||||
cellId,
|
||||
origin: canonicalOrigin(value.origin, `${cellId} origin`),
|
||||
region: String(value.region ?? 'us-central1'),
|
||||
zone: String(value.zone ?? ''),
|
||||
migName: String(value.mig_name ?? ''),
|
||||
instanceGroup: String(value.instance_group ?? ''),
|
||||
backendName: String(value.backend_name ?? ''),
|
||||
backendId: String(value.backend_id ?? ''),
|
||||
urlMapName: String(value.url_map_name ?? ''),
|
||||
generationIdentity: String(value.generation_identity ?? ''),
|
||||
image: String(value.image ?? ''),
|
||||
imageDigest: String(value.image ?? '').split('@')[1] ?? '',
|
||||
capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`),
|
||||
databasePoolMax: positiveInteger(
|
||||
value.database_pool_max ?? 10,
|
||||
`${cellId} database pool maximum`,
|
||||
100
|
||||
),
|
||||
connectionHardCap:
|
||||
value.connection_hard_cap === null || value.connection_hard_cap === undefined
|
||||
? undefined
|
||||
: positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`),
|
||||
connectionUnobservedBound:
|
||||
value.connection_unobserved_bound === null ||
|
||||
value.connection_unobserved_bound === undefined
|
||||
? undefined
|
||||
: nonnegativeInteger(
|
||||
value.connection_unobserved_bound,
|
||||
`${cellId} unobserved connection bound`
|
||||
),
|
||||
initiallyEnabled: value.initially_enabled,
|
||||
fenced: value.fenced,
|
||||
desiredTargetSize: value.desired_target_size
|
||||
}
|
||||
if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`)
|
||||
if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) {
|
||||
throw new Error(`${cellId} has an invalid region`)
|
||||
}
|
||||
for (const [name, resource] of [
|
||||
['MIG', expected.migName],
|
||||
['instance group', expected.instanceGroup],
|
||||
['backend', expected.backendName],
|
||||
['backend ID', expected.backendId]
|
||||
]) {
|
||||
if (!resource) throw new Error(`${cellId} has no ${name}`)
|
||||
}
|
||||
if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) {
|
||||
throw new Error(`${cellId} image is not digest-pinned`)
|
||||
}
|
||||
if (typeof expected.initiallyEnabled !== 'boolean') {
|
||||
throw new Error(`${cellId} has no initial admission state`)
|
||||
}
|
||||
if (
|
||||
(expected.connectionHardCap === undefined) !==
|
||||
(expected.connectionUnobservedBound === undefined) ||
|
||||
(expected.connectionHardCap !== undefined &&
|
||||
(!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) ||
|
||||
expected.connectionUnobservedBound >=
|
||||
expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE))
|
||||
) {
|
||||
throw new Error(`${cellId} has invalid connection capacity`)
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
export function assertDeploymentConnectionCapacity(expected, runtime, director) {
|
||||
if (expected.connectionHardCap === undefined) {
|
||||
if (runtime !== null || director !== null) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
return
|
||||
}
|
||||
const hardCap = expected.connectionHardCap
|
||||
const unobservedBound = expected.connectionUnobservedBound
|
||||
const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE
|
||||
const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound
|
||||
const matches = (capacity) =>
|
||||
capacity?.hardCap === hardCap &&
|
||||
capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE &&
|
||||
capacity.ordinaryConnectionLimit === ordinaryConnectionLimit &&
|
||||
capacity.unobservedBound === unobservedBound &&
|
||||
capacity.normalAdmissionPause === normalAdmissionPause
|
||||
if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) {
|
||||
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectDeployments(topology, sourceCellId, targetCellId) {
|
||||
if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ')
|
||||
const source = deployment(topology[sourceCellId], sourceCellId)
|
||||
const target = deployment(topology[targetCellId], targetCellId)
|
||||
for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) {
|
||||
if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`)
|
||||
}
|
||||
if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled')
|
||||
return { source, target }
|
||||
}
|
||||
|
||||
export function validateMig(mig, instances, expected) {
|
||||
if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`)
|
||||
const policy = mig.updatePolicy ?? {}
|
||||
if (
|
||||
policy.replacementMethod !== 'RECREATE' ||
|
||||
Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 ||
|
||||
Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1
|
||||
) {
|
||||
throw new Error(`${expected.cellId} MIG replacement policy is unsafe`)
|
||||
}
|
||||
const serving = instances.filter(
|
||||
(entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE'
|
||||
)
|
||||
if (instances.length !== 1 || serving.length !== 1) {
|
||||
throw new Error(`${expected.cellId} MIG must have one running endpoint`)
|
||||
}
|
||||
return serving[0].instance.split('/').at(-1)
|
||||
}
|
||||
|
||||
export function validateInstance(instance, expected, runtimeServiceAccount) {
|
||||
const publicConfigs = (instance.networkInterfaces ?? []).flatMap(
|
||||
(network) => network.accessConfigs ?? []
|
||||
)
|
||||
if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`)
|
||||
const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email)
|
||||
if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) {
|
||||
throw new Error(`${expected.cellId} runtime service account mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateBackend(backend, expected) {
|
||||
if (
|
||||
backend.protocol !== 'HTTP' ||
|
||||
Number(backend.timeoutSec) !== 86_400 ||
|
||||
(backend.backends ?? []).length !== 1 ||
|
||||
backend.backends[0].group !== expected.instanceGroup
|
||||
) {
|
||||
throw new Error(`${expected.cellId} backend topology mismatch`)
|
||||
}
|
||||
}
|
||||
|
||||
export function defaultCommandJson(args) {
|
||||
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
|
||||
}
|
||||
return JSON.parse(result.stdout)
|
||||
}
|
||||
|
||||
export function suppliedAdminIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'admin')
|
||||
}
|
||||
|
||||
export function suppliedFenceMutationIdentityToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN
|
||||
if (token === undefined) return null
|
||||
return validatedIdentityToken(token, 'fence mutation')
|
||||
}
|
||||
|
||||
function validatedIdentityToken(token, label) {
|
||||
// WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly.
|
||||
if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
||||
throw new Error(`invalid supplied ${label} identity token`)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
export function defaultIdentityToken(audience) {
|
||||
const supplied = suppliedAdminIdentityToken()
|
||||
if (supplied !== null) return supplied
|
||||
const result = spawnSync(
|
||||
'gcloud',
|
||||
['auth', 'print-identity-token', `--audiences=${audience}`],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
|
||||
)
|
||||
if (result.status !== 0) throw new Error('gcloud identity-token command failed')
|
||||
return result.stdout.trim()
|
||||
}
|
||||
|
||||
async function responseJson(response, label, cellId) {
|
||||
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
||||
if (isReserveCellRefusal(response.status, body)) throw reserveCellRollRefusal(cellId)
|
||||
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
export function createAdminPost(config, deps, token) {
|
||||
return async (origin, path, body) => {
|
||||
const requestToken = typeof token === 'function' ? token(path) : token
|
||||
for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) {
|
||||
let response
|
||||
try {
|
||||
response = await deps.fetch(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${requestToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
} catch (error) {
|
||||
if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error
|
||||
deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' })
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
if (
|
||||
RETRYABLE_ADMIN_PATHS.has(path) &&
|
||||
([502, 503, 504].includes(response.status) ||
|
||||
(path === '/v1/admin/evacuation-status' && response.status === 500)) &&
|
||||
attempt < ADMIN_RETRY_ATTEMPTS
|
||||
) {
|
||||
// These endpoints are read-only or transactionally idempotent, so a
|
||||
// lost response may be retried without widening deployment authority.
|
||||
deps.emit({
|
||||
event: 'candidate_admin_retry',
|
||||
path,
|
||||
attempt,
|
||||
reason: `http_${response.status}`
|
||||
})
|
||||
await response.arrayBuffer().catch(() => undefined)
|
||||
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
|
||||
continue
|
||||
}
|
||||
// The director names no cell, so a two-cell request names both.
|
||||
const cells =
|
||||
body?.cellId ?? [body?.sourceCellId, body?.targetCellId].filter(Boolean).join(' or ')
|
||||
return await responseJson(response, path, cells || null)
|
||||
}
|
||||
throw new Error(`${path} retry attempts exhausted`)
|
||||
}
|
||||
}
|
||||
|
||||
async function checkHttp(deps, origin, path) {
|
||||
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`)
|
||||
}
|
||||
|
||||
export async function inspectCell(config, deps, adminPost, expected) {
|
||||
const common = ['--project', config.project, '--zone', expected.zone, '--format=json']
|
||||
const mig = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'describe',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instances = deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'list-instances',
|
||||
expected.migName,
|
||||
...common
|
||||
])
|
||||
const instanceName = validateMig(mig, instances, expected)
|
||||
const instance = deps.commandJson([
|
||||
'compute',
|
||||
'instances',
|
||||
'describe',
|
||||
instanceName,
|
||||
...common
|
||||
])
|
||||
validateInstance(instance, expected, config.runtimeServiceAccount)
|
||||
const backend = deps.commandJson([
|
||||
'compute',
|
||||
'backend-services',
|
||||
'describe',
|
||||
expected.backendName,
|
||||
'--global',
|
||||
'--project',
|
||||
config.project,
|
||||
'--format=json'
|
||||
])
|
||||
validateBackend(backend, expected)
|
||||
await checkHttp(deps, expected.origin, '/health')
|
||||
await checkHttp(deps, expected.origin, '/ready')
|
||||
const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 })
|
||||
if (
|
||||
runtime.role !== 'cell' ||
|
||||
runtime.cellId !== expected.cellId ||
|
||||
runtime.cellUrl !== expected.origin ||
|
||||
(runtime.region ?? 'us-central1') !== expected.region ||
|
||||
runtime.imageDigest !== expected.imageDigest
|
||||
) {
|
||||
throw new Error(`${expected.cellId} served runtime does not match Terraform topology`)
|
||||
}
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: expected.cellId
|
||||
})
|
||||
if (
|
||||
status.status?.cellUrl !== expected.origin ||
|
||||
(status.status?.region ?? 'us-central1') !== expected.region ||
|
||||
status.status?.runtime?.cellUrl !== expected.origin ||
|
||||
status.status?.runtime?.ready !== true ||
|
||||
status.status?.runtime?.heartbeatFresh !== true
|
||||
) {
|
||||
throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`)
|
||||
}
|
||||
assertDeploymentConnectionCapacity(
|
||||
expected,
|
||||
runtime.connectionCapacity ?? null,
|
||||
status.status.connectionCapacity ?? null
|
||||
)
|
||||
return {
|
||||
...status.status,
|
||||
draining: runtime.draining === true,
|
||||
process: runtime.runtime ?? null,
|
||||
runtimeConnectionCapacity: runtime.connectionCapacity ?? null
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForMigration(config, deps, adminPost, completeReady) {
|
||||
const deadline = deps.now() + config.timeoutMs
|
||||
while (deps.now() < deadline) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: config.sourceCellId,
|
||||
targetCellId: config.targetCellId,
|
||||
completeReady
|
||||
})
|
||||
deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status })
|
||||
if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) {
|
||||
return status
|
||||
}
|
||||
if (
|
||||
completeReady &&
|
||||
status.targetRegistered === status.inProgress &&
|
||||
status.registeredSourceActive === 0 &&
|
||||
status.registeredCompletable === 0 &&
|
||||
status.registeredTargetInactive === status.inProgress
|
||||
) {
|
||||
// CI waiting cannot revive an offline desktop; keep its proven migration
|
||||
// pending until that target control reconnects.
|
||||
return status
|
||||
}
|
||||
await deps.wait(config.pollIntervalMs)
|
||||
}
|
||||
throw new Error('timed out waiting for candidate migration')
|
||||
}
|
||||
|
||||
export async function setCellState(config, adminPost, cellId, enabled) {
|
||||
const post = async (path, body) => await adminPost(config.directorOrigin, path, body)
|
||||
const inspected = await inspectAdmissionSelector(post)
|
||||
if (inspected.selector.generation > 0) {
|
||||
await transitionAdmissionSelector(post, {
|
||||
[cellId]: enabled ? 'general' : 'existing-only'
|
||||
})
|
||||
return
|
||||
}
|
||||
await post('/v1/admin/cell-state', { v: 1, cellId, enabled })
|
||||
}
|
||||
|
||||
function assertNoDurableActivity(status, operation) {
|
||||
const activity = [
|
||||
status.assignments,
|
||||
status.activityLeases,
|
||||
status.reservedRequests,
|
||||
status.outgoingMigrations,
|
||||
status.incomingMigrations
|
||||
]
|
||||
if (activity.some((value) => Number(value) !== 0)) {
|
||||
throw new Error(`${operation} requires zero durable activity`)
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
allowEmptyAdmissionRollback,
|
||||
selectorActive
|
||||
) {
|
||||
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
}).catch(() => null)
|
||||
if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
await setCellState(config, adminPost, target.cellId, false).catch(() => undefined)
|
||||
return
|
||||
}
|
||||
if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) {
|
||||
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
|
||||
deps.emit({
|
||||
event: 'candidate_rollback_waiting_for_lease_expiry',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: status.inProgress
|
||||
})
|
||||
return
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_forward_recovery_required',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
targetRegistered: status?.targetRegistered ?? null
|
||||
})
|
||||
}
|
||||
|
||||
export async function drainSource(
|
||||
config,
|
||||
deps,
|
||||
token,
|
||||
source,
|
||||
graceMs = config.drainGraceMs,
|
||||
traceValue
|
||||
) {
|
||||
const response = await deps.fetch(`${source.origin}/v1/admin/drain`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
...(traceValue ? { 'x-orca-drain-trace': traceValue } : {})
|
||||
},
|
||||
body: JSON.stringify({ v: 1, graceMs }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
await responseJson(response, 'source drain')
|
||||
return {
|
||||
backendStatus: response.status,
|
||||
backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
event,
|
||||
eventName = 'candidate_complete'
|
||||
) {
|
||||
const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: source.cellId
|
||||
})
|
||||
const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: target.cellId
|
||||
})
|
||||
if (
|
||||
finalSource.status.activityLeases !== 0 ||
|
||||
finalSource.status.reservedRequests !== 0 ||
|
||||
finalSource.status.outgoingMigrations !== 0 ||
|
||||
finalSource.status.runtime?.observedRequests !== 0 ||
|
||||
finalTarget.status.incomingMigrations !== 0 ||
|
||||
finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits
|
||||
) {
|
||||
throw new Error('aggregate post-migration counts are not reconciled')
|
||||
}
|
||||
event({
|
||||
event: eventName,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
dormantSourceAssignments: finalSource.status.assignments,
|
||||
targetAssignments: finalTarget.status.assignments,
|
||||
targetActivityLeases: finalTarget.status.activityLeases,
|
||||
targetReservedRequests: finalTarget.status.reservedRequests
|
||||
})
|
||||
}
|
||||
|
||||
export async function runCandidateDeployment(config, overrides = {}) {
|
||||
const deps = {
|
||||
commandJson: overrides.commandJson ?? defaultCommandJson,
|
||||
identityToken: overrides.identityToken ?? defaultIdentityToken,
|
||||
fetch: overrides.fetch ?? fetch,
|
||||
emit:
|
||||
overrides.emit ??
|
||||
((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
|
||||
now: overrides.now ?? Date.now,
|
||||
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
||||
random: overrides.random ?? Math.random
|
||||
}
|
||||
const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8'))
|
||||
const { source, target } = selectDeployments(
|
||||
topology,
|
||||
config.sourceCellId,
|
||||
config.targetCellId
|
||||
)
|
||||
const token = deps.identityToken(config.adminAudience)
|
||||
const adminPost = createAdminPost(config, deps, token)
|
||||
const selectorPost = async (path, body) =>
|
||||
await adminPost(config.directorOrigin, path, body)
|
||||
const selectorInspection = await inspectAdmissionSelector(selectorPost)
|
||||
const selectorActive = selectorInspection.selector.generation > 0
|
||||
const sourceStatus = await inspectCell(config, deps, adminPost, source)
|
||||
const targetStatus = await inspectCell(config, deps, adminPost, target)
|
||||
const sourceAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, source.cellId)
|
||||
: sourceStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
const targetAdmission = selectorActive
|
||||
? selectorCellState(selectorInspection.selector, target.cellId)
|
||||
: targetStatus.enabled
|
||||
? 'general'
|
||||
: 'existing-only'
|
||||
if (config.mode === 'audit') {
|
||||
const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
})
|
||||
// Forward recovery needs durable aggregate evidence without exposing assignment identities.
|
||||
deps.emit({
|
||||
event: 'candidate_audit',
|
||||
source: aggregateCellStatus(sourceStatus),
|
||||
target: aggregateCellStatus(targetStatus),
|
||||
migration
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'recover-forward') {
|
||||
if (
|
||||
selectorActive
|
||||
? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only'
|
||||
: sourceStatus.enabled || !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'forward recovery requires existing-only source and migration-only target'
|
||||
: 'forward recovery requires disabled source and enabled target'
|
||||
)
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
deps.emit({
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
inProgress: completion.inProgress,
|
||||
registeredSourceActive: completion.registeredSourceActive,
|
||||
registeredCompletable: completion.registeredCompletable,
|
||||
registeredTargetInactive: completion.registeredTargetInactive
|
||||
})
|
||||
throw new Error('forward recovery remains pending for inactive target controls')
|
||||
}
|
||||
await verifyCandidateCompletion(
|
||||
config,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
deps.emit,
|
||||
'candidate_forward_recovered'
|
||||
)
|
||||
return
|
||||
}
|
||||
if (config.mode === 'disable-cell') {
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
// Disabling new admission preserves origin-owned sessions and durable recovery work.
|
||||
deps.emit({
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled,
|
||||
assignments: targetStatus.assignments,
|
||||
activityLeases: targetStatus.activityLeases,
|
||||
reservedRequests: targetStatus.reservedRequests,
|
||||
outgoingMigrations: targetStatus.outgoingMigrations,
|
||||
incomingMigrations: targetStatus.incomingMigrations
|
||||
})
|
||||
return
|
||||
}
|
||||
// Repair is safe only before a candidate owns assignments or origin-scoped work.
|
||||
if (config.mode === 'reset-empty-candidate') {
|
||||
assertNoDurableActivity(targetStatus, 'candidate admission reset')
|
||||
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
|
||||
deps.emit({
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: target.cellId,
|
||||
changed: targetStatus.enabled
|
||||
})
|
||||
return
|
||||
}
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
assertNoDurableActivity(targetStatus, 'cell admission enable')
|
||||
if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) {
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false })
|
||||
return
|
||||
}
|
||||
}
|
||||
const continuingEvacuation = config.mode === 'continue-evacuation'
|
||||
if (continuingEvacuation) {
|
||||
if (
|
||||
selectorActive
|
||||
? targetAdmission !== 'migration-only'
|
||||
: !targetStatus.enabled
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive
|
||||
? 'continued evacuation requires migration-only target'
|
||||
: 'continued evacuation requires enabled target'
|
||||
)
|
||||
}
|
||||
} else if (!selectorActive && targetStatus.enabled) {
|
||||
throw new Error('candidate cell is already enabled')
|
||||
} else if (
|
||||
selectorActive &&
|
||||
!['migration-only', 'existing-only'].includes(targetAdmission)
|
||||
) {
|
||||
throw new Error('candidate cell must not be generally admitted')
|
||||
}
|
||||
const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId
|
||||
})
|
||||
if (capacity.requiredTargetUnits > capacity.availableTargetUnits) {
|
||||
throw new Error('candidate lacks survivor request-unit headroom')
|
||||
}
|
||||
deps.emit({
|
||||
event: 'candidate_preflight',
|
||||
mode: config.mode,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
sourceOrigin: source.origin,
|
||||
targetOrigin: target.origin,
|
||||
sourceMig: source.migName,
|
||||
targetMig: target.migName,
|
||||
sourceBackend: source.backendName,
|
||||
targetBackend: target.backendName,
|
||||
sourceDigest: source.imageDigest,
|
||||
targetDigest: target.imageDigest,
|
||||
sourceAssignments: capacity.sourceAssignments,
|
||||
requiredTargetUnits: capacity.requiredTargetUnits,
|
||||
availableTargetUnits: capacity.availableTargetUnits
|
||||
})
|
||||
if (config.mode === 'preflight') return
|
||||
if (config.mode === 'enable-empty-cell') {
|
||||
await setCellState(config, adminPost, target.cellId, true)
|
||||
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true })
|
||||
return
|
||||
}
|
||||
// Fresh execution starts from source-only admission; continuation preserves its target.
|
||||
if (
|
||||
!continuingEvacuation &&
|
||||
(selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled)
|
||||
) {
|
||||
throw new Error(
|
||||
selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled'
|
||||
)
|
||||
}
|
||||
if (selectorActive && targetAdmission !== 'migration-only') {
|
||||
throw new Error('target cell is not migration-only')
|
||||
}
|
||||
|
||||
let migrationsStarted = 0
|
||||
try {
|
||||
if (!selectorActive) {
|
||||
if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false)
|
||||
if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true)
|
||||
}
|
||||
for (;;) {
|
||||
const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', {
|
||||
v: 1,
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
limit: config.batchSize
|
||||
})
|
||||
migrationsStarted += result.started
|
||||
deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted })
|
||||
if (result.started === 0) break
|
||||
}
|
||||
} catch (error) {
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
!continuingEvacuation,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
try {
|
||||
if (selectorActive) {
|
||||
const currentSelector = await inspectAdmissionSelector(selectorPost)
|
||||
if (
|
||||
currentSelector.selector.generation !== selectorInspection.selector.generation ||
|
||||
JSON.stringify(currentSelector.selector.membership) !==
|
||||
JSON.stringify(selectorInspection.selector.membership)
|
||||
) {
|
||||
throw new Error('admission selector changed before drain')
|
||||
}
|
||||
}
|
||||
await drainSource(config, deps, token, source)
|
||||
await waitForMigration(config, deps, adminPost, false)
|
||||
const completion = await waitForMigration(config, deps, adminPost, true)
|
||||
if (completion.inProgress > 0) {
|
||||
throw new Error('candidate migration remains pending for inactive target controls')
|
||||
}
|
||||
} catch (error) {
|
||||
// A completion response can be lost after its transaction commits. Never
|
||||
// reverse admission here merely because no in-progress row remains.
|
||||
await recoverCandidateFailure(
|
||||
config,
|
||||
deps,
|
||||
adminPost,
|
||||
source,
|
||||
target,
|
||||
false,
|
||||
selectorActive
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
await verifyCandidateCompletion(config, adminPost, source, target, deps.emit)
|
||||
}
|
||||
|
||||
export function aggregateCellStatus(status) {
|
||||
return {
|
||||
cellId: status.cellId,
|
||||
enabled: status.enabled,
|
||||
assignments: status.assignments,
|
||||
activityLeases: status.activityLeases,
|
||||
activityRequestUnits: status.activityRequestUnits,
|
||||
reservedRequests: status.reservedRequests,
|
||||
outgoingMigrations: status.outgoingMigrations,
|
||||
incomingMigrations: status.incomingMigrations,
|
||||
runtimeReady: status.runtime?.ready ?? false,
|
||||
heartbeatFresh: status.runtime?.heartbeatFresh ?? false,
|
||||
observedRequests: status.runtime?.observedRequests ?? null
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
await runCandidateDeployment(parseArguments(argv))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,922 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertDeploymentConnectionCapacity,
|
||||
createAdminPost,
|
||||
deployment,
|
||||
parseArguments,
|
||||
runCandidateDeployment,
|
||||
selectDeployments,
|
||||
suppliedAdminIdentityToken,
|
||||
validateBackend,
|
||||
validateInstance,
|
||||
validateMig
|
||||
} from './deploy-relay-gce-candidate.mjs'
|
||||
|
||||
const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com'
|
||||
const digestA = `sha256:${'a'.repeat(64)}`
|
||||
const digestB = `sha256:${'b'.repeat(64)}`
|
||||
|
||||
test('retries evacuation-status HTTP 500 without widening other admin retries', async () => {
|
||||
const events = []
|
||||
let calls = 0
|
||||
const adminPost = createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return calls === 1
|
||||
? new Response(JSON.stringify({ error: 'transient' }), { status: 500 })
|
||||
: new Response(JSON.stringify({ ok: true }), { status: 200 })
|
||||
},
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')
|
||||
assert.deepEqual(
|
||||
await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}),
|
||||
{ ok: true }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(events, [{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'http_500'
|
||||
}])
|
||||
|
||||
calls = 0
|
||||
await assert.rejects(
|
||||
createAdminPost({}, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 })
|
||||
},
|
||||
emit: () => {},
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')('https://relay.example', '/v1/admin/runtime-status', {}),
|
||||
/runtime-status failed: persistent/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('a reserve-cell refusal names the flip back to db; other 404s keep their error', async () => {
|
||||
const post = (body) => createAdminPost({}, {
|
||||
fetch: async () => Response.json(body, { status: 404 }),
|
||||
emit: () => {},
|
||||
wait: async () => {},
|
||||
random: () => 0
|
||||
}, 'token')
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_in_reserve_mode' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: 'production-gce-c7' }
|
||||
),
|
||||
/production-gce-c7 is in reserve mode .*flip it to admitMode=db/
|
||||
)
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_not_found' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: 'production-gce-c7' }
|
||||
),
|
||||
/cell-status failed: cell_not_found/
|
||||
)
|
||||
await assert.rejects(
|
||||
post({ error: 'cell_in_reserve_mode' })(
|
||||
'https://relay.example',
|
||||
'/v1/admin/evacuation-status',
|
||||
{ v: 1, sourceCellId: 'production-gce-c7', targetCellId: 'production-gce-c8' }
|
||||
),
|
||||
/production-gce-c7 or production-gce-c8 is in reserve mode/
|
||||
)
|
||||
})
|
||||
|
||||
test('verifies a 1,000-cap cell against both runtime and director telemetry', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 1_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 1_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 840
|
||||
}
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
hardCap: 600,
|
||||
heartbeatFresh: true
|
||||
}),
|
||||
/differs from Terraform/
|
||||
)
|
||||
})
|
||||
|
||||
test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => {
|
||||
const expected = deployment(
|
||||
{
|
||||
...topology().target,
|
||||
connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
},
|
||||
'target'
|
||||
)
|
||||
const capacity = {
|
||||
hardCap: 3_000,
|
||||
controlRebindReserve: 100,
|
||||
ordinaryConnectionLimit: 2_900,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 2_840
|
||||
}
|
||||
|
||||
assert.doesNotThrow(() =>
|
||||
assertDeploymentConnectionCapacity(expected, capacity, {
|
||||
...capacity,
|
||||
heartbeatFresh: true
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
function topology() {
|
||||
return {
|
||||
source: {
|
||||
origin: 'https://c1.relay.example.com',
|
||||
zone: 'us-central1-b',
|
||||
mig_name: 'relay-c1',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c1',
|
||||
backend_name: 'relay-c1',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c1',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: true
|
||||
},
|
||||
target: {
|
||||
origin: 'https://c2.relay.example.com',
|
||||
zone: 'us-central1-c',
|
||||
mig_name: 'relay-c2',
|
||||
instance_group: 'https://compute.example/instanceGroups/relay-c2',
|
||||
backend_name: 'relay-c2',
|
||||
backend_id: 'https://compute.example/backendServices/relay-c2',
|
||||
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`,
|
||||
capacity_requests: 4_000,
|
||||
initially_enabled: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function withTopology(operation) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-'))
|
||||
const file = join(directory, 'topology.json')
|
||||
writeFileSync(file, JSON.stringify(topology()))
|
||||
return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true }))
|
||||
}
|
||||
|
||||
function config(topologyFile, mode = 'preflight') {
|
||||
return {
|
||||
project: 'test-project',
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
adminAudience: 'https://relay.example.com/v1/admin/drain',
|
||||
topologyFile,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
runtimeServiceAccount,
|
||||
mode,
|
||||
batchSize: 100,
|
||||
drainGraceMs: 120_000,
|
||||
pollIntervalMs: 1,
|
||||
timeoutMs: 1_000
|
||||
}
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function migrationResponse(body) {
|
||||
return response({
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0,
|
||||
...body
|
||||
})
|
||||
}
|
||||
|
||||
function fakeCommand(args) {
|
||||
const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1]
|
||||
if (args.includes('list-instances')) {
|
||||
return [
|
||||
{
|
||||
instance: `https://compute.example/instances/${name}-vm`,
|
||||
instanceStatus: 'RUNNING',
|
||||
currentAction: 'NONE'
|
||||
}
|
||||
]
|
||||
}
|
||||
if (args.includes('instance-groups')) {
|
||||
return {
|
||||
targetSize: 1,
|
||||
updatePolicy: {
|
||||
replacementMethod: 'RECREATE',
|
||||
maxSurge: { fixed: 0 },
|
||||
maxUnavailable: { fixed: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (args.includes('instances')) {
|
||||
return {
|
||||
networkInterfaces: [{ networkIP: '10.42.0.2' }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
}
|
||||
}
|
||||
const cell = name === 'relay-c1' ? topology().source : topology().target
|
||||
return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] }
|
||||
}
|
||||
|
||||
function harness({
|
||||
failTargetEnable = false,
|
||||
failDrain = false,
|
||||
failAfterRegistration = false,
|
||||
failCompletionResponse = false,
|
||||
sourceEnabled = true,
|
||||
targetEnabled = false,
|
||||
targetAssignments = 0,
|
||||
migrationInProgress = 0,
|
||||
migrationTargetRegistered = 0,
|
||||
migrationTargetInactive = 0,
|
||||
transientCompletionFailures = 0,
|
||||
dormantSourceAssignments = 0,
|
||||
selectorGeneration = 0
|
||||
} = {}) {
|
||||
const state = {
|
||||
source: {
|
||||
enabled: selectorGeneration > 0 ? false : sourceEnabled,
|
||||
assignments: 2,
|
||||
activityLeases: 2
|
||||
},
|
||||
target: {
|
||||
enabled: selectorGeneration > 0 ? true : targetEnabled,
|
||||
assignments: targetAssignments,
|
||||
activityLeases: targetAssignments
|
||||
}
|
||||
}
|
||||
const events = []
|
||||
const stateChanges = []
|
||||
let batch = 0
|
||||
let migrationCompleted = false
|
||||
let remainingTransientCompletionFailures = transientCompletionFailures
|
||||
const fetch = async (url, options = {}) => {
|
||||
const parsed = new URL(url)
|
||||
if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true })
|
||||
const body = JSON.parse(options.body ?? '{}')
|
||||
if (parsed.pathname === '/v1/admin/runtime-status') {
|
||||
const target = parsed.origin.includes('c2.')
|
||||
return response({
|
||||
v: 1,
|
||||
role: 'cell',
|
||||
cellId: target ? 'target' : 'source',
|
||||
cellUrl: parsed.origin,
|
||||
imageDigest: target ? digestB : digestA
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: selectorGeneration,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly:
|
||||
selectorGeneration > 0
|
||||
? ['source']
|
||||
: Object.keys(state).filter((cellId) => !state[cellId].enabled),
|
||||
migrationOnly: selectorGeneration > 0 ? ['target'] : [],
|
||||
general:
|
||||
selectorGeneration > 0
|
||||
? []
|
||||
: Object.keys(state).filter((cellId) => state[cellId].enabled)
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
const cell = state[body.cellId]
|
||||
return response({
|
||||
v: 1,
|
||||
status: {
|
||||
cellId: body.cellId,
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
enabled: cell.enabled,
|
||||
admissionState:
|
||||
selectorGeneration > 0
|
||||
? body.cellId === 'source'
|
||||
? 'existing-only'
|
||||
: 'migration-only'
|
||||
: cell.enabled
|
||||
? 'general'
|
||||
: 'existing-only',
|
||||
assignments: cell.assignments,
|
||||
activityLeases: cell.activityLeases,
|
||||
activityRequestUnits: cell.activityLeases,
|
||||
reservedRequests: cell.activityLeases,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtime: {
|
||||
cellUrl: topology()[body.cellId].origin,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: cell.activityLeases
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-capacity') {
|
||||
return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-state') {
|
||||
stateChanges.push([body.cellId, body.enabled])
|
||||
if (failTargetEnable && body.cellId === 'target' && body.enabled) {
|
||||
return response({ error: 'injected_enable_failure' }, 409)
|
||||
}
|
||||
state[body.cellId].enabled = body.enabled
|
||||
return response({ ok: true })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuate-cell') {
|
||||
if (failAfterRegistration && batch > 0) {
|
||||
return response({ error: 'injected_batch_failure' }, 503)
|
||||
}
|
||||
const started = batch++ === 0 ? 2 : 0
|
||||
return response({ v: 1, started })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/evacuation-status') {
|
||||
if (failAfterRegistration) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 1,
|
||||
registeredSourceActive: 1,
|
||||
completed: 0,
|
||||
blocked: 1
|
||||
})
|
||||
}
|
||||
if (failDrain) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (body.completeReady) {
|
||||
state.source.assignments = dormantSourceAssignments
|
||||
state.source.activityLeases = 0
|
||||
state.target.assignments = 2
|
||||
state.target.activityLeases = 2
|
||||
if (remainingTransientCompletionFailures > 0) {
|
||||
remainingTransientCompletionFailures--
|
||||
throw new TypeError('injected transient fetch failure')
|
||||
}
|
||||
if (migrationTargetInactive > 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationTargetInactive,
|
||||
targetRegistered: migrationTargetInactive,
|
||||
registeredTargetInactive: migrationTargetInactive,
|
||||
completed: 0,
|
||||
blocked: migrationTargetInactive
|
||||
})
|
||||
}
|
||||
migrationCompleted = true
|
||||
if (failCompletionResponse) {
|
||||
return response({ error: 'injected_completion_response_failure' }, 503)
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 2,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (migrationCompleted) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 0,
|
||||
targetRegistered: 0,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (batch === 0) {
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: migrationInProgress,
|
||||
targetRegistered: migrationTargetRegistered,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
return migrationResponse({
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredCompletable: 2,
|
||||
completed: 0,
|
||||
blocked: 0
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/drain') {
|
||||
return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true })
|
||||
}
|
||||
return response({ error: 'unexpected_request' }, 500)
|
||||
}
|
||||
return {
|
||||
overrides: {
|
||||
commandJson: fakeCommand,
|
||||
identityToken: () => 'secret-token-never-emitted',
|
||||
fetch,
|
||||
emit: (event) => events.push(event),
|
||||
wait: async () => undefined,
|
||||
random: () => 0
|
||||
},
|
||||
events,
|
||||
stateChanges,
|
||||
state
|
||||
}
|
||||
}
|
||||
|
||||
test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => {
|
||||
assert.throws(() => parseArguments([]), /missing --project/)
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArguments([
|
||||
'--project',
|
||||
'project',
|
||||
'--director-origin',
|
||||
'https://relay.example.com',
|
||||
'--admin-audience',
|
||||
'https://relay.example.com/not-drain',
|
||||
'--topology-file',
|
||||
'topology.json',
|
||||
'--source-cell-id',
|
||||
'source',
|
||||
'--target-cell-id',
|
||||
'target',
|
||||
'--runtime-service-account',
|
||||
runtimeServiceAccount,
|
||||
'--mode',
|
||||
'preflight'
|
||||
]),
|
||||
/director drain URL/
|
||||
)
|
||||
assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/)
|
||||
const overlapping = topology()
|
||||
overlapping.target.backend_id = overlapping.source.backend_id
|
||||
assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/)
|
||||
const enabled = topology()
|
||||
enabled.target.initially_enabled = true
|
||||
assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/)
|
||||
})
|
||||
|
||||
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
|
||||
assert.equal(suppliedAdminIdentityToken({}), null)
|
||||
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
|
||||
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
|
||||
assert.throws(() =>
|
||||
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => {
|
||||
const expected = selectDeployments(topology(), 'source', 'target').target
|
||||
assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/)
|
||||
assert.throws(
|
||||
() =>
|
||||
validateInstance(
|
||||
{
|
||||
networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }],
|
||||
serviceAccounts: [{ email: runtimeServiceAccount }]
|
||||
},
|
||||
expected,
|
||||
runtimeServiceAccount
|
||||
),
|
||||
/public IP/
|
||||
)
|
||||
assert.throws(
|
||||
() => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected),
|
||||
/topology mismatch/
|
||||
)
|
||||
})
|
||||
|
||||
test('preflights exact served digests and survivor headroom without mutating admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events[0].event, 'candidate_preflight')
|
||||
assert.equal(events[0].targetDigest, digestB)
|
||||
assert.equal(JSON.stringify(events).includes('secret-token'), false)
|
||||
})
|
||||
})
|
||||
|
||||
test('audits a partially committed migration without changing admission or completing rows', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'audit'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events, [
|
||||
{
|
||||
event: 'candidate_audit',
|
||||
source: {
|
||||
cellId: 'source',
|
||||
enabled: false,
|
||||
assignments: 2,
|
||||
activityLeases: 2,
|
||||
activityRequestUnits: 2,
|
||||
reservedRequests: 2,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 2
|
||||
},
|
||||
target: {
|
||||
cellId: 'target',
|
||||
enabled: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
activityRequestUnits: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtimeReady: true,
|
||||
heartbeatFresh: true,
|
||||
observedRequests: 1
|
||||
},
|
||||
migration: {
|
||||
v: 1,
|
||||
inProgress: 2,
|
||||
targetRegistered: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 0,
|
||||
completed: 0,
|
||||
blocked: 0,
|
||||
expiredUnregistered: 0,
|
||||
repairableExpiredUnregistered: 0,
|
||||
abortableExpiredUnregistered: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
blockedExpiredOnNewerTargetAssignment: 0
|
||||
}
|
||||
}
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('preflights with source admission disabled but still refuses execution', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.equal(events.at(-1).event, 'candidate_preflight')
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/source cell is not enabled/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly resets only an empty declared candidate to disabled admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_admission_reset',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to reset candidate admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('disables only new admission while preserving durable candidate activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'disable-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', false]])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_disabled',
|
||||
targetCellId: 'target',
|
||||
changed: true,
|
||||
assignments: 1,
|
||||
activityLeases: 1,
|
||||
reservedRequests: 1,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('explicitly enables only an empty preflighted cell for admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
|
||||
await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides)
|
||||
assert.deepEqual(stateChanges, [['target', true]])
|
||||
assert.equal(events.at(-2).event, 'candidate_preflight')
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'cell_admission_enabled',
|
||||
targetCellId: 'target',
|
||||
changed: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to enable cell admission while it owns durable activity', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ targetAssignments: 1 })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'enable-empty-cell'), overrides),
|
||||
/requires zero durable activity/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('executes target-first evacuation and verifies aggregate drained counts', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness()
|
||||
await runCandidateDeployment(config(file, 'execute'), overrides)
|
||||
assert.deepEqual(stateChanges.slice(0, 2), [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
assert.equal(events.at(-1).targetAssignments, 2)
|
||||
})
|
||||
})
|
||||
|
||||
test('executes within selector membership without legacy admission writes', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({ selectorGeneration: 1 })
|
||||
await runCandidateDeployment(config(file, 'execute'), testHarness.overrides)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
assert.equal(testHarness.state.target.enabled, true)
|
||||
})
|
||||
})
|
||||
|
||||
test('never restores legacy general admission after selector-era failure', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const testHarness = harness({
|
||||
selectorGeneration: 1,
|
||||
failAfterRegistration: true
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), testHarness.overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(testHarness.stateChanges, [])
|
||||
assert.equal(testHarness.state.source.enabled, false)
|
||||
})
|
||||
})
|
||||
|
||||
test('continues a partial evacuation without resetting target admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'continue-evacuation'), overrides)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_complete')
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses continued evacuation unless target admission is already enabled', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/requires enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves partial target admission when continued batching fails', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
failAfterRegistration: true,
|
||||
sourceEnabled: true,
|
||||
targetEnabled: true,
|
||||
targetAssignments: 1,
|
||||
migrationInProgress: 1,
|
||||
migrationTargetRegistered: 1
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [['source', false]])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
})
|
||||
})
|
||||
|
||||
test('resumes only a committed forward migration and preserves dormant source assignments', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
dormantSourceAssignments: 7
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_recovered',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
dormantSourceAssignments: 7,
|
||||
targetAssignments: 2,
|
||||
targetActivityLeases: 2,
|
||||
targetReservedRequests: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('retries a transient idempotent completion request without reversing admission', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
transientCompletionFailures: 1
|
||||
})
|
||||
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(
|
||||
events.filter(({ event }) => event === 'candidate_admin_retry'),
|
||||
[
|
||||
{
|
||||
event: 'candidate_admin_retry',
|
||||
path: '/v1/admin/evacuation-status',
|
||||
attempt: 1,
|
||||
reason: 'transport'
|
||||
}
|
||||
]
|
||||
)
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovered')
|
||||
})
|
||||
})
|
||||
|
||||
test('stops forward recovery promptly when only registered offline targets remain', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({
|
||||
sourceEnabled: false,
|
||||
targetEnabled: true,
|
||||
migrationInProgress: 2,
|
||||
migrationTargetRegistered: 2,
|
||||
migrationTargetInactive: 2
|
||||
})
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/pending for inactive target controls/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
assert.deepEqual(events.at(-1), {
|
||||
event: 'candidate_forward_pending',
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target',
|
||||
inProgress: 2,
|
||||
registeredSourceActive: 0,
|
||||
registeredCompletable: 0,
|
||||
registeredTargetInactive: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses forward recovery unless source and target admission match committed direction', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness()
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'recover-forward'), overrides),
|
||||
/requires disabled source and enabled target/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables an intact source when candidate admission fails before migration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, stateChanges } = harness({ failTargetEnable: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_enable_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true],
|
||||
['source', true],
|
||||
['target', false]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
test('re-enables the source and waits for lease rollback when no target registered', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failDrain: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_drain_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges.slice(-1), [['source', true]])
|
||||
assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry')
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves both routes for forward recovery after a target registration', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failAfterRegistration: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_batch_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 1)
|
||||
})
|
||||
})
|
||||
|
||||
test('does not reverse admission after completion commits but its response is lost', async () => {
|
||||
await withTopology(async (file) => {
|
||||
const { overrides, events, stateChanges } = harness({ failCompletionResponse: true })
|
||||
await assert.rejects(
|
||||
runCandidateDeployment(config(file, 'execute'), overrides),
|
||||
/injected_completion_response_failure/
|
||||
)
|
||||
assert.deepEqual(stateChanges, [
|
||||
['source', false],
|
||||
['target', true]
|
||||
])
|
||||
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
|
||||
assert.equal(events.at(-1).targetRegistered, 0)
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,136 +0,0 @@
|
||||
const REQUEST_TIMEOUT_MS = 15_000
|
||||
const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
|
||||
|
||||
export async function requestGitHubSmokeTokens(
|
||||
authOrigin,
|
||||
fetchImpl = fetch,
|
||||
environment = process.env,
|
||||
options = {}
|
||||
) {
|
||||
const origin = canonicalHttpsOrigin(authOrigin)
|
||||
const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable')
|
||||
const audience = `${origin}/v1/internal/github-smoke-token`
|
||||
const oidcUrl = new URL(requestUrl)
|
||||
oidcUrl.searchParams.set('audience', audience)
|
||||
const oidcResponse = await fetchImpl(oidcUrl, {
|
||||
headers: { authorization: `Bearer ${requestToken}` },
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const oidc = await readJson(oidcResponse, 'GitHub OIDC request')
|
||||
if (!oidcResponse.ok || !validJwt(oidc.value)) {
|
||||
throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`)
|
||||
}
|
||||
const exchangeResponse = await fetchImpl(audience, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${oidc.value}`,
|
||||
...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {})
|
||||
},
|
||||
...(options.relayAsiaLoad
|
||||
? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) }
|
||||
: {}),
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange')
|
||||
if (!exchangeResponse.ok) {
|
||||
throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`)
|
||||
}
|
||||
return {
|
||||
...parseAccessTokens(exchange.accessTokens),
|
||||
...(options.relayAsiaLoad
|
||||
? {
|
||||
relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals(
|
||||
exchange.relayAsiaLoadPrincipals,
|
||||
options.relayAsiaLoad.principalCount
|
||||
)
|
||||
}
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
// Staging auth mints Asia load principal sets for this many shards.
|
||||
export const RELAY_ASIA_LOAD_PRINCIPAL_SHARDS = 4
|
||||
|
||||
function parseRelayAsiaLoadOptions(value) {
|
||||
if (
|
||||
!value || typeof value !== 'object' ||
|
||||
!Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex >= RELAY_ASIA_LOAD_PRINCIPAL_SHARDS ||
|
||||
!Number.isSafeInteger(value.principalCount) || value.principalCount < 1 ||
|
||||
value.principalCount > 32
|
||||
) throw new Error('Relay Asia load principal request is invalid')
|
||||
return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount }
|
||||
}
|
||||
|
||||
function canonicalHttpsOrigin(value) {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) {
|
||||
throw new Error('auth origin must be canonical HTTPS')
|
||||
}
|
||||
return url.origin
|
||||
}
|
||||
|
||||
function validJwt(value) {
|
||||
return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value)
|
||||
}
|
||||
|
||||
function parseAccessTokens(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Orca smoke identity response is malformed')
|
||||
}
|
||||
const expected = ['outsider', 'owner', 'recipient']
|
||||
if (Object.keys(value).sort().join(',') !== expected.join(',')) {
|
||||
throw new Error('Orca smoke identity response principals are invalid')
|
||||
}
|
||||
return Object.fromEntries(
|
||||
expected.map((name) => {
|
||||
const principal = value[name]
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' ||
|
||||
!validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' ||
|
||||
principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) {
|
||||
throw new Error('Orca smoke identity response principal is malformed')
|
||||
}
|
||||
return [name, principal]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function parseRelayAsiaLoadPrincipals(value, expectedCount) {
|
||||
if (!Array.isArray(value) || value.length !== expectedCount) {
|
||||
throw new Error('Relay Asia load principal response is invalid')
|
||||
}
|
||||
const userIds = new Set()
|
||||
return value.map((principal, principalIndex) => {
|
||||
if (
|
||||
!principal || typeof principal !== 'object' ||
|
||||
principal.principalIndex !== principalIndex ||
|
||||
typeof principal.userId !== 'string' ||
|
||||
!/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) ||
|
||||
typeof principal.profileId !== 'string' ||
|
||||
principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') ||
|
||||
userIds.has(principal.userId) ||
|
||||
typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) ||
|
||||
typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() ||
|
||||
principal.expiresAt > Date.now() + 610_000
|
||||
) throw new Error('Relay Asia load principal response is malformed')
|
||||
userIds.add(principal.userId)
|
||||
return principal
|
||||
})
|
||||
}
|
||||
|
||||
async function readJson(response, label) {
|
||||
try {
|
||||
return await response.json()
|
||||
} catch {
|
||||
throw new Error(`${label} did not return JSON`)
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
|
||||
|
||||
const jwt = (value) => `${value}.${value}.${value}`
|
||||
|
||||
test('exchanges the runner OIDC token without returning request credentials', async () => {
|
||||
const requests = []
|
||||
const fetchImpl = async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
if (requests.length === 1) return Response.json({ value: jwt('github') })
|
||||
return Response.json({
|
||||
accessTokens: Object.fromEntries(
|
||||
['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])
|
||||
)
|
||||
})
|
||||
}
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
fetchImpl,
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
)
|
||||
assert.equal(result.owner.userId, 'usr_owner')
|
||||
assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/)
|
||||
assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token')
|
||||
assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`)
|
||||
})
|
||||
|
||||
test('fails with bounded errors and never includes credentials', async () => {
|
||||
await assert.rejects(
|
||||
requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => new Response('denied', { status: 403 }),
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token'
|
||||
}
|
||||
),
|
||||
(error) => {
|
||||
assert.doesNotMatch(String(error), /private-request-token|denied/)
|
||||
return true
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('requests and validates an exact Relay Asia principal batch', async () => {
|
||||
const requests = []
|
||||
const principals = Array.from({ length: 32 }, (_, principalIndex) => {
|
||||
const suffix = String(principalIndex).padStart(32, 'a')
|
||||
return {
|
||||
principalIndex,
|
||||
userId: `usr_relay_asia_load_${suffix}`,
|
||||
profileId: `prof_relay_asia_load_${suffix}`,
|
||||
accessToken: jwt(`load${principalIndex}`),
|
||||
expiresAt: Date.now() + 600_000
|
||||
}
|
||||
})
|
||||
const result = await requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
return requests.length === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: principals
|
||||
})
|
||||
},
|
||||
{
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
},
|
||||
{ relayAsiaLoad: { shardIndex: 3, principalCount: 32 } }
|
||||
)
|
||||
assert.equal(result.relayAsiaLoadPrincipals.length, 32)
|
||||
assert.deepEqual(JSON.parse(requests[1].init.body), {
|
||||
relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 }
|
||||
})
|
||||
assert.equal(requests[1].init.headers['content-type'], 'application/json')
|
||||
})
|
||||
|
||||
test('rejects malformed or duplicate Relay Asia principal batches', async () => {
|
||||
const environment = {
|
||||
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
|
||||
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
|
||||
}
|
||||
let request = 0
|
||||
await assert.rejects(requestGitHubSmokeTokens(
|
||||
'https://auth-staging.onorca.dev',
|
||||
async () => ++request === 1
|
||||
? Response.json({ value: jwt('github') })
|
||||
: Response.json({
|
||||
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
|
||||
name,
|
||||
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
|
||||
])),
|
||||
relayAsiaLoadPrincipals: []
|
||||
}),
|
||||
environment,
|
||||
{ relayAsiaLoad: { shardIndex: 0, principalCount: 32 } }
|
||||
), /principal response is invalid/)
|
||||
})
|
||||
@@ -1,146 +0,0 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs'
|
||||
|
||||
// The relay root keeps its historical path so every existing caller — 9 workflows, the fence
|
||||
// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps
|
||||
// roots stay in the private repository with the services they own.
|
||||
const ROOT_DIRECTORIES = {
|
||||
relay: join('infra', 'terraform')
|
||||
}
|
||||
|
||||
const command = process.argv[2]
|
||||
const environment = readEnvironment(process.argv.slice(3))
|
||||
const root = readRoot(process.argv.slice(3))
|
||||
const tool = process.env.IAC_TOOL || findTool()
|
||||
|
||||
if (!command || !['init', 'plan', 'apply'].includes(command)) {
|
||||
exitWithUsage()
|
||||
}
|
||||
|
||||
if (!environment) {
|
||||
exitWithUsage('Missing --env staging|production')
|
||||
}
|
||||
|
||||
if (!root) {
|
||||
exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`)
|
||||
}
|
||||
|
||||
const rootDirectory = ROOT_DIRECTORIES[root]
|
||||
const terraformDir = join(process.cwd(), rootDirectory)
|
||||
const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`)
|
||||
const varFile = join(terraformDir, 'environments', `${environment}.tfvars`)
|
||||
|
||||
if (!existsSync(backendConfig)) {
|
||||
throw new Error(`Backend config not found: ${backendConfig}`)
|
||||
}
|
||||
|
||||
if (!existsSync(varFile)) {
|
||||
throw new Error(`Variable file not found: ${varFile}`)
|
||||
}
|
||||
|
||||
const chdir = `-chdir=${rootDirectory}`
|
||||
|
||||
if (command === 'init') {
|
||||
run([chdir, 'init', `-backend-config=backend/${environment}.hcl`])
|
||||
} else if (command === 'plan') {
|
||||
run([
|
||||
chdir,
|
||||
'plan',
|
||||
`-var-file=environments/${environment}.tfvars`,
|
||||
`-out=${environment}.tfplan`
|
||||
])
|
||||
} else {
|
||||
// A normal staging apply must not implicitly wake or partially mutate a sleeping data plane.
|
||||
// Only the relay root can touch that data plane; the guard would refuse app work for no reason.
|
||||
if (environment === 'staging' && root === 'relay') assertStagingRelayAwake()
|
||||
run([chdir, 'apply', `${environment}.tfplan`])
|
||||
}
|
||||
|
||||
function readEnvironment(args) {
|
||||
const envIndex = args.indexOf('--env')
|
||||
if (envIndex >= 0) {
|
||||
return args[envIndex + 1]
|
||||
}
|
||||
|
||||
return process.env.ORCA_CLOUD_ENV
|
||||
}
|
||||
|
||||
function readRoot(args) {
|
||||
const rootIndex = args.indexOf('--root')
|
||||
const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay'
|
||||
return requested in ROOT_DIRECTORIES ? requested : undefined
|
||||
}
|
||||
|
||||
function findTool() {
|
||||
for (const candidate of ['tofu', 'terraform']) {
|
||||
try {
|
||||
execFileSync(candidate, ['version'], { stdio: 'ignore' })
|
||||
return candidate
|
||||
} catch {
|
||||
// Try the next compatible IaC binary.
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.')
|
||||
}
|
||||
|
||||
function run(args) {
|
||||
execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' })
|
||||
}
|
||||
|
||||
function terraformEnv() {
|
||||
if (
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GOOGLE_CREDENTIALS ||
|
||||
process.env.GOOGLE_OAUTH_ACCESS_TOKEN
|
||||
) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
const token = readGcloudAccessToken()
|
||||
if (!token) {
|
||||
return process.env
|
||||
}
|
||||
|
||||
// Local convenience: Terraform uses ADC, while engineers often only have
|
||||
// gcloud CLI auth. CI should use Workload Identity instead.
|
||||
return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token }
|
||||
}
|
||||
|
||||
function readGcloudAccessToken() {
|
||||
for (const candidate of gcloudCandidates()) {
|
||||
try {
|
||||
return execFileSync(candidate, ['auth', 'print-access-token'], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
}).trim()
|
||||
} catch {
|
||||
// Try the next common gcloud location.
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function gcloudCandidates() {
|
||||
return [
|
||||
process.env.GCLOUD_PATH,
|
||||
'gcloud',
|
||||
join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'),
|
||||
join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud')
|
||||
].filter(Boolean)
|
||||
}
|
||||
|
||||
function exitWithUsage(message) {
|
||||
if (message) {
|
||||
console.error(message)
|
||||
}
|
||||
|
||||
console.error(
|
||||
'Usage: pnpm infra:<init|plan|apply> --env staging|production [--root relay]'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const repository = fileURLToPath(new URL('../../', import.meta.url))
|
||||
const script = 'dev/scripts/infra.mjs'
|
||||
|
||||
// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects
|
||||
// is observable without running Terraform.
|
||||
function invoke(args) {
|
||||
return execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
}).trim()
|
||||
}
|
||||
|
||||
function rejects(args) {
|
||||
try {
|
||||
execFileSync('node', [script, ...args], {
|
||||
cwd: repository,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: { ...process.env, IAC_TOOL: 'echo' }
|
||||
})
|
||||
} catch (error) {
|
||||
return error.stderr
|
||||
}
|
||||
throw new Error(`expected ${args.join(' ')} to exit non-zero`)
|
||||
}
|
||||
|
||||
// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this
|
||||
// without --root. If the default ever moves off infra/terraform they break silently at the plan.
|
||||
test('omitting --root keeps every existing caller on the relay root', () => {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment]),
|
||||
`-chdir=infra/terraform init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
assert.equal(
|
||||
invoke(['plan', '--env', environment]),
|
||||
`-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// Only the relay root ships here; the foundation and apps roots stay in the private repository.
|
||||
test('each root name selects exactly its own directory', () => {
|
||||
const directories = { relay: 'infra/terraform' }
|
||||
for (const [root, directory] of Object.entries(directories)) {
|
||||
for (const environment of ['staging', 'production']) {
|
||||
assert.equal(
|
||||
invoke(['init', '--env', environment, '--root', root]),
|
||||
`-chdir=${directory} init -backend-config=backend/${environment}.hcl`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('an unknown root fails closed rather than falling back to the relay root', () => {
|
||||
const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay'])
|
||||
assert.match(stderr, /Unknown --root/)
|
||||
assert.doesNotMatch(stderr, /infra\/terraform /)
|
||||
})
|
||||
|
||||
test('a missing environment still fails before any root is resolved', () => {
|
||||
assert.match(rejects(['plan']), /Missing --env/)
|
||||
})
|
||||
|
||||
// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the
|
||||
// app or foundation roots would block work that never touches that data plane.
|
||||
test('the sleeping staging relay guard is scoped to the relay root', () => {
|
||||
const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8')
|
||||
assert.match(source, /environment === 'staging' && root === 'relay'/)
|
||||
})
|
||||
@@ -1,695 +0,0 @@
|
||||
import { createHash, createPrivateKey, createPublicKey } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { RelayLoadControlPeer } from './relay-load-control-peer.mjs'
|
||||
import { RELAY_ASIA_LOAD_PRINCIPAL_SHARDS, requestGitHubSmokeTokens } from './github-smoke-token.mjs'
|
||||
import { relayLoadFailureReason } from './relay-load-connection-failure.mjs'
|
||||
import {
|
||||
assertRelayLoadDirectorCapacityToken,
|
||||
waitForRelayLoadDirectorCapacity,
|
||||
waitForRelayLoadRequestUnits
|
||||
} from './relay-load-director-capacity-gate.mjs'
|
||||
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
|
||||
import {
|
||||
proveRelayLoadPlacementBoundary,
|
||||
proveRelayLoadRegionalFallback
|
||||
} from './relay-load-placement-boundary.mjs'
|
||||
import {
|
||||
proveRelayLoadRebindBoundary,
|
||||
waitForRelayLoadRebindGate
|
||||
} from './relay-load-rebind-boundary.mjs'
|
||||
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
|
||||
import { proveRelayLoadDuplicateAssign } from './relay-load-duplicate-assign.mjs'
|
||||
import {
|
||||
openRelayLoadInviteOffers,
|
||||
proveRelayLoadRequestUnitBoundary
|
||||
} from './relay-load-request-unit-boundary.mjs'
|
||||
import {
|
||||
assertRelayLoadRampAccepted,
|
||||
relayLoadRunHasDisallowedFailures,
|
||||
runRelayLoadWithShutdown
|
||||
} from './relay-load-run-lifecycle.mjs'
|
||||
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
|
||||
import { holdRelayLoadSplice, relayLoadSpliceReopenSummary } from './relay-load-splice-retry.mjs'
|
||||
import {
|
||||
installRelayLoadResumeDevice,
|
||||
relayLoadResumeSummary,
|
||||
resumeRelayLoadPhoneOnce
|
||||
} from './relay-load-resume-phone.mjs'
|
||||
import {
|
||||
parseRelayLoadArguments,
|
||||
relayLoadPrincipalIndex,
|
||||
relayLoadReaderEvidenceError,
|
||||
relayLoadResumePhoneIndexes,
|
||||
relayLoadSpliceIndexes,
|
||||
relayLoadSpliceProfile,
|
||||
relayLoadSpliceStartDelayMs
|
||||
} from './relay-load-profile.mjs'
|
||||
|
||||
function signingKey(path) {
|
||||
if (!path) return {}
|
||||
const key = createPrivateKey(readFileSync(path, 'utf8'))
|
||||
const signingKeyId = createHash('sha256')
|
||||
.update(createPublicKey(key).export({ type: 'spki', format: 'der' }))
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
return { signingKey: key, signingKeyId }
|
||||
}
|
||||
|
||||
function report(state, final = false) {
|
||||
const elapsedSeconds = Math.max(1, (Date.now() - state.startedAt) / 1000)
|
||||
const memory = process.memoryUsage()
|
||||
const cpu = process.cpuUsage(state.generatorBaselineCpu)
|
||||
const rssMiB = memory.rss / 1_048_576
|
||||
state.generatorPeakRssMiB = Math.max(state.generatorPeakRssMiB, rssMiB)
|
||||
const readerQueueEvidence = state.readerEvidence?.snapshot() ?? []
|
||||
const output = {
|
||||
event: final ? 'relay_load_complete' : 'relay_load_progress',
|
||||
controls: state.controls,
|
||||
shardCount: state.shardCount,
|
||||
shardIndex: state.shardIndex,
|
||||
configuredRampSeconds: state.rampMs / 1000,
|
||||
configuredSteadySeconds: state.durationMs / 1000,
|
||||
configuredSpliceHoldSeconds: state.spliceHoldMs / 1000,
|
||||
requiredLeaseHorizons: state.requiredLeaseHorizons,
|
||||
configuredSplices: state.splices,
|
||||
configuredSlowReaderSplices: state.slowReaderSplices,
|
||||
configuredWedgedReaderSplices: state.wedgedReaderSplices,
|
||||
active: state.active.size,
|
||||
peakActive: state.peakActive,
|
||||
steadyMinimumActive: state.steadyMinimumActive,
|
||||
connected: state.connected,
|
||||
connectionFailures: state.connectionFailures,
|
||||
rampConnectionFailures: state.rampConnectionFailures,
|
||||
steadyConnectionFailures: state.steadyConnectionFailures,
|
||||
transitionConnectionFailures: state.transitionConnectionFailures,
|
||||
connectionFailuresByReason: state.connectionFailuresByReason,
|
||||
closes: state.closes,
|
||||
unexpectedCloses: state.unexpectedCloses,
|
||||
unexpectedClosesByCode: state.unexpectedClosesByCode,
|
||||
drains: state.drains,
|
||||
pings: state.pings,
|
||||
pingRate: Number((state.pings / elapsedSeconds).toFixed(2)),
|
||||
tokens: state.tokens,
|
||||
tokenRate: Number((state.tokens / elapsedSeconds).toFixed(2)),
|
||||
refreshes: state.refreshes,
|
||||
refreshErrors: state.refreshErrors,
|
||||
protocolErrors: state.protocolErrors,
|
||||
socketErrors: state.socketErrors,
|
||||
rebindProbesOpened: state.rebindProbesOpened,
|
||||
rebindOverflowReason: state.rebindOverflowReason,
|
||||
placementOverflowReason: state.placementOverflowReason,
|
||||
regionalFallbacksProved: state.regionalFallbacksProved,
|
||||
oldClientUsFirstProved: state.oldClientUsFirstProved,
|
||||
stickyAssignmentProved: state.stickyAssignmentProved,
|
||||
duplicateAssign: state.duplicateAssign,
|
||||
requestUnitInvitesOpened: state.requestUnitInvitesOpened,
|
||||
requestUnitPrincipalCount: state.requestUnitPrincipalCount,
|
||||
relayAsiaLoadPrincipalCount: state.relayAsiaLoadPrincipalCount,
|
||||
requestUnitOverflowReason: state.requestUnitOverflowReason,
|
||||
requestUnitCleanupProved: state.requestUnitCleanupProved,
|
||||
phaseBarrierPassed: state.phaseBarrierPassed,
|
||||
activeSplices: state.activeSplices,
|
||||
peakActiveSplices: state.peakActiveSplices,
|
||||
completedSplices: state.completedSplices,
|
||||
failedSplices: state.failedSplices,
|
||||
spliceRetries: state.spliceRetries,
|
||||
spliceFailuresByReason: state.spliceFailuresByReason,
|
||||
spliceReopen: relayLoadSpliceReopenSummary(state.spliceReopenGapsMs, state.spliceFailedAt.size),
|
||||
slowReaderSplicesCompleted: state.slowReaderSplicesCompleted,
|
||||
wedgedReaderSplicesClosed: state.wedgedReaderSplicesClosed,
|
||||
readerQueueEvidence,
|
||||
readerQueuedBytesPeak: Math.max(
|
||||
0,
|
||||
...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes)
|
||||
),
|
||||
readerClosesByCode: state.readerClosesByCode,
|
||||
resumePhones: relayLoadResumeSummary(state.resumeAttempts),
|
||||
// Every cell a control connected to, so a canary can prove its load reached the target.
|
||||
assignedCellOrigins: [...state.assignedCellOrigins].sort(),
|
||||
controlHeadroom: Math.max(0, state.controls - state.active.size),
|
||||
generatorRssMiB: Number(rssMiB.toFixed(1)),
|
||||
generatorPeakRssMiB: Number(state.generatorPeakRssMiB.toFixed(1)),
|
||||
generatorRssGrowthMiB: Number(
|
||||
Math.max(0, state.generatorPeakRssMiB - state.generatorBaselineRssMiB).toFixed(1)
|
||||
),
|
||||
generatorHeapUsedMiB: Number((memory.heapUsed / 1_048_576).toFixed(1)),
|
||||
generatorCpuPercent: Number(
|
||||
(((cpu.user + cpu.system) / 1_000_000 / elapsedSeconds) * 100).toFixed(1)
|
||||
),
|
||||
generatorEventLoopP99Ms: Number((state.eventLoopDelay.percentile(99) / 1_000_000).toFixed(2)),
|
||||
shutdownEvidence: final ? state.shutdownEvidence : undefined,
|
||||
elapsedSeconds: Number(elapsedSeconds.toFixed(1))
|
||||
}
|
||||
console.log(JSON.stringify(output))
|
||||
return output
|
||||
}
|
||||
|
||||
const config = parseRelayLoadArguments(process.argv.slice(2))
|
||||
let accessToken = process.env.ORCA_RELAY_LOAD_ACCESS_TOKEN
|
||||
let accessTokenProviderForIndex
|
||||
const adminToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (
|
||||
config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0 ||
|
||||
config.slowReaderSplices + config.wedgedReaderSplices > 0 ||
|
||||
config.requestUnitOverflowProbes > 0 || config.requestUnitCleanupTimeoutMs > 0
|
||||
) {
|
||||
assertRelayLoadDirectorCapacityToken({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken
|
||||
}, Date.now,
|
||||
config.rampMs + config.durationMs + config.wedgedReaderHoldMs +
|
||||
(config.phaseBarrierDir ? 2 * config.phaseBarrierTimeoutMs : 0) +
|
||||
config.spliceRampMs + config.requestUnitCleanupTimeoutMs + 120_000)
|
||||
}
|
||||
const key = signingKey(config.signingKeyFile)
|
||||
if (!accessToken && !key.signingKey && process.env.ACTIONS_ID_TOKEN_REQUEST_URL &&
|
||||
process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) {
|
||||
let tokens
|
||||
let refresh
|
||||
const loadOptions = config.relayAsiaLoadPrincipalCount > 0
|
||||
? {
|
||||
relayAsiaLoad: {
|
||||
// Staging auth mints principal sets for shards 0-3 only; shards past that (a second
|
||||
// hosted runner) reuse a set, and their hosts stay distinct by peer index.
|
||||
shardIndex: config.shardIndex % RELAY_ASIA_LOAD_PRINCIPAL_SHARDS,
|
||||
principalCount: config.relayAsiaLoadPrincipalCount
|
||||
}
|
||||
}
|
||||
: undefined
|
||||
const smokeTokens = async () => {
|
||||
const expiresAt = config.relayAsiaLoadPrincipalCount > 0
|
||||
? tokens?.relayAsiaLoadPrincipals?.[0]?.expiresAt
|
||||
: tokens?.owner?.expiresAt
|
||||
if (expiresAt > Date.now() + 60_000) return tokens
|
||||
refresh ??= requestGitHubSmokeTokens(
|
||||
config.authOrigin,
|
||||
fetch,
|
||||
process.env,
|
||||
loadOptions
|
||||
)
|
||||
try {
|
||||
tokens = await refresh
|
||||
return tokens
|
||||
} finally {
|
||||
refresh = undefined
|
||||
}
|
||||
}
|
||||
accessTokenProviderForIndex = (index) => async () => {
|
||||
const current = await smokeTokens()
|
||||
return config.relayAsiaLoadPrincipalCount > 0
|
||||
? current.relayAsiaLoadPrincipals[
|
||||
relayLoadPrincipalIndex(
|
||||
index,
|
||||
config.shardCount,
|
||||
current.relayAsiaLoadPrincipals.length
|
||||
)
|
||||
].accessToken
|
||||
: current.owner.accessToken
|
||||
}
|
||||
await smokeTokens()
|
||||
}
|
||||
if (!accessToken && !accessTokenProviderForIndex && !key.signingKey) {
|
||||
throw new Error('provide GitHub OIDC, ORCA_RELAY_LOAD_ACCESS_TOKEN, or --signing-key-file')
|
||||
}
|
||||
const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 })
|
||||
eventLoopDelay.enable()
|
||||
const generatorBaselineRssMiB = process.memoryUsage().rss / 1_048_576
|
||||
const generatorBaselineCpu = process.cpuUsage()
|
||||
const state = {
|
||||
...config,
|
||||
startedAt: Date.now(),
|
||||
active: new Set(),
|
||||
peakActive: 0,
|
||||
steadyMinimumActive: null,
|
||||
steadyStarted: false,
|
||||
connected: 0,
|
||||
connectionFailures: 0,
|
||||
rampConnectionFailures: 0,
|
||||
steadyConnectionFailures: 0,
|
||||
transitionConnectionFailures: 0,
|
||||
connectionFailuresByReason: {},
|
||||
closes: 0,
|
||||
unexpectedCloses: 0,
|
||||
unexpectedClosesByCode: {},
|
||||
drains: 0,
|
||||
pings: 0,
|
||||
tokens: 0,
|
||||
refreshes: 0,
|
||||
refreshErrors: 0,
|
||||
protocolErrors: 0,
|
||||
socketErrors: 0,
|
||||
rebindProbesOpened: 0,
|
||||
rebindOverflowReason: null,
|
||||
placementOverflowReason: null,
|
||||
regionalFallbacksProved: 0,
|
||||
oldClientUsFirstProved: 0,
|
||||
stickyAssignmentProved: 0,
|
||||
duplicateAssign: null,
|
||||
requestUnitInvitesOpened: 0,
|
||||
requestUnitOverflowReason: null,
|
||||
requestUnitCleanupProved: 0,
|
||||
phaseBarrierPassed: false,
|
||||
activeSplices: 0,
|
||||
peakActiveSplices: 0,
|
||||
completedSplices: 0,
|
||||
failedSplices: 0,
|
||||
spliceRetries: 0,
|
||||
spliceFailuresByReason: {},
|
||||
spliceFailedAt: new Map(),
|
||||
resumeAttempts: [],
|
||||
spliceReopenGapsMs: [],
|
||||
slowReaderSplicesCompleted: 0,
|
||||
wedgedReaderSplicesClosed: 0,
|
||||
readerEvidence: null,
|
||||
readerClosesByCode: {},
|
||||
assignedCellOrigins: new Set(),
|
||||
generatorBaselineRssMiB,
|
||||
generatorBaselineCpu,
|
||||
generatorPeakRssMiB: generatorBaselineRssMiB,
|
||||
peerShutdowns: 0,
|
||||
shutdownEvidence: null,
|
||||
eventLoopDelay,
|
||||
stopping: false,
|
||||
transitionWindow: false
|
||||
}
|
||||
const peers = new Map()
|
||||
const reconnectTimers = new Set()
|
||||
// Duplicate-assign probe hosts: outside `peers`, so they never count toward the run's controls.
|
||||
const probePeers = []
|
||||
|
||||
async function readRuntimeQueuedBytes(origin) {
|
||||
const response = await fetch(`${origin}/v1/admin/runtime-status`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${adminToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(5_000)
|
||||
})
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new Error('reader evidence identity was rejected')
|
||||
}
|
||||
if (!response.ok) throw new Error(`reader runtime status returned ${response.status}`)
|
||||
const status = await response.json()
|
||||
const queuedBytes = status?.runtime?.queuedBytes
|
||||
if (!Number.isSafeInteger(queuedBytes) || queuedBytes < 0) {
|
||||
throw new Error('reader runtime queued bytes are invalid')
|
||||
}
|
||||
return queuedBytes
|
||||
}
|
||||
|
||||
async function observeReaderPressure(input) {
|
||||
if (!state.readerEvidence) throw new Error('reader evidence baseline is unavailable')
|
||||
await state.readerEvidence.observe(input)
|
||||
state.generatorPeakRssMiB = Math.max(
|
||||
state.generatorPeakRssMiB,
|
||||
process.memoryUsage().rss / 1_048_576
|
||||
)
|
||||
}
|
||||
|
||||
function recordSteadyMinimum() {
|
||||
if (!state.steadyStarted || state.stopping) return
|
||||
state.steadyMinimumActive = Math.min(state.steadyMinimumActive, state.active.size)
|
||||
}
|
||||
|
||||
function scheduleReconnect(peer) {
|
||||
if (state.stopping) return
|
||||
const timeout = setTimeout(() => {
|
||||
reconnectTimers.delete(timeout)
|
||||
void connect(peer)
|
||||
}, Math.floor(Math.random() * (config.reconnectMaxMs + 1)))
|
||||
reconnectTimers.add(timeout)
|
||||
}
|
||||
|
||||
function observe(type, detail) {
|
||||
if (type === 'connected') {
|
||||
state.active.add(detail.index)
|
||||
state.connected++
|
||||
const cellOrigin = peers.get(detail.index)?.assignedCellUrl()
|
||||
if (cellOrigin) state.assignedCellOrigins.add(cellOrigin)
|
||||
state.peakActive = Math.max(state.peakActive, state.active.size)
|
||||
recordSteadyMinimum()
|
||||
} else if (type === 'closed') {
|
||||
state.active.delete(detail.index)
|
||||
state.closes++
|
||||
if (!detail.stopped && !detail.expectedDrain) {
|
||||
state.unexpectedCloses++
|
||||
const code = String(detail.code)
|
||||
state.unexpectedClosesByCode[code] = (state.unexpectedClosesByCode[code] ?? 0) + 1
|
||||
}
|
||||
if (!detail.stopped) scheduleReconnect(peers.get(detail.index))
|
||||
recordSteadyMinimum()
|
||||
} else if (type === 'drain') state.drains++
|
||||
else if (type === 'ping') state.pings++
|
||||
else if (type === 'token') state.tokens++
|
||||
else if (type === 'refresh') state.refreshes++
|
||||
else if (type === 'refreshError') state.refreshErrors++
|
||||
else if (type === 'protocolError') state.protocolErrors++
|
||||
else if (type === 'socketError') state.socketErrors++
|
||||
else if (type === 'spliceOpened') {
|
||||
const failedAt = state.spliceFailedAt.get(detail.index)
|
||||
if (failedAt !== undefined) {
|
||||
state.spliceReopenGapsMs.push(Date.now() - failedAt)
|
||||
state.spliceFailedAt.delete(detail.index)
|
||||
}
|
||||
state.activeSplices++
|
||||
state.peakActiveSplices = Math.max(state.peakActiveSplices, state.activeSplices)
|
||||
} else if (type === 'spliceCompleted') {
|
||||
state.completedSplices++
|
||||
if (detail.readerMode === 'slow') state.slowReaderSplicesCompleted++
|
||||
} else if (type === 'spliceWedged') {
|
||||
state.wedgedReaderSplicesClosed++
|
||||
const code = String(detail.code)
|
||||
state.readerClosesByCode[code] = (state.readerClosesByCode[code] ?? 0) + 1
|
||||
} else if (type === 'spliceClosed') state.activeSplices--
|
||||
else if (type === 'spliceFailed') state.failedSplices++
|
||||
else if (type === 'shutdown') state.peerShutdowns++
|
||||
}
|
||||
|
||||
async function connect(peer) {
|
||||
try {
|
||||
await peer.connect()
|
||||
} catch (error) {
|
||||
state.connectionFailures++
|
||||
if (state.steadyStarted) state.steadyConnectionFailures++
|
||||
else if (state.transitionWindow) state.transitionConnectionFailures++
|
||||
else state.rampConnectionFailures++
|
||||
const reason = relayLoadFailureReason(error)
|
||||
state.connectionFailuresByReason[reason] =
|
||||
(state.connectionFailuresByReason[reason] ?? 0) + 1
|
||||
scheduleReconnect(peer)
|
||||
}
|
||||
}
|
||||
|
||||
const peerOptions = (index, overrides = {}) => ({
|
||||
...config,
|
||||
...key,
|
||||
accessToken,
|
||||
...(accessTokenProviderForIndex
|
||||
? { accessTokenProvider: accessTokenProviderForIndex(index) }
|
||||
: {}),
|
||||
seed: 0x4f524341 ^ config.shardIndex,
|
||||
...overrides
|
||||
})
|
||||
if (config.regionBehaviorProbes > 0) {
|
||||
const proofIndex = config.controls * config.shardCount + 10_000
|
||||
const regionProof = await proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: new RelayLoadControlPeer(
|
||||
proofIndex,
|
||||
peerOptions(proofIndex, { preferredRegion: undefined }),
|
||||
() => undefined
|
||||
),
|
||||
stickyPeer: new RelayLoadControlPeer(
|
||||
proofIndex + 1,
|
||||
peerOptions(proofIndex + 1, { preferredRegion: 'asia-east2' }),
|
||||
() => undefined
|
||||
),
|
||||
asiaOrigin: config.capacityCellOrigin
|
||||
})
|
||||
state.oldClientUsFirstProved = regionProof.oldClientUsFirst ? 1 : 0
|
||||
state.stickyAssignmentProved = regionProof.stickyAssignmentPreserved ? 1 : 0
|
||||
}
|
||||
const initialConnections = []
|
||||
for (let localIndex = 0; localIndex < config.controls; localIndex++) {
|
||||
const globalIndex = localIndex * config.shardCount + config.shardIndex
|
||||
const peer = new RelayLoadControlPeer(globalIndex, peerOptions(globalIndex), observe)
|
||||
peers.set(globalIndex, peer)
|
||||
const rampOffset =
|
||||
config.controls === 1 ? 0 : Math.floor((localIndex / (config.controls - 1)) * config.rampMs)
|
||||
const offset = config.rampStartDelayMs + rampOffset
|
||||
initialConnections.push(delay(offset).then(() => connect(peer)))
|
||||
}
|
||||
function recordResumeAttempt(peer, attempt) {
|
||||
state.resumeAttempts.push(attempt)
|
||||
console.log(JSON.stringify({ event: 'relay_load_resume_attempt', index: peer.index, ...attempt }))
|
||||
}
|
||||
// A paired phone that reconnects every interval until the run stops; every attempt is a line.
|
||||
async function runResumePhone(peer, localIndex, endsAt) {
|
||||
const relayDeviceId = `load-resume-device-${peer.index}`
|
||||
// Spread the phones across one interval.
|
||||
await delay(Math.floor((localIndex / Math.max(1, config.resumePhones)) * config.resumeIntervalMs))
|
||||
let token
|
||||
while (!state.stopping && Date.now() < endsAt) {
|
||||
const startedAt = Date.now()
|
||||
if (!token) {
|
||||
try {
|
||||
token = await installRelayLoadResumeDevice(peer, relayDeviceId)
|
||||
} catch (error) {
|
||||
recordResumeAttempt(peer, {
|
||||
at: startedAt,
|
||||
ok: false,
|
||||
failure: 'install',
|
||||
error: String(error.message).slice(0, 160)
|
||||
})
|
||||
}
|
||||
} else {
|
||||
recordResumeAttempt(peer, await resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, {
|
||||
holdMs: config.resumeHoldMs,
|
||||
stopping: () => state.stopping,
|
||||
delay
|
||||
}))
|
||||
}
|
||||
await delay(Math.min(
|
||||
Math.max(1_000, config.resumeIntervalMs - (Date.now() - startedAt)),
|
||||
Math.max(0, endsAt - Date.now())
|
||||
))
|
||||
}
|
||||
}
|
||||
const progressTimer = setInterval(() => report(state), 10_000)
|
||||
progressTimer.unref()
|
||||
await runRelayLoadWithShutdown(async () => {
|
||||
await Promise.all(initialConnections)
|
||||
assertRelayLoadRampAccepted(state.rampConnectionFailures, config.maxRampConnectionFailures)
|
||||
if (config.duplicateAssignProbes > 0) {
|
||||
const reconnects = { closes: 0, closesByCode: {}, reconnectFailures: 0 }
|
||||
// A probe host closed by a demotion re-assigns through the director, as a desktop would.
|
||||
const observeProbe = (peer) => (type, detail) => {
|
||||
if (type !== 'closed' || detail.stopped) return
|
||||
reconnects.closes++
|
||||
const code = String(detail.code)
|
||||
reconnects.closesByCode[code] = (reconnects.closesByCode[code] ?? 0) + 1
|
||||
if (state.stopping) return
|
||||
const timeout = setTimeout(() => {
|
||||
reconnectTimers.delete(timeout)
|
||||
void peer.connect().catch(() => {
|
||||
reconnects.reconnectFailures++
|
||||
})
|
||||
}, 1_000 + Math.floor(Math.random() * 5_000))
|
||||
reconnectTimers.add(timeout)
|
||||
}
|
||||
const base = config.controls * config.shardCount + 20_000
|
||||
const pairs = Array.from({ length: config.duplicateAssignProbes }, (_, probe) => {
|
||||
const index = base + probe * config.shardCount + config.shardIndex
|
||||
const primary = new RelayLoadControlPeer(index, peerOptions(index), (type, detail) =>
|
||||
observeProbe(primary)(type, detail)
|
||||
)
|
||||
const twin = new RelayLoadControlPeer(
|
||||
index,
|
||||
peerOptions(index, { keys: primary.keys }),
|
||||
() => undefined
|
||||
)
|
||||
probePeers.push(primary, twin)
|
||||
return { primary, twin }
|
||||
})
|
||||
state.duplicateAssign = {
|
||||
...(await proveRelayLoadDuplicateAssign({
|
||||
pairs,
|
||||
holdMs: config.duplicateAssignHoldMs,
|
||||
delay,
|
||||
failureReason: relayLoadFailureReason
|
||||
})),
|
||||
probeReconnects: reconnects
|
||||
}
|
||||
}
|
||||
if (
|
||||
config.rebindProbes > 0 || config.placementOverflowProbes > 0 ||
|
||||
config.regionalFallbackProbes > 0
|
||||
) {
|
||||
state.transitionWindow = config.rebindDelayMs > 0
|
||||
await waitForRelayLoadRebindGate({
|
||||
delay,
|
||||
delayMs: config.rebindDelayMs,
|
||||
activeCount: () => state.active.size,
|
||||
requiredCount: config.controls
|
||||
})
|
||||
state.transitionWindow = false
|
||||
}
|
||||
if (config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0) {
|
||||
const closesBeforeBoundary = state.closes
|
||||
await waitForRelayLoadDirectorCapacity({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
hardCap: config.capacityHardCap,
|
||||
unobservedBound: config.capacityUnobservedBound,
|
||||
requiredConnections: config.aggregateControls
|
||||
})
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed during the director capacity gate')
|
||||
}
|
||||
const overflowIndex = config.controls * config.shardCount + config.shardIndex
|
||||
if (config.placementOverflowProbes > 0) {
|
||||
state.placementOverflowReason = await proveRelayLoadPlacementBoundary({
|
||||
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), observe),
|
||||
failureReason: relayLoadFailureReason
|
||||
})
|
||||
}
|
||||
if (config.regionalFallbackProbes > 0) {
|
||||
await proveRelayLoadRegionalFallback({
|
||||
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), () => undefined),
|
||||
blockedOrigin: config.capacityCellOrigin
|
||||
})
|
||||
state.regionalFallbacksProved = 1
|
||||
}
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed during the placement boundary probe')
|
||||
}
|
||||
await waitForRelayLoadDirectorCapacity({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
hardCap: config.capacityHardCap,
|
||||
unobservedBound: config.capacityUnobservedBound,
|
||||
requiredConnections: config.aggregateControls,
|
||||
requiredSamples: 1
|
||||
})
|
||||
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
|
||||
throw new Error('ordinary controls changed before post-probe capacity verification')
|
||||
}
|
||||
}
|
||||
const rebindResult = await proveRelayLoadRebindBoundary({
|
||||
peers: [...state.active].map((index) => peers.get(index)),
|
||||
probeCount: config.rebindProbes,
|
||||
holdMs: config.rebindHoldMs,
|
||||
delay,
|
||||
failureReason: relayLoadFailureReason,
|
||||
requireOverflow: config.requireRebindOverflow
|
||||
})
|
||||
state.rebindProbesOpened = rebindResult.opened
|
||||
state.rebindOverflowReason = rebindResult.overflowReason
|
||||
if (config.requestUnitInvites > 0) {
|
||||
state.requestUnitInvitesOpened = await openRelayLoadInviteOffers({
|
||||
peers: [...state.active].sort((left, right) => left - right).map((index) => peers.get(index)),
|
||||
count: config.requestUnitInvites,
|
||||
ratePerSecond: config.requestUnitInvitesPerSecond
|
||||
})
|
||||
}
|
||||
if (config.requestUnitOverflowProbes > 0) {
|
||||
await waitForRelayLoadRequestUnits({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
capacityRequests: config.requestUnitCapacity,
|
||||
expectedRequestUnits: config.requestUnitCapacity,
|
||||
expectedActivityLeases: config.requestUnitCapacity
|
||||
})
|
||||
state.requestUnitOverflowReason = await proveRelayLoadRequestUnitBoundary(
|
||||
peers.get([...state.active][0])
|
||||
)
|
||||
}
|
||||
if (config.phaseBarrierDir) {
|
||||
await waitForRelayLoadPhaseBarrier({
|
||||
directory: config.phaseBarrierDir,
|
||||
shardCount: config.shardCount,
|
||||
shardIndex: config.shardIndex,
|
||||
timeoutMs: config.phaseBarrierTimeoutMs
|
||||
})
|
||||
state.phaseBarrierPassed = true
|
||||
}
|
||||
state.steadyStarted = true
|
||||
state.steadyMinimumActive = state.active.size
|
||||
const spliceIndexes = relayLoadSpliceIndexes(config)
|
||||
const readerOrigins = spliceIndexes.flatMap((index, spliceIndex) =>
|
||||
relayLoadSpliceProfile(config, spliceIndex).readerMode === 'normal'
|
||||
? []
|
||||
: [peers.get(index).lastAssignment.cellUrl]
|
||||
)
|
||||
state.readerEvidence = await createRelayLoadReaderEvidence(readerOrigins, {
|
||||
readQueuedBytes: readRuntimeQueuedBytes,
|
||||
delay
|
||||
})
|
||||
if (config.phaseBarrierDir) {
|
||||
await waitForRelayLoadPhaseBarrier({
|
||||
directory: `${config.phaseBarrierDir}-splices`,
|
||||
shardCount: config.shardCount,
|
||||
shardIndex: config.shardIndex,
|
||||
timeoutMs: config.phaseBarrierTimeoutMs
|
||||
})
|
||||
}
|
||||
const splicePromises = spliceIndexes.map((index, spliceIndex) =>
|
||||
delay(relayLoadSpliceStartDelayMs(config, spliceIndex)).then(() =>
|
||||
holdRelayLoadSplice({
|
||||
open: () =>
|
||||
peers.get(index).openSplice({
|
||||
payloadBytes: config.splicePayloadBytes,
|
||||
...relayLoadSpliceProfile(config, spliceIndex),
|
||||
observeReaderPressure,
|
||||
holdMs: config.spliceHoldMs
|
||||
}),
|
||||
retryMaxMs: config.spliceRetryMaxMs,
|
||||
stopping: () => state.stopping,
|
||||
onFailure: (reason) => {
|
||||
state.spliceRetries++
|
||||
state.spliceFailuresByReason[reason] = (state.spliceFailuresByReason[reason] ?? 0) + 1
|
||||
if (!state.spliceFailedAt.has(index)) state.spliceFailedAt.set(index, Date.now())
|
||||
},
|
||||
delay
|
||||
})
|
||||
)
|
||||
)
|
||||
const resumeEndsAt = Date.now() + config.durationMs
|
||||
const resumePromises = relayLoadResumePhoneIndexes(config).map((index, localIndex) =>
|
||||
runResumePhone(peers.get(index), localIndex, resumeEndsAt)
|
||||
)
|
||||
await Promise.all([...splicePromises, ...resumePromises, delay(config.durationMs)])
|
||||
}, async () => {
|
||||
state.stopping = true
|
||||
clearInterval(progressTimer)
|
||||
for (const timeout of reconnectTimers) clearTimeout(timeout)
|
||||
reconnectTimers.clear()
|
||||
await Promise.all([...peers.values(), ...probePeers].map((peer) => peer.shutdown()))
|
||||
eventLoopDelay.disable()
|
||||
state.shutdownEvidence = {
|
||||
peerShutdowns: state.peerShutdowns,
|
||||
activeControls: state.active.size,
|
||||
activeSplices: state.activeSplices,
|
||||
reconnectTimers: reconnectTimers.size
|
||||
}
|
||||
})
|
||||
if (config.requestUnitCleanupTimeoutMs > 0) {
|
||||
await waitForRelayLoadRequestUnits({
|
||||
directorOrigin: config.directorOrigin,
|
||||
adminToken,
|
||||
cellId: config.capacityCellId,
|
||||
capacityRequests: config.requestUnitCapacity,
|
||||
expectedRequestUnits: 0,
|
||||
expectedActivityLeases: 0,
|
||||
timeoutMs: config.requestUnitCleanupTimeoutMs
|
||||
})
|
||||
state.requestUnitCleanupProved = 1
|
||||
}
|
||||
const result = report(state, true)
|
||||
const minimumPeak = config.allowPartial ? 1 : Math.ceil(config.controls * 0.95)
|
||||
if (result.peakActive < minimumPeak) {
|
||||
throw new Error(`peak active controls ${result.peakActive} below required ${minimumPeak}`)
|
||||
}
|
||||
if (result.steadyMinimumActive < minimumPeak) {
|
||||
throw new Error(
|
||||
`steady minimum active controls ${result.steadyMinimumActive} below required ${minimumPeak}`
|
||||
)
|
||||
}
|
||||
if (relayLoadRunHasDisallowedFailures(result, config)) {
|
||||
throw new Error('relay load run observed connection, protocol, refresh, or socket errors')
|
||||
}
|
||||
const readerEvidenceError = relayLoadReaderEvidenceError(result, config)
|
||||
if (readerEvidenceError) throw new Error(readerEvidenceError)
|
||||
if (
|
||||
result.failedSplices > 0 ||
|
||||
result.completedSplices + result.wedgedReaderSplicesClosed !== config.splices ||
|
||||
result.shutdownEvidence.peerShutdowns !== config.controls ||
|
||||
result.shutdownEvidence.activeControls !== 0 ||
|
||||
result.shutdownEvidence.activeSplices !== 0 ||
|
||||
result.shutdownEvidence.reconnectTimers !== 0
|
||||
) {
|
||||
throw new Error('relay load run did not complete splices or shut down cleanly')
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
// Per-desktop disconnect gap for one drained cell, read from log lines that already ship:
|
||||
// the source cell's `control closed` line and the director's reconnect `assignment granted`
|
||||
// line (drains send `reconnect: true`, so every drained desktop's grant carries `hinted=true`).
|
||||
// Read-only: the operator exports both logs; this file never queries anything.
|
||||
|
||||
const CLOSE = /^\[orca-relay\] control closed host=(\S+) .*?\bsplices=(\d+) .*?\bcode=(\d+) reason=("(?:[^"\\]|\\.)*")/
|
||||
const GRANT = /^\[orca-relay\] assignment granted lane=\S+ hinted=true host=(\S+) cell=(\S+)$/
|
||||
// The cell's own drain close; a desktop closed this way had not moved yet, idle or not.
|
||||
const DRAIN_CLOSE_REASON = 'resolve configured director'
|
||||
|
||||
function entryText(entry) {
|
||||
if (typeof entry.textPayload === 'string') return entry.textPayload
|
||||
if (typeof entry.jsonPayload?.message === 'string') return entry.jsonPayload.message
|
||||
return null
|
||||
}
|
||||
|
||||
function entryTime(entry) {
|
||||
const at = Date.parse(entry.timestamp)
|
||||
if (!Number.isFinite(at)) throw new Error('log entry has no timestamp')
|
||||
return at
|
||||
}
|
||||
|
||||
export function readDrainCloses(entries) {
|
||||
const closes = []
|
||||
for (const entry of entries) {
|
||||
const match = CLOSE.exec(entryText(entry) ?? '')
|
||||
if (!match) continue
|
||||
closes.push({
|
||||
host: match[1],
|
||||
splices: Number(match[2]),
|
||||
code: Number(match[3]),
|
||||
reason: JSON.parse(match[4]),
|
||||
at: entryTime(entry)
|
||||
})
|
||||
}
|
||||
// gcloud exports newest first; the measurement needs each host's earliest close.
|
||||
return closes.sort((left, right) => left.at - right.at)
|
||||
}
|
||||
|
||||
export function readReconnectGrants(entries) {
|
||||
const grants = []
|
||||
for (const entry of entries) {
|
||||
const match = GRANT.exec(entryText(entry) ?? '')
|
||||
if (match) grants.push({ host: match[1], cellId: match[2], at: entryTime(entry) })
|
||||
}
|
||||
return grants.sort((left, right) => left.at - right.at)
|
||||
}
|
||||
|
||||
function percentile(sorted, fraction) {
|
||||
if (sorted.length === 0) return null
|
||||
return sorted[Math.min(sorted.length - 1, Math.ceil(fraction * sorted.length) - 1)]
|
||||
}
|
||||
|
||||
function summary(values) {
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
return {
|
||||
count: sorted.length,
|
||||
p50Ms: percentile(sorted, 0.5),
|
||||
p95Ms: percentile(sorted, 0.95),
|
||||
maxMs: sorted.at(-1) ?? null
|
||||
}
|
||||
}
|
||||
|
||||
// `controlsAtDrainStart` is the denominator: hosts the cell held when the drain began,
|
||||
// read from its runtime metrics line, so a desktop that never logged a close still counts.
|
||||
// `drainEndedAt` closes the window: after it the rolled cell's new container serves new sessions,
|
||||
// and their closes are not part of the drain. Grants after it still count, as the gap's far end.
|
||||
export function measureDrainDisconnectGap({
|
||||
sourceCellId,
|
||||
drainStartedAt,
|
||||
drainEndedAt,
|
||||
controlsAtDrainStart,
|
||||
closes,
|
||||
grants,
|
||||
cellRegions = {}
|
||||
}) {
|
||||
if (!Number.isFinite(drainStartedAt)) throw new Error('drain start time is invalid')
|
||||
if (!Number.isFinite(drainEndedAt) || drainEndedAt <= drainStartedAt) {
|
||||
throw new Error('drain end time is invalid')
|
||||
}
|
||||
const sourceRegion = cellRegions[sourceCellId]
|
||||
// First close per host after the drain began; later closes are the host's new sessions.
|
||||
const firstClose = new Map()
|
||||
for (const close of closes) {
|
||||
if (close.at < drainStartedAt || close.at > drainEndedAt || firstClose.has(close.host)) continue
|
||||
firstClose.set(close.host, close)
|
||||
}
|
||||
const grantsByHost = new Map()
|
||||
for (const grant of grants) {
|
||||
if (grant.at < drainStartedAt || grant.cellId === sourceCellId) continue
|
||||
grantsByHost.set(grant.host, [...(grantsByHost.get(grant.host) ?? []), grant])
|
||||
}
|
||||
const cutOffGapsMs = []
|
||||
const counts = { movedFirst: 0, cutOff: 0, cutOffUnresolved: 0, otherClose: 0 }
|
||||
const leftRegion = []
|
||||
let phoneSessionsDropped = 0
|
||||
for (const close of firstClose.values()) {
|
||||
phoneSessionsDropped += close.splices
|
||||
const hostGrants = grantsByHost.get(close.host) ?? []
|
||||
const first = hostGrants[0]
|
||||
if (first && first.at <= close.at) counts.movedFirst += 1
|
||||
else if (close.reason === DRAIN_CLOSE_REASON) {
|
||||
if (first) {
|
||||
counts.cutOff += 1
|
||||
cutOffGapsMs.push(first.at - close.at)
|
||||
} else counts.cutOffUnresolved += 1
|
||||
} else counts.otherClose += 1
|
||||
if (first && sourceRegion && cellRegions[first.cellId] && cellRegions[first.cellId] !== sourceRegion) {
|
||||
const back = hostGrants.find(
|
||||
(grant) => grant.at > first.at && cellRegions[grant.cellId] === sourceRegion
|
||||
)
|
||||
leftRegion.push(back ? back.at - first.at : null)
|
||||
}
|
||||
}
|
||||
const returned = leftRegion.filter((value) => value !== null)
|
||||
return {
|
||||
sourceCellId,
|
||||
controlsAtDrainStart,
|
||||
closedHosts: firstClose.size,
|
||||
// Below 1 means some drained desktops left no close line in the export; widen it.
|
||||
closeCoverage: controlsAtDrainStart > 0 ? firstClose.size / controlsAtDrainStart : null,
|
||||
...counts,
|
||||
// Lower bound until the target cells run an image that logs control activation.
|
||||
cutOffGap: summary(cutOffGapsMs),
|
||||
phoneSessionsDropped,
|
||||
leftSourceRegion: leftRegion.length,
|
||||
leftSourceRegionStillAway: leftRegion.length - returned.length,
|
||||
timeUntilBack: summary(returned)
|
||||
}
|
||||
}
|
||||
|
||||
function argument(name) {
|
||||
const index = process.argv.indexOf(`--${name}`)
|
||||
return index === -1 ? undefined : process.argv[index + 1]
|
||||
}
|
||||
|
||||
function required(name) {
|
||||
const value = argument(name)
|
||||
if (value === undefined) throw new Error(`--${name} is required`)
|
||||
return value
|
||||
}
|
||||
|
||||
function main() {
|
||||
const readJson = (path) => JSON.parse(readFileSync(path, 'utf8'))
|
||||
const cellRegionsPath = argument('cell-regions')
|
||||
const result = measureDrainDisconnectGap({
|
||||
sourceCellId: required('source-cell'),
|
||||
drainStartedAt: Date.parse(required('drain-started-at')),
|
||||
drainEndedAt: Date.parse(required('drain-ended-at')),
|
||||
controlsAtDrainStart: Number(required('controls')),
|
||||
closes: readDrainCloses(readJson(required('cell-log'))),
|
||||
grants: readReconnectGrants(readJson(required('director-log'))),
|
||||
cellRegions: cellRegionsPath ? readJson(cellRegionsPath) : {}
|
||||
})
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) main()
|
||||
@@ -1,158 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import {
|
||||
measureDrainDisconnectGap,
|
||||
readDrainCloses,
|
||||
readReconnectGrants
|
||||
} from './measure-relay-drain-disconnect-gap.mjs'
|
||||
|
||||
const start = Date.parse('2026-10-06T10:00:00Z')
|
||||
const at = (seconds) => new Date(start + seconds * 1000).toISOString()
|
||||
|
||||
function close(host, seconds, reason, splices = 0) {
|
||||
return {
|
||||
timestamp: at(seconds),
|
||||
jsonPayload: {
|
||||
message:
|
||||
`[orca-relay] control closed host=${host} gen=3 state=closed ageMs=100 app="1.4.0"` +
|
||||
` splices=${splices} pending=0 code=4001 reason=${JSON.stringify(reason)}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function grant(host, seconds, cellId) {
|
||||
return {
|
||||
timestamp: at(seconds),
|
||||
textPayload: `[orca-relay] assignment granted lane=drain-return hinted=true host=${host} cell=${cellId}`
|
||||
}
|
||||
}
|
||||
|
||||
describe('drain disconnect gap', () => {
|
||||
it('splits moved-first, cut-off and unresolved desktops and sums dropped phone sessions', () => {
|
||||
const closes = readDrainCloses([
|
||||
close('h-moved', 30, 'migration completed', 2),
|
||||
close('h-cut', 300, 'resolve configured director', 1),
|
||||
close('h-idle', 310, 'resolve configured director'),
|
||||
close('h-lost', 320, 'resolve configured director'),
|
||||
close('h-early', -5, 'resolve configured director'),
|
||||
// A later close of the same host is its next session, not the drain.
|
||||
close('h-cut', 900, 'resolve configured director', 7)
|
||||
])
|
||||
const grants = readReconnectGrants([
|
||||
grant('h-moved', 20, 'c2'),
|
||||
grant('h-cut', 304, 'c2'),
|
||||
grant('h-idle', 330, 'c3'),
|
||||
grant('h-other', 5, 'c2'),
|
||||
grant('h-cut', 290, 'c1')
|
||||
])
|
||||
const result = measureDrainDisconnectGap({
|
||||
sourceCellId: 'c1',
|
||||
drainStartedAt: start,
|
||||
drainEndedAt: start + 1_200_000,
|
||||
controlsAtDrainStart: 5,
|
||||
closes,
|
||||
grants
|
||||
})
|
||||
assert.equal(result.closedHosts, 4)
|
||||
assert.equal(result.closeCoverage, 0.8)
|
||||
assert.equal(result.movedFirst, 1)
|
||||
assert.equal(result.cutOff, 2)
|
||||
assert.equal(result.cutOffUnresolved, 1)
|
||||
assert.equal(result.otherClose, 0)
|
||||
assert.deepEqual(result.cutOffGap, { count: 2, p50Ms: 4000, p95Ms: 20000, maxMs: 20000 })
|
||||
assert.equal(result.phoneSessionsDropped, 3)
|
||||
})
|
||||
|
||||
it('times desktops that left the source region until a grant brings them back', () => {
|
||||
const result = measureDrainDisconnectGap({
|
||||
sourceCellId: 'a1',
|
||||
drainStartedAt: start,
|
||||
drainEndedAt: start + 1_200_000,
|
||||
controlsAtDrainStart: 2,
|
||||
closes: readDrainCloses([
|
||||
close('h-away', 100, 'resolve configured director'),
|
||||
close('h-stuck', 100, 'resolve configured director')
|
||||
]),
|
||||
grants: readReconnectGrants([
|
||||
grant('h-away', 101, 'u1'),
|
||||
grant('h-away', 3701, 'a2'),
|
||||
grant('h-stuck', 102, 'u1')
|
||||
]),
|
||||
cellRegions: { a1: 'asia-east2', a2: 'asia-east2', u1: 'us-central1' }
|
||||
})
|
||||
assert.equal(result.leftSourceRegion, 2)
|
||||
assert.equal(result.leftSourceRegionStillAway, 1)
|
||||
assert.equal(result.timeUntilBack.maxMs, 3_600_000)
|
||||
})
|
||||
|
||||
it('keeps each host earliest close when the export is newest first', () => {
|
||||
const result = measureDrainDisconnectGap({
|
||||
sourceCellId: 'c1',
|
||||
drainStartedAt: start,
|
||||
drainEndedAt: start + 1_200_000,
|
||||
controlsAtDrainStart: 1,
|
||||
closes: readDrainCloses([
|
||||
close('h', 300, 'resolve configured director'),
|
||||
close('h', 60, 'resolve configured director', 3)
|
||||
]),
|
||||
grants: readReconnectGrants([grant('h', 120, 'c2')])
|
||||
})
|
||||
assert.equal(result.movedFirst, 0)
|
||||
assert.equal(result.cutOff, 1)
|
||||
assert.deepEqual(result.cutOffGap, { count: 1, p50Ms: 60000, p95Ms: 60000, maxMs: 60000 })
|
||||
assert.equal(result.phoneSessionsDropped, 3)
|
||||
})
|
||||
|
||||
it('refuses an invalid drain start time', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
measureDrainDisconnectGap({
|
||||
sourceCellId: 'c1',
|
||||
drainStartedAt: Date.parse('not a time'),
|
||||
drainEndedAt: start,
|
||||
controlsAtDrainStart: 1,
|
||||
closes: [],
|
||||
grants: []
|
||||
}),
|
||||
/drain start time is invalid/
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves out closes after the drain ended and refuses a bad end time', () => {
|
||||
const input = {
|
||||
sourceCellId: 'c1',
|
||||
drainStartedAt: start,
|
||||
drainEndedAt: start + 600_000,
|
||||
controlsAtDrainStart: 1,
|
||||
closes: readDrainCloses([
|
||||
close('h-drained', 100, 'resolve configured director'),
|
||||
// The new container's session after the roll.
|
||||
close('h-new', 700, '', 4)
|
||||
]),
|
||||
grants: readReconnectGrants([grant('h-drained', 650, 'c2')])
|
||||
}
|
||||
const result = measureDrainDisconnectGap(input)
|
||||
assert.equal(result.closedHosts, 1)
|
||||
assert.equal(result.otherClose, 0)
|
||||
assert.equal(result.phoneSessionsDropped, 0)
|
||||
// A grant after the window still ends that host's gap.
|
||||
assert.equal(result.cutOffGap.maxMs, 550_000)
|
||||
for (const drainEndedAt of [Number.NaN, start]) {
|
||||
assert.throws(
|
||||
() => measureDrainDisconnectGap({ ...input, drainEndedAt }),
|
||||
/drain end time is invalid/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores lines that are not the two it reads', () => {
|
||||
assert.deepEqual(readDrainCloses([{ timestamp: at(0), textPayload: 'unrelated' }]), [])
|
||||
assert.deepEqual(
|
||||
readReconnectGrants([grant('h', 0, 'c1')].map((entry) => ({
|
||||
...entry,
|
||||
textPayload: entry.textPayload.replace('hinted=true', 'hinted=false')
|
||||
}))),
|
||||
[]
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,509 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
addExactMigrationCells,
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const SHAPES = {
|
||||
staging: {
|
||||
directorOrigin: 'https://relay-staging.onorca.dev',
|
||||
domain: 'relay-staging.onorca.dev',
|
||||
allCells: ['staging-gce-c4', 'staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7'],
|
||||
registrationWaves: [['staging-gce-c4'], ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7']],
|
||||
promotionWaves: [['staging-gce-c4'], ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7']]
|
||||
},
|
||||
production: {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
domain: 'relay.onorca.dev',
|
||||
allCells: [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34',
|
||||
'production-gce-c35'
|
||||
],
|
||||
// The launch set was registered together; each later cell registers alone beside it.
|
||||
registrationWaves: [
|
||||
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31'],
|
||||
['production-gce-c32'],
|
||||
['production-gce-c33'],
|
||||
['production-gce-c34'],
|
||||
['production-gce-c35']
|
||||
],
|
||||
promotionWaves: [
|
||||
['production-gce-c27'],
|
||||
['production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31'],
|
||||
['production-gce-c32'],
|
||||
['production-gce-c33'],
|
||||
['production-gce-c34'],
|
||||
['production-gce-c35']
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
const PRODUCTION_CANARY_CELL = 'production-gce-c27'
|
||||
const STAGING_PROOF_CELL = 'staging-gce-c4'
|
||||
const ASIA_REGION = 'asia-east2'
|
||||
const ASIA_CELL_SHAPE = { region: ASIA_REGION, capacityRequests: 6_000, connectionHardCap: 3_000 }
|
||||
// The US cells at the Asia shape live in the root region; every other cell here is Asia, except
|
||||
// the staging dead-cell guard cells, which keep c2's ordinary US shape (staging.tfvars).
|
||||
const STAGING_US_CELL_SHAPE = { region: 'us-central1', capacityRequests: 4_000, connectionHardCap: 600 }
|
||||
const CELL_SHAPES = {
|
||||
'production-gce-c32': { ...ASIA_CELL_SHAPE, region: 'us-central1' },
|
||||
'production-gce-c33': { ...ASIA_CELL_SHAPE, region: 'us-central1' },
|
||||
'staging-gce-c5': STAGING_US_CELL_SHAPE,
|
||||
'staging-gce-c6': STAGING_US_CELL_SHAPE,
|
||||
'staging-gce-c7': STAGING_US_CELL_SHAPE
|
||||
}
|
||||
|
||||
function cellShape(cellId) {
|
||||
return CELL_SHAPES[cellId] ?? ASIA_CELL_SHAPE
|
||||
}
|
||||
|
||||
function cellRegion(cellId) {
|
||||
return cellShape(cellId).region
|
||||
}
|
||||
|
||||
export function parseRelayAsiaAdmissionArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['environment', 'mode', 'cell-ids', 'image-digest']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(values['image-digest'])) {
|
||||
throw new Error('--image-digest is invalid')
|
||||
}
|
||||
if (![
|
||||
'inspect', 'initialize', 'verify', 'registered', 'register',
|
||||
'promote', 'recover-promotion', 'rollback'
|
||||
].includes(values.mode)) {
|
||||
throw new Error('--mode is invalid')
|
||||
}
|
||||
const expectedGeneration = values.mode === 'inspect'
|
||||
? undefined
|
||||
: Number(values['expected-generation'])
|
||||
if (
|
||||
values.mode !== 'inspect' &&
|
||||
(!Number.isSafeInteger(expectedGeneration) || expectedGeneration < 0)
|
||||
) {
|
||||
throw new Error('--expected-generation is invalid')
|
||||
}
|
||||
const shape = SHAPES[values.environment]
|
||||
if (!shape) throw new Error('--environment is invalid')
|
||||
const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean)
|
||||
const distinct = new Set(cells)
|
||||
if (distinct.size !== cells.length || cells.some((cell) => !shape.allCells.includes(cell))) {
|
||||
throw new Error('--cell-ids are invalid')
|
||||
}
|
||||
const exact = (expected) => JSON.stringify([...cells].sort()) === JSON.stringify([...expected].sort())
|
||||
const [launchWave] = shape.registrationWaves
|
||||
if (
|
||||
(['inspect', 'verify'].includes(values.mode) &&
|
||||
!exact(shape.allCells) && !shape.registrationWaves.some(exact)) ||
|
||||
// Generation zero predates every later cell, so the boundary freezes only the launch set.
|
||||
(values.mode === 'initialize' && !exact(launchWave)) ||
|
||||
(['register', 'registered'].includes(values.mode) && !shape.registrationWaves.some(exact)) ||
|
||||
(['promote', 'recover-promotion'].includes(values.mode) && !shape.promotionWaves.some(exact)) ||
|
||||
// Rollback takes any reviewed wave or the whole set, never a mixed partial set.
|
||||
(values.mode === 'rollback' && !exact(shape.allCells) &&
|
||||
![...shape.registrationWaves, ...shape.promotionWaves].some(exact))
|
||||
) throw new Error('--cell-ids do not match the reviewed admission wave')
|
||||
const attemptId = values['attempt-id']
|
||||
if (!['inspect', 'verify', 'registered'].includes(values.mode) &&
|
||||
!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
|
||||
throw new Error('--attempt-id is invalid')
|
||||
}
|
||||
return {
|
||||
environment: values.environment,
|
||||
mode: values.mode,
|
||||
cells,
|
||||
expectedGeneration,
|
||||
expectedMembershipSha256: values['expected-membership-sha256'],
|
||||
imageDigest: values['image-digest'],
|
||||
attemptId,
|
||||
token: process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? ''
|
||||
}
|
||||
}
|
||||
|
||||
function hostname(cellId) {
|
||||
return cellId.split('-').at(-1)
|
||||
}
|
||||
|
||||
function cellOrigin(shape, cellId) {
|
||||
return `https://${hostname(cellId)}.${shape.domain}`
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function defaultPost(fetchImpl, token) {
|
||||
return async (url, body) => await responseJson(await fetchImpl(url, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}), new URL(url).pathname)
|
||||
}
|
||||
|
||||
async function verifyRuntime(fetchImpl, post, shape, cellId, imageDigest, requireDirector) {
|
||||
const origin = cellOrigin(shape, cellId)
|
||||
const [health, ready, runtime] = await Promise.all([
|
||||
fetchImpl(`${origin}/health`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
|
||||
fetchImpl(`${origin}/ready`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
|
||||
post(`${origin}/v1/admin/runtime-status`, { v: 1 })
|
||||
])
|
||||
if (!health.ok || !ready.ok) throw new Error(`${cellId} is not ready`)
|
||||
if (
|
||||
runtime.cellId !== cellId ||
|
||||
runtime.cellUrl !== origin ||
|
||||
runtime.region !== cellRegion(cellId) ||
|
||||
runtime.imageDigest !== imageDigest ||
|
||||
runtime.draining !== false ||
|
||||
runtime.connectionCapacity?.hardCap !== cellShape(cellId).connectionHardCap ||
|
||||
runtime.connectionCapacity?.unobservedBound !== 60
|
||||
) throw new Error(`${cellId} runtime does not match its reviewed shape`)
|
||||
if (requireDirector) {
|
||||
const result = await post(`${shape.directorOrigin}/v1/admin/cell-status`, { v: 1, cellId })
|
||||
if (
|
||||
result.status?.cellUrl !== origin ||
|
||||
result.status?.runtime?.heartbeatFresh !== true ||
|
||||
result.status?.runtime?.ready !== true
|
||||
) throw new Error(`${cellId} has no fresh ready director heartbeat`)
|
||||
}
|
||||
}
|
||||
|
||||
function membershipStates(selector, cells) {
|
||||
return Object.fromEntries(cells.map((cellId) => [cellId, selectorCellState(selector, cellId)]))
|
||||
}
|
||||
|
||||
function inspectedMembershipStates(selector, cells) {
|
||||
const known = new Set([
|
||||
...selector.membership.existingOnly,
|
||||
...selector.membership.migrationOnly,
|
||||
...selector.membership.general
|
||||
])
|
||||
return Object.fromEntries(cells.map((cellId) => [
|
||||
cellId,
|
||||
known.has(cellId) ? selectorCellState(selector, cellId) : 'absent'
|
||||
]))
|
||||
}
|
||||
|
||||
function sameMembership(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right)
|
||||
}
|
||||
|
||||
function membershipSha256(membership) {
|
||||
return createHash('sha256').update(JSON.stringify(membership)).digest('hex')
|
||||
}
|
||||
|
||||
async function initializeAdmissionBoundary(post, selectorPost, shape, config, current) {
|
||||
if (config.expectedGeneration !== 0) {
|
||||
throw new Error('admission boundary initialization requires generation 0')
|
||||
}
|
||||
if (
|
||||
!/^[a-f0-9]{64}$/.test(config.expectedMembershipSha256 ?? '') ||
|
||||
membershipSha256(current.selector.membership) !== config.expectedMembershipSha256
|
||||
) {
|
||||
throw new Error('admission membership changed before boundary initialization')
|
||||
}
|
||||
const targetStates = inspectedMembershipStates(current.selector, config.cells)
|
||||
if (Object.values(targetStates).some((state) => state !== 'absent')) {
|
||||
throw new Error('Asia cell exists before admission boundary initialization')
|
||||
}
|
||||
const intendedMembership = current.intent?.previousMembership ?? current.selector.membership
|
||||
const exactCommitted = (inspection) =>
|
||||
inspection.intent?.state === 'committed' &&
|
||||
inspection.intent.expectedGeneration === 0 &&
|
||||
inspection.selector.generation === 1 &&
|
||||
inspection.selector.attemptId === config.attemptId &&
|
||||
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
|
||||
sameMembership(inspection.intent.membership, intendedMembership) &&
|
||||
sameMembership(inspection.selector.membership, intendedMembership)
|
||||
const exactUnchanged = (inspection) =>
|
||||
inspection.intent?.state === 'unchanged' &&
|
||||
inspection.intent.expectedGeneration === 0 &&
|
||||
inspection.selector.generation === 0 &&
|
||||
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
|
||||
sameMembership(inspection.intent.membership, intendedMembership) &&
|
||||
sameMembership(inspection.selector.membership, intendedMembership)
|
||||
if (exactCommitted(current)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: inspectedMembershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
if (current.intent && !exactUnchanged(current)) {
|
||||
throw new Error('admission boundary initialization attempt diverged')
|
||||
}
|
||||
const request = {
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: config.expectedMembershipSha256,
|
||||
membership: intendedMembership
|
||||
}
|
||||
let applyError
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
await post(`${shape.directorOrigin}/v1/admin/admission-selector/apply`, request)
|
||||
} catch (error) {
|
||||
applyError = error
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(selectorPost, config.attemptId)
|
||||
if (exactCommitted(verified)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: verified.selector.generation,
|
||||
states: targetStates,
|
||||
recovered: current.intent !== null || applyError !== undefined || attempt > 0
|
||||
}
|
||||
}
|
||||
if (!exactUnchanged(verified)) {
|
||||
throw new Error('admission boundary initialization did not commit exactly', {
|
||||
cause: applyError
|
||||
})
|
||||
}
|
||||
}
|
||||
throw new Error('admission boundary initialization remained unchanged after retry', {
|
||||
cause: applyError
|
||||
})
|
||||
}
|
||||
|
||||
export async function operateRelayAsiaAdmission(config, dependencies = {}) {
|
||||
const shape = SHAPES[config.environment]
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const post = dependencies.post ?? defaultPost(fetchImpl, config.token)
|
||||
// Only the C4 proof keeps the server-enforced C4-only route (the Asia proof identity can call
|
||||
// nothing else); other staging cells go through the general route, as production does.
|
||||
const stagingProof = config.environment === 'staging' &&
|
||||
config.cells.length === 1 && config.cells[0] === STAGING_PROOF_CELL
|
||||
const selectorPost = (path, body) => {
|
||||
if (!stagingProof || path !== '/v1/admin/admission-selector/apply') {
|
||||
return post(`${shape.directorOrigin}${path}`, body)
|
||||
}
|
||||
const state = selectorCellState({ membership: body.membership }, STAGING_PROOF_CELL)
|
||||
if (!['general', 'migration-only'].includes(state)) {
|
||||
throw new Error('staging proof can only transition C4 between reviewed states')
|
||||
}
|
||||
return post(`${shape.directorOrigin}/v1/admin/admission-selector/apply-staging-asia-proof`, {
|
||||
v: 1,
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
state
|
||||
})
|
||||
}
|
||||
const current = await inspectAdmissionSelector(
|
||||
selectorPost,
|
||||
['inspect', 'verify', 'registered'].includes(config.mode) ? undefined : config.attemptId
|
||||
)
|
||||
if (config.mode === 'inspect') {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
membership: current.selector.membership,
|
||||
membershipSha256: membershipSha256(current.selector.membership),
|
||||
states: inspectedMembershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (
|
||||
!current.intent &&
|
||||
current.selector.generation !== config.expectedGeneration
|
||||
) {
|
||||
throw new Error('admission selector generation changed')
|
||||
}
|
||||
if (current.intent && current.intent.expectedGeneration !== config.expectedGeneration) {
|
||||
throw new Error('admission attempt generation does not match')
|
||||
}
|
||||
if (config.mode === 'initialize') {
|
||||
return await initializeAdmissionBoundary(post, selectorPost, shape, config, current)
|
||||
}
|
||||
if (config.mode === 'recover-promotion') {
|
||||
if (config.cells.every(
|
||||
(cellId) => selectorCellState(current.selector, cellId) === 'migration-only'
|
||||
)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: false,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (!current.intent) {
|
||||
if (
|
||||
current.selector.generation !== config.expectedGeneration
|
||||
) throw new Error('promotion state changed without the reviewed attempt')
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: false,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
const expectedMembership = membershipWithStates(
|
||||
{ membership: current.intent.previousMembership },
|
||||
Object.fromEntries(config.cells.map((cellId) => [cellId, 'general']))
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
|
||||
config.cells.some((cellId) => selectorCellState(current.selector, cellId) !== 'general')
|
||||
) throw new Error('promotion attempt is not the current general state')
|
||||
return {
|
||||
mode: config.mode,
|
||||
promoted: true,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (config.mode === 'rollback') {
|
||||
for (const cellId of config.cells) {
|
||||
if (!['general', 'migration-only'].includes(selectorCellState(current.selector, cellId))) {
|
||||
throw new Error(`${cellId} cannot roll back to migration-only`)
|
||||
}
|
||||
}
|
||||
} else if (config.mode === 'register') {
|
||||
const known = new Set([
|
||||
...current.selector.membership.existingOnly,
|
||||
...current.selector.membership.migrationOnly,
|
||||
...current.selector.membership.general
|
||||
])
|
||||
if (!current.intent && config.cells.some((cellId) => known.has(cellId))) {
|
||||
throw new Error('Asia cell is already registered')
|
||||
}
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
|
||||
))
|
||||
} else if (config.mode !== 'registered') {
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, true)
|
||||
))
|
||||
}
|
||||
if (config.mode === 'registered') {
|
||||
if (config.cells.some(
|
||||
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
|
||||
)) throw new Error('Asia cells are not registered migration-only')
|
||||
await Promise.all(config.cells.map((cellId) =>
|
||||
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
|
||||
))
|
||||
}
|
||||
if (['verify', 'registered'].includes(config.mode)) {
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells)
|
||||
}
|
||||
}
|
||||
if (config.mode === 'register') {
|
||||
if (current.intent) {
|
||||
const expectedCells = new Set(config.cells)
|
||||
const addedCells = current.intent.membership.migrationOnly.filter(
|
||||
(cellId) => !current.intent.previousMembership.migrationOnly.includes(cellId)
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
addedCells.length !== expectedCells.size ||
|
||||
addedCells.some((cellId) => !expectedCells.has(cellId)) ||
|
||||
current.selector.generation !== config.expectedGeneration + 1 ||
|
||||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
|
||||
) {
|
||||
throw new Error('admission attempt does not match the requested Asia registration')
|
||||
}
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
const result = await addExactMigrationCells(
|
||||
selectorPost,
|
||||
{
|
||||
attemptId: config.attemptId,
|
||||
cells: config.cells.map((cellId) => ({
|
||||
cellId,
|
||||
cellUrl: cellOrigin(shape, cellId),
|
||||
region: cellRegion(cellId),
|
||||
capacityRequests: cellShape(cellId).capacityRequests,
|
||||
connectionHardCap: cellShape(cellId).connectionHardCap,
|
||||
connectionUnobservedBound: 60
|
||||
}))
|
||||
},
|
||||
{ expectedCurrentSelector: current.selector }
|
||||
)
|
||||
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
|
||||
}
|
||||
const desiredState = config.mode === 'promote' ? 'general' : 'migration-only'
|
||||
if (current.intent) {
|
||||
const expectedMembership = membershipWithStates(
|
||||
{ membership: current.intent.previousMembership },
|
||||
Object.fromEntries(config.cells.map((cellId) => [cellId, desiredState]))
|
||||
)
|
||||
if (
|
||||
current.intent.state !== 'committed' ||
|
||||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
|
||||
current.selector.generation !== config.expectedGeneration + 1 ||
|
||||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
|
||||
) {
|
||||
throw new Error('admission attempt does not match the requested Asia transition')
|
||||
}
|
||||
return {
|
||||
mode: config.mode,
|
||||
generation: current.selector.generation,
|
||||
states: membershipStates(current.selector, config.cells),
|
||||
recovered: true
|
||||
}
|
||||
}
|
||||
if (config.mode === 'promote' && config.cells.some(
|
||||
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
|
||||
)) throw new Error('Asia promotion requires migration-only cells')
|
||||
// The Asia launch order binds Asia cells only; a US cell is proven by its own canary.
|
||||
const asiaWave = config.cells.every((cellId) => cellRegion(cellId) === ASIA_REGION)
|
||||
if (
|
||||
config.mode === 'promote' &&
|
||||
asiaWave &&
|
||||
config.environment === 'production' &&
|
||||
!config.cells.includes(PRODUCTION_CANARY_CELL) &&
|
||||
selectorCellState(current.selector, PRODUCTION_CANARY_CELL) !== 'general'
|
||||
) {
|
||||
throw new Error('Asia expansion requires the C27 canary to be general')
|
||||
}
|
||||
const [launchWave] = shape.registrationWaves
|
||||
if (
|
||||
config.mode === 'promote' &&
|
||||
asiaWave &&
|
||||
!config.cells.some((cellId) => launchWave.includes(cellId)) &&
|
||||
launchWave.some((cellId) => selectorCellState(current.selector, cellId) !== 'general')
|
||||
) {
|
||||
throw new Error('a later Asia cell requires every launch cell to be general')
|
||||
}
|
||||
const result = await applyExactAdmissionSelector(
|
||||
selectorPost,
|
||||
membershipWithStates(current.selector, Object.fromEntries(
|
||||
config.cells.map((cellId) => [cellId, desiredState])
|
||||
)),
|
||||
{ attemptId: config.attemptId, expectedCurrentSelector: current.selector }
|
||||
)
|
||||
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const config = parseRelayAsiaAdmissionArguments(process.argv.slice(2))
|
||||
if (!config.token) throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required')
|
||||
console.log(JSON.stringify(await operateRelayAsiaAdmission(config)))
|
||||
}
|
||||
@@ -1,967 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
operateRelayAsiaAdmission,
|
||||
parseRelayAsiaAdmissionArguments
|
||||
} from './operate-relay-asia-admission.mjs'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const membershipDigest = (membership) =>
|
||||
createHash('sha256').update(JSON.stringify(membership)).digest('hex')
|
||||
|
||||
function harness(initialSelector, runtimeDigests = {}, runtimeRegions = {}, runtimeHardCaps = {}) {
|
||||
const initialMembership = structuredClone(initialSelector.membership)
|
||||
let selector = structuredClone(initialSelector)
|
||||
const intents = new Map()
|
||||
const requests = []
|
||||
let fetches = 0
|
||||
let failAfterIntent = false
|
||||
const post = async (url, body) => {
|
||||
const parsed = new URL(url)
|
||||
requests.push({ path: parsed.pathname, body })
|
||||
if (parsed.pathname === '/v1/admin/runtime-status') {
|
||||
const [cell, domain] = parsed.hostname.split(/\.(.*)/)
|
||||
const cellId = `${domain.startsWith('relay-staging.') ? 'staging' : 'production'}-gce-${cell}`
|
||||
return {
|
||||
cellId,
|
||||
cellUrl: parsed.origin,
|
||||
region: runtimeRegions[cellId] ?? 'asia-east2',
|
||||
imageDigest: runtimeDigests[cellId] ?? digest,
|
||||
draining: false,
|
||||
connectionCapacity: { hardCap: runtimeHardCaps[cellId] ?? 3_000, unobservedBound: 60 }
|
||||
}
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
return {
|
||||
status: {
|
||||
cellUrl: `https://${body.cellId.split('-').at(-1)}.${parsed.hostname}`,
|
||||
runtime: { heartbeatFresh: true, ready: true }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (parsed.pathname.endsWith('/status')) {
|
||||
return { selector, intent: body.attemptId ? intents.get(body.attemptId) ?? null : null }
|
||||
}
|
||||
if (parsed.pathname.endsWith('/add-migration-cells')) {
|
||||
selector = {
|
||||
generation: selector.generation + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: {
|
||||
...selector.membership,
|
||||
migrationOnly: [...selector.membership.migrationOnly, ...body.cells.map((cell) => cell.cellId)].sort()
|
||||
}
|
||||
}
|
||||
} else if (parsed.pathname.endsWith('/apply-staging-asia-proof')) {
|
||||
const membership = structuredClone(selector.membership)
|
||||
membership.migrationOnly = membership.migrationOnly.filter((cell) => cell !== 'staging-gce-c4')
|
||||
membership.general = membership.general.filter((cell) => cell !== 'staging-gce-c4')
|
||||
membership[body.state === 'general' ? 'general' : 'migrationOnly'].push('staging-gce-c4')
|
||||
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership }
|
||||
} else if (parsed.pathname.endsWith('/apply')) {
|
||||
if (
|
||||
body.expectedMembershipSha256 &&
|
||||
body.expectedMembershipSha256 !== membershipDigest(selector.membership)
|
||||
) throw new Error('admission_selector_membership_mismatch')
|
||||
if (failAfterIntent) {
|
||||
failAfterIntent = false
|
||||
intents.set(body.attemptId, {
|
||||
state: 'unchanged',
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
previousMembership: structuredClone(selector.membership),
|
||||
membership: structuredClone(body.membership)
|
||||
})
|
||||
throw new Error('failure after intent persistence')
|
||||
}
|
||||
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership: body.membership }
|
||||
} else throw new Error(`unexpected ${parsed.pathname}`)
|
||||
intents.set(body.attemptId, {
|
||||
state: 'committed', expectedGeneration: body.expectedGeneration,
|
||||
previousMembership: initialSelector.membership,
|
||||
membership: selector.membership
|
||||
})
|
||||
return { changed: true, selector }
|
||||
}
|
||||
const fetch = async () => {
|
||||
fetches++
|
||||
return new Response(null, { status: 200 })
|
||||
}
|
||||
const commitWithoutResponse = async (path, body) => {
|
||||
await post(`https://relay.onorca.dev${path}`, body)
|
||||
throw new Error('response lost after commit')
|
||||
}
|
||||
return {
|
||||
post, fetch, requests, commitWithoutResponse,
|
||||
failNextApplyAfterIntent: () => (failAfterIntent = true),
|
||||
apply: async (attemptId, membership) => await post(
|
||||
'https://relay.onorca.dev/v1/admin/admission-selector/apply',
|
||||
{ attemptId, expectedGeneration: selector.generation, membership }
|
||||
),
|
||||
fetchCount: () => fetches, selector: () => selector
|
||||
}
|
||||
}
|
||||
|
||||
const baseSelector = {
|
||||
generation: 7,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26'] }
|
||||
}
|
||||
|
||||
test('inspects generation zero without requiring target registration or making a mutation', async () => {
|
||||
const subject = harness({
|
||||
generation: 0,
|
||||
membership: {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'inspect', cells: ['staging-gce-c4'],
|
||||
imageDigest: digest, token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(result.generation, 0)
|
||||
assert.equal(result.states['staging-gce-c4'], 'absent')
|
||||
assert.deepEqual(result.membership, subject.selector().membership)
|
||||
assert.equal(result.membershipSha256, membershipDigest(subject.selector().membership))
|
||||
assert.deepEqual(subject.requests.map(({ path }) => path), [
|
||||
'/v1/admin/admission-selector/status'
|
||||
])
|
||||
})
|
||||
|
||||
test('initializes generation zero without changing membership', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_0', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/apply'))
|
||||
assert.deepEqual(request.body, {
|
||||
v: 1,
|
||||
attemptId: 'asia_boundary_0',
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
membership
|
||||
})
|
||||
assert.equal(result.generation, 1)
|
||||
assert.equal(result.states['staging-gce-c4'], 'absent')
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('retries the same fingerprint-bound initialization after intent persistence', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
subject.failNextApplyAfterIntent()
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_intent_retry', token: 'not-logged'
|
||||
}, subject)
|
||||
const applies = subject.requests.filter(({ path }) => path.endsWith('/apply'))
|
||||
assert.equal(applies.length, 2)
|
||||
assert.deepEqual(applies[1].body, applies[0].body)
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(result.generation, 1)
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('recovers a committed generation-zero initialization', async () => {
|
||||
const membership = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership })
|
||||
const config = {
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
attemptId: 'asia_boundary_retry', token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 0,
|
||||
expectedMembershipSha256: membershipDigest(membership),
|
||||
membership
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 1)
|
||||
assert.deepEqual(subject.selector().membership, membership)
|
||||
})
|
||||
|
||||
test('rejects generation-zero membership drift after inspect', async () => {
|
||||
const inspected = {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2']
|
||||
}
|
||||
const changed = {
|
||||
existingOnly: ['staging-gce-c2', 'staging-gce-c3'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c1']
|
||||
}
|
||||
const subject = harness({ generation: 0, membership: changed })
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 0, imageDigest: digest,
|
||||
expectedMembershipSha256: membershipDigest(inspected),
|
||||
attemptId: 'asia_boundary_drift', token: 'not-logged'
|
||||
}, subject), /membership changed/)
|
||||
assert.equal(subject.requests.some(({ path }) => path.endsWith('/apply')), false)
|
||||
})
|
||||
|
||||
test('registers all three Asia cells atomically with region and exact limits', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_7', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.equal(request.body.cells.length, 3)
|
||||
assert.ok(request.body.cells.every((cell) =>
|
||||
cell.region === 'asia-east2' && cell.capacityRequests === 6_000 &&
|
||||
cell.connectionHardCap === 3_000 && cell.connectionUnobservedBound === 60
|
||||
))
|
||||
assert.equal(result.generation, 8)
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
|
||||
})
|
||||
|
||||
test('promotes the canary only after runtime and director-heartbeat checks', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: { existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_8', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 2)
|
||||
assert.equal(result.states['production-gce-c27'], 'general')
|
||||
})
|
||||
|
||||
test('checks registered migration-only cells before director configuration without requiring heartbeat', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'registered',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 8, imageDigest: digest, token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 6)
|
||||
assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 0)
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
|
||||
})
|
||||
|
||||
test('rolls back admission without requiring an unhealthy runtime to answer', async () => {
|
||||
const subject = harness({
|
||||
generation: 9,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_9', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.equal(subject.fetchCount(), 0)
|
||||
assert.equal(result.states['production-gce-c27'], 'migration-only')
|
||||
})
|
||||
|
||||
test('uses the server-enforced C4-only route for staging proof transitions', async () => {
|
||||
const subject = harness({
|
||||
generation: 3,
|
||||
membership: {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2', 'staging-gce-c4']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'rollback', cells: ['staging-gce-c4'],
|
||||
expectedGeneration: 3, imageDigest: digest, attemptId: 'asia_staging_rollback',
|
||||
token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(
|
||||
({ path }) => path.endsWith('/apply-staging-asia-proof')
|
||||
)
|
||||
assert.deepEqual(request.body, {
|
||||
v: 1,
|
||||
attemptId: 'asia_staging_rollback',
|
||||
expectedGeneration: 3,
|
||||
state: 'migration-only'
|
||||
})
|
||||
assert.deepEqual(subject.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: ['staging-gce-c4'],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
assert.equal(result.states['staging-gce-c4'], 'migration-only')
|
||||
})
|
||||
|
||||
test('fails closed when the exact selector generation moved', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 6, imageDigest: digest, attemptId: 'asia_register_6', token: 'not-logged'
|
||||
}, subject), /generation changed/)
|
||||
assert.equal(subject.fetchCount(), 0)
|
||||
})
|
||||
|
||||
test('recovers a committed registration when the workflow retries the original generation', async () => {
|
||||
const subject = harness(baseSelector)
|
||||
const config = {
|
||||
environment: 'production', mode: 'register',
|
||||
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/add-migration-cells',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
cells: config.cells.map((cellId) => ({ cellId }))
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 8)
|
||||
})
|
||||
|
||||
test('recovers a committed promotion when the workflow retries the original generation', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [],
|
||||
general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 9)
|
||||
assert.equal(recovered.states['production-gce-c27'], 'general')
|
||||
})
|
||||
|
||||
test('inspects an ambiguous promotion without creating a new transition', async () => {
|
||||
const untouched = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_promote',
|
||||
token: 'not-logged'
|
||||
}
|
||||
const absent = await operateRelayAsiaAdmission(config, untouched)
|
||||
assert.equal(absent.promoted, false)
|
||||
assert.equal(untouched.requests.some(({ path }) => path.endsWith('/apply')), false)
|
||||
|
||||
const committed = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
await assert.rejects(committed.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, committed)
|
||||
assert.equal(recovered.promoted, true)
|
||||
assert.equal(recovered.generation, 9)
|
||||
})
|
||||
|
||||
test('treats an already rolled-back promotion as recovered', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_after_rollback',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
await subject.apply('later_rollback', {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
})
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.promoted, false)
|
||||
assert.equal(recovered.generation, 10)
|
||||
})
|
||||
|
||||
test('recovers a committed rollback when the workflow retries the original generation', async () => {
|
||||
const subject = harness({
|
||||
generation: 9,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_retry',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse(
|
||||
'/v1/admin/admission-selector/apply',
|
||||
{
|
||||
v: 1,
|
||||
attemptId: config.attemptId,
|
||||
expectedGeneration: config.expectedGeneration,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
}
|
||||
), /response lost after commit/)
|
||||
const recovered = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.equal(recovered.recovered, true)
|
||||
assert.equal(recovered.generation, 10)
|
||||
assert.equal(recovered.states['production-gce-c27'], 'migration-only')
|
||||
})
|
||||
|
||||
test('rejects a committed transition retry after a later selector change', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
|
||||
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_stale_promote',
|
||||
token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
|
||||
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
|
||||
membership: {
|
||||
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
|
||||
}
|
||||
}), /response lost after commit/)
|
||||
await subject.apply('later_rollback', {
|
||||
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
|
||||
})
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission(config, subject),
|
||||
/does not match the requested Asia transition/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires the C27 canary before promoting C28 and C29', async () => {
|
||||
const subject = harness({
|
||||
generation: 8,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
general: ['production-gce-c26']
|
||||
}
|
||||
})
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote',
|
||||
cells: ['production-gce-c28', 'production-gce-c29'], expectedGeneration: 8,
|
||||
imageDigest: digest, attemptId: 'asia_wave_before_canary', token: 'not-logged'
|
||||
}, subject), /C27 canary/)
|
||||
})
|
||||
|
||||
const launchCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
|
||||
|
||||
function admissionArguments(environment, mode, cellIds) {
|
||||
return [
|
||||
'--environment', environment, '--mode', mode, '--cell-ids', cellIds,
|
||||
'--image-digest', digest, '--expected-generation', '9', '--attempt-id', 'asia_wave_9'
|
||||
]
|
||||
}
|
||||
|
||||
test('accepts only reviewed Asia admission waves', () => {
|
||||
const accepted = [
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['inspect', 'production-gce-c30'],
|
||||
['inspect', 'production-gce-c31'],
|
||||
['verify', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['verify', 'production-gce-c30'],
|
||||
['verify', 'production-gce-c31'],
|
||||
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['register', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['register', 'production-gce-c30'],
|
||||
['registered', 'production-gce-c30'],
|
||||
['register', 'production-gce-c31'],
|
||||
['registered', 'production-gce-c31'],
|
||||
['promote', 'production-gce-c27'],
|
||||
['promote', 'production-gce-c28,production-gce-c29'],
|
||||
['promote', 'production-gce-c30'],
|
||||
['recover-promotion', 'production-gce-c30'],
|
||||
['promote', 'production-gce-c31'],
|
||||
['recover-promotion', 'production-gce-c31'],
|
||||
['rollback', 'production-gce-c27'],
|
||||
['rollback', 'production-gce-c28,production-gce-c29'],
|
||||
['rollback', 'production-gce-c30'],
|
||||
['rollback', 'production-gce-c31'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35'],
|
||||
...['production-gce-c32', 'production-gce-c33'].flatMap((cellId) => [
|
||||
'inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback'
|
||||
].map((mode) => [mode, cellId])),
|
||||
...['inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback']
|
||||
.map((mode) => [mode, 'production-gce-c34']),
|
||||
...['inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback']
|
||||
.map((mode) => [mode, 'production-gce-c35']),
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35']
|
||||
]
|
||||
for (const [mode, cellIds] of accepted) {
|
||||
assert.deepEqual(
|
||||
parseRelayAsiaAdmissionArguments(admissionArguments('production', mode, cellIds)).cells,
|
||||
cellIds.split(','),
|
||||
`${mode} ${cellIds}`
|
||||
)
|
||||
}
|
||||
const rejected = [
|
||||
['promote', 'production-gce-c34,production-gce-c35'],
|
||||
['promote', 'production-gce-c31,production-gce-c35'],
|
||||
['register', 'production-gce-c34,production-gce-c35'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'],
|
||||
['initialize', 'production-gce-c30'],
|
||||
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c29,production-gce-c30'],
|
||||
['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['initialize', 'production-gce-c32'],
|
||||
['register', 'production-gce-c32,production-gce-c33'],
|
||||
['register', 'production-gce-c31,production-gce-c32'],
|
||||
['promote', 'production-gce-c32,production-gce-c33'],
|
||||
['rollback', 'production-gce-c32,production-gce-c33'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['verify', 'production-gce-c30,production-gce-c31'],
|
||||
['verify', 'production-gce-c27,production-gce-c30'],
|
||||
['promote', 'production-gce-c27,production-gce-c30'],
|
||||
['promote', 'production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['promote', 'production-gce-c30,production-gce-c31'],
|
||||
['promote', 'production-gce-c33,production-gce-c34'],
|
||||
['promote', 'production-gce-c31,production-gce-c34'],
|
||||
['register', 'production-gce-c33,production-gce-c34'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33'],
|
||||
['rollback', 'production-gce-c27,production-gce-c30'],
|
||||
['rollback', 'production-gce-c30,production-gce-c31'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['rollback', 'production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['rollback', 'production-gce-c29'],
|
||||
['register', 'staging-gce-c4']
|
||||
]
|
||||
for (const [mode, cellIds] of rejected) {
|
||||
assert.throws(
|
||||
() => parseRelayAsiaAdmissionArguments(admissionArguments('production', mode, cellIds)),
|
||||
/--cell-ids/,
|
||||
`${mode} ${cellIds}`
|
||||
)
|
||||
}
|
||||
assert.deepEqual(
|
||||
parseRelayAsiaAdmissionArguments(admissionArguments('staging', 'promote', 'staging-gce-c4')).cells,
|
||||
['staging-gce-c4']
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayAsiaAdmissionArguments(admissionArguments('staging', 'promote', 'production-gce-c30')),
|
||||
/--cell-ids are invalid/
|
||||
)
|
||||
})
|
||||
|
||||
test('registers C30 alone beside the general launch cells', async () => {
|
||||
const subject = harness({
|
||||
generation: 9,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: [...launchCells] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: ['production-gce-c30'],
|
||||
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_register_c30', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId: 'production-gce-c30', cellUrl: 'https://c30.relay.onorca.dev', region: 'asia-east2',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { 'production-gce-c30': 'migration-only' })
|
||||
assert.deepEqual(subject.selector().membership.general, launchCells)
|
||||
})
|
||||
|
||||
test('registers C31 alone beside the general C27-C30', async () => {
|
||||
const general = [...launchCells, 'production-gce-c30']
|
||||
const subject = harness({
|
||||
generation: 11,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: [...general] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: ['production-gce-c31'],
|
||||
expectedGeneration: 11, imageDigest: digest, attemptId: 'asia_register_c31', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId: 'production-gce-c31', cellUrl: 'https://c31.relay.onorca.dev', region: 'asia-east2',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { 'production-gce-c31': 'migration-only' })
|
||||
assert.deepEqual(subject.selector().membership.general, general)
|
||||
})
|
||||
|
||||
test('registers the C34 spare alone as migration-only in asia-east2', async () => {
|
||||
const general = [...launchCells, 'production-gce-c30', 'production-gce-c31']
|
||||
const subject = harness({
|
||||
generation: 17,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: [...general, 'production-gce-c32', 'production-gce-c33']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: ['production-gce-c34'],
|
||||
expectedGeneration: 17, imageDigest: digest, attemptId: 'asia_register_c34', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId: 'production-gce-c34', cellUrl: 'https://c34.relay.onorca.dev', region: 'asia-east2',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { 'production-gce-c34': 'migration-only' })
|
||||
assert.equal(subject.selector().membership.general.includes('production-gce-c34'), false)
|
||||
})
|
||||
|
||||
test('registers the C35 spare alone as migration-only in asia-east2', async () => {
|
||||
const general = [...launchCells, 'production-gce-c30', 'production-gce-c31']
|
||||
const subject = harness({
|
||||
generation: 17,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: [...general, 'production-gce-c32', 'production-gce-c33', 'production-gce-c34']
|
||||
}
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: ['production-gce-c35'],
|
||||
expectedGeneration: 17, imageDigest: digest, attemptId: 'asia_register_c35', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId: 'production-gce-c35', cellUrl: 'https://c35.relay.onorca.dev', region: 'asia-east2',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { 'production-gce-c35': 'migration-only' })
|
||||
assert.equal(subject.selector().membership.general.includes('production-gce-c35'), false)
|
||||
})
|
||||
|
||||
test('promotes the registered C34 spare to general and leaves every other cell alone', async () => {
|
||||
const general = [
|
||||
...launchCells, 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33'
|
||||
]
|
||||
const membership = {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c34'],
|
||||
general: [...general]
|
||||
}
|
||||
const subject = harness({ generation: 345, membership })
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c34'],
|
||||
expectedGeneration: 345, imageDigest: digest, attemptId: 'asia_promote_c34', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.deepEqual(result.states, { 'production-gce-c34': 'general' })
|
||||
assert.deepEqual(subject.selector().membership.migrationOnly, ['production-gce-c17'])
|
||||
assert.deepEqual(subject.selector().membership.general, [...general, 'production-gce-c34'].sort())
|
||||
})
|
||||
|
||||
test('promotes the registered C35 spare to general and leaves every other cell alone', async () => {
|
||||
const general = [
|
||||
...launchCells, 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33',
|
||||
'production-gce-c34'
|
||||
]
|
||||
const membership = {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c35'],
|
||||
general: [...general]
|
||||
}
|
||||
const subject = harness({ generation: 345, membership })
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c35'],
|
||||
expectedGeneration: 345, imageDigest: digest, attemptId: 'asia_promote_c35', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.deepEqual(result.states, { 'production-gce-c35': 'general' })
|
||||
assert.deepEqual(subject.selector().membership.migrationOnly, ['production-gce-c17'])
|
||||
assert.deepEqual(subject.selector().membership.general, [...general, 'production-gce-c35'].sort())
|
||||
})
|
||||
|
||||
const usRegions = { 'production-gce-c32': 'us-central1', 'production-gce-c33': 'us-central1' }
|
||||
|
||||
test('registers C32 and C33 one at a time in us-central1 at the Asia shape', async () => {
|
||||
const general = [...launchCells, 'production-gce-c30', 'production-gce-c31']
|
||||
for (const [cellId, generation] of [['production-gce-c32', 13], ['production-gce-c33', 15]]) {
|
||||
const hostname = cellId.split('-').at(-1)
|
||||
const subject = harness({
|
||||
generation, membership: { existingOnly: [], migrationOnly: [], general: [...general] }
|
||||
}, {}, usRegions)
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: [cellId],
|
||||
expectedGeneration: generation, imageDigest: digest, attemptId: `us_register_${hostname}`,
|
||||
token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId, cellUrl: `https://${hostname}.relay.onorca.dev`, region: 'us-central1',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { [cellId]: 'migration-only' })
|
||||
// A US cell whose runtime reports Asia is the wrong cell, not a US one.
|
||||
await assert.rejects(operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: [cellId],
|
||||
expectedGeneration: generation, imageDigest: digest, attemptId: `us_register_${hostname}`,
|
||||
token: 'not-logged'
|
||||
}, harness({
|
||||
generation, membership: { existingOnly: [], migrationOnly: [], general: [...general] }
|
||||
})), new RegExp(`${cellId} runtime does not match`))
|
||||
}
|
||||
})
|
||||
|
||||
test('promotes a US cell without the Asia launch order, which still binds Asia cells', async () => {
|
||||
const selector = (cellId) => ({
|
||||
generation: 14,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: [cellId, 'production-gce-c27', 'production-gce-c28'].sort(),
|
||||
general: ['production-gce-c29']
|
||||
}
|
||||
})
|
||||
const config = (cellId) => ({
|
||||
environment: 'production', mode: 'promote', cells: [cellId],
|
||||
expectedGeneration: 14, imageDigest: digest, attemptId: 'us_promote_wave', token: 'not-logged'
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission(
|
||||
config('production-gce-c32'), harness(selector('production-gce-c32'), {}, usRegions)
|
||||
)
|
||||
assert.deepEqual(result.states, { 'production-gce-c32': 'general' })
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission(config('production-gce-c31'), harness(selector('production-gce-c31'))),
|
||||
/C27 canary/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires the C27 canary to be general before promoting C30', async () => {
|
||||
const selector = (general) => ({
|
||||
generation: 10,
|
||||
membership: {
|
||||
existingOnly: [],
|
||||
migrationOnly: ['production-gce-c30', ...launchCells.filter((cell) => !general.includes(cell))].sort(),
|
||||
general
|
||||
}
|
||||
})
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c30'],
|
||||
expectedGeneration: 10, imageDigest: digest, attemptId: 'asia_promote_c30', token: 'not-logged'
|
||||
}
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission(config, harness(selector(['production-gce-c28', 'production-gce-c29']))),
|
||||
/C27 canary/
|
||||
)
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission(config, harness(selector(['production-gce-c27', 'production-gce-c29']))),
|
||||
/every launch cell to be general/
|
||||
)
|
||||
const subject = harness(selector([...launchCells]))
|
||||
const result = await operateRelayAsiaAdmission(config, subject)
|
||||
assert.deepEqual(result.states, { 'production-gce-c30': 'general' })
|
||||
assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 1)
|
||||
})
|
||||
|
||||
test('promotes C30 on its own digest while the launch cells serve another', async () => {
|
||||
const c30Digest = `sha256:${'b'.repeat(64)}`
|
||||
const selector = {
|
||||
generation: 10,
|
||||
membership: { existingOnly: [], migrationOnly: ['production-gce-c30'], general: [...launchCells] }
|
||||
}
|
||||
const config = {
|
||||
environment: 'production', mode: 'promote', cells: ['production-gce-c30'],
|
||||
expectedGeneration: 10, imageDigest: c30Digest, attemptId: 'asia_promote_c30', token: 'not-logged'
|
||||
}
|
||||
const digests = { 'production-gce-c30': c30Digest }
|
||||
const result = await operateRelayAsiaAdmission(config, harness(selector, digests))
|
||||
assert.deepEqual(result.states, { 'production-gce-c30': 'general' })
|
||||
await assert.rejects(
|
||||
operateRelayAsiaAdmission({ ...config, imageDigest: digest }, harness(selector, digests)),
|
||||
/production-gce-c30 runtime does not match/
|
||||
)
|
||||
})
|
||||
|
||||
const admissionWorkflow = readRelayWorkflow('operate-relay-asia-admission.yml')
|
||||
|
||||
function workflowBlock(first, last) {
|
||||
const start = admissionWorkflow.indexOf(first)
|
||||
const end = admissionWorkflow.indexOf(last, start)
|
||||
assert.ok(start !== -1 && end !== -1, first)
|
||||
return admissionWorkflow.slice(start, end + last.length).replace(/^ {10}/gm, '')
|
||||
}
|
||||
|
||||
// Runs the workflow's own promotion input block, so each wave's canary routing is what ships.
|
||||
function promotionOutputs(cellIds) {
|
||||
const temp = mkdtempSync(join(tmpdir(), 'relay-admission-inputs-'))
|
||||
try {
|
||||
const output = join(temp, 'output')
|
||||
const result = spawnSync('bash', ['-c', workflowBlock(
|
||||
'set -euo pipefail\n test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"',
|
||||
'} >> "${GITHUB_OUTPUT}"'
|
||||
)], {
|
||||
env: {
|
||||
...process.env, GITHUB_OUTPUT: output, DEPLOY_WORKLOAD_IDENTITY_PROVIDER: 'provider',
|
||||
DEPLOY_SERVICE_ACCOUNT: 'account', IMAGE_DIGEST: digest, OPERATION_MODE: 'promote',
|
||||
EXPECTED_SELECTOR_GENERATION: '9', SELECTOR_ATTEMPT_ID: 'promote_wave_9',
|
||||
OPERATION_CONFIRMATION: 'PROMOTE_ASIA_GENERAL', TARGET_ENVIRONMENT: 'production',
|
||||
TARGET_CELL_IDS: cellIds, EXPECTED_SELECTOR_MEMBERSHIP_SHA256: '', DIRECTOR_IMAGE_DIGEST: '',
|
||||
EVIDENCE_RUN_ID: '', EVIDENCE_RUN_ATTEMPT: ''
|
||||
},
|
||||
encoding: 'utf8'
|
||||
})
|
||||
if (result.status !== 0) return null
|
||||
return Object.fromEntries(readFileSync(output, 'utf8').trim().split('\n')
|
||||
.map((line) => line.split('=')))
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
test('runs each later cell\'s own canary with load aimed at that cell\'s region', () => {
|
||||
for (const [cellId, region] of [
|
||||
['production-gce-c30', 'asia-east2'], ['production-gce-c31', 'asia-east2'],
|
||||
['production-gce-c32', 'us-central1'], ['production-gce-c33', 'us-central1'],
|
||||
['production-gce-c34', 'asia-east2'], ['production-gce-c35', 'asia-east2']
|
||||
]) {
|
||||
const outputs = promotionOutputs(cellId)
|
||||
assert.equal(outputs?.canary, 'true', cellId)
|
||||
assert.equal(outputs.canary_cell, cellId)
|
||||
assert.equal(outputs.canary_region, region, cellId)
|
||||
assert.equal(outputs.evidence_kind, 'none')
|
||||
}
|
||||
assert.equal(promotionOutputs('production-gce-c32,production-gce-c33'), null)
|
||||
assert.equal(promotionOutputs('production-gce-c33,production-gce-c34'), null)
|
||||
assert.equal(promotionOutputs('production-gce-c34,production-gce-c35'), null)
|
||||
const canaryStart = workflowBlock('case "${CANARY_CELL}" in', '\n esac')
|
||||
for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34', 'production-gce-c35']) {
|
||||
const result = spawnSync('bash', ['-euo', 'pipefail', '-c',
|
||||
`${canaryStart}\necho "\${verify_cells} \${expected_states}"`], {
|
||||
env: { ...process.env, CANARY_CELL: cellId }, encoding: 'utf8'
|
||||
})
|
||||
assert.equal(result.stdout.trim(), `${cellId} {"${cellId}":"general"}`)
|
||||
}
|
||||
assert.match(admissionWorkflow, /--preferred-region "\$\{CANARY_REGION\}"/)
|
||||
assert.match(admissionWorkflow, /CANARY_REGION: \$\{\{ steps\.inputs\.outputs\.canary_region \}\}/)
|
||||
})
|
||||
|
||||
const stagingDeadCellWave = ['staging-gce-c5', 'staging-gce-c6', 'staging-gce-c7']
|
||||
const stagingUsShape = Object.fromEntries(stagingDeadCellWave.map((cellId) => [cellId, 'us-central1']))
|
||||
const stagingUsCaps = Object.fromEntries(stagingDeadCellWave.map((cellId) => [cellId, 600]))
|
||||
|
||||
test('registers the staging dead-cell wave at its own US shape', async () => {
|
||||
const subject = harness({
|
||||
generation: 43,
|
||||
membership: { existingOnly: ['staging-gce-c3'], migrationOnly: [], general: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4'] }
|
||||
}, {}, stagingUsShape, stagingUsCaps)
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'register', cells: stagingDeadCellWave,
|
||||
expectedGeneration: 43, imageDigest: digest, attemptId: 'staging_c5_c7_register', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells.map((cell) => cell.cellUrl), [
|
||||
'https://c5.relay-staging.onorca.dev', 'https://c6.relay-staging.onorca.dev', 'https://c7.relay-staging.onorca.dev'
|
||||
])
|
||||
assert.ok(request.body.cells.every((cell) =>
|
||||
cell.region === 'us-central1' && cell.capacityRequests === 4_000 &&
|
||||
cell.connectionHardCap === 600 && cell.connectionUnobservedBound === 60
|
||||
))
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
|
||||
})
|
||||
|
||||
test('promotes the staging dead-cell wave through the general route, not the C4 proof route', async () => {
|
||||
const subject = harness({
|
||||
generation: 44,
|
||||
membership: {
|
||||
existingOnly: ['staging-gce-c3'],
|
||||
migrationOnly: [...stagingDeadCellWave],
|
||||
general: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4']
|
||||
}
|
||||
}, {}, stagingUsShape, stagingUsCaps)
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'staging', mode: 'promote', cells: stagingDeadCellWave,
|
||||
expectedGeneration: 44, imageDigest: digest, attemptId: 'staging_c5_c7_promote', token: 'not-logged'
|
||||
}, subject)
|
||||
assert.ok(subject.requests.some(({ path }) => path === '/v1/admin/admission-selector/apply'))
|
||||
assert.ok(!subject.requests.some(({ path }) => path.endsWith('/apply-staging-asia-proof')))
|
||||
assert.deepEqual(new Set(Object.values(result.states)), new Set(['general']))
|
||||
assert.deepEqual(subject.selector().membership.general, [
|
||||
'staging-gce-c1', 'staging-gce-c2', 'staging-gce-c4', ...stagingDeadCellWave
|
||||
])
|
||||
})
|
||||
|
||||
test('refuses a staging wave that was not reviewed', () => {
|
||||
assert.throws(() => parseRelayAsiaAdmissionArguments([
|
||||
'--environment', 'staging', '--mode', 'promote', '--cell-ids', 'staging-gce-c4,staging-gce-c5',
|
||||
'--expected-generation', '44', '--attempt-id', 'staging_mixed_wave', '--image-digest', digest
|
||||
]), /reviewed admission wave/)
|
||||
assert.throws(() => parseRelayAsiaAdmissionArguments([
|
||||
'--environment', 'staging', '--mode', 'register', '--cell-ids', 'staging-gce-c8',
|
||||
'--expected-generation', '44', '--attempt-id', 'staging_unknown', '--image-digest', digest
|
||||
]), /cell-ids are invalid/)
|
||||
})
|
||||
@@ -1,619 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Writes one cell's runtime switch object (cells/<cellId>.json in <project>-relay-control) and
|
||||
// reads back what the cell applied. One cell per run by construction: the object is per cell.
|
||||
//
|
||||
// dry-run: checks the cell and the current generation, prints the change, writes nothing.
|
||||
// write: the same checks, then a compare-and-swap write (ifGenerationMatch), then waits for
|
||||
// the cell's runtime status to report the new generation applied.
|
||||
import { parseArgs } from 'node:util'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
export const CELL_ID_PATTERN = /^(production|staging)-gce-c[0-9]{1,3}$/
|
||||
export const WRITE_CONFIRMATION = 'WRITE_RELAY_CELL_FLAGS'
|
||||
// The cell polls every 5 s (+20% jitter); 15 s leaves two polls of slack.
|
||||
export const READ_BACK_TIMEOUT_MS = 15_000
|
||||
const READ_BACK_INTERVAL_MS = 1_000
|
||||
const STORAGE_ROOT = 'https://storage.googleapis.com'
|
||||
|
||||
// Every switch a cell image may know, with what an absent key means on the cell. A cell
|
||||
// resets a key missing from its object to this default, so a write must carry every key it
|
||||
// does not mean to change: the tool reads the current object and changes only what is named.
|
||||
const enumOf = (...options) => (value) => (options.includes(value) ? value : undefined)
|
||||
const booleanValue = (value) => (value === 'true' ? true : value === 'false' ? false : undefined)
|
||||
const numberIn = (min, max, integer) => (value) => {
|
||||
const number = Number(value)
|
||||
if (value === '' || !Number.isFinite(number) || number < min || number > max) return undefined
|
||||
if (integer && !Number.isInteger(number)) return undefined
|
||||
return number
|
||||
}
|
||||
export const CELL_FLAG_SPECS = {
|
||||
readinessLocal: { parse: booleanValue, default: false },
|
||||
ticketCheck: { parse: enumOf('off', 'shadow', 'enforce'), default: 'off' },
|
||||
admitMode: { parse: enumOf('db', 'reserve'), default: 'db' },
|
||||
// Absent means the cell's region default.
|
||||
intakePerSec: {
|
||||
parse: (value) => {
|
||||
const rate = numberIn(0, 1_000, false)(value)
|
||||
return rate === 0 ? undefined : rate
|
||||
}
|
||||
},
|
||||
reserveDryRun: { parse: booleanValue, default: false },
|
||||
rejectionFence: { parse: booleanValue, default: true },
|
||||
// Absent keeps the cell's boot value.
|
||||
readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) },
|
||||
// Flip-back leases in flight; absent is a third of the cell's pool.
|
||||
reregisterInFlight: { parse: numberIn(1, 16, true) },
|
||||
// Step 6: acts only while the cell is reserve.
|
||||
phonePath: { parse: enumOf('db', 'memory'), default: 'db' }
|
||||
}
|
||||
|
||||
export const RESERVE_CONFIRMATION = 'RESERVE'
|
||||
export const BREAK_GLASS_CONFIRMATION = 'BREAK GLASS'
|
||||
// Longer than a director's cached read of the admit-mode table (5 s).
|
||||
export const RESERVE_MODE_CACHE_WAIT_MS = 10_000
|
||||
// How long a flip back may take to lease every control the cell admitted from memory: its
|
||||
// control count at a per-lane pace below the measured 0.38/s (170 ms trips), within the job.
|
||||
export const REREGISTER_WAIT_MIN_MS = 8 * 60_000
|
||||
export const REREGISTER_WAIT_MAX_MS = 50 * 60_000
|
||||
const REREGISTER_CONTROLS_PER_LANE_PER_SEC = 0.3
|
||||
// The cell takes a third of its database pool, and caps the switch at the pool less two
|
||||
// (host-session-registry.ts reregisterInFlightLimit). An image from before runtime-status
|
||||
// reported the pool falls back to the deployed sizes (US 10, Asia 16).
|
||||
const CELL_DATABASE_POOL_MAX = { 'asia-east2': 16 }
|
||||
const CELL_DATABASE_POOL_MAX_DEFAULT = 10
|
||||
|
||||
export function reregisterLanes(runtime) {
|
||||
const reported = runtime?.databasePoolMax
|
||||
const pool =
|
||||
Number.isInteger(reported) && reported > 0
|
||||
? reported
|
||||
: (CELL_DATABASE_POOL_MAX[runtime?.region] ?? CELL_DATABASE_POOL_MAX_DEFAULT)
|
||||
const flag = runtime?.flagsApplied?.flags?.reregisterInFlight
|
||||
if (flag === undefined) return Math.max(1, Math.floor(pool / 3))
|
||||
return Math.max(1, Math.min(Number(flag), pool - 2))
|
||||
}
|
||||
|
||||
export function reregisterWaitMs(runtime) {
|
||||
const controls = Number(runtime?.runtime?.controls ?? 0)
|
||||
const lanes = reregisterLanes(runtime)
|
||||
const estimate = (controls / (lanes * REREGISTER_CONTROLS_PER_LANE_PER_SEC)) * 1_000
|
||||
return Math.min(REREGISTER_WAIT_MAX_MS, Math.max(REREGISTER_WAIT_MIN_MS, Math.ceil(estimate)))
|
||||
}
|
||||
|
||||
// An image from before `supportedFlags` accepts the keys its applied flags list, and only
|
||||
// ticketCheck off/shadow: anything else is dropped there, or voids the whole object.
|
||||
function legacySupportedFlags(appliedFlags) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(appliedFlags ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
key === 'ticketCheck'
|
||||
? { type: 'enum', values: ['off', 'shadow'] }
|
||||
: { type: typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'number' : 'enum', values: [value] }
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
function accepts(spec, value) {
|
||||
if (spec?.type === 'boolean') return typeof value === 'boolean'
|
||||
if (spec?.type === 'enum') return spec.values.includes(value)
|
||||
if (spec?.type === 'number') {
|
||||
return (
|
||||
typeof value === 'number' &&
|
||||
(spec.min === undefined || value >= spec.min) &&
|
||||
(spec.max === undefined || value <= spec.max) &&
|
||||
(!spec.integer || Number.isInteger(value))
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The whole object as it will be written, not only the change: a bad key or value already in
|
||||
// the file would void or be dropped on every later write too.
|
||||
export function assertSupportedObject(flags, runtime) {
|
||||
const supported = runtime.supportedFlags ?? legacySupportedFlags(runtime.flagsApplied?.flags)
|
||||
for (const [key, value] of Object.entries(flags)) {
|
||||
if (!accepts(supported[key], value)) {
|
||||
throw new Error(
|
||||
`the cell's image does not support ${key}=${JSON.stringify(value)}; roll the image first`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A reserve cell's dead-man is fed only by director polls through the load balancer. With SQL
|
||||
// readiness the balancer drops the cell in a database outage, the polls fail, and the cell trips
|
||||
// (staging drill 2026-10-11), so reserve requires readinessLocal=true in the object it writes.
|
||||
export function assertReserveKeepsLocalReadiness(flags, changes) {
|
||||
if ((flags.admitMode ?? 'db') !== 'reserve' || flags.readinessLocal === true) return
|
||||
if ('readinessLocal' in changes && !('admitMode' in changes)) {
|
||||
throw new Error(
|
||||
'readinessLocal must stay true while the cell is reserve; flip it back with admitMode=db first, or in the same write'
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
'admitMode=reserve requires readinessLocal=true on the cell, already written or in the same write ' +
|
||||
'(e.g. --set readinessLocal=true,admitMode=reserve): with SQL readiness the load balancer drops the cell ' +
|
||||
"in a database outage and the directors' polls can no longer feed its dead-man"
|
||||
)
|
||||
}
|
||||
|
||||
// The cell ignores phonePath=memory outside reserve, so writing it there only arms a later
|
||||
// admitMode=reserve flip to turn phones over unnamed. Rolling back with admitMode=db stays allowed.
|
||||
export function assertPhonePathNeedsReserve(flags, changes) {
|
||||
if ((flags.phonePath ?? 'db') !== 'memory') return
|
||||
if ((flags.admitMode ?? 'db') === 'reserve') {
|
||||
if (changes.admitMode === 'reserve' && !('phonePath' in changes)) {
|
||||
throw new Error(
|
||||
'the object still says phonePath=memory: name phonePath in the same write ' +
|
||||
'(e.g. --set admitMode=reserve,phonePath=default), so a reserve flip never turns phones over unnamed'
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
if ('phonePath' in changes) {
|
||||
throw new Error("phonePath=memory requires admitMode=reserve on the cell, already written or in the same write")
|
||||
}
|
||||
}
|
||||
|
||||
// `key=value,key=value`; `key=default` removes the key so the cell uses its default.
|
||||
export function parseFlagChanges(text) {
|
||||
const changes = {}
|
||||
for (const part of (text ?? '').split(',').map((entry) => entry.trim()).filter(Boolean)) {
|
||||
const separator = part.indexOf('=')
|
||||
const key = separator < 0 ? part : part.slice(0, separator)
|
||||
const raw = separator < 0 ? '' : part.slice(separator + 1).trim()
|
||||
const spec = CELL_FLAG_SPECS[key]
|
||||
if (!spec) throw new Error(`unknown switch ${key}`)
|
||||
if (key in changes) throw new Error(`switch ${key} named twice`)
|
||||
if (raw === 'default') {
|
||||
changes[key] = null
|
||||
continue
|
||||
}
|
||||
const value = spec.parse(raw)
|
||||
if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`)
|
||||
// phonePath=db is the default, stored as no key: an image without the switch refuses the key.
|
||||
changes[key] = key === 'phonePath' && value === 'db' ? null : value
|
||||
}
|
||||
if (Object.keys(changes).length === 0) throw new Error('set names no switch to change')
|
||||
return changes
|
||||
}
|
||||
|
||||
// The switches as the cell will apply them: every known key, absent ones at their default.
|
||||
function effectiveFlags(flags) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(CELL_FLAG_SPECS).map(([key, spec]) => [key, flags?.[key] ?? spec.default])
|
||||
)
|
||||
}
|
||||
|
||||
export function parseCellFlagsRequest(values) {
|
||||
const mode = values.mode
|
||||
if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write')
|
||||
const cellId = values['cell-id'] ?? ''
|
||||
if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id')
|
||||
const changes = parseFlagChanges(values.set)
|
||||
// 0 means "no object yet": the write then creates it and refuses if one appeared.
|
||||
const expectedGeneration = values['expected-generation'] ?? ''
|
||||
if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) {
|
||||
throw new Error('expected-generation must be 0 or the exact current object generation')
|
||||
}
|
||||
if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) {
|
||||
throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
// Reserve moves this cell's hosts off the database: it is typed for this cell, every time.
|
||||
if (
|
||||
mode === 'write' &&
|
||||
changes.admitMode === 'reserve' &&
|
||||
values['confirm-reserve'] !== `${RESERVE_CONFIRMATION} ${cellId}`
|
||||
) {
|
||||
throw new Error(`admitMode=reserve requires --confirm-reserve "${RESERVE_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
// Break glass: a dead or unreachable reserve cell cannot report db, so nothing would ever
|
||||
// let the sweeps re-place its hosts. Only admitMode=db, typed for this cell.
|
||||
const breakGlass = values['break-glass'] ?? ''
|
||||
if (breakGlass !== '') {
|
||||
if (breakGlass !== `${BREAK_GLASS_CONFIRMATION} ${cellId}`) {
|
||||
throw new Error(`break glass requires --break-glass "${BREAK_GLASS_CONFIRMATION} ${cellId}"`)
|
||||
}
|
||||
if (mode !== 'write' || changes.admitMode !== null || Object.keys(changes).length !== 1) {
|
||||
throw new Error('break glass only writes admitMode=default (db), on its own')
|
||||
}
|
||||
}
|
||||
const directorOrigin = values['director-origin'] ?? ''
|
||||
if (changes.admitMode !== undefined && !/^https:\/\/relay(-staging)?\.onorca\.dev$/.test(directorOrigin)) {
|
||||
throw new Error('an admitMode change needs --director-origin, where Postgres records it')
|
||||
}
|
||||
const projectId = values['project-id'] ?? ''
|
||||
if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid')
|
||||
const cellOrigin = values['cell-origin'] ?? ''
|
||||
if (!/^https:\/\/c[0-9]{1,3}\.relay(-staging)?\.onorca\.dev$/.test(cellOrigin)) {
|
||||
throw new Error('cell origin is not a relay cell origin')
|
||||
}
|
||||
if (new URL(cellOrigin).hostname.split('.')[0] !== cellId.replace(/^[a-z]+-gce-/, '')) {
|
||||
throw new Error('cell origin names another cell')
|
||||
}
|
||||
return {
|
||||
mode,
|
||||
cellId,
|
||||
cellOrigin,
|
||||
directorOrigin,
|
||||
breakGlass: breakGlass !== '',
|
||||
projectId,
|
||||
expectedGeneration,
|
||||
changes
|
||||
}
|
||||
}
|
||||
|
||||
// Read-modify-write: the current object's switches with only the named ones changed.
|
||||
export function desiredObject(request, current) {
|
||||
if (current !== null && (current?.v !== 1 || current?.cellId !== request.cellId)) {
|
||||
throw new Error('the current object is not this cell\'s v1 switch object; fix it by hand')
|
||||
}
|
||||
const flags = { ...(current?.flags ?? {}) }
|
||||
for (const [key, value] of Object.entries(request.changes)) {
|
||||
if (value === null) delete flags[key]
|
||||
else flags[key] = value
|
||||
}
|
||||
return { v: 1, cellId: request.cellId, flags }
|
||||
}
|
||||
|
||||
function objectUrl(request, suffix = '') {
|
||||
const bucket = `${request.projectId}-relay-control`
|
||||
return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent(
|
||||
`cells/${request.cellId}.json`
|
||||
)}${suffix}`
|
||||
}
|
||||
|
||||
// One transient 5xx or network failure is retried once, as every relay admin step does.
|
||||
export async function readRuntime(fetchImpl, request, idToken, wait) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${request.cellOrigin}/v1/admin/runtime-status`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 })
|
||||
},
|
||||
{ wait, timeoutMs: 10_000, retryDelayMs: 1_000 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(`cell runtime status returned ${response.status}`)
|
||||
}
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
export async function readCurrentObject(fetchImpl, request, accessToken) {
|
||||
const headers = { authorization: `Bearer ${accessToken}` }
|
||||
const metadata = await fetchImpl(objectUrl(request), { headers })
|
||||
if (metadata.status === 404) {
|
||||
await metadata.body?.cancel().catch(() => undefined)
|
||||
return { generation: '0', object: null }
|
||||
}
|
||||
if (!metadata.ok) throw new Error(`object metadata read returned ${metadata.status}`)
|
||||
const { generation } = await metadata.json()
|
||||
const media = await fetchImpl(objectUrl(request, `?alt=media&ifGenerationMatch=${generation}`), {
|
||||
headers
|
||||
})
|
||||
if (!media.ok) throw new Error(`object read returned ${media.status}`)
|
||||
return { generation: String(generation), object: await media.json().catch(() => null) }
|
||||
}
|
||||
|
||||
function generationConflict(current, expected) {
|
||||
return Object.assign(new Error(`current generation is ${current}, not expected ${expected}`), {
|
||||
generationConflict: true
|
||||
})
|
||||
}
|
||||
|
||||
async function writeObject(fetchImpl, request, object, accessToken, audit) {
|
||||
const boundary = `relay-cell-flags-${Date.now()}`
|
||||
const metadata = {
|
||||
name: `cells/${request.cellId}.json`,
|
||||
contentType: 'application/json',
|
||||
cacheControl: 'no-store',
|
||||
// Who and which run, beside the generation the bucket's versioning already keeps.
|
||||
metadata: audit
|
||||
}
|
||||
const body = [
|
||||
`--${boundary}`,
|
||||
'Content-Type: application/json; charset=UTF-8',
|
||||
'',
|
||||
JSON.stringify(metadata),
|
||||
`--${boundary}`,
|
||||
'Content-Type: application/json',
|
||||
'',
|
||||
JSON.stringify(object),
|
||||
`--${boundary}--`,
|
||||
''
|
||||
].join('\r\n')
|
||||
const bucket = `${request.projectId}-relay-control`
|
||||
const response = await fetchImpl(
|
||||
`${STORAGE_ROOT}/upload/storage/v1/b/${bucket}/o?uploadType=multipart&ifGenerationMatch=${request.expectedGeneration}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${accessToken}`,
|
||||
'content-type': `multipart/related; boundary=${boundary}`
|
||||
},
|
||||
body
|
||||
}
|
||||
)
|
||||
// A 4xx means nothing was written; a 5xx or a lost reply may have been.
|
||||
if (response.status === 412) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), {
|
||||
notWritten: true,
|
||||
generationConflict: true
|
||||
})
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw Object.assign(new Error(`object write returned ${response.status}`), {
|
||||
notWritten: response.status < 500
|
||||
})
|
||||
}
|
||||
return String((await response.json()).generation)
|
||||
}
|
||||
|
||||
// The census's record (relay_cell_admit_modes, through a director). Without `admitMode` it only
|
||||
// reads. Read back on every write: sweeps act on this, not on the cell's switch file.
|
||||
async function directorAdmitMode(fetchImpl, request, idToken, wait, admitMode) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${request.directorOrigin}/v1/admin/cell-admit-mode`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, cellId: request.cellId, ...(admitMode ? { admitMode } : {}) })
|
||||
},
|
||||
{ wait, timeoutMs: 10_000, retryDelayMs: 1_000 }
|
||||
)
|
||||
const body = await response.json().catch(() => null)
|
||||
if (!response.ok) throw new Error(`director refused admitMode: ${response.status} ${body?.error ?? ''}`)
|
||||
if (admitMode && body?.admitMode !== admitMode) {
|
||||
throw new Error(`director recorded admitMode ${body?.admitMode}, not ${admitMode}`)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// Every known key, not just the ones this run changed: a cell that kept an old value of any
|
||||
// switch has not applied this object. A key with no fixed default (the cell reports its own
|
||||
// value) is compared only when the object names it.
|
||||
export function sameFlags(applied, desired) {
|
||||
const expected = effectiveFlags(desired)
|
||||
const actual = effectiveFlags(applied)
|
||||
return Object.entries(CELL_FLAG_SPECS).every(
|
||||
([key, spec]) =>
|
||||
(!('default' in spec) && desired?.[key] === undefined) || actual[key] === expected[key]
|
||||
)
|
||||
}
|
||||
|
||||
// Sweeps resume only once every control the cell admitted from memory holds a lease again.
|
||||
async function waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log }) {
|
||||
let waitMs
|
||||
let deadline = now() + REREGISTER_WAIT_MIN_MS
|
||||
while (now() < deadline) {
|
||||
const runtime = await readRuntime(fetchImpl, request, idToken, sleep).catch(() => null)
|
||||
// An image without the field never admitted from memory.
|
||||
if (runtime && (runtime.admitModeEffective ?? 'db') === 'db') return
|
||||
if (runtime && waitMs === undefined) {
|
||||
waitMs = reregisterWaitMs(runtime)
|
||||
deadline = now() + waitMs
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_reregistration_wait', cellId: request.cellId, waitMs }))
|
||||
}
|
||||
await sleep(5_000)
|
||||
}
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_reregistration_still_running', cellId: request.cellId }))
|
||||
throw new Error(
|
||||
`cell still re-registering after ${waitMs ?? REREGISTER_WAIT_MIN_MS} ms. Its connections are safe and Postgres ` +
|
||||
'keeps reserve, so sweeps stay off the cell. Run again with --set admitMode=db, which records db once ' +
|
||||
'runtime-status reports admitModeEffective=db; to speed it up, name both in one write: ' +
|
||||
'--set admitMode=db,reregisterInFlight=N'
|
||||
)
|
||||
}
|
||||
|
||||
// No cell read-back and no wait for its controls: the cell is not answering. The file says db
|
||||
// so a cell that comes back boots in db, and Postgres says db so the sweeps re-place its hosts.
|
||||
async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep }) {
|
||||
const { accessToken, idToken, audit } = dependencies
|
||||
// Only for a dead cell: one that answers, or still heartbeats, takes the ordinary flip back.
|
||||
const answered = await fetchImpl(`${request.cellOrigin}/v1/admin/runtime-status`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1 }),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
})
|
||||
.then(async (response) => {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
return response.ok
|
||||
})
|
||||
.catch(() => false)
|
||||
if (answered) throw new Error('the cell answers runtime-status: flip it back with --set admitMode=db, not break glass')
|
||||
const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
if (recorded?.heartbeatFresh !== false) {
|
||||
throw new Error(
|
||||
`the director ${recorded?.heartbeatFresh ? 'still sees heartbeats from' : 'cannot say whether it hears'} ` +
|
||||
'the cell: break glass is only for a cell that has stopped'
|
||||
)
|
||||
}
|
||||
const current = await readCurrentObject(fetchImpl, request, accessToken)
|
||||
if (current.generation !== request.expectedGeneration) {
|
||||
throw generationConflict(current.generation, request.expectedGeneration)
|
||||
}
|
||||
// phonePath goes too: a dead cell comes back with phones on the database path as well.
|
||||
const object = desiredObject({ ...request, changes: { ...request.changes, phonePath: null } }, current.object)
|
||||
const generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit }
|
||||
log(JSON.stringify(result))
|
||||
return { ...result, written: true }
|
||||
}
|
||||
|
||||
export async function operateCellFlags(request, dependencies) {
|
||||
const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies
|
||||
const now = dependencies.now ?? Date.now
|
||||
const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
if (request.breakGlass) return await breakGlassToDatabase(request, dependencies, { fetchImpl, log, sleep })
|
||||
const runtime = await readRuntime(fetchImpl, request, idToken, sleep)
|
||||
// Proves the origin is this cell, and that its image reads switch files at all.
|
||||
if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) {
|
||||
throw new Error(`origin answers as ${runtime.role}/${runtime.cellId}, not ${request.cellId}`)
|
||||
}
|
||||
if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)')
|
||||
const current = await readCurrentObject(fetchImpl, request, accessToken)
|
||||
const object = desiredObject(request, current.object)
|
||||
assertSupportedObject(object.flags, runtime)
|
||||
// A tripped dead-man: any new generation that keeps reserve re-arms it, so the cell trips
|
||||
// again and its re-registration restarts. Only a write that also says db is safe.
|
||||
if (
|
||||
runtime.flagsApplied.flags?.admitMode === 'reserve' &&
|
||||
runtime.admitModeEffective === 'db' &&
|
||||
(object.flags.admitMode ?? 'db') === 'reserve'
|
||||
) {
|
||||
throw new Error(
|
||||
'the cell tripped its dead-man (reserve configured, db effective); a write that keeps reserve re-arms it. ' +
|
||||
'Name admitMode=db in the same write, e.g. --set admitMode=db,reregisterInFlight=N'
|
||||
)
|
||||
}
|
||||
assertReserveKeepsLocalReadiness(object.flags, request.changes)
|
||||
assertPhonePathNeedsReserve(object.flags, request.changes)
|
||||
const plan = {
|
||||
event: 'orca_relay_cell_flags_plan',
|
||||
mode: request.mode,
|
||||
cellId: request.cellId,
|
||||
currentGeneration: current.generation,
|
||||
currentObject: current.object,
|
||||
appliedBefore: runtime.flagsApplied,
|
||||
changes: request.changes,
|
||||
desired: object
|
||||
}
|
||||
// Before the generation check, so a dry run always shows the generation to pass next.
|
||||
log(JSON.stringify(plan))
|
||||
if (current.generation !== request.expectedGeneration) {
|
||||
throw generationConflict(current.generation, request.expectedGeneration)
|
||||
}
|
||||
// The cell applies every readable write; a generation it did not apply was voided there. A
|
||||
// write of an object this image accepts whole (checked above) repairs it.
|
||||
if (current.generation !== '0' && String(runtime.flagsApplied.generation) !== current.generation) {
|
||||
log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_cell_flags_repairing_unapplied_object',
|
||||
cellId: request.cellId,
|
||||
appliedGeneration: runtime.flagsApplied.generation,
|
||||
currentGeneration: current.generation
|
||||
})
|
||||
)
|
||||
}
|
||||
const admitModeChange =
|
||||
request.changes.admitMode === undefined ? undefined : (request.changes.admitMode ?? 'db')
|
||||
if (admitModeChange !== undefined) {
|
||||
const recorded = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded_before', ...recorded }))
|
||||
}
|
||||
if (request.mode === 'dry-run') return { ...plan, written: false }
|
||||
// Sweeps skip the cell before it starts admitting from memory, never after.
|
||||
let generation
|
||||
if (admitModeChange === 'reserve') {
|
||||
const before = await directorAdmitMode(fetchImpl, request, idToken, sleep)
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve')
|
||||
// Every director's cached set expires (5 s) before the cell starts admitting from memory;
|
||||
// the second write re-runs the open-flow check over anything started meanwhile.
|
||||
await sleep(RESERVE_MODE_CACHE_WAIT_MS)
|
||||
try {
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'reserve')
|
||||
} catch (error) {
|
||||
// A drain, migration or rehome opened during the wait: Postgres goes back as it was.
|
||||
if (before?.admitMode === 'db') await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
} catch (error) {
|
||||
// The file was refused outright (412, 4xx): put Postgres back as it was. After a 5xx the
|
||||
// object may be written, so Postgres keeps reserve (sweeps stay off: the safe side).
|
||||
if (error?.notWritten && before?.admitMode === 'db') {
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
}
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation }))
|
||||
const deadline = now() + READ_BACK_TIMEOUT_MS
|
||||
let last = runtime.flagsApplied
|
||||
while (now() < deadline) {
|
||||
await sleep(READ_BACK_INTERVAL_MS)
|
||||
try {
|
||||
last = (await readRuntime(fetchImpl, request, idToken, sleep)).flagsApplied
|
||||
} catch {
|
||||
// The object is written; a failed poll only delays the read-back until the deadline.
|
||||
continue
|
||||
}
|
||||
if (String(last?.generation) !== generation) continue
|
||||
// The image predates a switch this object names: it applied the rest and dropped that one.
|
||||
if (Array.isArray(last.ignoredKeys) && last.ignoredKeys.length > 0) {
|
||||
throw new Error(
|
||||
`cell applied generation ${generation} but ignores ${last.ignoredKeys.join(', ')}; ` +
|
||||
'its image predates them, so flip the object back'
|
||||
)
|
||||
}
|
||||
if (sameFlags(last.flags, object.flags)) {
|
||||
const result = { ...plan, written: true, generation, applied: last }
|
||||
log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' }))
|
||||
if (admitModeChange === 'db') {
|
||||
await waitForLeasedControls(fetchImpl, request, idToken, { now, sleep, log })
|
||||
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
|
||||
log(JSON.stringify({ event: 'orca_relay_cell_admit_mode_recorded', cellId: request.cellId, admitMode: 'db' }))
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`cell did not apply generation ${generation} within ${READ_BACK_TIMEOUT_MS} ms ` +
|
||||
`(last applied ${JSON.stringify(last)}); the object is written`
|
||||
)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
||||
const { values } = parseArgs({
|
||||
options: Object.fromEntries(
|
||||
[
|
||||
'mode',
|
||||
'cell-id',
|
||||
'cell-origin',
|
||||
'project-id',
|
||||
'set',
|
||||
'expected-generation',
|
||||
'confirmation',
|
||||
'confirm-reserve',
|
||||
'break-glass',
|
||||
'director-origin'
|
||||
].map((name) => [name, { type: 'string' }])
|
||||
)
|
||||
})
|
||||
try {
|
||||
const request = parseCellFlagsRequest(values)
|
||||
const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!accessToken || !idToken) throw new Error('both tokens are required')
|
||||
await operateCellFlags(request, {
|
||||
accessToken,
|
||||
idToken,
|
||||
audit: {
|
||||
// The person who started this attempt, which differs from GITHUB_ACTOR on a re-run.
|
||||
writtenBy: process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown',
|
||||
runId: process.env.GITHUB_RUN_ID ?? 'local',
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0'
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
// On stdout, so the job summary's copy of the run carries the reason.
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_cell_flags_failed',
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,611 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertSupportedObject,
|
||||
desiredObject,
|
||||
operateCellFlags,
|
||||
parseCellFlagsRequest,
|
||||
parseFlagChanges,
|
||||
READ_BACK_TIMEOUT_MS,
|
||||
REREGISTER_WAIT_MAX_MS,
|
||||
REREGISTER_WAIT_MIN_MS,
|
||||
reregisterLanes,
|
||||
reregisterWaitMs,
|
||||
WRITE_CONFIRMATION
|
||||
} from './operate-relay-cell-flags.mjs'
|
||||
|
||||
const CELL = 'production-gce-c26'
|
||||
const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` }
|
||||
const values = (overrides = {}) => ({
|
||||
mode: 'write',
|
||||
'cell-id': CELL,
|
||||
'cell-origin': 'https://c26.relay.onorca.dev',
|
||||
'project-id': 'onorca-cloud',
|
||||
set: 'readinessLocal=true',
|
||||
'expected-generation': '0',
|
||||
confirmation: `${WRITE_CONFIRMATION} ${CELL}`,
|
||||
...overrides
|
||||
})
|
||||
|
||||
// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls.
|
||||
const SUPPORTED = {
|
||||
readinessLocal: { type: 'boolean' },
|
||||
ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] },
|
||||
admitMode: { type: 'enum', values: ['db', 'reserve'] },
|
||||
intakePerSec: { type: 'number', min: 0, max: 1_000 },
|
||||
reserveDryRun: { type: 'boolean' },
|
||||
rejectionFence: { type: 'boolean' },
|
||||
readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true },
|
||||
reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true },
|
||||
phonePath: { type: 'enum', values: ['db', 'memory'] }
|
||||
}
|
||||
|
||||
function fakeGoogleAndCell({
|
||||
stored = null,
|
||||
applyAfterPolls = 2,
|
||||
role = 'cell',
|
||||
cellId = CELL,
|
||||
supportedFlags = SUPPORTED,
|
||||
// Polls that still report admitModeEffective=reserve after a flip back.
|
||||
reregisteringPolls = 0,
|
||||
controls,
|
||||
// Forces admitModeEffective, as a tripped dead-man reports it.
|
||||
effectiveOverride,
|
||||
heartbeatFresh = false,
|
||||
// The director refuses this numbered reserve post (1-based) with this error.
|
||||
refuseReservePost
|
||||
} = {}) {
|
||||
const state = {
|
||||
stored,
|
||||
writes: [],
|
||||
polls: 0,
|
||||
// A cell that has applied whatever object is already there.
|
||||
applied: stored
|
||||
? { generation: Number(stored.generation), flags: stored.object.flags }
|
||||
: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } },
|
||||
directorAdmitMode: 'db',
|
||||
reservePosts: 0,
|
||||
order: [],
|
||||
reregisteringPolls
|
||||
}
|
||||
const fetchImpl = async (url, init = {}) => {
|
||||
const target = new URL(url)
|
||||
if (target.pathname === '/v1/admin/cell-admit-mode') {
|
||||
const body = JSON.parse(String(init.body))
|
||||
if (body.admitMode === 'reserve') {
|
||||
state.reservePosts += 1
|
||||
if (refuseReservePost?.post === state.reservePosts) {
|
||||
return Response.json({ error: refuseReservePost.error }, { status: 409 })
|
||||
}
|
||||
}
|
||||
if (body.admitMode) {
|
||||
state.directorAdmitMode = body.admitMode
|
||||
state.order.push(`director:${body.admitMode}`)
|
||||
}
|
||||
return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh })
|
||||
}
|
||||
if (target.pathname === '/v1/admin/runtime-status') {
|
||||
state.polls += 1
|
||||
if (state.stored && state.polls > applyAfterPolls) {
|
||||
state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags }
|
||||
}
|
||||
const reserveApplied = state.applied.flags.admitMode === 'reserve'
|
||||
const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db'
|
||||
return Response.json({
|
||||
v: 1,
|
||||
role,
|
||||
cellId,
|
||||
flagsApplied: state.applied,
|
||||
admitModeEffective: effectiveOverride ?? effective,
|
||||
...(controls === undefined ? {} : { runtime: { controls } }),
|
||||
...(supportedFlags ? { supportedFlags } : {})
|
||||
})
|
||||
}
|
||||
if (target.pathname.startsWith('/upload/')) {
|
||||
const expected = target.searchParams.get('ifGenerationMatch')
|
||||
if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 })
|
||||
const parts = String(init.body).split('\r\n')
|
||||
const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2])
|
||||
const metadata = JSON.parse(parts[3])
|
||||
const generation = String(Number(state.stored?.generation ?? 1000) + 1)
|
||||
state.stored = { generation, object }
|
||||
state.writes.push({ generation, object, metadata })
|
||||
state.order.push(`file:${object.flags.admitMode ?? 'db'}`)
|
||||
return Response.json({ generation })
|
||||
}
|
||||
if (!state.stored) return new Response(null, { status: 404 })
|
||||
if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object)
|
||||
return Response.json({ generation: state.stored.generation })
|
||||
}
|
||||
return { state, fetchImpl }
|
||||
}
|
||||
|
||||
const run = (request, fake, extra = {}) => {
|
||||
let clock = 0
|
||||
const lines = []
|
||||
return {
|
||||
lines,
|
||||
result: operateCellFlags(request, {
|
||||
fetchImpl: fake.fetchImpl,
|
||||
accessToken: 'access',
|
||||
idToken: 'id',
|
||||
audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' },
|
||||
log: (line) => lines.push(JSON.parse(line)),
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
clock += ms
|
||||
},
|
||||
...extra
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
test('refuses malformed requests before touching anything', () => {
|
||||
assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/)
|
||||
assert.throws(
|
||||
() => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })),
|
||||
/another cell/
|
||||
)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/)
|
||||
// A dry run needs no confirmation.
|
||||
assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run')
|
||||
})
|
||||
|
||||
test('a dry run reports the change and writes nothing', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
|
||||
const outcome = await result
|
||||
assert.equal(outcome.written, false)
|
||||
assert.equal(fake.state.writes.length, 0)
|
||||
assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
|
||||
assert.deepEqual(lines[0].desired.flags, { readinessLocal: true })
|
||||
})
|
||||
|
||||
test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const { result, lines } = run(parseCellFlagsRequest(values()), fake)
|
||||
const outcome = await result
|
||||
assert.equal(outcome.written, true)
|
||||
assert.equal(outcome.generation, '1001')
|
||||
assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } })
|
||||
assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' })
|
||||
assert.deepEqual(fake.state.writes[0].object, {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true }
|
||||
})
|
||||
assert.deepEqual(lines.map((line) => line.event), [
|
||||
'orca_relay_cell_flags_plan',
|
||||
'orca_relay_cell_flags_written',
|
||||
'orca_relay_cell_flags_applied_read_back'
|
||||
])
|
||||
// Flip back: the next write must name the generation just written.
|
||||
const back = run(
|
||||
parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })),
|
||||
fake
|
||||
)
|
||||
assert.equal((await back.result).generation, '1002')
|
||||
})
|
||||
|
||||
test('refuses a stale expected generation, and a concurrent writer between read and write', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
|
||||
})
|
||||
await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/)
|
||||
assert.equal(fake.state.writes.length, 0)
|
||||
const racing = fakeGoogleAndCell()
|
||||
const write = racing.fetchImpl
|
||||
racing.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} }
|
||||
return await write(url, init)
|
||||
}
|
||||
await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/)
|
||||
})
|
||||
|
||||
test('a dry run against a stale generation still shows the plan, then refuses', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
|
||||
})
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
|
||||
await assert.rejects(result, /current generation is 7/)
|
||||
assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
|
||||
assert.equal(lines[0].currentGeneration, '7')
|
||||
})
|
||||
|
||||
test('rides out a transient 5xx on the first read and during the read-back', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
let runtimeCalls = 0
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') {
|
||||
runtimeCalls += 1
|
||||
// The first call, and the first read-back poll after the write.
|
||||
if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 })
|
||||
}
|
||||
return await inner(url, init)
|
||||
}
|
||||
const outcome = await run(parseCellFlagsRequest(values()), fake).result
|
||||
assert.equal(outcome.written, true)
|
||||
assert.equal(outcome.generation, '1001')
|
||||
})
|
||||
|
||||
test('refuses an origin that is not the named cell, or an image without the flag channel', async () => {
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result,
|
||||
/not production-gce-c26/
|
||||
)
|
||||
const old = fakeGoogleAndCell()
|
||||
const fetchImpl = old.fetchImpl
|
||||
old.fetchImpl = async (url, init) =>
|
||||
new URL(url).pathname === '/v1/admin/runtime-status'
|
||||
? Response.json({ v: 1, role: 'cell', cellId: CELL })
|
||||
: await fetchImpl(url, init)
|
||||
await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/)
|
||||
})
|
||||
|
||||
test('fails loudly when the cell does not apply the write within the read-back window', async () => {
|
||||
const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS })
|
||||
await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/)
|
||||
assert.equal(fake.state.writes.length, 1)
|
||||
})
|
||||
|
||||
test('changes only the named switches and keeps every other key the object already has', () => {
|
||||
const current = {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 }
|
||||
}
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE })
|
||||
)
|
||||
assert.deepEqual(desiredObject(request, current), {
|
||||
v: 1,
|
||||
cellId: CELL,
|
||||
flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 }
|
||||
})
|
||||
assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 })
|
||||
assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/)
|
||||
assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false })
|
||||
assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 })
|
||||
assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/)
|
||||
})
|
||||
|
||||
test('a write keeps the switches it did not name, end to end', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
||||
})
|
||||
const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result
|
||||
assert.equal(outcome.written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true })
|
||||
})
|
||||
|
||||
test('the read-back compares every switch, not only the one this run changed', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
||||
})
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
// The cell reports the new generation but still runs the old ticketCheck.
|
||||
if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true }
|
||||
return Response.json(body)
|
||||
}
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result,
|
||||
/did not apply generation 8/
|
||||
)
|
||||
})
|
||||
|
||||
test('a switch the cell reports on its own is compared only when the object names it', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 }
|
||||
return Response.json(body)
|
||||
}
|
||||
assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true)
|
||||
})
|
||||
|
||||
test('fails the read-back when the cell ignores a switch the object names', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
const response = await inner(url, init)
|
||||
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
||||
const body = await response.json()
|
||||
if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode']
|
||||
return Response.json(body)
|
||||
}
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE })), fake).result,
|
||||
/ignores admitMode/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses a switch the cell image does not support, before writing anything', async () => {
|
||||
const old = fakeGoogleAndCell({ supportedFlags: null })
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result,
|
||||
/does not support ticketCheck="enforce"/
|
||||
)
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result,
|
||||
/does not support admitMode/
|
||||
)
|
||||
assert.equal(old.state.writes.length, 0)
|
||||
// An image from before supportedFlags still takes the two switches it always had.
|
||||
assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true)
|
||||
assert.throws(
|
||||
() => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }),
|
||||
/readTimeoutMarginMs/
|
||||
)
|
||||
assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED })
|
||||
})
|
||||
|
||||
test('refuses to carry a bad value already in the file', async () => {
|
||||
const bad = fakeGoogleAndCell({
|
||||
supportedFlags: null,
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } }
|
||||
})
|
||||
// The old image voided generation 7 and still runs generation 0.
|
||||
bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result,
|
||||
/does not support ticketCheck="enforce"/
|
||||
)
|
||||
assert.equal(bad.state.writes.length, 0)
|
||||
})
|
||||
|
||||
const DIRECTOR = { 'director-origin': RESERVE['director-origin'] }
|
||||
|
||||
test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => {
|
||||
assert.throws(
|
||||
() => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })),
|
||||
/--confirm-reserve "RESERVE production-gce-c26"/
|
||||
)
|
||||
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/)
|
||||
const fake = fakeGoogleAndCell()
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ ...DIRECTOR, set: 'readinessLocal=true,admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` })
|
||||
)
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
// Recorded, re-checked after every director's cache has expired, then the file.
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve'])
|
||||
})
|
||||
|
||||
test('a refused file write after Postgres says reserve puts Postgres back to db', async () => {
|
||||
const fake = fakeGoogleAndCell()
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 })
|
||||
return await inner(url, init)
|
||||
}
|
||||
const request = parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE }))
|
||||
await assert.rejects(run(request, fake).result, /returned 403/)
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db'])
|
||||
})
|
||||
|
||||
test('repairs an object the cell did not apply when the whole new object is valid', async () => {
|
||||
const voided = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
||||
})
|
||||
voided.state.applied = { generation: 3, flags: { readinessLocal: false } }
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided)
|
||||
assert.equal((await result).written, true)
|
||||
assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object'))
|
||||
})
|
||||
|
||||
test('a flip back records db in Postgres only after the cell has leased every control', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
||||
reregisteringPolls: 3
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
||||
assert.equal(fake.state.reregisteringPolls, -1)
|
||||
})
|
||||
|
||||
test('the flip-back wait scales with the control count and the switch-file pace', () => {
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS)
|
||||
// 1,500 controls at the default 3 lanes: 1,667 s.
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667)
|
||||
assert.equal(
|
||||
reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }),
|
||||
// The cell caps 10 at its US pool of 10 less two.
|
||||
625_000
|
||||
)
|
||||
assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS)
|
||||
})
|
||||
|
||||
test('the wait counts the lanes the cell actually uses', () => {
|
||||
assert.equal(reregisterLanes({ region: 'us-central1' }), 3)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2' }), 5)
|
||||
assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14)
|
||||
// The pool the cell reports wins over the deployed-size table.
|
||||
assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10)
|
||||
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5)
|
||||
})
|
||||
|
||||
test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => {
|
||||
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
||||
const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' })
|
||||
for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) {
|
||||
const fake = tripped()
|
||||
const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
|
||||
await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/)
|
||||
assert.deepEqual(fake.state.writes, [])
|
||||
}
|
||||
const fake = tripped()
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' })
|
||||
)
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 })
|
||||
})
|
||||
|
||||
test('reserve requires readinessLocal=true, already written or in the same write', async () => {
|
||||
const refused = fakeGoogleAndCell()
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), refused).result,
|
||||
/admitMode=reserve requires readinessLocal=true/
|
||||
)
|
||||
assert.deepEqual(refused.state.order, [])
|
||||
assert.deepEqual(refused.state.writes, [])
|
||||
|
||||
// Already written by an earlier run: reserve alone is enough.
|
||||
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
||||
const ready = fakeGoogleAndCell({ stored })
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })
|
||||
)
|
||||
assert.equal((await run(request, ready).result).written, true)
|
||||
assert.deepEqual(ready.state.writes[0].object.flags, { readinessLocal: true, admitMode: 'reserve' })
|
||||
})
|
||||
|
||||
test('a reserve cell refuses readinessLocal=false, but takes it together with the flip back', async () => {
|
||||
const stored = {
|
||||
generation: '7',
|
||||
object: { v: 1, cellId: CELL, flags: { readinessLocal: true, admitMode: 'reserve' } }
|
||||
}
|
||||
for (const set of ['readinessLocal=false', 'readinessLocal=default']) {
|
||||
const fake = fakeGoogleAndCell({ stored })
|
||||
const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
|
||||
await assert.rejects(run(request, fake).result, /readinessLocal must stay true while the cell is reserve/)
|
||||
assert.deepEqual(fake.state.writes, [])
|
||||
}
|
||||
const fake = fakeGoogleAndCell({ stored })
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(
|
||||
values({ ...DIRECTOR, set: 'admitMode=db,readinessLocal=false', 'expected-generation': '7' })
|
||||
)
|
||||
assert.equal((await run(request, fake).result).written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' })
|
||||
})
|
||||
|
||||
test('phonePath=memory needs reserve, and a reserve flip never turns phones over unnamed', async () => {
|
||||
const onDatabase = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
||||
const refused = fakeGoogleAndCell({ stored: onDatabase })
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })), refused).result,
|
||||
/phonePath=memory requires admitMode=reserve/
|
||||
)
|
||||
assert.deepEqual(refused.state.writes, [])
|
||||
|
||||
const together = fakeGoogleAndCell({ stored: onDatabase })
|
||||
const both = parseCellFlagsRequest(
|
||||
values({ set: 'admitMode=reserve,phonePath=memory', 'expected-generation': '7', ...RESERVE })
|
||||
)
|
||||
assert.equal((await run(both, together).result).written, true)
|
||||
assert.deepEqual(together.state.writes[0].object.flags, {
|
||||
readinessLocal: true,
|
||||
admitMode: 'reserve',
|
||||
phonePath: 'memory'
|
||||
})
|
||||
|
||||
const reserve = { readinessLocal: true, admitMode: 'reserve' }
|
||||
const onReserve = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: reserve } } })
|
||||
const alone = parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' }))
|
||||
assert.equal((await run(alone, onReserve).result).written, true)
|
||||
|
||||
// Rolling back with admitMode=db leaves phonePath=memory in the object, which the cell ignores.
|
||||
const memory = { ...reserve, phonePath: 'memory' }
|
||||
const rollback = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
|
||||
rollback.state.directorAdmitMode = 'reserve'
|
||||
const toDatabase = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=db', 'expected-generation': '7' }))
|
||||
assert.equal((await run(toDatabase, rollback).result).written, true)
|
||||
assert.deepEqual(rollback.state.writes[0].object.flags, { ...memory, admitMode: 'db' })
|
||||
|
||||
// phonePath=db removes the key, so an image without the switch still takes later writes.
|
||||
const off = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
|
||||
assert.equal((await run(parseCellFlagsRequest(values({ set: 'phonePath=db', 'expected-generation': '7' })), off).result).written, true)
|
||||
assert.deepEqual(off.state.writes[0].object.flags, reserve)
|
||||
|
||||
const lingering = { readinessLocal: true, admitMode: 'db', phonePath: 'memory' }
|
||||
const unnamed = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: lingering } } })
|
||||
await assert.rejects(
|
||||
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })), unnamed)
|
||||
.result,
|
||||
/still says phonePath=memory/
|
||||
)
|
||||
assert.deepEqual(unnamed.state.writes, [])
|
||||
})
|
||||
|
||||
test('a migration opened during the cache wait puts Postgres back to db', async () => {
|
||||
const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } })
|
||||
const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' }))
|
||||
await assert.rejects(run(request, fake).result, /cell_has_open_migration/)
|
||||
assert.deepEqual(fake.state.order, ['director:reserve', 'director:db'])
|
||||
assert.deepEqual(fake.state.writes, [])
|
||||
})
|
||||
|
||||
test('break glass refuses a cell that answers or that the director still hears', async () => {
|
||||
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` }
|
||||
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
||||
const answering = fakeGoogleAndCell({ stored })
|
||||
await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/)
|
||||
const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true })
|
||||
const inner = heard.fetchImpl
|
||||
heard.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
||||
return await inner(url, init)
|
||||
}
|
||||
await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/)
|
||||
assert.deepEqual(heard.state.writes, [])
|
||||
assert.deepEqual(answering.state.writes, [])
|
||||
})
|
||||
|
||||
test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => {
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
||||
reregisteringPolls: Number.POSITIVE_INFINITY,
|
||||
controls: 1_500
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
||||
const { lines, result } = run(request, fake)
|
||||
await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/)
|
||||
assert.deepEqual(fake.state.order, ['file:db'])
|
||||
assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667)
|
||||
})
|
||||
|
||||
test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => {
|
||||
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' }
|
||||
assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/)
|
||||
assert.throws(
|
||||
() => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })),
|
||||
/on its own/
|
||||
)
|
||||
const fake = fakeGoogleAndCell({
|
||||
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }
|
||||
})
|
||||
fake.state.directorAdmitMode = 'reserve'
|
||||
const inner = fake.fetchImpl
|
||||
fake.fetchImpl = async (url, init) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
||||
return await inner(url, init)
|
||||
}
|
||||
const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake)
|
||||
assert.equal((await result).written, true)
|
||||
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true })
|
||||
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
||||
assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass')
|
||||
})
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
import { inspectAdmissionSelector } from './relay-admission-selector.mjs'
|
||||
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
const MODES = new Set(['inspect', 'enable', 'pause', 'disable', 'recover-enable'])
|
||||
// c1-c99 with no leading zero; a fixed ceiling here once refused a newly general cell.
|
||||
const PRODUCTION_CELL_ID = /^production-gce-c[1-9][0-9]?$/
|
||||
|
||||
function canonicalCells(value) {
|
||||
if (value === 'none') return []
|
||||
const cells = value.split(',').map((cell) => cell.trim()).filter(Boolean).sort()
|
||||
if (
|
||||
cells.length === 0 ||
|
||||
new Set(cells).size !== cells.length ||
|
||||
cells.some((cell) => !PRODUCTION_CELL_ID.test(cell))
|
||||
) throw new Error('selector membership is invalid')
|
||||
return cells
|
||||
}
|
||||
|
||||
function integer(value, name, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new Error(`${name} is invalid`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['mode', 'director-origin', 'expected-control-generation']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (!MODES.has(values.mode)) throw new Error('--mode is invalid')
|
||||
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
|
||||
throw new Error('--director-origin must be the production Relay origin')
|
||||
}
|
||||
const recovery = values.mode === 'recover-enable'
|
||||
const mutation = values.mode !== 'inspect' && !recovery
|
||||
const mutationKeys = [
|
||||
'not-before',
|
||||
'rate-per-minute',
|
||||
'preference-max-age-ms',
|
||||
'host-cooldown-ms',
|
||||
'drain-grace-ms',
|
||||
'confirmation'
|
||||
]
|
||||
if (mutation && mutationKeys.some((key) => values[key] === undefined)) {
|
||||
throw new Error('mutations require the complete durable control shape')
|
||||
}
|
||||
if (!mutation && !recovery && mutationKeys.some((key) => values[key] !== undefined)) {
|
||||
throw new Error('inspect cannot carry mutation arguments')
|
||||
}
|
||||
const selectorKeys = [
|
||||
'expected-selector-generation',
|
||||
'expected-existing-only-cells',
|
||||
'expected-migration-only-cells',
|
||||
'expected-general-cells'
|
||||
]
|
||||
if (!recovery && selectorKeys.some((key) => values[key] === undefined)) {
|
||||
throw new Error('operation requires exact selector state')
|
||||
}
|
||||
if (recovery && selectorKeys.some((key) => values[key] !== undefined)) {
|
||||
throw new Error('enable recovery cannot depend on selector diagnostics')
|
||||
}
|
||||
const expectedConfirmation = {
|
||||
enable: 'ENABLE_REGIONAL_REHOMING',
|
||||
pause: 'PAUSE_REGIONAL_REHOMING',
|
||||
disable: 'DISABLE_REGIONAL_REHOMING'
|
||||
}[values.mode]
|
||||
if (mutation && values.confirmation !== expectedConfirmation) {
|
||||
throw new Error('confirmation does not match the requested control action')
|
||||
}
|
||||
if (recovery && values.confirmation !== 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE') {
|
||||
throw new Error('confirmation does not authorize failed-enable recovery')
|
||||
}
|
||||
if (
|
||||
recovery &&
|
||||
mutationKeys
|
||||
.filter((key) => key !== 'confirmation')
|
||||
.some((key) => values[key] !== undefined)
|
||||
) throw new Error('enable recovery cannot carry durable control shape arguments')
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
return {
|
||||
mode: values.mode,
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
...(!recovery
|
||||
? {
|
||||
expectedSelectorGeneration: integer(
|
||||
values['expected-selector-generation'],
|
||||
'--expected-selector-generation'
|
||||
),
|
||||
expectedMembership: {
|
||||
existingOnly: canonicalCells(values['expected-existing-only-cells']),
|
||||
migrationOnly: canonicalCells(values['expected-migration-only-cells']),
|
||||
general: canonicalCells(values['expected-general-cells'])
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
expectedControlGeneration: integer(
|
||||
values['expected-control-generation'],
|
||||
'--expected-control-generation'
|
||||
),
|
||||
...(mutation
|
||||
? {
|
||||
notBefore: integer(values['not-before'], '--not-before'),
|
||||
ratePerMinute: integer(values['rate-per-minute'], '--rate-per-minute', {
|
||||
minimum: 1,
|
||||
maximum: 120
|
||||
}),
|
||||
preferenceMaxAgeMs: integer(
|
||||
values['preference-max-age-ms'],
|
||||
'--preference-max-age-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
hostCooldownMs: integer(
|
||||
values['host-cooldown-ms'],
|
||||
'--host-cooldown-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
|
||||
minimum: 60_000,
|
||||
maximum: 60 * 60_000
|
||||
})
|
||||
}
|
||||
: {}),
|
||||
token
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}: ${body.error ?? 'unknown'}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function exactMembership(actual, expected) {
|
||||
return ['existingOnly', 'migrationOnly', 'general'].every(
|
||||
(key) => JSON.stringify(actual[key]) === JSON.stringify(expected[key])
|
||||
)
|
||||
}
|
||||
|
||||
function assertControl(control, expected) {
|
||||
if (
|
||||
(expected.generation !== undefined && control?.generation !== expected.generation) ||
|
||||
typeof control.enabled !== 'boolean' ||
|
||||
!Number.isSafeInteger(control.observationStartedAt) ||
|
||||
!Number.isSafeInteger(control.notBefore) ||
|
||||
!Number.isSafeInteger(control.ratePerMinute) ||
|
||||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
|
||||
// A director predating the per-host cooldown does not report it. Reading
|
||||
// the control and both emergency brakes must keep working against that
|
||||
// image; only enable requires the field.
|
||||
(control.hostCooldownMs !== undefined &&
|
||||
!Number.isSafeInteger(control.hostCooldownMs)) ||
|
||||
!Number.isSafeInteger(control.drainGraceMs)
|
||||
) throw new Error('director returned an invalid regional rehome control')
|
||||
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
|
||||
throw new Error('regional rehome enabled state does not match')
|
||||
}
|
||||
return control
|
||||
}
|
||||
|
||||
// Echo the cooldown only when the director already reports it: a legacy
|
||||
// director rejects the unknown key outright and would refuse every brake.
|
||||
function cooldownField(before, value) {
|
||||
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
|
||||
}
|
||||
|
||||
async function verifiedDisabledControl(post, generation) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, { generation, enabled: false })
|
||||
}
|
||||
|
||||
async function applyDisabledControl(post, before) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: before.generation,
|
||||
enabled: false,
|
||||
notBefore: before.notBefore,
|
||||
ratePerMinute: before.ratePerMinute,
|
||||
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
|
||||
...cooldownField(before, before.hostCooldownMs),
|
||||
drainGraceMs: before.drainGraceMs,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})).control, { generation: before.generation + 1, enabled: false })
|
||||
}
|
||||
|
||||
async function resolveAmbiguousDisable(post, before, firstError) {
|
||||
const observed = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {})
|
||||
if (observed.generation === before.generation + 1 && !observed.enabled) {
|
||||
return observed
|
||||
}
|
||||
if (observed.generation !== before.generation || !observed.enabled) {
|
||||
throw new AggregateError(
|
||||
[firstError],
|
||||
'failed-enable recovery reached an unexpected control generation'
|
||||
)
|
||||
}
|
||||
try {
|
||||
return await applyDisabledControl(post, before)
|
||||
} catch (retryError) {
|
||||
try {
|
||||
return await verifiedDisabledControl(post, before.generation + 1)
|
||||
} catch (readbackError) {
|
||||
throw new AggregateError(
|
||||
[firstError, retryError, readbackError],
|
||||
'failed-enable recovery exhausted two bounded CAS attempts'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function recoverRegionalRehomeEnable(config, post) {
|
||||
const before = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {})
|
||||
if (
|
||||
before.generation < config.expectedControlGeneration ||
|
||||
(before.generation === config.expectedControlGeneration && before.enabled)
|
||||
) throw new Error('durable control cannot belong to the failed enable attempt')
|
||||
if (!before.enabled) {
|
||||
const verified = await verifiedDisabledControl(post, before.generation)
|
||||
return { mode: config.mode, recovered: false, control: verified }
|
||||
}
|
||||
let applied
|
||||
try {
|
||||
applied = await applyDisabledControl(post, before)
|
||||
} catch (error) {
|
||||
applied = await resolveAmbiguousDisable(post, before, error)
|
||||
}
|
||||
const verified = await verifiedDisabledControl(post, applied.generation)
|
||||
return { mode: config.mode, recovered: true, control: verified }
|
||||
}
|
||||
|
||||
export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const post = dependencies.post ?? (async (path, body) => await responseJson(
|
||||
// Generation-guarded writes make a retry a no-op or an explicit mismatch, never a double apply.
|
||||
await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${config.directorOrigin}${path}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.token}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
},
|
||||
{ wait: dependencies.wait }
|
||||
),
|
||||
path
|
||||
))
|
||||
if (config.mode === 'recover-enable') {
|
||||
return await recoverRegionalRehomeEnable(config, post)
|
||||
}
|
||||
const selector = (await inspectAdmissionSelector(post)).selector
|
||||
if (
|
||||
selector.generation !== config.expectedSelectorGeneration ||
|
||||
!exactMembership(selector.membership, config.expectedMembership)
|
||||
) throw new Error('admission selector does not match the reviewed generation and membership')
|
||||
|
||||
const inspected = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})
|
||||
const before = assertControl(inspected.control, {
|
||||
generation: config.expectedControlGeneration
|
||||
})
|
||||
if (config.mode === 'inspect') return { mode: config.mode, selector, control: before }
|
||||
if (config.mode === 'enable' && before.enabled) {
|
||||
throw new Error('regional rehome is already enabled; inspect before changing its rate')
|
||||
}
|
||||
if (config.mode === 'pause' && !before.enabled) {
|
||||
throw new Error('regional rehome is already paused')
|
||||
}
|
||||
const enabled = config.mode === 'enable'
|
||||
if (enabled && before.hostCooldownMs === undefined) {
|
||||
throw new Error(
|
||||
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
|
||||
)
|
||||
}
|
||||
const applied = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: config.expectedControlGeneration,
|
||||
enabled,
|
||||
notBefore: config.notBefore,
|
||||
ratePerMinute: config.ratePerMinute,
|
||||
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
|
||||
...cooldownField(before, config.hostCooldownMs),
|
||||
drainGraceMs: config.drainGraceMs,
|
||||
confirmation: enabled
|
||||
? 'ENABLE_REGIONAL_REHOMING'
|
||||
: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
const after = assertControl(applied.control, {
|
||||
generation: config.expectedControlGeneration + 1,
|
||||
enabled
|
||||
})
|
||||
const verified = assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})).control, {
|
||||
generation: after.generation,
|
||||
enabled
|
||||
})
|
||||
return { mode: config.mode, selector, control: verified }
|
||||
}
|
||||
|
||||
export async function main(
|
||||
argv = process.argv.slice(2),
|
||||
environment = process.env,
|
||||
dependencies = {},
|
||||
write = (value) => process.stdout.write(value)
|
||||
) {
|
||||
const result = await operateRegionalRehome(
|
||||
parseRegionalRehomeArguments(argv, environment),
|
||||
dependencies
|
||||
)
|
||||
write(`${JSON.stringify({ event: 'relay_regional_rehome_control', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
main,
|
||||
operateRegionalRehome,
|
||||
parseRegionalRehomeArguments,
|
||||
recoverRegionalRehomeEnable
|
||||
} from './operate-relay-regional-rehome.mjs'
|
||||
|
||||
const membership = {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c2'],
|
||||
general: ['production-gce-c7', 'production-gce-c27']
|
||||
}
|
||||
|
||||
function argumentsFor(mode, confirmation) {
|
||||
return [
|
||||
'--mode', mode,
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-selector-generation', '11',
|
||||
'--expected-existing-only-cells', membership.existingOnly.join(','),
|
||||
'--expected-migration-only-cells', membership.migrationOnly.join(','),
|
||||
'--expected-general-cells', membership.general.join(','),
|
||||
'--expected-control-generation', '4',
|
||||
...(mode === 'inspect' ? [] : [
|
||||
'--not-before', '2000000000000',
|
||||
'--rate-per-minute', '10',
|
||||
'--preference-max-age-ms', '86400000',
|
||||
'--host-cooldown-ms', '604800000',
|
||||
'--drain-grace-ms', '60000',
|
||||
'--confirmation', confirmation
|
||||
])
|
||||
]
|
||||
}
|
||||
|
||||
function control(generation, enabled) {
|
||||
return {
|
||||
generation,
|
||||
enabled,
|
||||
observationStartedAt: 1,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000
|
||||
}
|
||||
}
|
||||
|
||||
// The control a director predating the per-host cooldown reports.
|
||||
function legacyControl(generation, enabled) {
|
||||
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
|
||||
return rest
|
||||
}
|
||||
|
||||
function legacyDirector(controls) {
|
||||
const requests = []
|
||||
const post = async (path, body) => {
|
||||
requests.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return { selector: { generation: 11, membership } }
|
||||
}
|
||||
return { v: 1, control: controls.shift() }
|
||||
}
|
||||
return { requests, post }
|
||||
}
|
||||
|
||||
test('parses exact selector and typed control confirmation', () => {
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
assert.equal(parsed.expectedSelectorGeneration, 11)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.ratePerMinute, 10)
|
||||
assert.equal(parsed.hostCooldownMs, 604_800_000)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
|
||||
(value, index, all) =>
|
||||
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
|
||||
),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/complete durable control shape/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/confirmation/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect').concat('--rate-per-minute', '10'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/inspect cannot/
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts every one- and two-digit production cell and rejects malformed membership', () => {
|
||||
const parseGeneral = (general) => parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect').map((value, index, all) =>
|
||||
all[index - 1] === '--expected-general-cells' ? general : value
|
||||
),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
).expectedMembership.general
|
||||
assert.deepEqual(
|
||||
parseGeneral('production-gce-c30,production-gce-c9,production-gce-c29'),
|
||||
['production-gce-c29', 'production-gce-c30', 'production-gce-c9']
|
||||
)
|
||||
assert.deepEqual(parseGeneral('production-gce-c99'), ['production-gce-c99'])
|
||||
for (const general of [
|
||||
'production-gce-c0',
|
||||
'production-gce-c100',
|
||||
'production-gce-c09',
|
||||
'production-gce-c30,production-gce-c30',
|
||||
'staging-gce-c30',
|
||||
','
|
||||
]) {
|
||||
assert.throws(() => parseGeneral(general), /selector membership is invalid/, general)
|
||||
}
|
||||
})
|
||||
|
||||
test('binds enable to exact selector and durable control generations', async () => {
|
||||
const requests = []
|
||||
const controls = [
|
||||
{ generation: 4, enabled: false },
|
||||
{ generation: 5, enabled: true },
|
||||
{ generation: 5, enabled: true }
|
||||
].map((control) => ({
|
||||
observationStartedAt: 1,
|
||||
notBefore: 0,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
...control
|
||||
}))
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
const result = await operateRegionalRehome(config, {
|
||||
post: async (path, body) => {
|
||||
requests.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return { selector: { generation: 11, membership } }
|
||||
}
|
||||
return { v: 1, control: controls.shift() }
|
||||
}
|
||||
})
|
||||
assert.equal(result.control.generation, 5)
|
||||
assert.equal(result.control.hostCooldownMs, 604_800_000)
|
||||
assert.deepEqual(requests[2].body, {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: 4,
|
||||
enabled: true,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('inspects a director that predates the per-host cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, true)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 4)
|
||||
assert.equal(result.control.hostCooldownMs, undefined)
|
||||
})
|
||||
|
||||
for (const [mode, confirmation, enabledBefore] of [
|
||||
['pause', 'PAUSE_REGIONAL_REHOMING', true],
|
||||
['disable', 'DISABLE_REGIONAL_REHOMING', false]
|
||||
]) {
|
||||
test(`${mode} still brakes a director that predates the cooldown`, async () => {
|
||||
const director = legacyDirector([
|
||||
legacyControl(4, enabledBefore),
|
||||
legacyControl(5, false),
|
||||
legacyControl(5, false)
|
||||
])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor(mode, confirmation),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 5)
|
||||
// The unknown key would be refused by that director's strict schema.
|
||||
assert.equal('hostCooldownMs' in director.requests[2].body, false)
|
||||
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
|
||||
})
|
||||
}
|
||||
|
||||
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
|
||||
const requests = []
|
||||
let current = legacyControl(7, true)
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
current = legacyControl(8, false)
|
||||
return { control: current }
|
||||
})
|
||||
|
||||
assert.equal(result.control.generation, 8)
|
||||
assert.equal('hostCooldownMs' in requests[1], false)
|
||||
})
|
||||
|
||||
test('refuses to enable a director that does not report the cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, false)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, { post: director.post }),
|
||||
/per-host rehome cooldown/
|
||||
)
|
||||
// Read-only: selector status and the control inspect, and nothing else.
|
||||
assert.equal(director.requests.length, 2)
|
||||
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
|
||||
})
|
||||
|
||||
test('fails closed on selector drift before reading or mutating control', async () => {
|
||||
let calls = 0
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('disable', 'DISABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, {
|
||||
post: async () => {
|
||||
calls += 1
|
||||
return { selector: { generation: 12, membership } }
|
||||
}
|
||||
}),
|
||||
/selector/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('failed-enable recovery CAS-disables an advanced enabled generation', async () => {
|
||||
const requests = []
|
||||
let current = control(7, true)
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
assert.equal(body.expectedGeneration, 7)
|
||||
current = control(8, false)
|
||||
throw new Error('enable recovery response was lost')
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.deepEqual(result.control, control(8, false))
|
||||
assert.deepEqual(requests[1], {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
expectedGeneration: 7,
|
||||
enabled: false,
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('failed-enable recovery retries once when the first CAS never commits', async () => {
|
||||
let current = control(7, true)
|
||||
const applyRequests = []
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
applyRequests.push(body)
|
||||
if (applyRequests.length === 1) {
|
||||
throw new Error('disable request was lost before commit')
|
||||
}
|
||||
current = control(8, false)
|
||||
throw new Error('retry response was lost after commit')
|
||||
})
|
||||
assert.equal(applyRequests.length, 2)
|
||||
assert.deepEqual(applyRequests[1], applyRequests[0])
|
||||
assert.equal(result.recovered, true)
|
||||
assert.deepEqual(result.control, control(8, false))
|
||||
})
|
||||
|
||||
test('failed-enable recovery stops after two uncommitted CAS attempts', async () => {
|
||||
let applyCalls = 0
|
||||
await assert.rejects(
|
||||
recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
if (body.action === 'inspect') return { control: control(7, true) }
|
||||
applyCalls += 1
|
||||
throw new Error(`disable attempt ${applyCalls} was lost before commit`)
|
||||
}),
|
||||
/exhausted two bounded CAS attempts/
|
||||
)
|
||||
assert.equal(applyCalls, 2)
|
||||
})
|
||||
|
||||
test('failed-enable recovery is a verified no-op before enable and after cleanup', async () => {
|
||||
for (const current of [control(4, false), control(8, false)]) {
|
||||
const requests = []
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
return { control: current }
|
||||
})
|
||||
assert.equal(result.recovered, false)
|
||||
assert.equal(result.control.enabled, false)
|
||||
assert.deepEqual(requests.map(({ action }) => action), ['inspect', 'inspect'])
|
||||
}
|
||||
})
|
||||
|
||||
test('failed-enable recovery rejects an unchanged pre-existing enabled state', async () => {
|
||||
await assert.rejects(
|
||||
recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async () => ({ control: control(4, true) })),
|
||||
/cannot belong to the failed enable attempt/
|
||||
)
|
||||
})
|
||||
|
||||
test('parses recovery without depending on selector diagnostics', () => {
|
||||
const recoveryArguments = [
|
||||
'--mode', 'recover-enable',
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-control-generation', '4',
|
||||
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
|
||||
]
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
recoveryArguments,
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.expectedMembership, undefined)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
recoveryArguments.concat('--not-before', '2000000000000'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/cannot carry durable control shape/
|
||||
)
|
||||
})
|
||||
|
||||
test('main executes recovery mode and emits verified disabled control', async () => {
|
||||
let current = control(5, true)
|
||||
let output = ''
|
||||
await main([
|
||||
'--mode', 'recover-enable',
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--expected-control-generation', '4',
|
||||
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
|
||||
], { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }, {
|
||||
post: async (_path, body) => {
|
||||
if (body.action === 'apply') current = control(6, false)
|
||||
return { control: current }
|
||||
}
|
||||
}, (value) => {
|
||||
output += value
|
||||
})
|
||||
assert.deepEqual(JSON.parse(output), {
|
||||
event: 'relay_regional_rehome_control',
|
||||
mode: 'recover-enable',
|
||||
recovered: true,
|
||||
control: control(6, false)
|
||||
})
|
||||
})
|
||||
|
||||
test('retries a transient 503 on the director control endpoint', async () => {
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
const paths = []
|
||||
let selectorCalls = 0
|
||||
const result = await operateRegionalRehome(config, {
|
||||
wait: async () => {},
|
||||
fetch: async (url) => {
|
||||
const path = new URL(url).pathname
|
||||
paths.push(path)
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
selectorCalls += 1
|
||||
// The first read of each admin path 503s the way a warming instance does.
|
||||
if (selectorCalls === 1) return new Response('warming up', { status: 503 })
|
||||
return Response.json({ selector: { generation: 11, membership } })
|
||||
}
|
||||
if (paths.filter((value) => value === path).length === 1) {
|
||||
return new Response('warming up', { status: 503 })
|
||||
}
|
||||
return Response.json({ v: 1, control: control(4, false) })
|
||||
}
|
||||
})
|
||||
assert.equal(result.control.generation, 4)
|
||||
assert.deepEqual(paths, [
|
||||
'/v1/admin/admission-selector/status',
|
||||
'/v1/admin/admission-selector/status',
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'/v1/admin/regional-rehome-control'
|
||||
])
|
||||
})
|
||||
|
||||
test('fails when both attempts at the director control endpoint return 503', async () => {
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, {
|
||||
wait: async () => {},
|
||||
fetch: async () => {
|
||||
calls += 1
|
||||
return new Response('warming up', { status: 503 })
|
||||
}
|
||||
}),
|
||||
/returned 503/
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
@@ -1,537 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
inspectAdmissionSelector,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const PROJECT = 'onorca-cloud-staging'
|
||||
const REGION = 'us-central1'
|
||||
const DIRECTOR_ORIGIN = 'https://relay-staging.onorca.dev'
|
||||
const ADMIN_AUDIENCE = `${DIRECTOR_ORIGIN}/v1/admin/drain`
|
||||
const SQL_INSTANCE = 'orca-cloud-staging-auth-db'
|
||||
const CLOUD_RUN_SERVICES = [
|
||||
{ name: 'orca-cloud-relay-staging', healthOrigin: DIRECTOR_ORIGIN },
|
||||
{ name: 'orca-cloud-auth-staging', healthOrigin: 'https://auth-staging.onorca.dev' }
|
||||
]
|
||||
const POLL_INTERVAL_MS = 5_000
|
||||
const WAKE_TIMEOUT_MS = 12 * 60 * 1_000
|
||||
|
||||
export function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
|
||||
const name = key.slice(2)
|
||||
if (!['mode', 'wake-cells', 'topology-file'].includes(name)) {
|
||||
throw new Error(`unsupported argument --${name}`)
|
||||
}
|
||||
values[name] = value
|
||||
}
|
||||
if (!['status', 'sleep', 'wake'].includes(values.mode)) {
|
||||
throw new Error('--mode must be status, sleep, or wake')
|
||||
}
|
||||
if (!['configured', 'all'].includes(values['wake-cells'])) {
|
||||
throw new Error('--wake-cells must be configured or all')
|
||||
}
|
||||
if (!values['topology-file']) throw new Error('missing --topology-file')
|
||||
return {
|
||||
mode: values.mode,
|
||||
wakeCells: values['wake-cells'],
|
||||
topologyFile: values['topology-file']
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalStagingCell(cellId, value) {
|
||||
if (!/^staging-gce-[a-z0-9-]+$/.test(cellId)) throw new Error(`unsafe staging cell ID ${cellId}`)
|
||||
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
|
||||
const cell = {
|
||||
cellId,
|
||||
migName: String(value.mig_name ?? ''),
|
||||
zone: String(value.zone ?? ''),
|
||||
origin: String(value.origin ?? ''),
|
||||
initiallyEnabled: value.initially_enabled
|
||||
}
|
||||
if (!/^orca-cloud-staging-relay-gce-[a-z0-9-]+$/.test(cell.migName)) {
|
||||
throw new Error(`${cellId} has an unsafe MIG name`)
|
||||
}
|
||||
if (!/^(?:us-central1|asia-east2)-[a-z]$/.test(cell.zone)) {
|
||||
throw new Error(`${cellId} has an unsafe zone`)
|
||||
}
|
||||
const origin = new URL(cell.origin)
|
||||
if (
|
||||
origin.protocol !== 'https:' ||
|
||||
origin.origin !== cell.origin ||
|
||||
!origin.hostname.endsWith('.relay-staging.onorca.dev')
|
||||
) {
|
||||
throw new Error(`${cellId} has an unsafe origin`)
|
||||
}
|
||||
if (typeof cell.initiallyEnabled !== 'boolean') {
|
||||
throw new Error(`${cellId} has no initial admission state`)
|
||||
}
|
||||
return cell
|
||||
}
|
||||
|
||||
export function readStagingTopology(file) {
|
||||
const parsed = JSON.parse(readFileSync(file, 'utf8'))
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error('staging topology must be an object')
|
||||
}
|
||||
const cells = Object.entries(parsed)
|
||||
.map(([cellId, value]) => canonicalStagingCell(cellId, value))
|
||||
.sort((left, right) => left.cellId.localeCompare(right.cellId))
|
||||
if (cells.length < 2 || cells.length > 10) {
|
||||
throw new Error('staging topology must contain 2..10 cells')
|
||||
}
|
||||
if (cells.filter((cell) => cell.initiallyEnabled).length < 2) {
|
||||
throw new Error('staging topology must retain two configured admission cells')
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function defaultCommand(args, json) {
|
||||
const result = spawnSync('gcloud', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`)
|
||||
}
|
||||
return json ? JSON.parse(result.stdout) : result.stdout.trim()
|
||||
}
|
||||
|
||||
function suppliedAdminToken(environment = process.env) {
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
|
||||
throw new Error('workflow did not supply a valid masked staging admin token')
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
|
||||
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function createAdminPost(deps) {
|
||||
const token = deps.adminToken()
|
||||
return async (origin, path, body) =>
|
||||
await responseJson(
|
||||
await deps.fetch(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
)
|
||||
}
|
||||
|
||||
async function waitUntil(deps, label, operation, timeoutMs = WAKE_TIMEOUT_MS) {
|
||||
const deadline = deps.now() + timeoutMs
|
||||
let lastError
|
||||
while (deps.now() < deadline) {
|
||||
try {
|
||||
const result = await operation()
|
||||
if (result) return result
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
}
|
||||
await deps.wait(POLL_INTERVAL_MS)
|
||||
}
|
||||
const detail = lastError instanceof Error ? `: ${lastError.message}` : ''
|
||||
throw new Error(`timed out waiting for ${label}${detail}`)
|
||||
}
|
||||
|
||||
async function checkHealth(deps, origin, path) {
|
||||
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
|
||||
const body = await response.json().catch(() => ({}))
|
||||
return response.ok && body.ok === true
|
||||
}
|
||||
|
||||
function sqlActivationPolicy(instance) {
|
||||
return String(instance.settings?.activationPolicy ?? '')
|
||||
}
|
||||
|
||||
function describeSql(deps) {
|
||||
return deps.commandJson([
|
||||
'sql',
|
||||
'instances',
|
||||
'describe',
|
||||
SQL_INSTANCE,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
async function ensureSqlPolicy(deps, policy) {
|
||||
if (sqlActivationPolicy(describeSql(deps)) === policy) return false
|
||||
deps.command([
|
||||
'sql',
|
||||
'instances',
|
||||
'patch',
|
||||
SQL_INSTANCE,
|
||||
'--project',
|
||||
PROJECT,
|
||||
`--activation-policy=${policy}`,
|
||||
'--quiet'
|
||||
])
|
||||
await waitUntil(deps, `Cloud SQL activation policy ${policy}`, () =>
|
||||
sqlActivationPolicy(describeSql(deps)) === policy
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
function describeMig(deps, cell) {
|
||||
return deps.commandJson([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'describe',
|
||||
cell.migName,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--zone',
|
||||
cell.zone,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
async function setMigSize(deps, cell, size) {
|
||||
const before = describeMig(deps, cell)
|
||||
if (Number(before.targetSize) !== size) {
|
||||
deps.command([
|
||||
'compute',
|
||||
'instance-groups',
|
||||
'managed',
|
||||
'resize',
|
||||
cell.migName,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--zone',
|
||||
cell.zone,
|
||||
`--size=${size}`,
|
||||
'--quiet'
|
||||
])
|
||||
}
|
||||
await waitUntil(deps, `${cell.cellId} size ${size}`, () => {
|
||||
const current = describeMig(deps, cell)
|
||||
return Number(current.targetSize) === size && current.status?.isStable === true
|
||||
})
|
||||
}
|
||||
|
||||
function activeRevisionName(service) {
|
||||
const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0)
|
||||
if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) {
|
||||
throw new Error('Cloud Run service must have exactly one active revision')
|
||||
}
|
||||
return active[0].revisionName
|
||||
}
|
||||
|
||||
function describeRunService(deps, name) {
|
||||
return deps.commandJson([
|
||||
'run',
|
||||
'services',
|
||||
'describe',
|
||||
name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
function describeRunRevision(deps, name) {
|
||||
return deps.commandJson([
|
||||
'run',
|
||||
'revisions',
|
||||
'describe',
|
||||
name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--format=json'
|
||||
])
|
||||
}
|
||||
|
||||
function revisionMinimum(revision) {
|
||||
return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0)
|
||||
}
|
||||
|
||||
async function ensureCloudRunScaleToZero(deps, service) {
|
||||
const before = describeRunService(deps, service.name)
|
||||
const activeRevision = describeRunRevision(deps, activeRevisionName(before))
|
||||
if (revisionMinimum(activeRevision) === 0) return false
|
||||
|
||||
const latestName = before.status?.latestReadyRevisionName
|
||||
const latest = latestName ? describeRunRevision(deps, latestName) : null
|
||||
if (!latest || revisionMinimum(latest) !== 0) {
|
||||
deps.command([
|
||||
'run',
|
||||
'services',
|
||||
'update',
|
||||
service.name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--min-instances=0',
|
||||
'--quiet'
|
||||
])
|
||||
}
|
||||
deps.command([
|
||||
'run',
|
||||
'services',
|
||||
'update-traffic',
|
||||
service.name,
|
||||
'--project',
|
||||
PROJECT,
|
||||
'--region',
|
||||
REGION,
|
||||
'--to-latest',
|
||||
'--quiet'
|
||||
])
|
||||
await waitUntil(deps, `${service.name} scale-to-zero revision`, async () => {
|
||||
const current = describeRunService(deps, service.name)
|
||||
const revision = describeRunRevision(deps, activeRevisionName(current))
|
||||
return revisionMinimum(revision) === 0 && (await checkHealth(deps, service.healthOrigin, '/health'))
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
async function cellStatus(adminPost, cell) {
|
||||
const response = await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: cell.cellId
|
||||
})
|
||||
if (!response.status || response.status.cellId !== cell.cellId) {
|
||||
throw new Error(`${cell.cellId} returned an invalid status`)
|
||||
}
|
||||
return response.status
|
||||
}
|
||||
|
||||
function assertQuiescent(status) {
|
||||
const active = {
|
||||
activityLeases: status.activityLeases,
|
||||
activityRequestUnits: status.activityRequestUnits,
|
||||
outgoingMigrations: status.outgoingMigrations,
|
||||
incomingMigrations: status.incomingMigrations,
|
||||
observedRequests: status.runtime?.observedRequests ?? 0
|
||||
}
|
||||
if (Object.values(active).some((value) => Number(value) !== 0)) {
|
||||
throw new Error(`${status.cellId} still has active Relay work: ${JSON.stringify(active)}`)
|
||||
}
|
||||
}
|
||||
|
||||
// `undefined` when the director predates the step-5 reserve census (its router 404s the route).
|
||||
async function reserveCells(deps, cells) {
|
||||
const token = deps.adminToken()
|
||||
const reserve = []
|
||||
for (const cell of cells) {
|
||||
const response = await deps.fetch(`${DIRECTOR_ORIGIN}/v1/admin/cell-admit-mode`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, cellId: cell.cellId }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
const result = await response.json().catch(() => null)
|
||||
if (response.status === 404 && result === null) return undefined
|
||||
if (!response.ok || result?.v !== 1 || !['db', 'reserve'].includes(result.admitMode)) {
|
||||
throw new Error(`${cell.cellId} admit mode read failed: ${result?.error ?? response.status}`)
|
||||
}
|
||||
if (result.admitMode === 'reserve') reserve.push(cell.cellId)
|
||||
}
|
||||
return reserve
|
||||
}
|
||||
|
||||
// Under the monotonic selector, admission belongs to the selector, and the selector-era wake
|
||||
// restores it; so this sleep proves the fleet empty and never writes a cell's admission.
|
||||
async function sleepUnderSelector(deps, cells, adminPost, selector) {
|
||||
const reserve = await reserveCells(deps, cells)
|
||||
if (reserve?.length) {
|
||||
throw new Error(`staging sleep refuses reserve-mode cells: ${reserve.join(', ')}`)
|
||||
}
|
||||
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
|
||||
const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of final) assertQuiescent(status)
|
||||
const after = await inspectAdmissionSelector(
|
||||
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
|
||||
)
|
||||
if (after.selector.generation !== selector.generation) {
|
||||
throw new Error('admission selector changed during staging sleep')
|
||||
}
|
||||
// The flag workflow runs outside the staging lock, so a reserve flip can land mid-sleep.
|
||||
const lateReserve = await reserveCells(deps, cells)
|
||||
if (lateReserve?.length) {
|
||||
throw new Error(`staging sleep refuses reserve-mode cells: ${lateReserve.join(', ')}`)
|
||||
}
|
||||
await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0)))
|
||||
await ensureSqlPolicy(deps, 'NEVER')
|
||||
deps.emit({
|
||||
event: 'staging_relay_slept',
|
||||
stoppedCells: cells.map((cell) => cell.cellId),
|
||||
selectorGeneration: selector.generation
|
||||
})
|
||||
}
|
||||
|
||||
async function setCellState(adminPost, cell, enabled) {
|
||||
await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-state', {
|
||||
v: 1,
|
||||
cellId: cell.cellId,
|
||||
enabled
|
||||
})
|
||||
}
|
||||
|
||||
async function stagingStatus(deps, cells) {
|
||||
return {
|
||||
event: 'staging_relay_power_status',
|
||||
project: PROJECT,
|
||||
sqlActivationPolicy: sqlActivationPolicy(describeSql(deps)),
|
||||
cells: cells.map((cell) => ({
|
||||
cellId: cell.cellId,
|
||||
initiallyEnabled: cell.initiallyEnabled,
|
||||
targetSize: Number(describeMig(deps, cell).targetSize)
|
||||
})),
|
||||
cloudRun: CLOUD_RUN_SERVICES.map((service) => {
|
||||
const described = describeRunService(deps, service.name)
|
||||
const revision = describeRunRevision(deps, activeRevisionName(described))
|
||||
return { service: service.name, activeRevisionMinimum: revisionMinimum(revision) }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function sleepStaging(deps, cells) {
|
||||
const sqlPolicy = sqlActivationPolicy(describeSql(deps))
|
||||
if (sqlPolicy === 'NEVER') {
|
||||
const runningCells = cells.filter((cell) => Number(describeMig(deps, cell).targetSize) !== 0)
|
||||
if (runningCells.length > 0) {
|
||||
// SQL-off plus running workers is an unknown partial state; never kill those workers blindly.
|
||||
throw new Error(
|
||||
`staging is partially asleep with running cells: ${runningCells.map((cell) => cell.cellId).join(', ')}`
|
||||
)
|
||||
}
|
||||
deps.emit({ event: 'staging_relay_sleep_reconciled', alreadyAsleep: true })
|
||||
return
|
||||
}
|
||||
|
||||
const adminPost = createAdminPost(deps)
|
||||
const initial = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of initial) assertQuiescent(status)
|
||||
const selector = await inspectAdmissionSelector(
|
||||
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
|
||||
)
|
||||
if (selector.selector.generation > 0) {
|
||||
await sleepUnderSelector(deps, cells, adminPost, selector.selector)
|
||||
return
|
||||
}
|
||||
const previouslyEnabled = new Set(initial.filter((status) => status.enabled).map((status) => status.cellId))
|
||||
|
||||
for (const cell of cells) await setCellState(adminPost, cell, false)
|
||||
await deps.wait(15_000)
|
||||
try {
|
||||
const disabled = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of disabled) {
|
||||
if (status.enabled) throw new Error(`${status.cellId} admission did not disable`)
|
||||
assertQuiescent(status)
|
||||
}
|
||||
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
|
||||
const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
|
||||
for (const status of final) assertQuiescent(status)
|
||||
} catch (error) {
|
||||
for (const cell of cells.filter((candidate) => previouslyEnabled.has(candidate.cellId))) {
|
||||
await setCellState(adminPost, cell, true).catch(() => undefined)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0)))
|
||||
await ensureSqlPolicy(deps, 'NEVER')
|
||||
deps.emit({ event: 'staging_relay_slept', stoppedCells: cells.map((cell) => cell.cellId) })
|
||||
}
|
||||
|
||||
async function wakeStaging(deps, cells, wakeCells) {
|
||||
await ensureSqlPolicy(deps, 'ALWAYS')
|
||||
await waitUntil(deps, 'staging director health', () => checkHealth(deps, DIRECTOR_ORIGIN, '/health'))
|
||||
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
|
||||
|
||||
const adminPost = createAdminPost(deps)
|
||||
const selector = await inspectAdmissionSelector(
|
||||
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
|
||||
)
|
||||
const selectorActive = selector.selector.generation > 0
|
||||
// Existing-only cells may still own live or dormant assignments, so a
|
||||
// selector-era wake restores the complete retained topology.
|
||||
const selected = selectorActive
|
||||
? cells
|
||||
: cells.filter((cell) => wakeCells === 'all' || cell.initiallyEnabled)
|
||||
await Promise.all(
|
||||
cells.map((cell) => setMigSize(deps, cell, selected.includes(cell) ? 1 : 0))
|
||||
)
|
||||
for (const cell of selected) {
|
||||
await waitUntil(deps, `${cell.cellId} health`, () => checkHealth(deps, cell.origin, '/health'))
|
||||
await waitUntil(deps, `${cell.cellId} readiness`, () => checkHealth(deps, cell.origin, '/ready'))
|
||||
}
|
||||
|
||||
for (const cell of cells) {
|
||||
if (selectorActive) {
|
||||
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
|
||||
const current = await cellStatus(adminPost, cell)
|
||||
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
|
||||
})
|
||||
if (status.admissionState !== selectorCellState(selector.selector, cell.cellId)) {
|
||||
throw new Error(`${cell.cellId} admission does not match selector`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!selected.includes(cell)) {
|
||||
await setCellState(adminPost, cell, false)
|
||||
continue
|
||||
}
|
||||
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
|
||||
const current = await cellStatus(adminPost, cell)
|
||||
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
|
||||
})
|
||||
if (cell.initiallyEnabled && !status.enabled) await setCellState(adminPost, cell, true)
|
||||
if (!cell.initiallyEnabled && status.enabled) await setCellState(adminPost, cell, false)
|
||||
}
|
||||
deps.emit({
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: selected.map((cell) => cell.cellId),
|
||||
admissionCells: selectorActive
|
||||
? selector.selector.membership.general
|
||||
: selected.filter((cell) => cell.initiallyEnabled).map((cell) => cell.cellId)
|
||||
})
|
||||
}
|
||||
|
||||
export async function runStagingRelayPower(config, overrides = {}) {
|
||||
const deps = {
|
||||
command: overrides.command ?? ((args) => defaultCommand(args, false)),
|
||||
commandJson: overrides.commandJson ?? ((args) => defaultCommand(args, true)),
|
||||
fetch: overrides.fetch ?? fetch,
|
||||
adminToken: overrides.adminToken ?? suppliedAdminToken,
|
||||
emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
|
||||
now: overrides.now ?? Date.now,
|
||||
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
}
|
||||
const cells = readStagingTopology(config.topologyFile)
|
||||
if (config.mode === 'status') deps.emit(await stagingStatus(deps, cells))
|
||||
else if (config.mode === 'sleep') await sleepStaging(deps, cells)
|
||||
else await wakeStaging(deps, cells, config.wakeCells)
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
await runStagingRelayPower(parseArguments(argv))
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,427 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseArguments,
|
||||
readStagingTopology,
|
||||
runStagingRelayPower
|
||||
} from './power-staging-relay.mjs'
|
||||
|
||||
function topologyFile(overrides = {}) {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'staging-relay-power-'))
|
||||
const topology = {
|
||||
'staging-gce-c1': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c1',
|
||||
zone: 'us-central1-b',
|
||||
origin: 'https://c1.relay-staging.onorca.dev',
|
||||
initially_enabled: true
|
||||
},
|
||||
'staging-gce-c2': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c2',
|
||||
zone: 'us-central1-c',
|
||||
origin: 'https://c2.relay-staging.onorca.dev',
|
||||
initially_enabled: true
|
||||
},
|
||||
'staging-gce-c3': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c3',
|
||||
zone: 'us-central1-a',
|
||||
origin: 'https://c3.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
},
|
||||
'staging-gce-c4': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c4',
|
||||
zone: 'asia-east2-a',
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
},
|
||||
...overrides
|
||||
}
|
||||
const file = join(directory, 'topology.json')
|
||||
writeFileSync(file, JSON.stringify(topology))
|
||||
return file
|
||||
}
|
||||
|
||||
function argumentConfig(file, mode, wakeCells = 'configured') {
|
||||
return parseArguments([
|
||||
'--mode',
|
||||
mode,
|
||||
'--wake-cells',
|
||||
wakeCells,
|
||||
'--topology-file',
|
||||
file
|
||||
])
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function harness({
|
||||
sqlPolicy = 'ALWAYS',
|
||||
migSize = 1,
|
||||
observedRequests = 0,
|
||||
selectorGeneration = 0,
|
||||
admitModes
|
||||
} = {}) {
|
||||
const cells = new Map(
|
||||
['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'].map((cellId, index) => [
|
||||
cellId,
|
||||
{
|
||||
enabled: index < 2,
|
||||
targetSize: migSize,
|
||||
observedRequests,
|
||||
initiallyEnabled: index < 2
|
||||
}
|
||||
])
|
||||
)
|
||||
const revisions = new Map([
|
||||
['orca-cloud-relay-staging', { active: 'relay-00001', latest: 'relay-00001', min: 1 }],
|
||||
['orca-cloud-auth-staging', { active: 'auth-00001', latest: 'auth-00001', min: 1 }]
|
||||
])
|
||||
const revisionMinimums = new Map([
|
||||
['relay-00001', 1],
|
||||
['auth-00001', 1]
|
||||
])
|
||||
const commands = []
|
||||
const events = []
|
||||
const posts = []
|
||||
let activationPolicy = sqlPolicy
|
||||
let clock = 0
|
||||
|
||||
function cellForMig(name) {
|
||||
return [...cells.entries()].find(([, value], index) => {
|
||||
const suffix = `c${index + 1}`
|
||||
return name.endsWith(suffix) && value
|
||||
})
|
||||
}
|
||||
|
||||
function commandJson(args) {
|
||||
if (args[0] === 'sql') return { settings: { activationPolicy } }
|
||||
if (args[0] === 'compute') {
|
||||
const entry = cellForMig(args[4])
|
||||
return { targetSize: entry[1].targetSize, status: { isStable: true } }
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services') {
|
||||
const state = revisions.get(args[3])
|
||||
return {
|
||||
status: {
|
||||
latestReadyRevisionName: state.latest,
|
||||
traffic: [{ percent: 100, revisionName: state.active }]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'revisions') {
|
||||
return {
|
||||
metadata: {
|
||||
annotations: {
|
||||
'autoscaling.knative.dev/minScale': String(revisionMinimums.get(args[3]) ?? 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`unexpected JSON command ${args.join(' ')}`)
|
||||
}
|
||||
|
||||
function command(args) {
|
||||
commands.push(args)
|
||||
if (args[0] === 'sql') {
|
||||
activationPolicy = args.find((arg) => arg.startsWith('--activation-policy='))?.split('=')[1]
|
||||
return
|
||||
}
|
||||
if (args[0] === 'compute') {
|
||||
const entry = cellForMig(args[4])
|
||||
entry[1].targetSize = Number(args.find((arg) => arg.startsWith('--size='))?.split('=')[1])
|
||||
return
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update') {
|
||||
const state = revisions.get(args[3])
|
||||
state.latest = `${args[3]}-power`
|
||||
revisionMinimums.set(state.latest, 0)
|
||||
return
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update-traffic') {
|
||||
const state = revisions.get(args[3])
|
||||
state.active = state.latest
|
||||
return
|
||||
}
|
||||
throw new Error(`unexpected command ${args.join(' ')}`)
|
||||
}
|
||||
|
||||
async function fetchImpl(url, options = {}) {
|
||||
const parsed = new URL(url)
|
||||
if (!options.method) return response({ ok: true })
|
||||
const body = JSON.parse(options.body)
|
||||
posts.push(parsed.pathname)
|
||||
if (parsed.pathname === '/v1/admin/cell-admit-mode') {
|
||||
// A pre-census director's router answers an unknown route with a plain-text 404.
|
||||
if (!admitModes) return new Response('404 Not Found', { status: 404 })
|
||||
return response({ v: 1, cellId: body.cellId, admitMode: admitModes[body.cellId] ?? 'db' })
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-status') {
|
||||
const state = cells.get(body.cellId)
|
||||
const index = [...cells.keys()].indexOf(body.cellId)
|
||||
return response({
|
||||
v: 1,
|
||||
status: {
|
||||
cellId: body.cellId,
|
||||
enabled: state.enabled,
|
||||
admissionState:
|
||||
selectorGeneration > 0
|
||||
? index === 0
|
||||
? 'existing-only'
|
||||
: index === 1
|
||||
? 'migration-only'
|
||||
: index === 2
|
||||
? 'general'
|
||||
: 'migration-only'
|
||||
: state.enabled
|
||||
? 'general'
|
||||
: 'existing-only',
|
||||
assignments: 0,
|
||||
reservedRequests: 0,
|
||||
activityLeases: 0,
|
||||
activityRequestUnits: 0,
|
||||
outgoingMigrations: 0,
|
||||
incomingMigrations: 0,
|
||||
runtime: {
|
||||
ready: state.targetSize === 1,
|
||||
heartbeatFresh: state.targetSize === 1,
|
||||
observedRequests: state.observedRequests
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: selectorGeneration,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly:
|
||||
selectorGeneration > 0
|
||||
? ['staging-gce-c1']
|
||||
: [...cells]
|
||||
.filter(([, state]) => !state.enabled)
|
||||
.map(([cellId]) => cellId),
|
||||
migrationOnly:
|
||||
selectorGeneration > 0 ? ['staging-gce-c2', 'staging-gce-c4'] : [],
|
||||
general:
|
||||
selectorGeneration > 0
|
||||
? ['staging-gce-c3']
|
||||
: [...cells]
|
||||
.filter(([, state]) => state.enabled)
|
||||
.map(([cellId]) => cellId)
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
}
|
||||
if (parsed.pathname === '/v1/admin/cell-state') {
|
||||
cells.get(body.cellId).enabled = body.enabled
|
||||
return response({ ok: true })
|
||||
}
|
||||
throw new Error(`unexpected fetch ${parsed.pathname}`)
|
||||
}
|
||||
|
||||
return {
|
||||
cells,
|
||||
commands,
|
||||
events,
|
||||
posts,
|
||||
admitModes,
|
||||
deps: {
|
||||
command,
|
||||
commandJson,
|
||||
fetch: fetchImpl,
|
||||
adminToken: () => 'header.payload.signature',
|
||||
emit: (event) => events.push(event),
|
||||
now: () => clock,
|
||||
wait: async (ms) => {
|
||||
clock += ms
|
||||
}
|
||||
},
|
||||
sqlPolicy: () => activationPolicy
|
||||
}
|
||||
}
|
||||
|
||||
test('accepts only explicit staging power arguments and topology', () => {
|
||||
const file = topologyFile()
|
||||
assert.equal(argumentConfig(file, 'status').mode, 'status')
|
||||
assert.equal(readStagingTopology(file).at(-1).zone, 'asia-east2-a')
|
||||
assert.throws(() => argumentConfig(file, 'destroy'))
|
||||
assert.throws(() => parseArguments(['--mode', 'sleep', '--wake-cells', 'configured']))
|
||||
|
||||
const unsafe = topologyFile({
|
||||
'staging-gce-c1': {
|
||||
mig_name: 'orca-cloud-relay-gce-c1',
|
||||
zone: 'us-central1-a',
|
||||
origin: 'https://c1.relay.onorca.dev',
|
||||
initially_enabled: true
|
||||
}
|
||||
})
|
||||
assert.throws(() => readStagingTopology(unsafe), /unsafe/)
|
||||
|
||||
const unreviewedRegion = topologyFile({
|
||||
'staging-gce-c4': {
|
||||
mig_name: 'orca-cloud-staging-relay-gce-c4',
|
||||
zone: 'europe-west1-b',
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
initially_enabled: false
|
||||
}
|
||||
})
|
||||
assert.throws(() => readStagingTopology(unreviewedRegion), /unsafe zone/)
|
||||
})
|
||||
|
||||
test('refuses sleep before changing admission when a cell has active requests', async () => {
|
||||
const testHarness = harness({ observedRequests: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
|
||||
/still has active Relay work/
|
||||
)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.equal(testHarness.cells.get('staging-gce-c1').enabled, true)
|
||||
})
|
||||
|
||||
test('sleeps only after disabling admission and proving zero active work', async () => {
|
||||
const testHarness = harness()
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'NEVER')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0])
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [false, false, false, false])
|
||||
assert.equal(testHarness.events.at(-1).event, 'staging_relay_slept')
|
||||
assert.equal(
|
||||
testHarness.commands.filter((args) => args[0] === 'run' && args[2] === 'update').length,
|
||||
2
|
||||
)
|
||||
})
|
||||
|
||||
test('sleeps under the selector without writing any cell admission', async () => {
|
||||
for (const admitModes of [undefined, {}]) {
|
||||
const testHarness = harness({ selectorGeneration: 3, admitModes })
|
||||
const before = [...testHarness.cells.values()].map((cell) => cell.enabled)
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'NEVER')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0])
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), before)
|
||||
assert.equal(testHarness.posts.includes('/v1/admin/cell-state'), false)
|
||||
assert.equal(testHarness.posts.includes('/v1/admin/cell-admit-mode'), true)
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_slept',
|
||||
stoppedCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'],
|
||||
selectorGeneration: 3
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
test('refuses a selector-era sleep while any cell is reserve or busy', async () => {
|
||||
const reserve = harness({ selectorGeneration: 3, admitModes: { 'staging-gce-c2': 'reserve' } })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), reserve.deps),
|
||||
/refuses reserve-mode cells: staging-gce-c2/
|
||||
)
|
||||
assert.equal(reserve.commands.length, 0)
|
||||
|
||||
// A flip that lands after the first read, while Cloud Run scales down, still stops the resize.
|
||||
const late = harness({ selectorGeneration: 3, admitModes: {} })
|
||||
const run = late.deps.command
|
||||
late.deps.command = (args) => {
|
||||
if (args[0] === 'run' && args[2] === 'update-traffic') late.admitModes['staging-gce-c4'] = 'reserve'
|
||||
return run(args)
|
||||
}
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), late.deps),
|
||||
/refuses reserve-mode cells: staging-gce-c4/
|
||||
)
|
||||
assert.deepEqual([...late.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
assert.equal(late.sqlPolicy(), 'ALWAYS')
|
||||
|
||||
const busy = harness({ selectorGeneration: 3, observedRequests: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), busy.deps),
|
||||
/still has active Relay work/
|
||||
)
|
||||
assert.equal(busy.commands.length, 0)
|
||||
assert.deepEqual([...busy.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
})
|
||||
|
||||
test('a selector-era sleep then wake restores the selector membership', async () => {
|
||||
const testHarness = harness({ selectorGeneration: 3 })
|
||||
const file = topologyFile()
|
||||
await runStagingRelayPower(argumentConfig(file, 'sleep'), testHarness.deps)
|
||||
for (const state of testHarness.cells.values()) state.observedRequests = 0
|
||||
await runStagingRelayPower(argumentConfig(file, 'wake'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'ALWAYS')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
assert.equal(testHarness.posts.includes('/v1/admin/cell-state'), false)
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'],
|
||||
admissionCells: ['staging-gce-c3']
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses to terminate workers from an unknown partially asleep state', async () => {
|
||||
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 1 })
|
||||
await assert.rejects(
|
||||
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
|
||||
/partially asleep with running cells/
|
||||
)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
})
|
||||
|
||||
test('wakes SQL and configured cells while leaving the candidate off and disabled', async () => {
|
||||
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 0 })
|
||||
for (const cell of testHarness.cells.values()) cell.enabled = false
|
||||
for (const state of testHarness.cells.values()) state.observedRequests = 0
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
|
||||
|
||||
assert.equal(testHarness.sqlPolicy(), 'ALWAYS')
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 0, 0])
|
||||
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [true, true, false, false])
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: ['staging-gce-c1', 'staging-gce-c2'],
|
||||
admissionCells: ['staging-gce-c1', 'staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('wakes every retained cell without rewriting selector-era admission', async () => {
|
||||
const testHarness = harness({
|
||||
sqlPolicy: 'NEVER',
|
||||
migSize: 0,
|
||||
selectorGeneration: 1
|
||||
})
|
||||
const states = [...testHarness.cells.values()]
|
||||
states[0].enabled = false
|
||||
states[1].enabled = true
|
||||
states[2].enabled = true
|
||||
states[3].enabled = true
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
|
||||
|
||||
assert.deepEqual(states.map((cell) => cell.targetSize), [1, 1, 1, 1])
|
||||
assert.deepEqual(states.map((cell) => cell.enabled), [false, true, true, true])
|
||||
assert.deepEqual(testHarness.events.at(-1), {
|
||||
event: 'staging_relay_woke',
|
||||
runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'],
|
||||
admissionCells: ['staging-gce-c3']
|
||||
})
|
||||
})
|
||||
|
||||
test('status is read-only and reports the current billable floor controls', async () => {
|
||||
const testHarness = harness()
|
||||
await runStagingRelayPower(argumentConfig(topologyFile(), 'status'), testHarness.deps)
|
||||
assert.equal(testHarness.commands.length, 0)
|
||||
assert.equal(testHarness.events[0].project, 'onorca-cloud-staging')
|
||||
assert.equal(testHarness.events[0].sqlActivationPolicy, 'ALWAYS')
|
||||
assert.equal(testHarness.events[0].cells.length, 4)
|
||||
})
|
||||
@@ -1,86 +0,0 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
function argumentsFrom(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['current-json', 'topology-json', 'output', 'cell-ids', 'image-digest']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// Production Asia pools sit 176 ms from Cloud SQL and run at 16; staging C4 and the production
|
||||
// US cells at the Asia shape keep the default 10.
|
||||
const DATABASE_POOL_MAX = {
|
||||
production: { 'asia-east2': 16, 'us-central1': 10 },
|
||||
staging: { 'asia-east2': 10 }
|
||||
}
|
||||
|
||||
export function prepareRelayAsiaDirectorCells({ currentCells, topology, cellIds, imageDigest }) {
|
||||
if (!Array.isArray(currentCells) || !topology || Array.isArray(topology)) {
|
||||
throw new Error('director inputs are invalid')
|
||||
}
|
||||
const additions = cellIds.split(',').map((value) => value.trim()).filter(Boolean)
|
||||
if (
|
||||
additions.length === 0 ||
|
||||
new Set(additions).size !== additions.length
|
||||
) throw new Error('director Asia additions are invalid')
|
||||
const currentIds = new Set(currentCells.map((cell) => cell.id))
|
||||
if (currentIds.size !== currentCells.length) throw new Error('current director cells contain duplicates')
|
||||
const normalizedCurrent = currentCells.map((cell) => ({
|
||||
...cell,
|
||||
region: cell.region ?? 'us-central1'
|
||||
}))
|
||||
const desiredCells = additions.map((cellId) => {
|
||||
const cell = topology[cellId]
|
||||
const pools = DATABASE_POOL_MAX[cellId.split('-')[0]] ?? {}
|
||||
if (
|
||||
!cell ||
|
||||
!Object.hasOwn(pools, cell.region) ||
|
||||
cell.capacity_requests !== 6_000 ||
|
||||
cell.database_pool_max !== pools[cell.region] ||
|
||||
cell.connection_hard_cap !== 3_000 ||
|
||||
cell.connection_unobserved_bound !== 60 ||
|
||||
cell.initially_enabled !== false ||
|
||||
cell.image?.split('@')[1] !== imageDigest
|
||||
) throw new Error(`${cellId} state output does not match the reviewed Asia shape`)
|
||||
return {
|
||||
id: cellId,
|
||||
url: cell.origin,
|
||||
capacityRequests: cell.capacity_requests,
|
||||
region: cell.region,
|
||||
initiallyEnabled: false,
|
||||
connectionHardCap: cell.connection_hard_cap,
|
||||
connectionUnobservedBound: cell.connection_unobserved_bound
|
||||
}
|
||||
})
|
||||
const desiredById = new Map(desiredCells.map((cell) => [cell.id, cell]))
|
||||
for (const current of normalizedCurrent) {
|
||||
const desired = desiredById.get(current.id)
|
||||
if (!desired) continue
|
||||
for (const [key, value] of Object.entries(desired)) {
|
||||
if (current[key] !== value) {
|
||||
throw new Error(`${current.id} director configuration differs from the reviewed Asia shape`)
|
||||
}
|
||||
}
|
||||
}
|
||||
const configured = new Set(normalizedCurrent.map((cell) => cell.id))
|
||||
return [...normalizedCurrent, ...desiredCells.filter((cell) => !configured.has(cell.id))]
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const values = argumentsFrom(process.argv.slice(2))
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: JSON.parse(readFileSync(values['current-json'], 'utf8')),
|
||||
topology: JSON.parse(readFileSync(values['topology-json'], 'utf8')),
|
||||
cellIds: values['cell-ids'],
|
||||
imageDigest: values['image-digest']
|
||||
})
|
||||
writeFileSync(values.output, `${JSON.stringify(result)}\n`, { mode: 0o600 })
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { prepareRelayAsiaDirectorCells } from './prepare-relay-asia-director-cells.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const topologyCell = (ordinal, zone) => ({
|
||||
origin: `https://c${ordinal}.relay.onorca.dev`, region: 'asia-east2', zone,
|
||||
capacity_requests: 6_000, database_pool_max: 16,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60,
|
||||
initially_enabled: false,
|
||||
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${digest}`
|
||||
})
|
||||
|
||||
test('preserves current order, defaults predecessor regions, and appends exact Asia cells', () => {
|
||||
const current = [{
|
||||
id: 'production-gce-c1', url: 'https://c1.relay.onorca.dev',
|
||||
capacityRequests: 4_000, initiallyEnabled: false
|
||||
}]
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: current,
|
||||
topology: {
|
||||
'production-gce-c27': topologyCell(27, 'asia-east2-a'),
|
||||
'production-gce-c28': topologyCell(28, 'asia-east2-b'),
|
||||
'production-gce-c29': topologyCell(29, 'asia-east2-c')
|
||||
},
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
imageDigest: digest
|
||||
})
|
||||
assert.equal(result[0].region, 'us-central1')
|
||||
assert.deepEqual(result.slice(1).map(({ id }) => id), [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
|
||||
])
|
||||
assert.ok(result.slice(1).every((cell) =>
|
||||
cell.region === 'asia-east2' && cell.initiallyEnabled === false &&
|
||||
cell.connectionHardCap === 3_000
|
||||
))
|
||||
})
|
||||
|
||||
test('is idempotent for an exact existing Asia cell and rejects director drift', () => {
|
||||
const topology = { 'production-gce-c27': topologyCell(27, 'asia-east2-a') }
|
||||
const current = prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
})
|
||||
assert.deepEqual(prepareRelayAsiaDirectorCells({
|
||||
currentCells: current, topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), current)
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [{ ...current[0], capacityRequests: 5_999 }],
|
||||
topology, cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), /director configuration differs/)
|
||||
})
|
||||
|
||||
test('appends C30 after the configured launch cells without touching them', () => {
|
||||
const launch = prepareRelayAsiaDirectorCells({
|
||||
currentCells: [],
|
||||
topology: {
|
||||
'production-gce-c27': topologyCell(27, 'asia-east2-a'),
|
||||
'production-gce-c28': topologyCell(28, 'asia-east2-b'),
|
||||
'production-gce-c29': topologyCell(29, 'asia-east2-c')
|
||||
},
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
imageDigest: digest
|
||||
})
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: launch,
|
||||
topology: { 'production-gce-c30': topologyCell(30, 'asia-east2-a') },
|
||||
cellIds: 'production-gce-c30',
|
||||
imageDigest: digest
|
||||
})
|
||||
assert.deepEqual(result.slice(0, 3), launch)
|
||||
assert.deepEqual(result[3], {
|
||||
id: 'production-gce-c30', url: 'https://c30.relay.onorca.dev', capacityRequests: 6_000,
|
||||
region: 'asia-east2', initiallyEnabled: false, connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
})
|
||||
|
||||
test('checks C30 against its own digest, not the launch cells\' digest', () => {
|
||||
const c30Digest = `sha256:${'b'.repeat(64)}`
|
||||
const c30 = {
|
||||
...topologyCell(30, 'asia-east2-a'),
|
||||
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${c30Digest}`
|
||||
}
|
||||
const launch = [27, 28, 29].map((ordinal) => ({
|
||||
id: `production-gce-c${ordinal}`, url: `https://c${ordinal}.relay.onorca.dev`,
|
||||
capacityRequests: 6_000, region: 'asia-east2', initiallyEnabled: false,
|
||||
connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}))
|
||||
assert.equal(prepareRelayAsiaDirectorCells({
|
||||
currentCells: launch, topology: { 'production-gce-c30': c30 },
|
||||
cellIds: 'production-gce-c30', imageDigest: c30Digest
|
||||
}).length, 4)
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: launch, topology: { 'production-gce-c30': c30 },
|
||||
cellIds: 'production-gce-c30', imageDigest: digest
|
||||
}), /does not match/)
|
||||
})
|
||||
|
||||
test('pins the Asia pool per environment', () => {
|
||||
const staging = { ...topologyCell(4, 'asia-east2-a'), database_pool_max: 10 }
|
||||
assert.equal(prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology: { 'staging-gce-c4': staging },
|
||||
cellIds: 'staging-gce-c4', imageDigest: digest
|
||||
}).length, 1)
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology: { 'staging-gce-c4': { ...staging, database_pool_max: 16 } },
|
||||
cellIds: 'staging-gce-c4', imageDigest: digest
|
||||
}), /does not match/)
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [],
|
||||
topology: { 'production-gce-c30': { ...topologyCell(30, 'asia-east2-a'), database_pool_max: 10 } },
|
||||
cellIds: 'production-gce-c30', imageDigest: digest
|
||||
}), /does not match/)
|
||||
})
|
||||
|
||||
test('rejects a mismatching topology state output', () => {
|
||||
const wrong = topologyCell(27, 'asia-east2-a')
|
||||
wrong.database_pool_max = 20
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology: { 'production-gce-c27': wrong },
|
||||
cellIds: 'production-gce-c27', imageDigest: digest
|
||||
}), /does not match/)
|
||||
})
|
||||
|
||||
test('appends a US cell at the Asia shape with the default pool, never the Asia one', () => {
|
||||
const usCell = {
|
||||
...topologyCell(32, 'us-central1-a'), region: 'us-central1', database_pool_max: 10
|
||||
}
|
||||
const current = [{
|
||||
id: 'production-gce-c26', url: 'https://c26.relay.onorca.dev', capacityRequests: 4_000,
|
||||
initiallyEnabled: false, connectionHardCap: 1_000, connectionUnobservedBound: 60
|
||||
}]
|
||||
const result = prepareRelayAsiaDirectorCells({
|
||||
currentCells: current, topology: { 'production-gce-c32': usCell },
|
||||
cellIds: 'production-gce-c32', imageDigest: digest
|
||||
})
|
||||
assert.deepEqual(result.at(-1), {
|
||||
id: 'production-gce-c32', url: 'https://c32.relay.onorca.dev', capacityRequests: 6_000,
|
||||
region: 'us-central1', initiallyEnabled: false, connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
for (const drift of [{ database_pool_max: 16 }, { region: 'europe-west1' }]) {
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: current, topology: { 'production-gce-c32': { ...usCell, ...drift } },
|
||||
cellIds: 'production-gce-c32', imageDigest: digest
|
||||
}), /reviewed Asia shape/, JSON.stringify(drift))
|
||||
}
|
||||
// Staging has no US cell at this shape.
|
||||
assert.throws(() => prepareRelayAsiaDirectorCells({
|
||||
currentCells: [], topology: { 'staging-gce-c4': { ...usCell, database_pool_max: 10 } },
|
||||
cellIds: 'staging-gce-c4', imageDigest: digest
|
||||
}), /reviewed Asia shape/)
|
||||
})
|
||||
@@ -1,140 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const BOOT_IMAGE = 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13'
|
||||
const SHAPES = {
|
||||
staging: {
|
||||
project: 'onorca-cloud-staging',
|
||||
databasePoolMax: { 'asia-east2': 10 },
|
||||
cells: { 'staging-gce-c4': 'asia-east2-a' },
|
||||
waves: [['staging-gce-c4']]
|
||||
},
|
||||
production: {
|
||||
project: 'onorca-cloud',
|
||||
// 16 covers asia-east2's round trip to us-central1 Postgres; US cells keep the default.
|
||||
databasePoolMax: { 'asia-east2': 16, 'us-central1': 10 },
|
||||
cells: {
|
||||
'production-gce-c27': 'asia-east2-a',
|
||||
'production-gce-c28': 'asia-east2-b',
|
||||
'production-gce-c29': 'asia-east2-c',
|
||||
'production-gce-c30': 'asia-east2-a',
|
||||
'production-gce-c31': 'asia-east2-b',
|
||||
'production-gce-c32': 'us-central1-a',
|
||||
'production-gce-c33': 'us-central1-b',
|
||||
'production-gce-c34': 'asia-east2-c',
|
||||
'production-gce-c35': 'asia-east2-a'
|
||||
},
|
||||
// The launch set, then each later additive cell; a plan targets one wave, never live cells.
|
||||
waves: [
|
||||
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31'],
|
||||
// Declared together, so they plan together: a lone C32 plan would hit C33's missing template.
|
||||
['production-gce-c32', 'production-gce-c33'],
|
||||
['production-gce-c34'],
|
||||
['production-gce-c35']
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
function argumentsFrom(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['existing-json', 'environment', 'cell-ids', 'image']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
function canonical(value) {
|
||||
if (Array.isArray(value)) return value.map(canonical)
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]))
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function prepareRelayAsiaTopologyInput({
|
||||
existingCells,
|
||||
existingAdditionalRegions,
|
||||
environment,
|
||||
cellIds,
|
||||
image
|
||||
}) {
|
||||
const shape = SHAPES[environment]
|
||||
if (!shape) throw new Error('invalid environment')
|
||||
const requested = cellIds.split(',').map((value) => value.trim()).filter(Boolean).sort()
|
||||
const expected = shape.waves
|
||||
.map((wave) => [...wave].sort())
|
||||
.find((wave) => JSON.stringify(wave) === JSON.stringify(requested))
|
||||
if (new Set(requested).size !== requested.length || !expected) {
|
||||
throw new Error('cell IDs do not match the reviewed Asia topology')
|
||||
}
|
||||
const prefix = `us-central1-docker.pkg.dev/${shape.project}/orca-cloud/relay@sha256:`
|
||||
if (!image.startsWith(prefix) || !/sha256:[a-f0-9]{64}$/.test(image)) {
|
||||
throw new Error('image is not the environment Relay image pinned by digest')
|
||||
}
|
||||
if (!existingCells || Array.isArray(existingCells) || typeof existingCells !== 'object') {
|
||||
throw new Error('existing Relay cells must be an object')
|
||||
}
|
||||
if (
|
||||
!existingAdditionalRegions ||
|
||||
Array.isArray(existingAdditionalRegions) ||
|
||||
typeof existingAdditionalRegions !== 'object' ||
|
||||
JSON.stringify(canonical(existingAdditionalRegions)) !==
|
||||
JSON.stringify(canonical({ 'asia-east2': '10.42.1.0/24' }))
|
||||
) {
|
||||
throw new Error('Asia subnet must be committed before topology planning')
|
||||
}
|
||||
const additions = Object.fromEntries(expected.map((cellId) => {
|
||||
const hostname = cellId.split('-').at(-1)
|
||||
const zone = shape.cells[cellId]
|
||||
const region = zone.slice(0, zone.lastIndexOf('-'))
|
||||
return [cellId, {
|
||||
hostname,
|
||||
region,
|
||||
zone,
|
||||
machine_type: 'e2-standard-4',
|
||||
boot_disk_gb: 30,
|
||||
boot_image: BOOT_IMAGE,
|
||||
capacity_requests: 6_000,
|
||||
database_pool_max: shape.databasePoolMax[region],
|
||||
image,
|
||||
initially_enabled: false,
|
||||
connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
}]
|
||||
}))
|
||||
for (const cellId of expected) {
|
||||
if (!existingCells[cellId]) {
|
||||
throw new Error('Asia cells must be committed before topology planning')
|
||||
}
|
||||
if (
|
||||
JSON.stringify(canonical(existingCells[cellId])) !==
|
||||
JSON.stringify(canonical(additions[cellId]))
|
||||
) {
|
||||
throw new Error('committed Asia cell differs from the reviewed topology')
|
||||
}
|
||||
}
|
||||
return {
|
||||
relay_gce_additional_region_subnetwork_cidrs: existingAdditionalRegions,
|
||||
relay_gce_cells: existingCells
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const values = argumentsFrom(process.argv.slice(2))
|
||||
const existing = JSON.parse(readFileSync(values['existing-json'], 'utf8'))
|
||||
prepareRelayAsiaTopologyInput({
|
||||
existingCells: existing.relay_gce_cells,
|
||||
existingAdditionalRegions: existing.relay_gce_additional_region_subnetwork_cidrs,
|
||||
environment: values.environment,
|
||||
cellIds: values['cell-ids'],
|
||||
image: values.image
|
||||
})
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { prepareRelayAsiaTopologyInput } from './prepare-relay-asia-topology-input.mjs'
|
||||
|
||||
const image = `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`
|
||||
|
||||
const additionalRegions = { 'asia-east2': '10.42.1.0/24' }
|
||||
|
||||
const productionCells = () => Object.fromEntries([
|
||||
[27, 'asia-east2-a'],
|
||||
[28, 'asia-east2-b'],
|
||||
[29, 'asia-east2-c'],
|
||||
[30, 'asia-east2-a'],
|
||||
[31, 'asia-east2-b'],
|
||||
[32, 'us-central1-a'],
|
||||
[33, 'us-central1-b'],
|
||||
[34, 'asia-east2-c'],
|
||||
[35, 'asia-east2-a']
|
||||
].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, {
|
||||
hostname: `c${ordinal}`, region: zone.slice(0, -2), zone,
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13',
|
||||
capacity_requests: 6_000, database_pool_max: zone.startsWith('asia-') ? 16 : 10, image,
|
||||
initially_enabled: false,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60
|
||||
}]))
|
||||
|
||||
test('accepts the exact production topology only after it is durably committed', () => {
|
||||
const existing = {
|
||||
'production-gce-c26': { hostname: 'c26', image: 'existing' },
|
||||
...productionCells()
|
||||
}
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: existing,
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image })
|
||||
assert.equal(result.relay_gce_cells, existing)
|
||||
assert.equal(result.relay_gce_additional_region_subnetwork_cidrs, additionalRegions)
|
||||
assert.deepEqual(existing['production-gce-c27'], {
|
||||
hostname: 'c27', region: 'asia-east2', zone: 'asia-east2-a',
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13',
|
||||
capacity_requests: 6_000, database_pool_max: 16, image, initially_enabled: false,
|
||||
connection_hard_cap: 3_000, connection_unobserved_bound: 60
|
||||
})
|
||||
})
|
||||
|
||||
test('accepts the additive C30 wave without re-planning the launch cells', () => {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c30', image })
|
||||
assert.equal(result.relay_gce_cells['production-gce-c30'].zone, 'asia-east2-a')
|
||||
})
|
||||
|
||||
test('accepts the additive C31 wave in the next zone of the rotation', () => {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c31', image })
|
||||
assert.equal(result.relay_gce_cells['production-gce-c31'].zone, 'asia-east2-b')
|
||||
})
|
||||
|
||||
test('accepts the additive C34 spare wave in asia-east2-c at the Asia pool', () => {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c34', image })
|
||||
assert.equal(result.relay_gce_cells['production-gce-c34'].zone, 'asia-east2-c')
|
||||
assert.equal(result.relay_gce_cells['production-gce-c34'].database_pool_max, 16)
|
||||
})
|
||||
|
||||
test('accepts the additive C35 spare wave in asia-east2-a at the Asia pool', () => {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c35', image })
|
||||
assert.equal(result.relay_gce_cells['production-gce-c35'].zone, 'asia-east2-a')
|
||||
assert.equal(result.relay_gce_cells['production-gce-c35'].database_pool_max, 16)
|
||||
})
|
||||
|
||||
test('accepts the additive US C32+C33 wave at the default pool only', () => {
|
||||
const cellIds = 'production-gce-c32,production-gce-c33'
|
||||
for (const [cellId, zone] of [
|
||||
['production-gce-c32', 'us-central1-a'], ['production-gce-c33', 'us-central1-b']
|
||||
]) {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image })
|
||||
assert.equal(result.relay_gce_cells[cellId].zone, zone)
|
||||
assert.equal(result.relay_gce_cells[cellId].region, 'us-central1')
|
||||
// Copying the Asia pool onto a US cell is drift, not the reviewed shape.
|
||||
const asiaPool = productionCells()
|
||||
asiaPool[cellId].database_pool_max = 16
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({ existingCells: asiaPool,
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image
|
||||
}), /differs from the reviewed topology/)
|
||||
const asiaRegion = productionCells()
|
||||
asiaRegion[cellId].region = 'asia-east2'
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({ existingCells: asiaRegion,
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production', cellIds, image
|
||||
}), /differs from the reviewed topology/)
|
||||
}
|
||||
})
|
||||
|
||||
// Reads the committed file so a reviewed-shape constant cannot drift from what the plan reads.
|
||||
test('matches every committed production Asia cell entry', () => {
|
||||
const tfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const committed = {}
|
||||
for (const cellId of Object.keys(productionCells())) {
|
||||
const start = tfvars.indexOf(` "${cellId}" = {`)
|
||||
assert.notEqual(start, -1, `${cellId} is not committed`)
|
||||
const body = tfvars.slice(start, tfvars.indexOf('\n }', start))
|
||||
const cell = {}
|
||||
for (const [, key, raw] of body.matchAll(/^\s+([a-z_]+)\s+=\s+("[^"]*"|\S+)/gm)) {
|
||||
cell[key] = raw.startsWith('"') ? raw.slice(1, -1)
|
||||
: raw === 'true' || raw === 'false' ? raw === 'true' : Number(raw)
|
||||
}
|
||||
committed[cellId] = cell
|
||||
}
|
||||
// Each wave is pinned on its own: C30-C33 launch on the director's digest, not C27's.
|
||||
for (const wave of [
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
'production-gce-c30',
|
||||
'production-gce-c31',
|
||||
'production-gce-c32,production-gce-c33',
|
||||
'production-gce-c34',
|
||||
'production-gce-c35'
|
||||
]) {
|
||||
const committedImage = committed[wave.split(',')[0]].image
|
||||
assert.doesNotThrow(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: committed, existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production', cellIds: wave, image: committedImage
|
||||
}), wave)
|
||||
}
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: committed, existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production', cellIds: 'production-gce-c30',
|
||||
image
|
||||
}), /differs from the reviewed topology/)
|
||||
// The US cells and the C34 spare launch on the newest cell digest, the one C31 launched on.
|
||||
for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34']) {
|
||||
assert.equal(committed[cellId].image, committed['production-gce-c31'].image, cellId)
|
||||
}
|
||||
})
|
||||
|
||||
test('accepts the one exact committed staging Asia cell', () => {
|
||||
const stagingImage = image.replace('onorca-cloud/', 'onorca-cloud-staging/')
|
||||
const stagingCell = {
|
||||
hostname: 'c4', region: 'asia-east2', zone: 'asia-east2-a',
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-121-18867-638-13',
|
||||
capacity_requests: 6_000, database_pool_max: 10, image: stagingImage,
|
||||
initially_enabled: false, connection_hard_cap: 3_000,
|
||||
connection_unobserved_bound: 60
|
||||
}
|
||||
const result = prepareRelayAsiaTopologyInput({
|
||||
existingCells: { 'staging-gce-c3': { hostname: 'c3' }, 'staging-gce-c4': stagingCell },
|
||||
existingAdditionalRegions: additionalRegions,
|
||||
environment: 'staging',
|
||||
cellIds: 'staging-gce-c4',
|
||||
image: stagingImage
|
||||
})
|
||||
assert.equal(result.relay_gce_cells['staging-gce-c4'].zone, 'asia-east2-a')
|
||||
assert.equal(result.relay_gce_cells['staging-gce-c4'].image, stagingImage)
|
||||
})
|
||||
|
||||
test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drift', () => {
|
||||
for (const cellIds of [
|
||||
'production-gce-c27',
|
||||
'production-gce-c27,production-gce-c30',
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30',
|
||||
'production-gce-c30,production-gce-c30',
|
||||
'production-gce-c30,production-gce-c31',
|
||||
'production-gce-c32',
|
||||
'production-gce-c33',
|
||||
'production-gce-c33,production-gce-c34',
|
||||
'production-gce-c34,production-gce-c35',
|
||||
'production-gce-c36'
|
||||
]) {
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production', cellIds, image
|
||||
}), /cell IDs/, cellIds)
|
||||
}
|
||||
const poolDrift = productionCells()
|
||||
poolDrift['production-gce-c30'].database_pool_max = 10
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: poolDrift, existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production', cellIds: 'production-gce-c30', image
|
||||
}), /differs from the reviewed topology/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
|
||||
environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
image: image.replace('onorca-cloud/', 'other-project/')
|
||||
}), /environment Relay image/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: {}, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /subnet must be committed/)
|
||||
const missing = productionCells()
|
||||
delete missing['production-gce-c29']
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: missing, existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /cells must be committed/)
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: { ...productionCells(), 'production-gce-c27': {} },
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
|
||||
}), /differs from the reviewed topology/)
|
||||
})
|
||||
@@ -1,183 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
function parseArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-id', 'mode']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
const origin = new URL(values['director-origin'])
|
||||
if (origin.protocol !== 'https:' || origin.origin !== values['director-origin']) {
|
||||
throw new Error('--director-origin must be a canonical HTTPS origin')
|
||||
}
|
||||
if (!['isolate', 'activate', 'restore-fallback', 'restore'].includes(values.mode)) {
|
||||
throw new Error('--mode must be isolate, activate, restore-fallback, or restore')
|
||||
}
|
||||
const cellOrigin = values['cell-origin'] ? new URL(values['cell-origin']) : null
|
||||
if (
|
||||
values.mode === 'isolate' &&
|
||||
(!cellOrigin || cellOrigin.protocol !== 'https:' || cellOrigin.origin !== values['cell-origin'])
|
||||
) {
|
||||
throw new Error('--cell-origin must be a canonical HTTPS origin for isolate mode')
|
||||
}
|
||||
const restoreGeneralCellIds = values['general-cell-ids']?.split(',').filter(Boolean) ?? []
|
||||
if (
|
||||
['restore-fallback', 'restore'].includes(values.mode) &&
|
||||
restoreGeneralCellIds.length === 0
|
||||
) {
|
||||
throw new Error('--general-cell-ids is required for restore modes')
|
||||
}
|
||||
if (values.mode === 'restore' && !restoreGeneralCellIds.includes(values['cell-id'])) {
|
||||
throw new Error('--general-cell-ids must include the canary for restore mode')
|
||||
}
|
||||
if (values.mode === 'restore-fallback' && restoreGeneralCellIds.includes(values['cell-id'])) {
|
||||
throw new Error('--general-cell-ids cannot include the canary for fallback restore')
|
||||
}
|
||||
if (new Set(restoreGeneralCellIds).size !== restoreGeneralCellIds.length) {
|
||||
throw new Error('--general-cell-ids must be distinct')
|
||||
}
|
||||
return {
|
||||
directorOrigin: origin.origin,
|
||||
cellOrigin: cellOrigin?.origin,
|
||||
cellId: values['cell-id'],
|
||||
mode: values.mode,
|
||||
restoreGeneralCellIds
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
function sameMembership(left, right) {
|
||||
return JSON.stringify(left) === JSON.stringify(right)
|
||||
}
|
||||
|
||||
async function legacyCellState(post, cellId) {
|
||||
const result = await post('/v1/admin/cell-status', { v: 1, cellId })
|
||||
if (result.status?.cellId !== cellId) throw new Error('legacy admission status is invalid')
|
||||
const state = result.status.admissionState
|
||||
if (!['existing-only', 'migration-only', 'general'].includes(state)) {
|
||||
throw new Error('legacy admission status is invalid')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
async function applyLegacyStates(post, before, states, order) {
|
||||
const expected = membershipWithStates(before.selector, states)
|
||||
let changed = false
|
||||
for (const cellId of order) {
|
||||
const desired = states[cellId]
|
||||
const current = await legacyCellState(post, cellId)
|
||||
if (current === 'existing-only' && desired !== current) {
|
||||
throw new Error(`legacy admission cannot re-enable existing-only cell ${cellId}`)
|
||||
}
|
||||
if (current === desired) continue
|
||||
let cause
|
||||
try {
|
||||
await post('/v1/admin/cell-state', { v: 1, cellId, state: desired })
|
||||
} catch (error) {
|
||||
cause = error
|
||||
}
|
||||
if ((await legacyCellState(post, cellId)) !== desired) {
|
||||
const detail = cause instanceof Error ? `: ${cause.message}` : ''
|
||||
throw new Error(`legacy admission did not commit ${cellId} exactly${detail}`, { cause })
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post)
|
||||
if (verified.selector.generation !== 0 ||
|
||||
!sameMembership(verified.selector.membership, expected)) {
|
||||
throw new Error('legacy admission membership changed unexpectedly')
|
||||
}
|
||||
return { changed, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function prepareCapacityCanary(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const postAt = async (origin, path, body) =>
|
||||
await responseJson(
|
||||
await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
}),
|
||||
path
|
||||
)
|
||||
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const state = selectorCellState(before.selector, config.cellId)
|
||||
if (state === 'existing-only' && config.mode !== 'restore-fallback') {
|
||||
throw new Error('capacity canary cannot restore existing-only admission')
|
||||
}
|
||||
const states =
|
||||
config.mode === 'isolate'
|
||||
? { [config.cellId]: 'migration-only' }
|
||||
: config.mode === 'activate'
|
||||
? Object.fromEntries([
|
||||
...before.selector.membership.general.map((cellId) => [
|
||||
cellId,
|
||||
cellId === config.cellId ? 'general' : 'migration-only'
|
||||
]),
|
||||
[config.cellId, 'general']
|
||||
])
|
||||
: Object.fromEntries([
|
||||
...config.restoreGeneralCellIds.map((cellId) => [cellId, 'general']),
|
||||
...(config.mode === 'restore-fallback'
|
||||
? [[config.cellId, state === 'existing-only' ? 'existing-only' : 'migration-only']]
|
||||
: [])
|
||||
])
|
||||
const membership = membershipWithStates(before.selector, states)
|
||||
const legacyOrder =
|
||||
config.mode === 'activate'
|
||||
? [config.cellId, ...before.selector.membership.general.filter((id) => id !== config.cellId)]
|
||||
: config.mode === 'restore-fallback'
|
||||
? [...config.restoreGeneralCellIds, config.cellId]
|
||||
: Object.keys(states)
|
||||
const result = before.selector.generation === 0
|
||||
? await applyLegacyStates(post, before, states, legacyOrder)
|
||||
: sameMembership(membership, before.selector.membership)
|
||||
? { changed: false, selector: before.selector }
|
||||
: await applyExactAdmissionSelector(post, membership, {
|
||||
expectedCurrentSelector: before.selector
|
||||
})
|
||||
if (config.mode === 'isolate') {
|
||||
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
|
||||
}
|
||||
return {
|
||||
changed: result.changed,
|
||||
generation: result.selector.generation,
|
||||
...(config.mode === 'isolate' ? { drained: true } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const config = parseArguments(argv)
|
||||
const result = await prepareCapacityCanary(config)
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ event: 'relay_capacity_canary_admission', cellId: config.cellId, mode: config.mode, ...result })}\n`
|
||||
)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
import { prepareCapacityCanary } from './prepare-relay-capacity-canary.mjs'
|
||||
|
||||
function harness(initialState, options = {}) {
|
||||
const {
|
||||
generation = 4,
|
||||
ambiguousCellState = false,
|
||||
rejectCellState = false,
|
||||
fallbackState = 'general',
|
||||
extraGeneralCellIds = []
|
||||
} = options
|
||||
let selector = {
|
||||
generation,
|
||||
attemptId: 'initial',
|
||||
membership: {
|
||||
existingOnly: [
|
||||
'staging-gce-c1',
|
||||
...(initialState === 'existing-only' ? ['staging-gce-c3'] : [])
|
||||
],
|
||||
migrationOnly: [
|
||||
...(fallbackState === 'migration-only' ? ['staging-gce-c2'] : []),
|
||||
...(initialState === 'migration-only' ? ['staging-gce-c3'] : [])
|
||||
],
|
||||
general: [
|
||||
...extraGeneralCellIds,
|
||||
...(fallbackState === 'general' ? ['staging-gce-c2'] : []),
|
||||
...(initialState === 'general' ? ['staging-gce-c3'] : [])
|
||||
].sort()
|
||||
}
|
||||
}
|
||||
let intent = null
|
||||
let applies = 0
|
||||
let drains = 0
|
||||
const cellStateChanges = []
|
||||
const fetch = async (url, options) => {
|
||||
const path = new URL(url).pathname
|
||||
const body = JSON.parse(options.body)
|
||||
if (path === '/v1/admin/drain') {
|
||||
assert.deepEqual(body, { v: 1, graceMs: 0 })
|
||||
drains++
|
||||
return Response.json({ ok: true })
|
||||
}
|
||||
if (path === '/v1/admin/cell-status') {
|
||||
const state = selector.membership.existingOnly.includes(body.cellId)
|
||||
? 'existing-only'
|
||||
: selector.membership.migrationOnly.includes(body.cellId)
|
||||
? 'migration-only'
|
||||
: 'general'
|
||||
return Response.json({ status: { cellId: body.cellId, admissionState: state } })
|
||||
}
|
||||
if (path === '/v1/admin/cell-state') {
|
||||
assert.equal(selector.generation, 0)
|
||||
if (rejectCellState) {
|
||||
return Response.json({ error: 'invalid_token' }, { status: 401 })
|
||||
}
|
||||
const keys = {
|
||||
'existing-only': 'existingOnly',
|
||||
'migration-only': 'migrationOnly',
|
||||
general: 'general'
|
||||
}
|
||||
for (const cells of Object.values(selector.membership)) {
|
||||
const index = cells.indexOf(body.cellId)
|
||||
if (index !== -1) cells.splice(index, 1)
|
||||
}
|
||||
selector.membership[keys[body.state]].push(body.cellId)
|
||||
for (const cells of Object.values(selector.membership)) cells.sort()
|
||||
cellStateChanges.push({ cellId: body.cellId, state: body.state })
|
||||
if (ambiguousCellState) throw new Error('response lost')
|
||||
return Response.json({ ok: true })
|
||||
}
|
||||
if (path.endsWith('/status')) return Response.json({ selector, intent })
|
||||
applies++
|
||||
selector = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
intent = {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership,
|
||||
state: 'committed'
|
||||
}
|
||||
return Response.json({ changed: true, selector })
|
||||
}
|
||||
return {
|
||||
fetch,
|
||||
selector: () => selector,
|
||||
applies: () => applies,
|
||||
drains: () => drains,
|
||||
cellStateChanges
|
||||
}
|
||||
}
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.example.com',
|
||||
cellOrigin: 'https://c3.relay.example.com',
|
||||
cellId: 'staging-gce-c3',
|
||||
mode: 'isolate',
|
||||
restoreGeneralCellIds: []
|
||||
}
|
||||
|
||||
test('isolates a general canary as migration-only', async () => {
|
||||
const testHarness = harness('general')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
|
||||
{ changed: true, generation: 5, drained: true }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership.migrationOnly, [config.cellId])
|
||||
assert.equal(testHarness.applies(), 1)
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('activates the canary as the only general cell', async () => {
|
||||
const testHarness = harness('migration-only')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: true, generation: 5 }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: ['staging-gce-c2'],
|
||||
general: [config.cellId]
|
||||
})
|
||||
assert.equal(testHarness.applies(), 1)
|
||||
})
|
||||
|
||||
test('restores the reviewed staging general membership', async () => {
|
||||
const testHarness = harness('migration-only')
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore',
|
||||
restoreGeneralCellIds: ['staging-gce-c2', config.cellId]
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: true, generation: 5 }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2', config.cellId]
|
||||
})
|
||||
})
|
||||
|
||||
test('restores the fallback without promoting a possibly drained canary', async () => {
|
||||
const testHarness = harness('general')
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [config.cellId],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('restores the fallback while preserving an irreversible canary', async () => {
|
||||
const testHarness = harness('existing-only')
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1', config.cellId],
|
||||
migrationOnly: [],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('uses exact legacy admission writes before the selector boundary', async () => {
|
||||
const testHarness = harness('general', { generation: 0 })
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
|
||||
{ changed: true, generation: 0, drained: true }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'migration-only' }
|
||||
])
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('promotes a legacy canary before demoting its fallback', async () => {
|
||||
const testHarness = harness('migration-only', { generation: 0 })
|
||||
await prepareCapacityCanary(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'general' },
|
||||
{ cellId: 'staging-gce-c2', state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('makes the canary sole general with the live legacy membership shape', async () => {
|
||||
const extraGeneralCellIds = ['combined', 'staging-c1', 'staging-c2']
|
||||
const testHarness = harness('migration-only', { generation: 0, extraGeneralCellIds })
|
||||
const activate = { ...config, mode: 'activate' }
|
||||
await prepareCapacityCanary(activate, { fetch: testHarness.fetch, token: 'masked' })
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: config.cellId, state: 'general' },
|
||||
{ cellId: 'combined', state: 'migration-only' },
|
||||
{ cellId: 'staging-c1', state: 'migration-only' },
|
||||
{ cellId: 'staging-c2', state: 'migration-only' },
|
||||
{ cellId: 'staging-gce-c2', state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('restores a legacy fallback before demoting the target', async () => {
|
||||
const testHarness = harness('general', {
|
||||
generation: 0,
|
||||
fallbackState: 'migration-only'
|
||||
})
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [
|
||||
{ cellId: 'staging-gce-c2', state: 'general' },
|
||||
{ cellId: config.cellId, state: 'migration-only' }
|
||||
])
|
||||
})
|
||||
|
||||
test('keeps an already restored legacy fallback unchanged', async () => {
|
||||
const testHarness = harness('migration-only', { generation: 0 })
|
||||
assert.deepEqual(
|
||||
await prepareCapacityCanary(
|
||||
{
|
||||
...config,
|
||||
mode: 'restore-fallback',
|
||||
restoreGeneralCellIds: ['staging-gce-c2']
|
||||
},
|
||||
{ fetch: testHarness.fetch, token: 'masked' }
|
||||
),
|
||||
{ changed: false, generation: 0 }
|
||||
)
|
||||
assert.deepEqual(testHarness.cellStateChanges, [])
|
||||
assert.deepEqual(testHarness.selector().membership, {
|
||||
existingOnly: ['staging-gce-c1'],
|
||||
migrationOnly: [config.cellId],
|
||||
general: ['staging-gce-c2']
|
||||
})
|
||||
})
|
||||
|
||||
test('recovers an ambiguous legacy admission response by exact readback', async () => {
|
||||
const testHarness = harness('general', { generation: 0, ambiguousCellState: true })
|
||||
await assert.doesNotReject(
|
||||
prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
|
||||
)
|
||||
assert.equal(testHarness.drains(), 1)
|
||||
})
|
||||
|
||||
test('reports a rejected legacy admission write without draining', async () => {
|
||||
const testHarness = harness('general', { generation: 0, rejectCellState: true })
|
||||
const operation = prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
|
||||
await assert.rejects(operation, /cell-state returned 401/)
|
||||
assert.equal(testHarness.drains(), 0)
|
||||
})
|
||||
|
||||
test('the staging workflow supplies every required capacity transition argument', () => {
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const verifyCalls = workflow.match(
|
||||
/node dev\/scripts\/verify-relay-capacity-transition\.mjs[\s\S]*?(?=\n\s*\n|\n\s*- name:)/g
|
||||
)
|
||||
assert.ok(verifyCalls?.length >= 5)
|
||||
for (const call of verifyCalls) {
|
||||
for (const flag of ['--cell-origin', '--heartbeat', '--admission', '--draining', '--activity']) {
|
||||
assert.match(call, new RegExp(flag))
|
||||
}
|
||||
}
|
||||
const isolate = workflow.match(
|
||||
/node dev\/scripts\/prepare-relay-capacity-canary\.mjs[\s\S]*?--mode isolate/
|
||||
)?.[0]
|
||||
assert.match(isolate, /--cell-origin/)
|
||||
})
|
||||
@@ -1,196 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
import {
|
||||
applyExactAdmissionSelector,
|
||||
inspectAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorCellState
|
||||
} from './relay-admission-selector.mjs'
|
||||
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
|
||||
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
export const PRODUCTION_CAPACITY_CELL_IDS = [
|
||||
'production-gce-c7',
|
||||
'production-gce-c8',
|
||||
'production-gce-c9',
|
||||
'production-gce-c10',
|
||||
'production-gce-c13',
|
||||
'production-gce-c14',
|
||||
'production-gce-c15',
|
||||
'production-gce-c16',
|
||||
'production-gce-c19',
|
||||
'production-gce-c20',
|
||||
'production-gce-c21',
|
||||
'production-gce-c22',
|
||||
'production-gce-c23',
|
||||
'production-gce-c24',
|
||||
'production-gce-c25',
|
||||
'production-gce-c26'
|
||||
]
|
||||
|
||||
function cellOrigin(cellId) {
|
||||
return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev`
|
||||
}
|
||||
|
||||
// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
|
||||
const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
|
||||
|
||||
// Matches the cell's own cap on /v1/admin/drain.
|
||||
const MAX_PACE_WINDOW_MS = 20 * 60 * 1_000
|
||||
// Every paced image before the cap rose rejects a longer window with the same 400 an unpaced
|
||||
// image gives, and an unpaced drain is the opposite of what a slower window asked for.
|
||||
const LEGACY_MAX_PACE_WINDOW_MS = 5 * 60 * 1_000
|
||||
|
||||
export function parseProductionCapacityCellArguments(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
if (!['pace-check', 'isolate', 'drain', 'activate'].includes(values.mode)) {
|
||||
throw new Error('--mode must be pace-check, isolate, drain, or activate')
|
||||
}
|
||||
const approvedList = values['approved-cells']
|
||||
if (approvedList !== undefined && !APPROVED_CELL_LISTS[approvedList]) {
|
||||
throw new Error('--approved-cells is not a known allowlist')
|
||||
}
|
||||
const approvedCellIds = approvedList === undefined
|
||||
? PRODUCTION_CAPACITY_CELL_IDS
|
||||
: APPROVED_CELL_LISTS[approvedList]
|
||||
const cellId = values['cell-id']
|
||||
if (!approvedCellIds.includes(cellId)) {
|
||||
throw new Error('production capacity target is not approved')
|
||||
}
|
||||
const expectedCellOrigin = cellOrigin(cellId)
|
||||
if (
|
||||
values['director-origin'] !== DIRECTOR_ORIGIN ||
|
||||
values['cell-origin'] !== expectedCellOrigin
|
||||
) {
|
||||
throw new Error('production capacity target origin is not exact')
|
||||
}
|
||||
const paceWindowMs = values['pace-window-ms'] === undefined
|
||||
? 0
|
||||
: Number(values['pace-window-ms'])
|
||||
if (
|
||||
!Number.isSafeInteger(paceWindowMs) ||
|
||||
paceWindowMs < 0 ||
|
||||
paceWindowMs > MAX_PACE_WINDOW_MS
|
||||
) {
|
||||
throw new Error('--pace-window-ms must be an integer between 0 and 1200000')
|
||||
}
|
||||
return {
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
cellOrigin: expectedCellOrigin,
|
||||
cellId,
|
||||
mode: values.mode,
|
||||
paceWindowMs
|
||||
}
|
||||
}
|
||||
|
||||
async function responseJson(response, label) {
|
||||
const body = await response.json().catch(() => ({}))
|
||||
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
// Read-only, and run before the isolate: an image that does not advertise its cap has the
|
||||
// legacy one, so a slower pace stops here with the cell untouched instead of isolated.
|
||||
async function checkDrainPace(fetchImpl, config, wait) {
|
||||
const health = await responseJson(
|
||||
await fetchAdminOnceMore(fetchImpl, `${config.cellOrigin}/health`, {}, { wait }),
|
||||
'cell health'
|
||||
)
|
||||
if (health.ok !== true) throw new Error('cell health is not ok')
|
||||
const maxPaceWindowMs = Number.isSafeInteger(health.drainPaceWindowMaxMs)
|
||||
? health.drainPaceWindowMaxMs
|
||||
: LEGACY_MAX_PACE_WINDOW_MS
|
||||
if (config.paceWindowMs > maxPaceWindowMs) {
|
||||
throw new Error(
|
||||
`cell accepts drain paces up to ${maxPaceWindowMs} ms, not ${config.paceWindowMs} ms; ` +
|
||||
'it was not isolated'
|
||||
)
|
||||
}
|
||||
return { paceWindowMs: config.paceWindowMs, maxPaceWindowMs }
|
||||
}
|
||||
|
||||
export async function prepareProductionCapacityCell(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
if (config.mode === 'pace-check') return await checkDrainPace(fetchImpl, config, overrides.wait)
|
||||
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
|
||||
const postRaw = async (origin, path, body) =>
|
||||
await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${origin}${path}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
},
|
||||
{ wait: overrides.wait }
|
||||
)
|
||||
const postAt = async (origin, path, body) =>
|
||||
await responseJson(await postRaw(origin, path, body), path)
|
||||
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
|
||||
if (config.mode === 'drain') {
|
||||
const paceWindowMs = config.paceWindowMs ?? 0
|
||||
if (paceWindowMs > 0) {
|
||||
const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', {
|
||||
v: 1,
|
||||
graceMs: 0,
|
||||
paceWindowMs
|
||||
})
|
||||
if (paced.ok) {
|
||||
await paced.json().catch(() => ({}))
|
||||
return { changed: false, drained: true, paceWindowMs }
|
||||
}
|
||||
// A cell still on an image without paced drain rejects the unknown field outright.
|
||||
// An unpaced drain is the behaviour that cell already has, so fall back to it.
|
||||
if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`)
|
||||
await paced.json().catch(() => ({}))
|
||||
if (paceWindowMs > LEGACY_MAX_PACE_WINDOW_MS) {
|
||||
throw new Error(
|
||||
`cell rejected a ${paceWindowMs} ms drain pace; refusing to drain faster than asked`
|
||||
)
|
||||
}
|
||||
}
|
||||
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
|
||||
return { changed: false, drained: true, paceWindowMs: 0 }
|
||||
}
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const state = selectorCellState(before.selector, config.cellId)
|
||||
if (state === 'existing-only') throw new Error('production capacity target is irreversible')
|
||||
const desiredState = config.mode === 'isolate' ? 'migration-only' : 'general'
|
||||
const membership = membershipWithStates(before.selector, { [config.cellId]: desiredState })
|
||||
const result = await applyExactAdmissionSelector(post, membership, {
|
||||
expectedCurrentSelector: before.selector,
|
||||
// The stamp that tells the director this cell is parked for a restart rather
|
||||
// than held back as capacity, so it may re-place the hosts still on it. The
|
||||
// activate branch omits it, and moving to 'general' clears it in the same
|
||||
// statement that writes the state.
|
||||
...(config.mode === 'isolate' ? { rollIsolatedCells: [config.cellId] } : {})
|
||||
})
|
||||
return {
|
||||
changed: result.changed,
|
||||
generation: result.selector.generation,
|
||||
admissionState: desiredState,
|
||||
rollIsolated: config.mode === 'isolate'
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const config = parseProductionCapacityCellArguments(argv)
|
||||
const result = await prepareProductionCapacityCell(config)
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ event: 'relay_production_capacity_canary', cellId: config.cellId, mode: config.mode, ...result })}\n`
|
||||
)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,391 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import {
|
||||
parseProductionCapacityCellArguments,
|
||||
prepareProductionCapacityCell
|
||||
} from './prepare-relay-production-capacity-canary.mjs'
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
cellOrigin: 'https://c26.relay.onorca.dev',
|
||||
cellId: 'production-gce-c26'
|
||||
}
|
||||
|
||||
const membership = {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17'],
|
||||
general: ['production-gce-c25', 'production-gce-c26']
|
||||
}
|
||||
|
||||
function response(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
function canaryFetch() {
|
||||
let selector = { generation: 20, attemptId: null, membership }
|
||||
const calls = []
|
||||
const fetch = async (url, init) => {
|
||||
const path = new URL(url).pathname
|
||||
const body = JSON.parse(init.body)
|
||||
calls.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return response({
|
||||
v: 1,
|
||||
selector,
|
||||
intent: body.attemptId
|
||||
? {
|
||||
attemptId: body.attemptId,
|
||||
state: 'committed',
|
||||
expectedGeneration: selector.generation - 1,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership
|
||||
}
|
||||
: null
|
||||
})
|
||||
}
|
||||
if (path === '/v1/admin/admission-selector/apply') {
|
||||
selector = {
|
||||
generation: selector.generation + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
return response({ v: 1, changed: true, selector })
|
||||
}
|
||||
if (path === '/v1/admin/drain') return response({ v: 1, draining: true })
|
||||
throw new Error(`unexpected ${path}`)
|
||||
}
|
||||
return { calls, fetch, selector: () => selector }
|
||||
}
|
||||
|
||||
describe('production Relay capacity cell admission', () => {
|
||||
it('allows only the serving rollout cells', () => {
|
||||
assert.deepEqual(parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c7.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--mode', 'isolate'
|
||||
]), {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
cellOrigin: 'https://c7.relay.onorca.dev',
|
||||
cellId: 'production-gce-c7',
|
||||
mode: 'isolate',
|
||||
paceWindowMs: 0
|
||||
})
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c17.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c17',
|
||||
'--mode', 'isolate'
|
||||
]), /not approved/)
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c8.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--mode', 'isolate'
|
||||
]), /origin is not exact/)
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c27.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c27',
|
||||
'--mode', 'isolate'
|
||||
]), /not approved/)
|
||||
})
|
||||
|
||||
it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => {
|
||||
for (const cellId of [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
'production-gce-c31', 'production-gce-c34', 'production-gce-c35',
|
||||
// Migration-only canaries: the US-only capacity rollout never touches them either.
|
||||
'production-gce-c17', 'production-gce-c18',
|
||||
// US cells at the Asia shape: the 1,000-cap capacity rollout never touches them.
|
||||
'production-gce-c32', 'production-gce-c33'
|
||||
]) {
|
||||
const hostname = cellId.slice('production-gce-'.length)
|
||||
assert.deepEqual(parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', `https://${hostname}.relay.onorca.dev`,
|
||||
'--cell-id', cellId,
|
||||
'--approved-cells', 'same-cap',
|
||||
'--mode', 'isolate'
|
||||
]), {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
cellOrigin: `https://${hostname}.relay.onorca.dev`,
|
||||
cellId,
|
||||
mode: 'isolate',
|
||||
paceWindowMs: 0
|
||||
})
|
||||
}
|
||||
for (const cellId of ['production-gce-c12', 'production-gce-c36']) {
|
||||
const hostname = cellId.slice('production-gce-'.length)
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', `https://${hostname}.relay.onorca.dev`,
|
||||
'--cell-id', cellId,
|
||||
'--approved-cells', 'same-cap',
|
||||
'--mode', 'isolate'
|
||||
]), /not approved/)
|
||||
}
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c27.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c27',
|
||||
'--approved-cells', 'every-cell',
|
||||
'--mode', 'isolate'
|
||||
]), /not a known allowlist/)
|
||||
})
|
||||
|
||||
it('isolates only the selected cell without depending on its runtime', async () => {
|
||||
const fake = canaryFetch()
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.equal(result.admissionState, 'migration-only')
|
||||
assert.deepEqual(fake.selector().membership, {
|
||||
existingOnly: ['production-gce-c1'],
|
||||
migrationOnly: ['production-gce-c17', 'production-gce-c26'],
|
||||
general: ['production-gce-c25']
|
||||
})
|
||||
assert.doesNotMatch(fake.calls.map(({ path }) => path).join(','), /\/v1\/admin\/drain/)
|
||||
})
|
||||
|
||||
it('stamps the roll isolation on isolate and never on activate', async () => {
|
||||
// The stamp is what lets the director tell a cell parked for a restart from
|
||||
// an evacuation target or an Asia rollback, both of which must keep their
|
||||
// hosts. Only this call site may send it.
|
||||
const isolate = canaryFetch()
|
||||
await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch: isolate.fetch, token: 'token' }
|
||||
)
|
||||
const isolateApply = isolate.calls.find(
|
||||
({ path }) => path === '/v1/admin/admission-selector/apply'
|
||||
)
|
||||
assert.deepEqual(isolateApply.body.rollIsolatedCells, [config.cellId])
|
||||
|
||||
// Restore has to be a real apply, not the no-op an already-general cell
|
||||
// takes, or the assertion below proves nothing.
|
||||
isolate.calls.length = 0
|
||||
const restored = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: isolate.fetch, token: 'token' }
|
||||
)
|
||||
assert.equal(restored.admissionState, 'general')
|
||||
const restoreApply = isolate.calls.find(
|
||||
({ path }) => path === '/v1/admin/admission-selector/apply'
|
||||
)
|
||||
assert.ok(restoreApply, 'restore must issue an apply')
|
||||
assert.equal(restoreApply.body.rollIsolatedCells, undefined)
|
||||
assert.ok(restoreApply.body.membership.general.includes(config.cellId))
|
||||
})
|
||||
|
||||
it('drains the selected cell independently after durable isolation', async () => {
|
||||
const fake = canaryFetch()
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
|
||||
assert.deepEqual(fake.calls, [{
|
||||
path: '/v1/admin/drain',
|
||||
body: { v: 1, graceMs: 0 }
|
||||
}])
|
||||
})
|
||||
|
||||
it('paces the drain send when the roll asks for a window', async () => {
|
||||
const fake = canaryFetch()
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 })
|
||||
assert.deepEqual(fake.calls, [{
|
||||
path: '/v1/admin/drain',
|
||||
body: { v: 1, graceMs: 0, paceWindowMs: 120_000 }
|
||||
}])
|
||||
})
|
||||
|
||||
it('drains unpaced when the cell image rejects the pacing field', async () => {
|
||||
const bodies = []
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
|
||||
{
|
||||
token: 'token',
|
||||
wait: async () => {},
|
||||
fetch: async (url, init) => {
|
||||
assert.equal(new URL(url).pathname, '/v1/admin/drain')
|
||||
const body = JSON.parse(init.body)
|
||||
bodies.push(body)
|
||||
if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400)
|
||||
return response({ v: 1, draining: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
|
||||
assert.deepEqual(bodies, [
|
||||
{ v: 1, graceMs: 0, paceWindowMs: 120_000 },
|
||||
{ v: 1, graceMs: 0 }
|
||||
])
|
||||
})
|
||||
|
||||
it('refuses to fall back to an unpaced drain when the cell rejects a slower window', async () => {
|
||||
for (const paceWindowMs of [900_000, 1_200_000]) {
|
||||
const bodies = []
|
||||
await assert.rejects(
|
||||
prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain', paceWindowMs },
|
||||
{
|
||||
token: 'token',
|
||||
wait: async () => {},
|
||||
fetch: async (_url, init) => {
|
||||
bodies.push(JSON.parse(init.body))
|
||||
return response({ error: 'invalid_request' }, 400)
|
||||
}
|
||||
}
|
||||
),
|
||||
/refusing to drain faster than asked/
|
||||
)
|
||||
assert.deepEqual(bodies, [{ v: 1, graceMs: 0, paceWindowMs }])
|
||||
}
|
||||
})
|
||||
|
||||
it('stops a slower pace than an old-cap cell accepts before anything is isolated', async () => {
|
||||
const check = async (health, paceWindowMs) => {
|
||||
const requests = []
|
||||
const result = prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'pace-check', paceWindowMs },
|
||||
{
|
||||
wait: async () => {},
|
||||
fetch: async (url, init) => {
|
||||
requests.push({ path: new URL(url).pathname, method: init.method ?? 'GET' })
|
||||
return response(health)
|
||||
}
|
||||
}
|
||||
)
|
||||
return { result, requests }
|
||||
}
|
||||
// Every image before the cap rose answers /health without it.
|
||||
const old = { ok: true, connectionCapacityProtocol: 2 }
|
||||
for (const pace of [900_000, 1_200_000]) {
|
||||
const { result, requests } = await check(old, pace)
|
||||
await assert.rejects(result, /up to 300000 ms, not \d+ ms; it was not isolated/)
|
||||
assert.deepEqual(requests, [{ path: '/health', method: 'GET' }])
|
||||
}
|
||||
assert.deepEqual(await (await check(old, 300_000)).result, {
|
||||
paceWindowMs: 300_000, maxPaceWindowMs: 300_000
|
||||
})
|
||||
const current = { ...old, drainPaceWindowMaxMs: 1_200_000 }
|
||||
assert.deepEqual(await (await check(current, 1_200_000)).result, {
|
||||
paceWindowMs: 1_200_000, maxPaceWindowMs: 1_200_000
|
||||
})
|
||||
await assert.rejects((await check({ ok: false }, 300_000)).result, /cell health is not ok/)
|
||||
})
|
||||
|
||||
it('fails a paced drain that the cell rejects for any other reason', async () => {
|
||||
await assert.rejects(
|
||||
prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
|
||||
{
|
||||
token: 'token',
|
||||
wait: async () => {},
|
||||
fetch: async () => response({ error: 'invalid_token' }, 401)
|
||||
}
|
||||
),
|
||||
/returned 401/
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses a pacing window that is not a bounded integer', () => {
|
||||
const argv = (value) => [
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-origin', 'https://c26.relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c26',
|
||||
'--mode', 'drain',
|
||||
'--pace-window-ms', value
|
||||
]
|
||||
for (const value of ['-1', '1200001', '1.5', 'soon']) {
|
||||
assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/)
|
||||
}
|
||||
for (const value of [300_000, 900_000, 1_200_000]) {
|
||||
assert.equal(parseProductionCapacityCellArguments(argv(String(value))).paceWindowMs, value)
|
||||
}
|
||||
})
|
||||
|
||||
it('restores only the selected cell to general admission', async () => {
|
||||
const fake = canaryFetch()
|
||||
await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'activate' },
|
||||
{ fetch: fake.fetch, token: 'token' }
|
||||
)
|
||||
assert.equal(result.admissionState, 'general')
|
||||
assert.deepEqual(fake.selector().membership, membership)
|
||||
})
|
||||
|
||||
it('refuses an irreversible existing-only target', async () => {
|
||||
const fetch = async () => response({
|
||||
v: 1,
|
||||
selector: {
|
||||
generation: 20,
|
||||
attemptId: null,
|
||||
membership: {
|
||||
existingOnly: ['production-gce-c26'],
|
||||
migrationOnly: ['production-gce-c17'],
|
||||
general: ['production-gce-c25']
|
||||
}
|
||||
},
|
||||
intent: null
|
||||
})
|
||||
await assert.rejects(
|
||||
prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'isolate' },
|
||||
{ fetch, token: 'token' }
|
||||
),
|
||||
/irreversible/
|
||||
)
|
||||
})
|
||||
|
||||
it('retries a transient 503 on the cell drain endpoint', async () => {
|
||||
let calls = 0
|
||||
const result = await prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain' },
|
||||
{
|
||||
token: 'token',
|
||||
wait: async () => {},
|
||||
fetch: async (url) => {
|
||||
assert.equal(new URL(url).pathname, '/v1/admin/drain')
|
||||
calls += 1
|
||||
if (calls === 1) return response({ error: 'warming up' }, 503)
|
||||
return response({ v: 1, draining: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
|
||||
})
|
||||
|
||||
it('fails when both drain attempts return a transient 503', async () => {
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
prepareProductionCapacityCell(
|
||||
{ ...config, mode: 'drain' },
|
||||
{
|
||||
token: 'token',
|
||||
wait: async () => {},
|
||||
fetch: async () => {
|
||||
calls += 1
|
||||
return response({ error: 'warming up' }, 503)
|
||||
}
|
||||
}
|
||||
),
|
||||
/returned 503/
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
})
|
||||
@@ -1,73 +0,0 @@
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const WRONG_CELL = 4409
|
||||
const DRAINING = 4503
|
||||
|
||||
function once(socket, event, listener) {
|
||||
if (typeof socket.once === 'function') {
|
||||
socket.once(event, listener)
|
||||
return
|
||||
}
|
||||
if (typeof socket.addEventListener !== 'function') {
|
||||
throw new Error('WebSocket event API is unavailable')
|
||||
}
|
||||
socket.addEventListener(event, (value) => {
|
||||
if (event === 'close') listener(value.code)
|
||||
else if (event === 'error') listener(value.error ?? new Error(value.message))
|
||||
else listener()
|
||||
}, { once: true })
|
||||
}
|
||||
|
||||
export function parseLegacyAdmissionProbeArguments(argv) {
|
||||
if (argv.length !== 2 || argv[0] !== '--cell-origin') throw new Error('invalid arguments')
|
||||
const origin = new URL(argv[1])
|
||||
if (origin.protocol !== 'https:' || origin.origin !== argv[1]) {
|
||||
throw new Error('--cell-origin must be a canonical HTTPS origin')
|
||||
}
|
||||
return { cellOrigin: origin.origin }
|
||||
}
|
||||
|
||||
export async function probeLegacyAdmission(config, overrides = {}) {
|
||||
const Socket = overrides.WebSocket ?? globalThis.WebSocket
|
||||
if (typeof Socket !== 'function') throw new Error('WebSocket is unavailable')
|
||||
const random = overrides.randomBytes ?? randomBytes
|
||||
const timeoutMs = overrides.timeoutMs ?? 15_000
|
||||
const hostId = random(12).toString('base64url')
|
||||
const credential = random(32).toString('base64url')
|
||||
const url = `${config.cellOrigin.replace('https://', 'wss://')}/v1/connect/${hostId}`
|
||||
await new Promise((resolve, reject) => {
|
||||
const socket = new Socket(url)
|
||||
const timer = setTimeout(() => {
|
||||
if (typeof socket.terminate === 'function') socket.terminate()
|
||||
else socket.close()
|
||||
reject(new Error('legacy admission probe timed out'))
|
||||
}, timeoutMs)
|
||||
once(socket, 'open', () => {
|
||||
socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
|
||||
})
|
||||
once(socket, 'close', (code) => {
|
||||
clearTimeout(timer)
|
||||
if (code === WRONG_CELL) resolve()
|
||||
else if (code === DRAINING) reject(new Error('legacy cell is draining'))
|
||||
else reject(new Error(`legacy admission probe closed with ${code}`))
|
||||
})
|
||||
once(socket, 'error', (error) => {
|
||||
clearTimeout(timer)
|
||||
reject(new Error(`legacy admission probe failed: ${error.message}`))
|
||||
})
|
||||
})
|
||||
return { accepting: true }
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await probeLegacyAdmission(parseLegacyAdmissionProbeArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_admission_verified', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseLegacyAdmissionProbeArguments,
|
||||
probeLegacyAdmission
|
||||
} from './probe-relay-legacy-admission.mjs'
|
||||
|
||||
function socketClosingWith(code, observed) {
|
||||
return class extends EventEmitter {
|
||||
constructor(url) {
|
||||
super()
|
||||
observed.url = url
|
||||
queueMicrotask(() => this.emit('open'))
|
||||
}
|
||||
|
||||
send(payload) {
|
||||
observed.payload = JSON.parse(payload)
|
||||
queueMicrotask(() => this.emit('close', code))
|
||||
}
|
||||
|
||||
terminate() {}
|
||||
}
|
||||
}
|
||||
|
||||
function nativeSocketClosingWith(code) {
|
||||
return class extends EventTarget {
|
||||
constructor() {
|
||||
super()
|
||||
queueMicrotask(() => this.dispatchEvent(new Event('open')))
|
||||
}
|
||||
|
||||
send() {
|
||||
const event = new Event('close')
|
||||
Object.defineProperty(event, 'code', { value: code })
|
||||
queueMicrotask(() => this.dispatchEvent(event))
|
||||
}
|
||||
|
||||
close() {}
|
||||
}
|
||||
}
|
||||
|
||||
const config = { cellOrigin: 'https://c2.relay.example.com' }
|
||||
const random = (length) => Buffer.alloc(length, length)
|
||||
|
||||
test('accepts only a canonical cell origin', () => {
|
||||
assert.deepEqual(
|
||||
parseLegacyAdmissionProbeArguments(['--cell-origin', config.cellOrigin]),
|
||||
config
|
||||
)
|
||||
assert.throws(
|
||||
() => parseLegacyAdmissionProbeArguments(['--cell-origin', `${config.cellOrigin}/path`]),
|
||||
/canonical/
|
||||
)
|
||||
})
|
||||
|
||||
test('proves admission with a synthetic invalid credential and exposes no identifier', async () => {
|
||||
const observed = {}
|
||||
assert.deepEqual(
|
||||
await probeLegacyAdmission(config, {
|
||||
WebSocket: socketClosingWith(4409, observed),
|
||||
randomBytes: random
|
||||
}),
|
||||
{ accepting: true }
|
||||
)
|
||||
assert.match(observed.url, /^wss:\/\/c2\.relay\.example\.com\/v1\/connect\/[A-Za-z0-9_-]{16}$/)
|
||||
assert.deepEqual(Object.keys(observed.payload).sort(), ['credential', 'mode', 'type', 'v'])
|
||||
})
|
||||
|
||||
test('uses the dependency-free Node WebSocket event API', async () => {
|
||||
await assert.doesNotReject(
|
||||
probeLegacyAdmission(config, {
|
||||
WebSocket: nativeSocketClosingWith(4409),
|
||||
randomBytes: random
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects the legacy draining close and any unknown outcome', async () => {
|
||||
for (const [code, message] of [[4503, /draining/], [4401, /closed with 4401/]]) {
|
||||
await assert.rejects(
|
||||
probeLegacyAdmission(config, {
|
||||
WebSocket: socketClosingWith(code, {}),
|
||||
randomBytes: random
|
||||
}),
|
||||
message
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -1,112 +0,0 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
// Every cell that carries the rehome identity: the eighteen US cells and the
|
||||
// seven asia-east2 cells that drain mis-homed hosts back the other way.
|
||||
const PRODUCTION_CELL =
|
||||
/^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
const SOURCE_TRANSIENT_RETRIES = 12
|
||||
const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000
|
||||
|
||||
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
|
||||
throw new Error('--director-origin must be the production Relay origin')
|
||||
}
|
||||
if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved')
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
||||
values['cell-incarnation']
|
||||
)) throw new Error('--cell-incarnation is invalid')
|
||||
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) {
|
||||
throw new Error('admin identity token is unavailable')
|
||||
}
|
||||
return {
|
||||
directorOrigin: DIRECTOR_ORIGIN,
|
||||
cellId: values['cell-id'],
|
||||
cellIncarnation: values['cell-incarnation'],
|
||||
token
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeRehomeTrust(config, dependencies = {}) {
|
||||
const fetchImpl = dependencies.fetch ?? fetch
|
||||
const request = () => fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.token}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
sourceCellId: config.cellId,
|
||||
sourceCellIncarnation: config.cellIncarnation
|
||||
})
|
||||
},
|
||||
{ wait: dependencies.wait }
|
||||
)
|
||||
const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
let response = await request()
|
||||
let body = await response.json().catch(() => ({}))
|
||||
// The director wraps source HTTP failures in 409; retry only explicit transient statuses. The
|
||||
// load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20).
|
||||
for (
|
||||
let retry = 0;
|
||||
retry < SOURCE_TRANSIENT_RETRIES &&
|
||||
response.status === 409 &&
|
||||
/^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '');
|
||||
retry++
|
||||
) {
|
||||
await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS)
|
||||
response = await request()
|
||||
body = await response.json().catch(() => ({}))
|
||||
}
|
||||
if (!response.ok) {
|
||||
const safeReasons = new Set([
|
||||
'invalid_token', 'director_only', 'invalid_request',
|
||||
'regional_rehome_trust_not_configured',
|
||||
'regional_rehome_trust_probe_source_unavailable',
|
||||
'regional_rehome_trust_probe_source_invalid_response',
|
||||
'regional_rehome_trust_probe_not_proven',
|
||||
...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504]
|
||||
.map((status) => `regional_rehome_trust_probe_source_${status}`)
|
||||
])
|
||||
const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error'
|
||||
throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`)
|
||||
}
|
||||
if (
|
||||
body.v !== 1 ||
|
||||
body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' ||
|
||||
body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' ||
|
||||
body.dedicatedIdentity?.accepted !== true ||
|
||||
body.dedicatedIdentity?.idempotent !== true ||
|
||||
body.sharedRuntimeIdentityRejected !== true ||
|
||||
body.proven !== true
|
||||
) throw new Error('application-mediated rehome trust proof is incomplete')
|
||||
return body
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv))
|
||||
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
parseRehomeTrustProbeArguments,
|
||||
probeRehomeTrust
|
||||
} from './probe-relay-rehome-trust.mjs'
|
||||
|
||||
const argv = [
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
'--cell-id', 'production-gce-c7',
|
||||
'--cell-incarnation', '11111111-1111-4111-8111-111111111111'
|
||||
]
|
||||
const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }
|
||||
|
||||
test('binds the application-mediated probe to an exact approved cell incarnation', () => {
|
||||
assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7')
|
||||
assert.throws(() => parseRehomeTrustProbeArguments(
|
||||
argv.with(1, 'https://other.example.test'),
|
||||
environment
|
||||
))
|
||||
assert.throws(() => parseRehomeTrustProbeArguments(argv, {
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token'
|
||||
}))
|
||||
})
|
||||
|
||||
test('requires complete aggregate application-mediated trust proof', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
const result = await probeRehomeTrust(config, {
|
||||
fetch: async (url, init) => {
|
||||
assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe')
|
||||
assert.deepEqual(JSON.parse(init.body), {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
return Response.json({
|
||||
v: 1,
|
||||
dedicatedIdentity: {
|
||||
firstOutcome: 'host-not-connected',
|
||||
secondOutcome: 'host-not-connected',
|
||||
accepted: true,
|
||||
idempotent: true
|
||||
},
|
||||
sharedRuntimeIdentityRejected: true,
|
||||
proven: true
|
||||
})
|
||||
}
|
||||
})
|
||||
assert.equal(result.proven, true)
|
||||
})
|
||||
|
||||
test('rejects partial or mismatched proof', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
await assert.rejects(
|
||||
probeRehomeTrust(config, {
|
||||
fetch: async () => Response.json({
|
||||
v: 1,
|
||||
dedicatedIdentity: {
|
||||
firstOutcome: 'host-not-connected',
|
||||
secondOutcome: 'host-not-connected',
|
||||
accepted: true,
|
||||
idempotent: true
|
||||
},
|
||||
sharedRuntimeIdentityRejected: false,
|
||||
proven: false
|
||||
})
|
||||
}),
|
||||
/incomplete/
|
||||
)
|
||||
})
|
||||
|
||||
const provenProbe = {
|
||||
v: 1,
|
||||
dedicatedIdentity: {
|
||||
firstOutcome: 'host-not-connected',
|
||||
secondOutcome: 'host-not-connected',
|
||||
accepted: true,
|
||||
idempotent: true
|
||||
},
|
||||
sharedRuntimeIdentityRejected: true,
|
||||
proven: true
|
||||
}
|
||||
|
||||
test('retries a transient 503 on the trust probe and proves on the second answer', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
let calls = 0
|
||||
const result = await probeRehomeTrust(config, {
|
||||
wait: async () => {},
|
||||
fetch: async () => {
|
||||
calls += 1
|
||||
if (calls === 1) return new Response('warming up', { status: 503 })
|
||||
return Response.json(provenProbe)
|
||||
}
|
||||
})
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(result.proven, true)
|
||||
})
|
||||
|
||||
test('fails when both trust-probe attempts return a transient 503', async () => {
|
||||
const config = parseRehomeTrustProbeArguments(argv, environment)
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
probeRehomeTrust(config, {
|
||||
wait: async () => {},
|
||||
fetch: async () => {
|
||||
calls += 1
|
||||
return new Response('warming up', { status: 503 })
|
||||
}
|
||||
}),
|
||||
/returned 503/
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
|
||||
test('approves the asia-east2 and US 3,000 rehome sources and still rejects unlisted cells', () => {
|
||||
for (const cellId of [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34',
|
||||
'production-gce-c35'
|
||||
]) {
|
||||
const parsed = parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
)
|
||||
assert.equal(parsed.cellId, cellId)
|
||||
}
|
||||
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c36']) {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
),
|
||||
/--cell-id is not approved/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('retries one director-wrapped source 503 without relaxing the proof', async () => {
|
||||
let calls = 0
|
||||
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
|
||||
wait: async () => {},
|
||||
fetch: async () => ++calls === 1
|
||||
? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
|
||||
: Response.json(provenProbe)
|
||||
})
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(result.proven, true)
|
||||
})
|
||||
|
||||
test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => {
|
||||
for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) {
|
||||
let calls = 0
|
||||
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
|
||||
wait: async () => { throw new Error('must not retry') },
|
||||
fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) }
|
||||
}), error => {
|
||||
assert.match(error.message, /returned 409/)
|
||||
assert.ok(!error.message.includes('secret-token-example'))
|
||||
if (reason.endsWith('_403')) assert.match(error.message, /source_403/)
|
||||
return true
|
||||
})
|
||||
assert.equal(calls, 1)
|
||||
}
|
||||
})
|
||||
|
||||
test('outlasts a load-balancer warm-up of consecutive wrapped source 503s', async () => {
|
||||
let calls = 0
|
||||
const waits = []
|
||||
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
|
||||
wait: async (ms) => { waits.push(ms) },
|
||||
fetch: async () => ++calls <= 6
|
||||
? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
|
||||
: Response.json(provenProbe)
|
||||
})
|
||||
assert.equal(calls, 7)
|
||||
assert.deepEqual(waits, Array(6).fill(10_000))
|
||||
assert.equal(result.proven, true)
|
||||
})
|
||||
|
||||
test('stops after two minutes of wrapped transient failures', async () => {
|
||||
let calls = 0
|
||||
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
|
||||
wait: async () => {},
|
||||
fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
|
||||
}), /returned 409.*source_503/)
|
||||
assert.equal(calls, 13)
|
||||
})
|
||||
|
||||
// The probe allowlist is the committed rehome source list, so a declared source can always probe.
|
||||
test('approves exactly the committed production rehome source cells', () => {
|
||||
const tfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const start = tfvars.indexOf('relay_region_rehome_source_cell_ids = [')
|
||||
const sources = new Set(
|
||||
[...tfvars.slice(start, tfvars.indexOf(']', start)).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell)
|
||||
)
|
||||
assert.ok(sources.has('production-gce-c35'))
|
||||
for (let ordinal = 1; ordinal <= 40; ordinal++) {
|
||||
const cellId = `production-gce-c${ordinal}`
|
||||
const approved = (() => {
|
||||
try {
|
||||
parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
})()
|
||||
assert.equal(approved, sources.has(cellId), cellId)
|
||||
}
|
||||
})
|
||||
@@ -1,85 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const production = source('infra/terraform/environments/production.tfvars')
|
||||
const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml')
|
||||
const jobWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
|
||||
|
||||
const RELAY_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
|
||||
|
||||
function source(path) {
|
||||
return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
// Slice each "<cell-id>" = { ... } entry out of relay_gce_cells.
|
||||
function productionCells() {
|
||||
const block = production.slice(production.indexOf('relay_gce_cells = {'))
|
||||
const cells = new Map()
|
||||
for (const match of block.matchAll(/"(production-gce-c\d+)" = \{([\s\S]*?)\n {2}\}/g)) {
|
||||
cells.set(match[1], match[2])
|
||||
}
|
||||
assert.ok(cells.size > 0, 'relay_gce_cells parsed empty')
|
||||
return cells
|
||||
}
|
||||
|
||||
function hardCap(body) {
|
||||
const match = body.match(/connection_hard_cap\s*=\s*(\d+)/)
|
||||
return match ? Number(match[1]) : undefined
|
||||
}
|
||||
|
||||
function imageDigest(body) {
|
||||
const match = body.match(/^\s*image\s*=\s*"([^"]+)"/m)
|
||||
assert.ok(match, 'cell entry has no image')
|
||||
const [repository, digest] = match[1].split('@')
|
||||
assert.equal(repository, RELAY_REPOSITORY)
|
||||
assert.match(digest, /^sha256:[0-9a-f]{64}$/)
|
||||
return digest
|
||||
}
|
||||
|
||||
function workflowPin(workflow, name) {
|
||||
const match = workflow.match(new RegExp(`${name}: (sha256:[0-9a-f]{64})`))
|
||||
assert.ok(match, `${name} is missing or not a full digest`)
|
||||
return match[1]
|
||||
}
|
||||
|
||||
test('every production cell pins a full relay image digest', () => {
|
||||
for (const [cellId, body] of productionCells()) {
|
||||
assert.match(imageDigest(body), /^sha256:[0-9a-f]{64}$/, `${cellId} image digest`)
|
||||
}
|
||||
})
|
||||
|
||||
test('the 1,000-cap cells are exactly the canonical capacity set', () => {
|
||||
const thousandCap = [...productionCells()]
|
||||
.filter(([, body]) => hardCap(body) === 1000)
|
||||
.map(([cellId]) => cellId)
|
||||
assert.deepEqual([...thousandCap].sort(), [...PRODUCTION_CAPACITY_CELL_IDS].sort())
|
||||
})
|
||||
|
||||
test('the 1,000-cap cells all serve one image digest', () => {
|
||||
const digests = new Map()
|
||||
for (const [cellId, body] of productionCells()) {
|
||||
if (hardCap(body) !== 1000) continue
|
||||
const digest = imageDigest(body)
|
||||
if (!digests.has(digest)) digests.set(digest, [])
|
||||
digests.get(digest).push(cellId)
|
||||
}
|
||||
assert.equal(
|
||||
digests.size,
|
||||
1,
|
||||
`1,000-cap cells split across digests: ${JSON.stringify([...digests])}`
|
||||
)
|
||||
assert.equal([...digests.values()][0].length, PRODUCTION_CAPACITY_CELL_IDS.length)
|
||||
})
|
||||
|
||||
// COMPATIBLE_CELL_IMAGE_DIGEST is one half of a reviewed (director, cell) skew pair, not a
|
||||
// claim about what the fleet serves; it is re-derived by hand for each capacity wave. So it
|
||||
// is deliberately NOT tied to the tfvars digest — only to its twin in the dispatch workflow.
|
||||
test('both capacity workflows declare the same reviewed image pins', () => {
|
||||
for (const name of ['PREDECESSOR_IMAGE_DIGEST', 'COMPATIBLE_CELL_IMAGE_DIGEST']) {
|
||||
assert.equal(workflowPin(dispatchWorkflow, name), workflowPin(jobWorkflow, name), name)
|
||||
}
|
||||
workflowPin(jobWorkflow, 'COMPATIBLE_DIRECTOR_IMAGE_DIGEST')
|
||||
})
|
||||
@@ -1,182 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
LEASED_WORKFLOWS,
|
||||
LOCK_GROUPS,
|
||||
NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
PRODUCTION_LEASE,
|
||||
SELECTABLE_LEASE,
|
||||
STAGING_LEASE,
|
||||
concurrencyBlocks,
|
||||
entrypointsFor,
|
||||
jobIf,
|
||||
jobNeeds,
|
||||
jobs,
|
||||
leaseSteps,
|
||||
leaseStepsByJob,
|
||||
mutatesSharedInstance,
|
||||
readWorkflow,
|
||||
reusableCalls,
|
||||
revisionMintingScripts,
|
||||
workflowFiles
|
||||
} from './cloud-sql-rollout-lock-census.mjs'
|
||||
|
||||
const expectedLease = { production: PRODUCTION_LEASE, staging: STAGING_LEASE, selectable: SELECTABLE_LEASE }
|
||||
const leasedFiles = Object.keys(LEASED_WORKFLOWS)
|
||||
|
||||
function contractFiles(file) {
|
||||
return [file, ...(LEASED_WORKFLOWS[file].leaseFiles ?? [])]
|
||||
}
|
||||
|
||||
test('every locked workflow declares exactly one lock group that never cancels', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const blocks = concurrencyBlocks(readWorkflow(file))
|
||||
assert.equal(blocks.length, 1, `${file} must declare exactly one concurrency block`)
|
||||
assert.equal(blocks[0].group, LEASED_WORKFLOWS[file].group, file)
|
||||
assert.equal(blocks[0].cancelInProgress, 'false', file)
|
||||
}
|
||||
})
|
||||
|
||||
test('every locked workflow takes the lease for its own environment', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const lease = expectedLease[LEASED_WORKFLOWS[file].env]
|
||||
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
|
||||
assert.ok(steps.length > 0, `${file} must use the Cloud SQL rollout lease action`)
|
||||
for (const step of steps) {
|
||||
assert.equal(step.bucket, lease.bucket, file)
|
||||
assert.equal(step.object, lease.object, file)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('the lease step runs after the credential that authorizes it', () => {
|
||||
for (const file of leasedFiles) {
|
||||
for (const member of contractFiles(file)) {
|
||||
const text = readWorkflow(member)
|
||||
const lines = text.split('\n')
|
||||
for (const step of leaseSteps(text)) {
|
||||
const before = lines.slice(0, step.line - 1)
|
||||
const gcloud = before.lastIndexOf(' - uses: google-github-actions/setup-gcloud@v2')
|
||||
assert.notEqual(gcloud, -1, `${member}: lease step at line ${step.line} has no setup-gcloud before it`)
|
||||
assert.ok(
|
||||
before.lastIndexOf(' - uses: actions/checkout@v4') !== -1,
|
||||
`${member}: lease step at line ${step.line} runs before the local action is checked out`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('multi-wave workflows hold one lease per run and free it exactly once', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const entry = LEASED_WORKFLOWS[file]
|
||||
const waveFiles = entry.leaseFiles ?? []
|
||||
const callCount = waveFiles.reduce(
|
||||
(total, member) => total + (reusableCalls(readWorkflow(file)).get(member) ?? 0),
|
||||
0
|
||||
)
|
||||
if (!entry.reentrant) {
|
||||
assert.ok(callCount <= 1, `${file} calls a leased reusable job ${callCount} times; it needs a release job`)
|
||||
for (const member of contractFiles(file)) {
|
||||
for (const step of leaseSteps(readWorkflow(member))) {
|
||||
assert.equal(step.release, undefined, `${member} must leave release at its default`)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
assert.ok(callCount > 1, `${file} no longer calls its reusable job more than once`)
|
||||
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
|
||||
const released = steps.filter((step) => step.release === "'true'")
|
||||
assert.equal(released.length, 1, `${file} must free the run lease exactly once`)
|
||||
for (const step of steps) {
|
||||
if (step === released[0]) continue
|
||||
assert.equal(step.release, "'false'", `${file} wave jobs must hold the lease`)
|
||||
}
|
||||
|
||||
const callerJobs = jobs(readWorkflow(file))
|
||||
const releaseJob = leaseStepsByJob(file).find((job) =>
|
||||
job.steps.some((step) => step.release === "'true'")
|
||||
)
|
||||
assert.ok(releaseJob, `${file} must free the lease from its own job`)
|
||||
const guard = jobIf(callerJobs.find((job) => job.id === releaseJob.id).text)
|
||||
assert.match(guard, /always\(\)/, `${file}: the release job must run on failure and cancellation`)
|
||||
|
||||
const holders = callerJobs
|
||||
.filter(
|
||||
(job) =>
|
||||
job.id !== releaseJob.id &&
|
||||
(waveFiles.some((member) => job.text.includes(`uses: ./.github/workflows/${member}`)) ||
|
||||
leaseStepsByJob(file).find((entry) => entry.id === job.id)?.steps.length > 0)
|
||||
)
|
||||
.map((job) => job.id)
|
||||
const needs = jobNeeds(callerJobs.find((job) => job.id === releaseJob.id).text)
|
||||
for (const holder of holders) {
|
||||
assert.ok(needs.includes(holder), `${file}: the release job must need ${holder}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('workflows with two lease-holding jobs can never run them together', () => {
|
||||
for (const file of leasedFiles) {
|
||||
const entry = LEASED_WORKFLOWS[file]
|
||||
if (entry.reentrant) continue
|
||||
const holding = leaseStepsByJob(file).filter((job) => job.steps.length > 0)
|
||||
if (holding.length <= 1) continue
|
||||
assert.ok(entry.exclusiveBy, `${file} has ${holding.length} lease-holding jobs and no exclusivity guard`)
|
||||
const guards = holding.map((job) => jobIf(jobs(readWorkflow(file)).find((j) => j.id === job.id).text))
|
||||
assert.equal(
|
||||
guards.filter((guard) => guard.includes(entry.exclusiveBy)).length,
|
||||
1,
|
||||
`${file}: exactly one job may run when ${entry.exclusiveBy}`
|
||||
)
|
||||
assert.equal(
|
||||
guards.filter((guard) => guard.includes(entry.exclusiveBy.replace('==', '!='))).length,
|
||||
guards.length - 1,
|
||||
`${file}: every other lease-holding job must be excluded when ${entry.exclusiveBy}`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('census: no workflow rolls out against the shared instance outside the lease', () => {
|
||||
const minters = revisionMintingScripts()
|
||||
assert.ok(minters.size > 0, 'the revision-minting script scan found nothing and is vacuous')
|
||||
const flagged = new Map()
|
||||
for (const file of workflowFiles()) {
|
||||
const reason = mutatesSharedInstance(readWorkflow(file), minters)
|
||||
if (reason) flagged.set(file, reason)
|
||||
}
|
||||
assert.ok(flagged.size > 0, 'the rollout census found no candidates and is vacuous')
|
||||
|
||||
for (const [file, reason] of flagged) {
|
||||
for (const entrypoint of entrypointsFor(file)) {
|
||||
assert.ok(
|
||||
entrypoint in LEASED_WORKFLOWS || entrypoint in NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
`${entrypoint} reaches ${file} (${reason}) but is neither leased nor recorded as a non-candidate`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of workflowFiles()) {
|
||||
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
|
||||
if (!groups.some((group) => LOCK_GROUPS.has(group))) continue
|
||||
assert.ok(
|
||||
file in LEASED_WORKFLOWS || file in NOT_A_CLOUD_SQL_CANDIDATE,
|
||||
`${file} sits in a Cloud SQL lock group but is neither leased nor recorded as a non-candidate`
|
||||
)
|
||||
}
|
||||
|
||||
for (const [file, reason] of Object.entries(NOT_A_CLOUD_SQL_CANDIDATE)) {
|
||||
assert.ok(typeof reason === 'string' && reason.length > 40, `${file} needs a real reason`)
|
||||
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
|
||||
assert.ok(
|
||||
flagged.has(file) || groups.some((group) => LOCK_GROUPS.has(group)),
|
||||
`${file} is recorded as a non-candidate but nothing would have flagged it`
|
||||
)
|
||||
}
|
||||
|
||||
for (const file of leasedFiles) {
|
||||
assert.doesNotThrow(() => readWorkflow(file), `${file} is leased but does not exist`)
|
||||
assert.ok(!(file in NOT_A_CLOUD_SQL_CANDIDATE), `${file} cannot be both leased and a non-candidate`)
|
||||
}
|
||||
})
|
||||
@@ -1,140 +0,0 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints.
|
||||
const path = process.env.MODEL_STATE
|
||||
const state = JSON.parse(readFileSync(path, 'utf8'))
|
||||
const args = process.argv.slice(2)
|
||||
const option = (name) => args[args.indexOf(name) + 1]
|
||||
const has = (name) => args.includes(name)
|
||||
const fail = (message) => {
|
||||
throw new Error(message)
|
||||
}
|
||||
const persist = () => writeFileSync(path, JSON.stringify(state))
|
||||
const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value))
|
||||
const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`)
|
||||
const traffic = () => [
|
||||
{ revisionName: state.serving, percent: 100 },
|
||||
...Object.entries(state.tags).map(([tag, name]) => ({
|
||||
tag,
|
||||
revisionName: name,
|
||||
url: `https://${tag}.test`
|
||||
}))
|
||||
]
|
||||
state.trace.push(args.join(' '))
|
||||
try {
|
||||
if (args[0] === 'curl') {
|
||||
if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) {
|
||||
fail('public check failed')
|
||||
}
|
||||
const url = args.find((arg) => arg.startsWith('https://'))
|
||||
const tag = new URL(url).hostname.split('.')[0]
|
||||
const name = state.tags[tag] ?? state.serving
|
||||
if (has('-w')) {
|
||||
output('200')
|
||||
} else {
|
||||
output({
|
||||
ok: true,
|
||||
deliveryProtocol: 2,
|
||||
mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation')
|
||||
? 'validation'
|
||||
: 'active'
|
||||
})
|
||||
}
|
||||
} else if (args.slice(0, 2).join(' ') === 'run deploy') {
|
||||
const name = `${option('deploy')}-${option('--revision-suffix')}`
|
||||
if (state.failure === 'deploy-before') {
|
||||
fail('deploy failed before create')
|
||||
}
|
||||
const item = structuredClone(revision(state.latest))
|
||||
item.metadata.name = name
|
||||
item.spec.containers[0].image = option('--image')
|
||||
item.status.imageDigest = option('--image')
|
||||
item.spec.containers[0].env = item.spec.containers[0].env.filter(
|
||||
(entry) => entry.name !== 'ORCA_PUSH_MODE'
|
||||
)
|
||||
if (has('--update-env-vars')) {
|
||||
item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' })
|
||||
}
|
||||
state.revisions[name] = item
|
||||
state.latest = name
|
||||
if (has('--tag')) {
|
||||
state.tags[option('--tag')] = name
|
||||
}
|
||||
state.peak = Math.max(state.peak, Object.keys(state.revisions).length)
|
||||
if (state.peak > 3) {
|
||||
fail('three-revision budget exceeded')
|
||||
}
|
||||
if (state.failure === 'deploy-after') {
|
||||
fail('deploy failed after create')
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run services describe') {
|
||||
if (state.failure === 'describe') {
|
||||
fail('describe failed')
|
||||
}
|
||||
if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) {
|
||||
output(state.latest)
|
||||
} else {
|
||||
const template = structuredClone(revision(state.latest))
|
||||
delete template.spec.containers[0].name
|
||||
output({
|
||||
spec: { template },
|
||||
status: { latestCreatedRevisionName: state.latest, traffic: traffic() }
|
||||
})
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run services update-traffic') {
|
||||
if (has('--to-revisions')) {
|
||||
const name = option('--to-revisions').split('=')[0]
|
||||
revision(name)
|
||||
state.serving = name
|
||||
}
|
||||
if (has('--remove-tags')) {
|
||||
for (const tag of option('--remove-tags').split(',')) {
|
||||
delete state.tags[tag]
|
||||
}
|
||||
}
|
||||
if (has('--clear-tags')) {
|
||||
state.tags = {}
|
||||
}
|
||||
if (state.failure === 'traffic-after') {
|
||||
fail('traffic changed but response failed')
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions list') {
|
||||
const names = Object.keys(state.revisions)
|
||||
output(
|
||||
(has('--filter')
|
||||
? names.filter((name) => name === option('--filter').split('=')[1])
|
||||
: names
|
||||
).join('\n')
|
||||
)
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions describe') {
|
||||
const item = revision(args[3])
|
||||
const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format')
|
||||
if (format.includes('minScale')) {
|
||||
output(item.metadata.annotations['autoscaling.knative.dev/minScale'])
|
||||
} else if (format.includes('maxScale')) {
|
||||
output(item.metadata.annotations['autoscaling.knative.dev/maxScale'])
|
||||
} else {
|
||||
output(item)
|
||||
}
|
||||
} else if (args.slice(0, 3).join(' ') === 'run revisions delete') {
|
||||
const name = args[3]
|
||||
if (name === state.latest) {
|
||||
fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted')
|
||||
}
|
||||
if (name === state.serving || Object.values(state.tags).includes(name)) {
|
||||
fail('revision has traffic or tags')
|
||||
}
|
||||
if (state.failure === 'delete') {
|
||||
fail('delete failed')
|
||||
}
|
||||
revision(name)
|
||||
delete state.revisions[name]
|
||||
} else {
|
||||
fail(`unmodeled gcloud call ${args.join(' ')}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error.message)
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
persist()
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import test from 'node:test'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readRelayWorkflow('push-deploy.yml')
|
||||
function step(name) {
|
||||
const start = workflow.indexOf(` - name: ${name}\n`)
|
||||
assert.notEqual(start, -1)
|
||||
const end = workflow.indexOf('\n - ', start + 1)
|
||||
const block = workflow.slice(start, end === -1 ? undefined : end)
|
||||
return block
|
||||
.slice(block.indexOf(' run: |\n') + ' run: |\n'.length)
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith(' '))
|
||||
.map((line) => line.slice(10))
|
||||
.join('\n')
|
||||
}
|
||||
const names = {
|
||||
preflight: 'Record the serving revision and require its Terraform-owned scaling',
|
||||
candidate: 'Deploy the candidate revision with no traffic',
|
||||
activate: 'Retire inert validation and activate the verified image',
|
||||
shift: 'Shift all traffic to the verified candidate',
|
||||
public: 'Verify the public origin after the shift',
|
||||
rollback: 'Roll traffic back to the previous revision',
|
||||
restore: 'Restore the known-good service template',
|
||||
promoteRecovery: 'Promote and verify the known-good recovery revision',
|
||||
cleanup: 'Delete the rejected candidate revision',
|
||||
retire: 'Retire previous consumers after public checks'
|
||||
}
|
||||
const image = `registry/push@sha256:${'a'.repeat(64)}`
|
||||
const spec = {
|
||||
serviceAccountName: 'runtime@test',
|
||||
containerConcurrency: 40,
|
||||
containers: [
|
||||
{
|
||||
image,
|
||||
name: 'push-test-1',
|
||||
env: [
|
||||
{
|
||||
name: 'ORCA_PUSH_DATABASE_URL',
|
||||
valueFrom: { secretKeyRef: { name: 'database', key: '7' } }
|
||||
},
|
||||
{ name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
const prior = {
|
||||
metadata: {
|
||||
name: 'push-test-old',
|
||||
annotations: {
|
||||
'autoscaling.knative.dev/minScale': '1',
|
||||
'autoscaling.knative.dev/maxScale': '2'
|
||||
}
|
||||
},
|
||||
spec,
|
||||
status: { imageDigest: image }
|
||||
}
|
||||
const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url))
|
||||
const options = { skip: process.platform === 'win32' }
|
||||
function exercise(callback) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'push-workflow-'))
|
||||
const statePath = join(dir, 'state.json')
|
||||
writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({
|
||||
revisions: { 'push-test-old': prior },
|
||||
latest: 'push-test-old',
|
||||
serving: 'push-test-old',
|
||||
tags: {},
|
||||
peak: 1,
|
||||
trace: []
|
||||
})
|
||||
)
|
||||
writeFileSync(join(dir, 'env'), '')
|
||||
const env = {
|
||||
...process.env,
|
||||
SERVICE_NAME: 'push-test',
|
||||
GCP_PROJECT_ID: 'test',
|
||||
GCP_REGION: 'test',
|
||||
GITHUB_RUN_ID: '123',
|
||||
GITHUB_RUN_ATTEMPT: '1',
|
||||
IMAGE: `registry/push@sha256:${'b'.repeat(64)}`,
|
||||
PUSH_MIN_INSTANCES: '1',
|
||||
PUSH_MAX_INSTANCES: '2',
|
||||
PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test',
|
||||
PUSH_ORIGIN: 'https://public.test',
|
||||
MODEL_STATE: statePath,
|
||||
MODEL_SCRIPT: model,
|
||||
RUNNER_TEMP: dir,
|
||||
GITHUB_ENV: join(dir, 'env'),
|
||||
GITHUB_STEP_SUMMARY: join(dir, 'summary')
|
||||
}
|
||||
const state = () => JSON.parse(readFileSync(statePath, 'utf8'))
|
||||
const change = (edit) => {
|
||||
const value = state()
|
||||
edit(value)
|
||||
writeFileSync(statePath, JSON.stringify(value))
|
||||
}
|
||||
const run = (key, ok = true, extra = '') => {
|
||||
const result = spawnSync(
|
||||
'bash',
|
||||
[
|
||||
'-c',
|
||||
`
|
||||
set -a
|
||||
source "$GITHUB_ENV"
|
||||
gcloud() { node "$MODEL_SCRIPT" "$@"; }
|
||||
curl() { node "$MODEL_SCRIPT" curl "$@"; }
|
||||
sleep() { :; }
|
||||
${extra}
|
||||
${names[key] ? step(names[key]) : key}
|
||||
`
|
||||
],
|
||||
{ cwd: dir, env, encoding: 'utf8', timeout: 30000 }
|
||||
)
|
||||
assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`)
|
||||
return result
|
||||
}
|
||||
try {
|
||||
callback({ run, state, change, dir })
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
function recover(h) {
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('restore')
|
||||
h.run('promoteRecovery')
|
||||
h.run('cleanup')
|
||||
h.run('retire')
|
||||
const state = h.state()
|
||||
assert.equal(state.serving, 'push-test-r123-1')
|
||||
assert.equal(state.latest, state.serving)
|
||||
assert.deepEqual(Object.keys(state.revisions), [state.serving])
|
||||
assert.equal(state.revisions[state.serving].spec.containers[0].image, image)
|
||||
assert.ok(state.peak <= 3)
|
||||
}
|
||||
|
||||
test(
|
||||
'the executable Cloud Run model rejects deleting latest even without tags or traffic',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags')
|
||||
const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false)
|
||||
assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'success creates successor before retirement and repeated rollouts retain one consumer',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
for (const attempt of ['1', '2']) {
|
||||
if (attempt === '2') {
|
||||
writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n')
|
||||
}
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) {
|
||||
h.run(key)
|
||||
}
|
||||
const state = h.state()
|
||||
assert.equal(state.serving, `push-test-a123-${attempt}`)
|
||||
assert.deepEqual(Object.keys(state.revisions), [state.serving])
|
||||
assert.equal(state.peak, 3)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
for (const failure of ['deploy-before', 'deploy-after', 'describe']) {
|
||||
test(`validation ${failure} recovers without deleting latest`, options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.change((state) => {
|
||||
state.failure = failure
|
||||
})
|
||||
h.run('candidate', false)
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
}
|
||||
for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) {
|
||||
test(
|
||||
`activation ${failure} frees validation slot before recovery and stays within three`,
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
state.failure = failure
|
||||
})
|
||||
h.run('activate', false)
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
test(
|
||||
'ambiguous traffic shift records intent before mutation, rolls back and recovers',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('activate')
|
||||
h.change((state) => {
|
||||
state.failure = 'traffic-after'
|
||||
})
|
||||
h.run('shift', false)
|
||||
assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/)
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('rollback')
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
|
||||
test('failed public check rolls back and recovers', options, () =>
|
||||
exercise((h) => {
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift']) {
|
||||
h.run(key)
|
||||
}
|
||||
h.change((state) => {
|
||||
state.failure = 'public'
|
||||
})
|
||||
h.run('public', false)
|
||||
h.change((state) => {
|
||||
delete state.failure
|
||||
})
|
||||
h.run('rollback')
|
||||
recover(h)
|
||||
})
|
||||
)
|
||||
|
||||
for (const defect of [
|
||||
'runtime',
|
||||
'secret',
|
||||
'mode',
|
||||
'image',
|
||||
'traffic',
|
||||
'scaling',
|
||||
'deploy-before',
|
||||
'deploy-after'
|
||||
]) {
|
||||
test(`recovery rejects ${defect} and preserves partial-create state`, options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
const revision = state.revisions[state.latest]
|
||||
if (defect === 'runtime') {
|
||||
revision.spec.serviceAccountName = 'wrong@test'
|
||||
}
|
||||
if (defect === 'secret') {
|
||||
revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8'
|
||||
}
|
||||
if (defect === 'scaling') {
|
||||
revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3'
|
||||
}
|
||||
if (defect === 'traffic') {
|
||||
state.serving = state.latest
|
||||
}
|
||||
if (defect.startsWith('deploy-')) {
|
||||
state.failure = defect
|
||||
}
|
||||
})
|
||||
// Corrupt the recovery response after the modeled deploy while keeping real jq assertions.
|
||||
const extra = ['mode', 'image'].includes(defect)
|
||||
? `
|
||||
gcloud() {
|
||||
node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $?
|
||||
if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then
|
||||
jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out"
|
||||
else cat "$RUNNER_TEMP/out"; fi
|
||||
}`
|
||||
: ''
|
||||
h.run('restore', false, extra)
|
||||
const recorded = readFileSync(join(h.dir, 'env'), 'utf8')
|
||||
assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/)
|
||||
assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/)
|
||||
assert.ok(h.state().peak <= 3)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
test('failed validation retirement blocks a fourth revision during recovery', options, () =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.change((state) => {
|
||||
state.failure = 'delete'
|
||||
})
|
||||
h.run('activate', false)
|
||||
h.run('restore', false)
|
||||
assert.equal(h.state().peak, 3)
|
||||
assert.equal(h.state().revisions['push-test-r123-1'], undefined)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'failed retirement after public checks leaves verified serving and blocks the next run',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) {
|
||||
h.run(key)
|
||||
}
|
||||
h.change((state) => {
|
||||
state.failure = 'delete'
|
||||
})
|
||||
h.run('retire', false)
|
||||
assert.equal(h.state().serving, 'push-test-a123-1')
|
||||
h.run('preflight', false)
|
||||
assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/)
|
||||
})
|
||||
)
|
||||
|
||||
test(
|
||||
'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout',
|
||||
options,
|
||||
() =>
|
||||
exercise((h) => {
|
||||
h.run('preflight')
|
||||
h.run('candidate')
|
||||
h.run('restore')
|
||||
h.change((state) => {
|
||||
state.failure = 'public'
|
||||
})
|
||||
h.run('promoteRecovery', false)
|
||||
assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/)
|
||||
h.run('preflight', false)
|
||||
})
|
||||
)
|
||||
|
||||
const capability = step('Require image support for inert validation')
|
||||
for (const [label, source, ok] of [
|
||||
['old image', 'export function loadPushConfig() { return {}; }', false],
|
||||
[
|
||||
'invalid mode accepted',
|
||||
'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }',
|
||||
false
|
||||
],
|
||||
[
|
||||
'validation supported',
|
||||
`export function loadPushConfig(env) {
|
||||
if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode');
|
||||
return { mode: 'validation' };
|
||||
}`,
|
||||
true
|
||||
]
|
||||
]) {
|
||||
test(`pre-production image smoke: ${label}`, options, () =>
|
||||
exercise((h) => {
|
||||
const dist = join(h.dir, 'apps', 'push', 'dist')
|
||||
mkdirSync(dist, { recursive: true })
|
||||
writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}')
|
||||
writeFileSync(join(dist, 'config.js'), source)
|
||||
h.run(capability, ok, 'docker() { node "${@: -3}"; }')
|
||||
})
|
||||
)
|
||||
}
|
||||
@@ -1,329 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
concurrencyBlocks,
|
||||
jobIf,
|
||||
jobs,
|
||||
LEASE_ACTION,
|
||||
leaseSteps
|
||||
} from './cloud-sql-rollout-lock-census.mjs'
|
||||
import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'
|
||||
|
||||
// Why: the push gateway holds the APNs key and is the only thing standing between a paired
|
||||
// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the
|
||||
// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone.
|
||||
const WORKFLOW = 'push-deploy.yml'
|
||||
const workflow = readRelayWorkflow(WORKFLOW)
|
||||
const deploy = () => {
|
||||
const job = jobs(workflow).find((entry) => entry.id === 'deploy')
|
||||
assert.ok(job, 'the workflow no longer declares a deploy job')
|
||||
return job
|
||||
}
|
||||
|
||||
function terraform(file) {
|
||||
return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8')
|
||||
}
|
||||
|
||||
// The ordered step names; every assertion below reads positions out of this list rather than
|
||||
// restating them, so a reordering that breaks the no-traffic guarantee fails here.
|
||||
const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1])
|
||||
|
||||
const indexOfStep = (name) => {
|
||||
const index = stepNames().indexOf(name)
|
||||
assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`)
|
||||
return index
|
||||
}
|
||||
|
||||
test('the whole surface stays inert until the owner enables cloud operations', () => {
|
||||
const guard = jobIf(deploy().text)
|
||||
assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard)
|
||||
assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard)
|
||||
assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate')
|
||||
})
|
||||
|
||||
test('it authenticates through Workload Identity and holds no repository secret', () => {
|
||||
assert.match(workflow, /uses: google-github-actions\/auth@v2/)
|
||||
assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/)
|
||||
assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/)
|
||||
assert.match(workflow, /environment: production/)
|
||||
for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
|
||||
assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`)
|
||||
}
|
||||
})
|
||||
|
||||
// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the
|
||||
// matching tfvars-independent list entry would fail authentication at dispatch time only.
|
||||
test('Terraform trusts this exact workflow file on the production deploy provider', () => {
|
||||
assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/)
|
||||
assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/)
|
||||
assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml')
|
||||
})
|
||||
|
||||
test('the rollout is serialized and leases its dedicated push rollout lock', () => {
|
||||
const blocks = concurrencyBlocks(workflow)
|
||||
assert.equal(blocks.length, 1)
|
||||
assert.equal(blocks[0].group, 'production-push-rollout')
|
||||
assert.equal(blocks[0].cancelInProgress, 'false')
|
||||
const steps = leaseSteps(workflow)
|
||||
assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run')
|
||||
assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state')
|
||||
assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock')
|
||||
assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run')
|
||||
})
|
||||
|
||||
// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and
|
||||
// pipefail only applies under an explicit bash shell. Every multi-line body here opts in.
|
||||
test('every multi-line command runs under bash with pipefail', () => {
|
||||
const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)]
|
||||
assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`)
|
||||
for (const match of bodies) {
|
||||
assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`)
|
||||
assert.match(match[2], /^ {10}set -euo pipefail$/m)
|
||||
}
|
||||
})
|
||||
|
||||
test('the candidate revision takes no traffic and is addressed by its own tag', () => {
|
||||
assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/)
|
||||
assert.match(workflow, /^ {12}--no-traffic \\$/m)
|
||||
assert.match(workflow, /--tag "\$\{tag\}"/)
|
||||
assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/)
|
||||
assert.ok(
|
||||
indexOfStep('Record the serving revision and require its Terraform-owned scaling') <
|
||||
indexOfStep('Deploy the candidate revision with no traffic'),
|
||||
'the rollback target must be captured before the candidate exists'
|
||||
)
|
||||
})
|
||||
|
||||
// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a
|
||||
// deploy that passed --max-instances would revert a later push_max_instances raise on every run.
|
||||
// The workflow asserts the shape instead of writing it, on the serving revision before the
|
||||
// candidate exists and on the candidate that inherits it.
|
||||
test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => {
|
||||
assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field')
|
||||
assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field')
|
||||
// The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to
|
||||
// the two instances the Cloud SQL connection budget leaves room for.
|
||||
assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m)
|
||||
assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m)
|
||||
assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/)
|
||||
assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m)
|
||||
const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling')
|
||||
assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic'))
|
||||
assert.match(workflow, /autoscaling\.knative\.dev\/minScale/)
|
||||
assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/)
|
||||
assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
|
||||
assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
|
||||
})
|
||||
|
||||
// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization
|
||||
// point. A build inside it blocks every relay deploy and rehome for its duration.
|
||||
test('the image is built before the rollout lease is taken', () => {
|
||||
const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`)
|
||||
assert.notEqual(lease, -1)
|
||||
const build = workflow.indexOf('- name: Build and publish the immutable gateway image')
|
||||
const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic')
|
||||
assert.ok(build < lease, 'the build must finish before the run takes the lease')
|
||||
assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift')
|
||||
})
|
||||
|
||||
// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout
|
||||
// lease can only account for a pool it declares. Leaving it at the application default hid it.
|
||||
test('the database pool size is Terraform-owned and bounded at plan time', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/)
|
||||
assert.match(source, /value = tostring\(var\.push_database_pool_max\)/)
|
||||
assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/)
|
||||
const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source)
|
||||
assert.ok(block, 'the push service no longer declares a lifecycle block')
|
||||
assert.match(
|
||||
block[1],
|
||||
/var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/,
|
||||
'instances x pool must be bounded at plan time'
|
||||
)
|
||||
assert.match(
|
||||
readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'),
|
||||
/ORCA_PUSH_DATABASE_POOL_MAX/,
|
||||
'the gateway must read the variable Terraform sets'
|
||||
)
|
||||
})
|
||||
|
||||
test('the candidate is probed on its own URL before any traffic moves', () => {
|
||||
const probe = indexOfStep('Probe the candidate readiness endpoint')
|
||||
assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic'))
|
||||
assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/)
|
||||
assert.match(workflow, /test "\$\{code\}" = 200/)
|
||||
assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health'))
|
||||
assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness')
|
||||
})
|
||||
|
||||
// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be
|
||||
// validate-only, must use a token that cannot exist, and must treat a denied credential as the
|
||||
// failure. Accepting PERMISSION_DENIED would make the whole step decorative.
|
||||
test('the FCM probe is validate-only and separates a bad token from a bad credential', () => {
|
||||
const fcm = indexOfStep('Prove the runtime identity can reach FCM')
|
||||
assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint'))
|
||||
assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.match(workflow, /"validate_only":true/)
|
||||
assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/)
|
||||
assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m)
|
||||
assert.match(workflow, /orca-push-deploy-probe-invalid-token/)
|
||||
assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/)
|
||||
assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/)
|
||||
// Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so
|
||||
// it is retried rather than read as either verdict. A denied credential still fails at once.
|
||||
assert.match(workflow, /for attempt in \$\(seq 1 5\); do/)
|
||||
const probe = workflow.slice(
|
||||
workflow.indexOf('- name: Prove the runtime identity can reach FCM'),
|
||||
workflow.indexOf('- name: Shift all traffic to the verified candidate')
|
||||
)
|
||||
assert.match(probe, /for attempt in \$\(seq 1 5\); do/)
|
||||
assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/,
|
||||
'the probe must exercise the runtime credential, not the deploy identity'
|
||||
)
|
||||
// Why: that token reads the Apple signing key. Masking it means a later `set -x` or a
|
||||
// debug re-run cannot print it into a public log.
|
||||
assert.match(
|
||||
probe,
|
||||
/test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/,
|
||||
'the impersonated token must be masked before anything else runs'
|
||||
)
|
||||
assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/)
|
||||
})
|
||||
|
||||
// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the
|
||||
// previous one. Terraform reverting the service to 100% LATEST would undo either silently.
|
||||
test('Terraform does not own the image or the traffic split', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n lifecycle \{([\s\S]*?)\n \}/.exec(source)
|
||||
assert.ok(block, 'the push service no longer declares a lifecycle block')
|
||||
assert.match(block[1], /template\[0\]\.containers\[0\]\.image/)
|
||||
assert.match(block[1], /^\s*traffic$/m)
|
||||
})
|
||||
|
||||
test('impersonating the runtime identity is a Terraform-declared grant', () => {
|
||||
const source = terraform('push-gateway.tf')
|
||||
assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/)
|
||||
assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/)
|
||||
assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/)
|
||||
})
|
||||
|
||||
test('the traffic shift is all-or-nothing and is verified after the fact', () => {
|
||||
const shift = indexOfStep('Shift all traffic to the verified candidate')
|
||||
assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/)
|
||||
assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/)
|
||||
assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/)
|
||||
assert.ok(shift < indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/)
|
||||
assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/)
|
||||
})
|
||||
|
||||
// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise
|
||||
// roll a healthy deploy back. It retries on the same schedule as the candidate probe.
|
||||
test('the post-shift origin check retries like the candidate probe', () => {
|
||||
const check = workflow.slice(
|
||||
workflow.indexOf('- name: Verify the public origin after the shift'),
|
||||
workflow.indexOf('- name: Roll traffic back to the previous revision')
|
||||
)
|
||||
assert.match(check, /for attempt in \$\(seq 1 30\); do/)
|
||||
assert.match(check, /sleep 5/)
|
||||
assert.match(check, /test "\$\{code\}" = 200/)
|
||||
})
|
||||
|
||||
// Why: the summary carries the rollback target. Writing it after the origin check meant the one
|
||||
// run that needed it, the run whose check failed, was the one run that never got it.
|
||||
test('the summary is written before anything that can fail after the shift', () => {
|
||||
const summary = indexOfStep('Publish the rollout summary')
|
||||
assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate'))
|
||||
assert.ok(summary < indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /Known-good image:/)
|
||||
assert.match(workflow, /GITHUB_STEP_SUMMARY/)
|
||||
})
|
||||
|
||||
// Why: everything after the shift runs with production on the candidate, so a failure there is a
|
||||
// live gateway that has to go back. The marker is what separates that case from a failure before
|
||||
// the shift, where production never moved and the candidate is the thing to clean up.
|
||||
test('a failure after the shift rolls production back automatically', () => {
|
||||
const rollback = indexOfStep('Roll traffic back to the previous revision')
|
||||
assert.ok(rollback > indexOfStep('Verify the public origin after the shift'))
|
||||
assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/)
|
||||
const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate')
|
||||
assert.ok(
|
||||
workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift,
|
||||
'the success marker follows the shift step'
|
||||
)
|
||||
const body = workflow.slice(
|
||||
workflow.indexOf('- name: Roll traffic back to the previous revision'),
|
||||
workflow.indexOf('- name: Delete the rejected candidate revision')
|
||||
)
|
||||
assert.match(
|
||||
body,
|
||||
/if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/,
|
||||
'the rollback must be conditioned on both failure and the shift marker'
|
||||
)
|
||||
assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/)
|
||||
assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/)
|
||||
assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/)
|
||||
assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary')
|
||||
})
|
||||
|
||||
// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its
|
||||
// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names.
|
||||
test('verified recovery authorizes rejected candidate deletion', () => {
|
||||
const body = workflow.slice(
|
||||
workflow.indexOf('- name: Delete the rejected candidate revision'),
|
||||
workflow.indexOf('- name: Drop the candidate traffic tag')
|
||||
)
|
||||
assert.match(
|
||||
body,
|
||||
/env\.RECOVERY_VERIFIED == 'true'/,
|
||||
'cleanup must wait for verified recovery traffic and public checks'
|
||||
)
|
||||
assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/)
|
||||
assert.ok(
|
||||
body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'),
|
||||
'the tag must come off before the revision is deleted'
|
||||
)
|
||||
assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/)
|
||||
})
|
||||
|
||||
test('the run always drops its traffic tag', () => {
|
||||
const cleanup = indexOfStep('Drop the candidate traffic tag')
|
||||
assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step')
|
||||
assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/)
|
||||
const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag'))
|
||||
assert.match(body, /if: always\(\)/)
|
||||
assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/)
|
||||
})
|
||||
|
||||
test('push credentials cannot assume the shared Relay deploy identity', () => {
|
||||
const source = terraform('push-deploy-identity.tf')
|
||||
assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/)
|
||||
assert.doesNotMatch(source, /"attribute.repository"\s*=/)
|
||||
assert.match(source, /attribute\.push_deploy\/production/)
|
||||
assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/)
|
||||
assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/)
|
||||
})
|
||||
|
||||
test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => {
|
||||
const service = terraform('push-gateway.tf')
|
||||
const database = terraform('push-dedicated-database.tf')
|
||||
assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/)
|
||||
assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/)
|
||||
assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/)
|
||||
assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/)
|
||||
assert.match(database, /tier\s*= "db-custom-2-7680"/)
|
||||
assert.match(database, /availability_type = "REGIONAL"/)
|
||||
assert.match(database, /deletion_protection\s*= true/)
|
||||
assert.match(database, /deletion_protection_enabled = true/)
|
||||
const identity = terraform('push-deploy-identity.tf')
|
||||
const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1]
|
||||
assert.ok(lease)
|
||||
assert.match(lease, /member = local\.push_deploy_member/)
|
||||
assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/)
|
||||
assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/)
|
||||
})
|
||||
@@ -1,31 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const CAPACITY_IDENTITY_NAME = 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT'
|
||||
|
||||
export function readProductionCapacityIdentity(revision) {
|
||||
const env = revision?.spec?.containers?.[0]?.env
|
||||
if (!Array.isArray(env)) throw new Error('director revision environment is missing')
|
||||
const matches = env.filter((entry) => entry?.name === CAPACITY_IDENTITY_NAME)
|
||||
if (matches.length === 0) return null
|
||||
if (matches.length !== 1) throw new Error('duplicate capacity identity')
|
||||
const value = matches[0]?.value
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new Error('capacity identity is not a literal string')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function main() {
|
||||
const revision = JSON.parse(readFileSync(0, 'utf8'))
|
||||
process.stdout.write(`${JSON.stringify(readProductionCapacityIdentity(revision))}\n`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
main()
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { readProductionCapacityIdentity } from './read-relay-production-capacity-identity.mjs'
|
||||
|
||||
function revision(env) {
|
||||
return { spec: { containers: [{ env }] } }
|
||||
}
|
||||
|
||||
test('reads absent, exact, and foreign literal capacity identities', () => {
|
||||
assert.equal(readProductionCapacityIdentity(revision([])), null)
|
||||
assert.equal(
|
||||
readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'capacity@example.test' }
|
||||
])),
|
||||
'capacity@example.test'
|
||||
)
|
||||
assert.equal(
|
||||
readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'foreign@example.test' }
|
||||
])),
|
||||
'foreign@example.test'
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects malformed or duplicate capacity identity entries', () => {
|
||||
for (const entry of [
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: null },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: '' },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', valueSource: { secretKeyRef: {} } }
|
||||
]) {
|
||||
assert.throws(
|
||||
() => readProductionCapacityIdentity(revision([entry])),
|
||||
/not a literal string/
|
||||
)
|
||||
}
|
||||
assert.throws(
|
||||
() => readProductionCapacityIdentity(revision([
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'one@example.test' },
|
||||
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'two@example.test' }
|
||||
])),
|
||||
/duplicate capacity identity/
|
||||
)
|
||||
assert.throws(() => readProductionCapacityIdentity({}), /environment is missing/)
|
||||
})
|
||||
@@ -1,140 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const SECRET = 'orca-cloud-relay-regional-placement-enabled'
|
||||
|
||||
function validate(input) {
|
||||
for (const key of ['project', 'region', 'service', 'bootstrap_version']) {
|
||||
if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) {
|
||||
throw new Error(`invalid ${key}`)
|
||||
}
|
||||
}
|
||||
if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service')
|
||||
if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) {
|
||||
throw new Error('invalid bootstrap_version')
|
||||
}
|
||||
}
|
||||
|
||||
export function classifyRelayServiceDescribeFailure(args, stderr) {
|
||||
const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe'
|
||||
if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) {
|
||||
return 'NOT_FOUND'
|
||||
}
|
||||
return 'GCLOUD_FAILED'
|
||||
}
|
||||
|
||||
function defaultRun(args) {
|
||||
const result = spawnSync('gcloud', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
maxBuffer: 10 * 1024 * 1024
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
const error = new Error('gcloud read failed')
|
||||
error.code = classifyRelayServiceDescribeFailure(args, result.stderr)
|
||||
throw error
|
||||
}
|
||||
return JSON.parse(result.stdout)
|
||||
}
|
||||
|
||||
function gcloudArguments(kind, input, revision) {
|
||||
return [
|
||||
'run', kind, 'describe', revision ?? input.service,
|
||||
'--project', input.project,
|
||||
'--region', input.region,
|
||||
'--format=json'
|
||||
]
|
||||
}
|
||||
|
||||
export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) {
|
||||
validate(input)
|
||||
const run = dependencies.run ?? defaultRun
|
||||
let service
|
||||
try {
|
||||
service = run(gcloudArguments('services', input))
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_FOUND') {
|
||||
return { version: input.bootstrap_version, cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const serving = (service.status?.traffic ?? []).filter(
|
||||
(entry) => Number(entry.percent ?? 0) > 0
|
||||
)
|
||||
if (
|
||||
serving.length !== 1 ||
|
||||
Number(serving[0].percent) !== 100 ||
|
||||
typeof serving[0].revisionName !== 'string'
|
||||
) {
|
||||
throw new Error('Relay director must have exactly one revision serving 100% traffic')
|
||||
}
|
||||
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
|
||||
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter((environment) =>
|
||||
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
|
||||
)
|
||||
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
|
||||
(typeof cohortSettings[0].value !== 'string' ||
|
||||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
|
||||
throw new Error('serving region correction cohort is invalid')
|
||||
}
|
||||
const cohort_percent = cohortSettings[0]?.value ?? '0'
|
||||
// Set by the director deploy (placement) or by hand (shadow cells), never by Terraform: carried
|
||||
// so an apply keeps them, and empty when the serving revision has none.
|
||||
const reserve_placement = servingPlainSetting(revision, 'ORCA_RELAY_RESERVE_PLACEMENT', /^(?:off|dry-run|on)$/)
|
||||
const shadow_seat_feed_cells = servingPlainSetting(
|
||||
revision,
|
||||
'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS',
|
||||
/^(?:all|[A-Za-z0-9_-]+(?:, ?[A-Za-z0-9_-]+)*)$/
|
||||
)
|
||||
const carried = { cohort_percent, reserve_placement, shadow_seat_feed_cells }
|
||||
const references = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter(
|
||||
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
)
|
||||
)
|
||||
if (references.length === 0) return { version: input.bootstrap_version, ...carried }
|
||||
const reference = normalizeSecretReference(references[0])
|
||||
if (
|
||||
references.length !== 1 ||
|
||||
reference?.secret !== SECRET ||
|
||||
!/^[1-9][0-9]*$/.test(reference?.version ?? '')
|
||||
) {
|
||||
throw new Error('serving regional placement secret reference is invalid')
|
||||
}
|
||||
return { version: reference.version, ...carried }
|
||||
}
|
||||
|
||||
// One plain value, or none; anything else (a secret, a duplicate, a malformed value) fails the
|
||||
// plan rather than letting an apply drop or reset the setting.
|
||||
function servingPlainSetting(revision, name, valid) {
|
||||
const settings = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter((environment) => environment.name === name)
|
||||
)
|
||||
if (settings.length === 0) return ''
|
||||
if (settings.length > 1 || typeof settings[0].value !== 'string' || !valid.test(settings[0].value)) {
|
||||
throw new Error(`serving ${name} is invalid`)
|
||||
}
|
||||
return settings[0].value
|
||||
}
|
||||
|
||||
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
|
||||
// `gcloud run revisions describe --format=json` emits the Knative v1 shape
|
||||
// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path.
|
||||
// A `key` of "latest" is deliberately left invalid: the director's serving
|
||||
// version must be a pinned integer for this data source to mean anything.
|
||||
export function normalizeSecretReference(environment) {
|
||||
const v2 = environment?.valueSource?.secretKeyRef
|
||||
if (v2) return { secret: v2.secret, version: v2.version }
|
||||
const v1 = environment?.valueFrom?.secretKeyRef
|
||||
if (!v1) return undefined
|
||||
const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined
|
||||
return { secret, version: v1.key }
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const chunks = []
|
||||
for await (const chunk of process.stdin) chunks.push(chunk)
|
||||
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`)
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
classifyRelayServiceDescribeFailure,
|
||||
readRelayServingRegionalPlacementVersion
|
||||
} from './read-relay-serving-regional-placement-version.mjs'
|
||||
|
||||
const input = {
|
||||
project: 'onorca-cloud',
|
||||
region: 'us-central1',
|
||||
service: 'orca-cloud-relay',
|
||||
bootstrap_version: '7'
|
||||
}
|
||||
|
||||
function revision(version = '11') {
|
||||
return {
|
||||
spec: {
|
||||
containers: [{
|
||||
env: [{
|
||||
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
||||
valueSource: {
|
||||
secretKeyRef: {
|
||||
secret: 'orca-cloud-relay-regional-placement-enabled',
|
||||
version
|
||||
}
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the
|
||||
// secret lives in `name` and the version in `key`, and `name` may be the full resource path.
|
||||
function v1Revision(name, key) {
|
||||
return {
|
||||
spec: {
|
||||
containers: [{
|
||||
env: [{
|
||||
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
|
||||
valueFrom: { secretKeyRef: { name, key } }
|
||||
}]
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serving() {
|
||||
return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
}
|
||||
|
||||
test('reads the exact version from the sole traffic-serving revision', () => {
|
||||
const calls = []
|
||||
const result = readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => {
|
||||
calls.push(args)
|
||||
return calls.length === 1
|
||||
? {
|
||||
status: {
|
||||
traffic: [
|
||||
{ revisionName: 'relay-failed-latest', tag: 'candidate' },
|
||||
{ revisionName: 'relay-serving', percent: 100 }
|
||||
]
|
||||
}
|
||||
}
|
||||
: revision()
|
||||
}
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { version: '11', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
assert.equal(calls[1][3], 'relay-serving')
|
||||
})
|
||||
|
||||
test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => {
|
||||
for (const name of [
|
||||
'orca-cloud-relay-regional-placement-enabled',
|
||||
'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled'
|
||||
]) {
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
|
||||
}), { version: '1', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects a v1 reference that names another secret or a floating version', () => {
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? serving()
|
||||
: v1Revision('projects/120364513935/secrets/some-other-secret', '1')
|
||||
}), /secret reference is invalid/)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? serving()
|
||||
: v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest')
|
||||
}), /secret reference is invalid/)
|
||||
})
|
||||
|
||||
test('falls back only when the service or setting is absent', () => {
|
||||
const notFound = new Error('not found')
|
||||
notFound.code = 'NOT_FOUND'
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw notFound }
|
||||
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: { spec: { containers: [{ env: [] }] } }
|
||||
}), { version: '7', cohort_percent: '0', reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
})
|
||||
|
||||
test('classifies real absent-service stderr without weakening revision failures', () => {
|
||||
const serviceArgs = ['run', 'services', 'describe', 'missing-service']
|
||||
const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]'
|
||||
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND')
|
||||
assert.equal(
|
||||
classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr),
|
||||
'GCLOUD_FAILED'
|
||||
)
|
||||
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED')
|
||||
})
|
||||
|
||||
test('rejects ambiguous traffic, malformed references, and read failures', () => {
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => ({
|
||||
status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] }
|
||||
})
|
||||
}), /exactly one revision/)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: revision('latest')
|
||||
}), /secret reference is invalid/)
|
||||
const denied = new Error('denied')
|
||||
denied.code = 'GCLOUD_FAILED'
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw denied }
|
||||
}), denied)
|
||||
})
|
||||
|
||||
|
||||
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
|
||||
for (const value of ['0', '1', '17', '100']) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), { version: '11', cohort_percent: value, reserve_placement: '', shadow_seat_feed_cells: '' })
|
||||
}
|
||||
})
|
||||
|
||||
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
|
||||
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
||||
const cases = [
|
||||
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
|
||||
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
|
||||
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
|
||||
[{ name, value: '1' }, { name, value: '2' }]
|
||||
]
|
||||
for (const settings of cases) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(...settings)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), /cohort is invalid/)
|
||||
}
|
||||
})
|
||||
|
||||
test('carries the deploy-owned placement and shadow cells, so an apply keeps them', () => {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(
|
||||
{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' },
|
||||
{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'production-gce-c1,production-gce-c2' }
|
||||
)
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), {
|
||||
version: '11',
|
||||
cohort_percent: '0',
|
||||
reserve_placement: 'on',
|
||||
shadow_seat_feed_cells: 'production-gce-c1,production-gce-c2'
|
||||
})
|
||||
})
|
||||
|
||||
test('fails closed on a malformed, secret-backed or duplicate placement or shadow-cell setting', () => {
|
||||
const cases = [
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'yes' }],
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'on' }, { name: 'ORCA_RELAY_RESERVE_PLACEMENT', value: 'off' }],
|
||||
[{ name: 'ORCA_RELAY_RESERVE_PLACEMENT', valueFrom: { secretKeyRef: { name: 'x', key: '1' } } }],
|
||||
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: 'c1;c2' }],
|
||||
[{ name: 'ORCA_RELAY_SHADOW_SEAT_FEED_CELLS', value: '' }]
|
||||
]
|
||||
for (const settings of cases) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(...settings)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), /is invalid/)
|
||||
}
|
||||
})
|
||||
|
||||
test('Terraform sets the carried settings only when the serving revision has them', () => {
|
||||
const terraform = readFileSync(fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)), 'utf8')
|
||||
const block = /dynamic "env" \{([\s\S]*?)\n {6}\}/.exec(terraform)?.[1] ?? ''
|
||||
assert.match(block, /ORCA_RELAY_RESERVE_PLACEMENT *= data\.external\.relay_serving_regional_placement_version\.result\.reserve_placement/)
|
||||
assert.match(block, /ORCA_RELAY_SHADOW_SEAT_FEED_CELLS *= data\.external\.relay_serving_regional_placement_version\.result\.shadow_seat_feed_cells/)
|
||||
assert.match(block, /if value != ""/)
|
||||
})
|
||||
@@ -1,137 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { createServer } from 'node:http'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const WORKFLOWS = [
|
||||
'deploy-relay-production-same-cap-job.yml',
|
||||
'operate-relay-production-rehome-job.yml'
|
||||
]
|
||||
|
||||
function workflow(name) {
|
||||
return readFileSync(fileURLToPath(relayWorkflowUrl(name)), 'utf8')
|
||||
}
|
||||
|
||||
const VERIFY_STEP = 'name: Verify new incarnation, exact image, protocol, and durable safety'
|
||||
const STALL_WINDOW = 'local out="${RUNNER_TEMP}/$1.json" window=60 delay=5 status'
|
||||
|
||||
function verifyStep() {
|
||||
return workflow('deploy-relay-production-same-cap-job.yml')
|
||||
.split(VERIFY_STEP)[1]
|
||||
.split('\n - name:')[0]
|
||||
}
|
||||
|
||||
function verifyAdminPost() {
|
||||
const step = verifyStep()
|
||||
const start = step.indexOf('admin_post() {')
|
||||
return step.slice(start, step.indexOf('\n }', start) + '\n }'.length)
|
||||
}
|
||||
|
||||
// curl 7.81 (the Ubuntu 22.04 runners) never retries under --fail-with-body, so the
|
||||
// remaining --retry flags are inert and only the verify loop below really retries.
|
||||
// What holds for every admin curl: it is bounded and never retries a final 4xx.
|
||||
test('every admin curl is bounded and never retries a final 4xx', () => {
|
||||
for (const name of WORKFLOWS) {
|
||||
for (const invocation of workflow(name).split(/\bcurl\b/).slice(1)) {
|
||||
const flags = invocation.split('\n }')[0]
|
||||
assert.match(flags, /--max-time 30/, name)
|
||||
// --retry-all-errors would also retry 401, 403, and 409, which are final.
|
||||
assert.doesNotMatch(flags, /--retry-all-errors/, name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('every retried admin request captures only the final attempt body', () => {
|
||||
const job = workflow('deploy-relay-production-same-cap-job.yml')
|
||||
// --fail-with-body writes every failed attempt to stdout, so a retried
|
||||
// request must land in a file curl truncates per attempt.
|
||||
assert.match(job, /--output "\$\{out\}"/)
|
||||
assert.equal(job.split('admin_post() {').length - 1, 2)
|
||||
for (const call of [
|
||||
/CURRENT_RUNTIME="\$\(admin_post current-runtime/,
|
||||
/CURRENT_DIRECTOR_STATUS="\$\(admin_post current-cell-status/,
|
||||
/TARGET_RUNTIME="\$\(admin_post target-runtime/,
|
||||
/TARGET_DIRECTOR_STATUS="\$\(admin_post target-cell-status/
|
||||
]) assert.match(job, call)
|
||||
// The verify loop captures only the status code; its body goes to the file.
|
||||
assert.doesNotMatch(job.replace(verifyAdminPost(), ''), /\$\(curl /)
|
||||
})
|
||||
|
||||
// c18, 2026-10-07 18:49Z: a DB stall made the fresh cell answer 503 "no healthy upstream";
|
||||
// three 2 s retries gave up and the cell was healthy 16 s later.
|
||||
// 60 s covers a 7 s stall, 16 s readiness grace and two 10 s LB health checks.
|
||||
test('the post-roll verify outlasts one DB stall and its health-check recovery', () => {
|
||||
assert.ok(verifyAdminPost().includes(STALL_WINDOW))
|
||||
assert.match(verifyAdminPost(), /--max-time 30/)
|
||||
})
|
||||
|
||||
// Runs the step's own admin_post, with its retry clock scaled down twelve-fold.
|
||||
async function runVerifyAdminPost(respond) {
|
||||
const scaled = verifyAdminPost().replace('window=60 delay=5', 'window=5 delay=1')
|
||||
assert.notEqual(scaled, verifyAdminPost())
|
||||
let calls = 0
|
||||
const startedAt = Date.now()
|
||||
const server = createServer((request, response) => {
|
||||
calls += 1
|
||||
const { status, body, drop } = respond(Date.now() - startedAt)
|
||||
if (drop) return void request.socket.destroy()
|
||||
response.writeHead(status, { 'content-type': 'application/json' })
|
||||
response.end(body)
|
||||
})
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const runnerTemp = mkdtempSync(join(tmpdir(), 'relay-verify-admin-post-'))
|
||||
try {
|
||||
const url = `http://127.0.0.1:${server.address().port}/v1/admin/runtime-status`
|
||||
const child = spawn(
|
||||
'bash',
|
||||
['-euo', 'pipefail', '-c', `${scaled}\nadmin_post target-runtime "${url}" '{"v":1}'`],
|
||||
{ env: { ...process.env, RUNNER_TEMP: runnerTemp, ORCA_RELAY_ADMIN_ID_TOKEN: 't' } }
|
||||
)
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
child.stdout.on('data', (chunk) => { stdout += chunk })
|
||||
child.stderr.on('data', (chunk) => { stderr += chunk })
|
||||
const code = await new Promise((resolve) => child.on('close', resolve))
|
||||
return { code, stdout, stderr, calls, elapsedMs: Date.now() - startedAt }
|
||||
} finally {
|
||||
server.close()
|
||||
rmSync(runnerTemp, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
const UNHEALTHY = { status: 503, body: 'no healthy upstream' }
|
||||
|
||||
test('the verify read recovers when the cell comes back inside the retry window', async () => {
|
||||
const result = await runVerifyAdminPost((elapsedMs) =>
|
||||
elapsedMs < 2_500 ? UNHEALTHY : { status: 200, body: '{"ok":true}' }
|
||||
)
|
||||
assert.equal(result.code, 0, result.stderr)
|
||||
assert.equal(result.stdout, '{"ok":true}')
|
||||
// The old three-retry budget would have given up here.
|
||||
assert.ok(result.calls > 3, `calls ${result.calls}`)
|
||||
})
|
||||
|
||||
test('the verify read still fails a cell that stays down past the window', async () => {
|
||||
const result = await runVerifyAdminPost(() => UNHEALTHY)
|
||||
assert.notEqual(result.code, 0)
|
||||
assert.ok(result.elapsedMs >= 3_000, `gave up after ${result.elapsedMs} ms: ${result.stderr}`)
|
||||
assert.ok(result.elapsedMs < 8_000, `bounded at ${result.elapsedMs} ms`)
|
||||
})
|
||||
|
||||
test('the verify read fails a final 4xx without waiting out the window', async () => {
|
||||
const result = await runVerifyAdminPost(() => ({ status: 409, body: '{"error":"generation"}' }))
|
||||
assert.notEqual(result.code, 0)
|
||||
assert.equal(result.calls, 1)
|
||||
})
|
||||
|
||||
test('a final connection failure does not report the previous attempt body', async () => {
|
||||
const result = await runVerifyAdminPost((elapsedMs) => (elapsedMs < 1_000 ? UNHEALTHY : { drop: true }))
|
||||
assert.notEqual(result.code, 0)
|
||||
assert.match(result.stderr, /admin_post target-runtime: HTTP 000/)
|
||||
assert.doesNotMatch(result.stderr, /no healthy upstream/)
|
||||
})
|
||||
@@ -1,29 +0,0 @@
|
||||
// A single transient 5xx (load-balancer warm-up behind a fresh instance) must not fail a
|
||||
// deploy step. 4xx is never retried: auth and generation-mismatch answers are final.
|
||||
const TRANSIENT_STATUSES = [500, 502, 503, 504]
|
||||
const RETRY_DELAY_MS = 2_000
|
||||
const REQUEST_TIMEOUT_MS = 30_000
|
||||
|
||||
export function isTransientAdminStatus(status) {
|
||||
return TRANSIENT_STATUSES.includes(status)
|
||||
}
|
||||
|
||||
// Each attempt gets its own timeout budget, so a reused signal cannot abort the retry.
|
||||
export async function fetchAdminOnceMore(fetchImpl, url, init, overrides = {}) {
|
||||
const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
const timeoutMs = overrides.timeoutMs ?? REQUEST_TIMEOUT_MS
|
||||
const retryDelayMs = overrides.retryDelayMs ?? RETRY_DELAY_MS
|
||||
const attempt = async () =>
|
||||
await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) })
|
||||
let response
|
||||
try {
|
||||
response = await attempt()
|
||||
} catch {
|
||||
await wait(retryDelayMs)
|
||||
return await attempt()
|
||||
}
|
||||
if (!isTransientAdminStatus(response.status)) return response
|
||||
await response.arrayBuffer?.().catch(() => undefined)
|
||||
await wait(retryDelayMs)
|
||||
return await attempt()
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
const url = 'https://relay.onorca.dev/v1/admin/cell-status'
|
||||
const init = { method: 'POST', body: '{"v":1}' }
|
||||
|
||||
function recordingWait(waits) {
|
||||
return async (ms) => { waits.push(ms) }
|
||||
}
|
||||
|
||||
test('a single transient 5xx is retried and the second answer is returned', async () => {
|
||||
const waits = []
|
||||
const statuses = [503, 200]
|
||||
let calls = 0
|
||||
const response = await fetchAdminOnceMore(
|
||||
async () => {
|
||||
calls += 1
|
||||
const status = statuses.shift()
|
||||
return new Response(JSON.stringify({ ok: status === 200 }), { status })
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: recordingWait(waits) }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(response.status, 200)
|
||||
assert.deepEqual(waits, [2_000])
|
||||
assert.deepEqual(await response.json(), { ok: true })
|
||||
})
|
||||
|
||||
test('a connection failure is retried and the second answer is returned', async () => {
|
||||
const waits = []
|
||||
let calls = 0
|
||||
const response = await fetchAdminOnceMore(
|
||||
async () => {
|
||||
calls += 1
|
||||
if (calls === 1) throw new TypeError('fetch failed')
|
||||
return Response.json({ ok: true })
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: recordingWait(waits) }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(response.status, 200)
|
||||
assert.deepEqual(waits, [2_000])
|
||||
})
|
||||
|
||||
test('two transient failures surface the second answer without a third attempt', async () => {
|
||||
let calls = 0
|
||||
const response = await fetchAdminOnceMore(
|
||||
async () => {
|
||||
calls += 1
|
||||
return new Response('down', { status: 503 })
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: async () => {} }
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.equal(response.status, 503)
|
||||
})
|
||||
|
||||
test('two connection failures rethrow the second error', async () => {
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
fetchAdminOnceMore(
|
||||
async () => {
|
||||
calls += 1
|
||||
throw new TypeError(`fetch failed ${calls}`)
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: async () => {} }
|
||||
),
|
||||
/fetch failed 2/
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
|
||||
test('4xx is final: auth and generation-mismatch answers are never retried', async () => {
|
||||
for (const status of [400, 401, 403, 404, 409, 429]) {
|
||||
let calls = 0
|
||||
const response = await fetchAdminOnceMore(
|
||||
async () => {
|
||||
calls += 1
|
||||
return new Response('no', { status })
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: async () => { throw new Error('must not wait') } }
|
||||
)
|
||||
assert.equal(calls, 1, `status ${status} must not be retried`)
|
||||
assert.equal(response.status, status)
|
||||
}
|
||||
})
|
||||
|
||||
test('each attempt carries its own unexpired timeout signal', async () => {
|
||||
const signals = []
|
||||
await fetchAdminOnceMore(
|
||||
async (_url, attemptInit) => {
|
||||
signals.push(attemptInit.signal)
|
||||
return new Response('down', { status: 502 })
|
||||
},
|
||||
url,
|
||||
init,
|
||||
{ wait: async () => {}, timeoutMs: 30_000 }
|
||||
)
|
||||
assert.equal(signals.length, 2)
|
||||
assert.notEqual(signals[0], signals[1])
|
||||
assert.equal(signals[1].aborted, false)
|
||||
})
|
||||
|
||||
test('the caller init is forwarded unchanged apart from the signal', async () => {
|
||||
let seen
|
||||
await fetchAdminOnceMore(
|
||||
async (seenUrl, attemptInit) => {
|
||||
seen = { seenUrl, attemptInit }
|
||||
return Response.json({})
|
||||
},
|
||||
url,
|
||||
{ method: 'POST', headers: { authorization: 'Bearer t' }, body: '{"v":1}' },
|
||||
{ wait: async () => {} }
|
||||
)
|
||||
assert.equal(seen.seenUrl, url)
|
||||
assert.equal(seen.attemptInit.method, 'POST')
|
||||
assert.deepEqual(seen.attemptInit.headers, { authorization: 'Bearer t' })
|
||||
assert.equal(seen.attemptInit.body, '{"v":1}')
|
||||
})
|
||||
@@ -1,280 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
const STATES = ['existing-only', 'migration-only', 'general']
|
||||
|
||||
function normalizeMembership(input) {
|
||||
const membership = {
|
||||
existingOnly: [...input.existingOnly].sort(),
|
||||
migrationOnly: [...input.migrationOnly].sort(),
|
||||
general: [...input.general].sort()
|
||||
}
|
||||
const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general]
|
||||
if (new Set(all).size !== all.length) throw new Error('selector membership contains duplicates')
|
||||
return membership
|
||||
}
|
||||
|
||||
function encodedMembership(membership) {
|
||||
return JSON.stringify(normalizeMembership(membership))
|
||||
}
|
||||
|
||||
function membershipSha256(membership) {
|
||||
return createHash('sha256').update(encodedMembership(membership)).digest('hex')
|
||||
}
|
||||
|
||||
function normalizeMigrationCells(input) {
|
||||
const cells = [...input]
|
||||
.map((cell) => ({
|
||||
cellId: cell.cellId,
|
||||
cellUrl: cell.cellUrl,
|
||||
capacityRequests: cell.capacityRequests,
|
||||
...(cell.region ? { region: cell.region } : {}),
|
||||
connectionHardCap: cell.connectionHardCap,
|
||||
connectionUnobservedBound: cell.connectionUnobservedBound
|
||||
}))
|
||||
.sort((left, right) => left.cellId.localeCompare(right.cellId))
|
||||
if (
|
||||
cells.length === 0 ||
|
||||
new Set(cells.map(({ cellId }) => cellId)).size !== cells.length ||
|
||||
new Set(cells.map(({ cellUrl }) => cellUrl)).size !== cells.length
|
||||
) {
|
||||
throw new Error('migration cell registration must contain distinct cells')
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function membershipWithMigrationCells(membership, cells) {
|
||||
const known = new Set([
|
||||
...membership.existingOnly,
|
||||
...membership.migrationOnly,
|
||||
...membership.general
|
||||
])
|
||||
if (cells.some(({ cellId }) => known.has(cellId))) {
|
||||
throw new Error('migration cell registration contains an existing selector cell')
|
||||
}
|
||||
return normalizeMembership({
|
||||
existingOnly: membership.existingOnly,
|
||||
migrationOnly: [...membership.migrationOnly, ...cells.map(({ cellId }) => cellId)],
|
||||
general: membership.general
|
||||
})
|
||||
}
|
||||
|
||||
function assertSelector(value) {
|
||||
if (
|
||||
!value ||
|
||||
!Number.isSafeInteger(value.generation) ||
|
||||
value.generation < 0 ||
|
||||
!value.membership
|
||||
) {
|
||||
throw new Error('director returned an invalid admission selector')
|
||||
}
|
||||
return {
|
||||
generation: value.generation,
|
||||
attemptId: value.attemptId ?? null,
|
||||
membership: normalizeMembership(value.membership)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectorAttemptId(expectedGeneration, membership) {
|
||||
const digest = createHash('sha256')
|
||||
.update(`${expectedGeneration}:${encodedMembership(membership)}`)
|
||||
.digest('hex')
|
||||
.slice(0, 24)
|
||||
return `selector_${expectedGeneration}_${digest}`
|
||||
}
|
||||
|
||||
export function membershipWithStates(selector, states) {
|
||||
const byCell = new Map()
|
||||
for (const [state, key] of [
|
||||
['existing-only', 'existingOnly'],
|
||||
['migration-only', 'migrationOnly'],
|
||||
['general', 'general']
|
||||
]) {
|
||||
for (const cellId of selector.membership[key]) byCell.set(cellId, state)
|
||||
}
|
||||
for (const [cellId, state] of Object.entries(states)) {
|
||||
if (!byCell.has(cellId)) throw new Error(`selector does not contain ${cellId}`)
|
||||
if (!STATES.includes(state)) throw new Error(`invalid admission state for ${cellId}`)
|
||||
if (byCell.get(cellId) === 'existing-only' && state !== 'existing-only') {
|
||||
throw new Error(`selector cannot re-enable existing-only cell ${cellId}`)
|
||||
}
|
||||
byCell.set(cellId, state)
|
||||
}
|
||||
return normalizeMembership({
|
||||
existingOnly: [...byCell].filter(([, state]) => state === 'existing-only').map(([id]) => id),
|
||||
migrationOnly: [...byCell].filter(([, state]) => state === 'migration-only').map(([id]) => id),
|
||||
general: [...byCell].filter(([, state]) => state === 'general').map(([id]) => id)
|
||||
})
|
||||
}
|
||||
|
||||
export async function inspectAdmissionSelector(post, attemptId) {
|
||||
const result = await post('/v1/admin/admission-selector/status', {
|
||||
v: 1,
|
||||
...(attemptId ? { attemptId } : {})
|
||||
})
|
||||
return {
|
||||
selector: assertSelector(result.selector),
|
||||
intent: result.intent
|
||||
? {
|
||||
...result.intent,
|
||||
previousMembership: result.intent.previousMembership
|
||||
? normalizeMembership(result.intent.previousMembership)
|
||||
: undefined,
|
||||
membership: normalizeMembership(result.intent.membership)
|
||||
}
|
||||
: null
|
||||
}
|
||||
}
|
||||
|
||||
function exactSelector(actual, expected) {
|
||||
return (
|
||||
actual.generation === expected.generation &&
|
||||
encodedMembership(actual.membership) === encodedMembership(expected.membership)
|
||||
)
|
||||
}
|
||||
|
||||
export async function applyExactAdmissionSelector(post, membership, options = {}) {
|
||||
const before = await inspectAdmissionSelector(post)
|
||||
const desired = normalizeMembership(membership)
|
||||
if (
|
||||
options.expectedCurrentSelector &&
|
||||
!exactSelector(before.selector, options.expectedCurrentSelector)
|
||||
) {
|
||||
throw new Error('admission selector changed before exact apply')
|
||||
}
|
||||
if (options.requireBoundary !== false && before.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
if (encodedMembership(before.selector.membership) === encodedMembership(desired)) {
|
||||
return { changed: false, selector: before.selector }
|
||||
}
|
||||
const attemptId =
|
||||
options.attemptId ?? selectorAttemptId(before.selector.generation, desired)
|
||||
const expected = {
|
||||
generation: before.selector.generation + 1,
|
||||
membership: desired
|
||||
}
|
||||
let result
|
||||
try {
|
||||
result = await post('/v1/admin/admission-selector/apply', {
|
||||
v: 1,
|
||||
attemptId,
|
||||
expectedGeneration: before.selector.generation,
|
||||
...(before.selector.generation === 0
|
||||
? { expectedMembershipSha256: membershipSha256(before.selector.membership) }
|
||||
: {}),
|
||||
// Only a same-cap roll's isolate passes this; every other caller omits it
|
||||
// and leaves the cell's hosts pinned, which is the pre-existing behaviour.
|
||||
...(options.rollIsolatedCells ? { rollIsolatedCells: options.rollIsolatedCells } : {}),
|
||||
membership: desired
|
||||
})
|
||||
} catch (error) {
|
||||
const inspected = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
inspected.intent?.state === 'committed' &&
|
||||
exactSelector(inspected.selector, expected)
|
||||
) {
|
||||
return { changed: true, selector: inspected.selector, recovered: true }
|
||||
}
|
||||
if (
|
||||
inspected.intent?.state === 'unchanged' &&
|
||||
exactSelector(inspected.selector, before.selector)
|
||||
) {
|
||||
throw new Error('admission selector apply remained unchanged after an ambiguous response', {
|
||||
cause: error
|
||||
})
|
||||
}
|
||||
throw new Error('admission selector apply diverged after an ambiguous response', {
|
||||
cause: error
|
||||
})
|
||||
}
|
||||
const applied = assertSelector(result.selector)
|
||||
if (!exactSelector(applied, expected)) {
|
||||
throw new Error('admission selector apply returned unexpected membership')
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
verified.intent?.state !== 'committed' ||
|
||||
!exactSelector(verified.selector, expected)
|
||||
) {
|
||||
throw new Error('admission selector commit could not be verified')
|
||||
}
|
||||
return { changed: result.changed === true, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function addExactMigrationCells(post, input, options = {}) {
|
||||
const cells = normalizeMigrationCells(input.cells)
|
||||
const attemptId = input.attemptId
|
||||
if (!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
|
||||
throw new Error('migration cell registration requires an exact attempt ID')
|
||||
}
|
||||
const before = await inspectAdmissionSelector(post, attemptId)
|
||||
let expectedGeneration
|
||||
let expectedMembership
|
||||
if (before.intent) {
|
||||
expectedGeneration = before.intent.expectedGeneration
|
||||
expectedMembership = normalizeMembership(before.intent.membership)
|
||||
} else {
|
||||
if (before.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
if (
|
||||
options.expectedCurrentSelector &&
|
||||
!exactSelector(before.selector, options.expectedCurrentSelector)
|
||||
) {
|
||||
throw new Error('admission selector changed before cell registration')
|
||||
}
|
||||
expectedGeneration = before.selector.generation
|
||||
expectedMembership = membershipWithMigrationCells(before.selector.membership, cells)
|
||||
}
|
||||
const expected = {
|
||||
generation: expectedGeneration + 1,
|
||||
membership: expectedMembership
|
||||
}
|
||||
let result
|
||||
try {
|
||||
result = await post('/v1/admin/admission-selector/add-migration-cells', {
|
||||
v: 1,
|
||||
attemptId,
|
||||
expectedGeneration,
|
||||
cells
|
||||
})
|
||||
} catch (error) {
|
||||
const inspected = await inspectAdmissionSelector(post, attemptId)
|
||||
if (
|
||||
!before.intent &&
|
||||
inspected.intent?.state === 'committed' &&
|
||||
exactSelector(inspected.selector, expected)
|
||||
) {
|
||||
return { changed: true, selector: inspected.selector, recovered: true }
|
||||
}
|
||||
throw new Error('migration cell registration did not commit exactly', { cause: error })
|
||||
}
|
||||
const applied = assertSelector(result.selector)
|
||||
if (!exactSelector(applied, expected)) {
|
||||
throw new Error('migration cell registration returned unexpected membership')
|
||||
}
|
||||
const verified = await inspectAdmissionSelector(post, attemptId)
|
||||
if (verified.intent?.state !== 'committed' || !exactSelector(verified.selector, expected)) {
|
||||
throw new Error('migration cell registration commit could not be verified')
|
||||
}
|
||||
return { changed: result.changed === true, selector: verified.selector }
|
||||
}
|
||||
|
||||
export async function transitionAdmissionSelector(post, states, options = {}) {
|
||||
const current = await inspectAdmissionSelector(post)
|
||||
if (current.selector.generation < 1) {
|
||||
throw new Error('admission selector boundary is not active')
|
||||
}
|
||||
return await applyExactAdmissionSelector(
|
||||
post,
|
||||
membershipWithStates(current.selector, states),
|
||||
options
|
||||
)
|
||||
}
|
||||
|
||||
export function selectorCellState(selector, cellId) {
|
||||
if (selector.membership.existingOnly.includes(cellId)) return 'existing-only'
|
||||
if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only'
|
||||
if (selector.membership.general.includes(cellId)) return 'general'
|
||||
throw new Error(`selector does not contain ${cellId}`)
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
addExactMigrationCells,
|
||||
applyExactAdmissionSelector,
|
||||
membershipWithStates,
|
||||
selectorAttemptId,
|
||||
transitionAdmissionSelector
|
||||
} from './relay-admission-selector.mjs'
|
||||
|
||||
const initialMembership = {
|
||||
existingOnly: ['legacy'],
|
||||
migrationOnly: ['target'],
|
||||
general: ['general']
|
||||
}
|
||||
|
||||
function selectorHarness({ ambiguous = null, generation = 1 } = {}) {
|
||||
let selector = { generation, attemptId: 'initial', membership: initialMembership }
|
||||
const intents = new Map()
|
||||
const requests = []
|
||||
let applies = 0
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
|
||||
}
|
||||
}
|
||||
applies++
|
||||
requests.push(body)
|
||||
const before = selector
|
||||
const committed = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: body.membership
|
||||
}
|
||||
intents.set(body.attemptId, {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: committed.generation,
|
||||
previousMembership: before.membership,
|
||||
membership: body.membership,
|
||||
state: ambiguous === 'unchanged' ? 'unchanged' : 'committed'
|
||||
})
|
||||
if (ambiguous !== 'unchanged') selector = committed
|
||||
if (ambiguous) throw new Error('lost selector response')
|
||||
return { changed: true, selector }
|
||||
}
|
||||
return { post, selector: () => selector, applies: () => applies, requests }
|
||||
}
|
||||
|
||||
test('derives deterministic attempts and applies exact selector transitions', async () => {
|
||||
const harness = selectorHarness()
|
||||
const desired = membershipWithStates(harness.selector(), { target: 'general' })
|
||||
assert.equal(
|
||||
selectorAttemptId(1, desired),
|
||||
selectorAttemptId(1, {
|
||||
existingOnly: ['legacy'],
|
||||
migrationOnly: [],
|
||||
general: ['target', 'general']
|
||||
})
|
||||
)
|
||||
const result = await transitionAdmissionSelector(harness.post, { target: 'general' })
|
||||
assert.equal(result.selector.generation, 2)
|
||||
assert.deepEqual(result.selector.membership.general, ['general', 'target'])
|
||||
})
|
||||
|
||||
test('accepts only an exact committed result after an ambiguous response', async () => {
|
||||
const harness = selectorHarness({ ambiguous: 'committed' })
|
||||
const result = await applyExactAdmissionSelector(harness.post, {
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(harness.applies(), 1)
|
||||
assert.equal(result.selector.generation, 2)
|
||||
})
|
||||
|
||||
test('binds a generation-zero cutover to the inspected membership', async () => {
|
||||
const harness = selectorHarness({ generation: 0 })
|
||||
await applyExactAdmissionSelector(
|
||||
harness.post,
|
||||
{
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
},
|
||||
{ requireBoundary: false }
|
||||
)
|
||||
assert.equal(
|
||||
harness.requests[0].expectedMembershipSha256,
|
||||
createHash('sha256').update(JSON.stringify(initialMembership)).digest('hex')
|
||||
)
|
||||
})
|
||||
|
||||
test('stops on an unchanged ambiguous result without replaying', async () => {
|
||||
const harness = selectorHarness({ ambiguous: 'unchanged' })
|
||||
await assert.rejects(
|
||||
applyExactAdmissionSelector(harness.post, {
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
}),
|
||||
/remained unchanged/
|
||||
)
|
||||
assert.equal(harness.applies(), 1)
|
||||
assert.equal(harness.selector().generation, 1)
|
||||
})
|
||||
|
||||
test('never restores an existing-only cell', () => {
|
||||
assert.throws(
|
||||
() => membershipWithStates({ membership: initialMembership }, { legacy: 'general' }),
|
||||
/cannot re-enable/
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses an exact apply after the inspected selector changes', async () => {
|
||||
const harness = selectorHarness()
|
||||
await assert.rejects(
|
||||
applyExactAdmissionSelector(
|
||||
harness.post,
|
||||
{
|
||||
existingOnly: ['legacy', 'target'],
|
||||
migrationOnly: [],
|
||||
general: ['general']
|
||||
},
|
||||
{
|
||||
expectedCurrentSelector: {
|
||||
generation: 0,
|
||||
membership: {
|
||||
existingOnly: ['target'],
|
||||
migrationOnly: [],
|
||||
general: ['general', 'legacy']
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
/changed before exact apply/
|
||||
)
|
||||
assert.equal(harness.applies(), 0)
|
||||
})
|
||||
|
||||
test('adds exact migration cells and recovers a committed response loss', async () => {
|
||||
let selector = { generation: 1, attemptId: 'initial', membership: initialMembership }
|
||||
const intents = new Map()
|
||||
let additions = 0
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
|
||||
}
|
||||
}
|
||||
additions++
|
||||
selector = {
|
||||
generation: body.expectedGeneration + 1,
|
||||
attemptId: body.attemptId,
|
||||
membership: {
|
||||
...selector.membership,
|
||||
migrationOnly: [
|
||||
...selector.membership.migrationOnly,
|
||||
...body.cells.map(({ cellId }) => cellId)
|
||||
].sort()
|
||||
}
|
||||
}
|
||||
intents.set(body.attemptId, {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: body.expectedGeneration,
|
||||
intendedGeneration: selector.generation,
|
||||
membership: selector.membership,
|
||||
state: 'committed'
|
||||
})
|
||||
throw new Error('lost cell registration response')
|
||||
}
|
||||
const result = await addExactMigrationCells(post, {
|
||||
attemptId: 'add_cells_exact',
|
||||
cells: [
|
||||
{
|
||||
cellId: 'target-2',
|
||||
cellUrl: 'https://target-2.example.com',
|
||||
capacityRequests: 4_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
})
|
||||
assert.equal(result.recovered, true)
|
||||
assert.equal(additions, 1)
|
||||
assert.deepEqual(result.selector.membership.migrationOnly, ['target', 'target-2'])
|
||||
})
|
||||
|
||||
test('does not recover an attempt owned by another selector operation', async () => {
|
||||
const selector = {
|
||||
generation: 2,
|
||||
attemptId: 'selector_collision',
|
||||
membership: initialMembership
|
||||
}
|
||||
const post = async (path, body) => {
|
||||
if (path.endsWith('/status')) {
|
||||
return {
|
||||
selector,
|
||||
intent: body.attemptId
|
||||
? {
|
||||
attemptId: body.attemptId,
|
||||
expectedGeneration: 1,
|
||||
intendedGeneration: 2,
|
||||
membership: initialMembership,
|
||||
state: 'committed'
|
||||
}
|
||||
: null
|
||||
}
|
||||
}
|
||||
throw new Error('admission_selector_attempt_mismatch')
|
||||
}
|
||||
await assert.rejects(
|
||||
addExactMigrationCells(post, {
|
||||
attemptId: 'selector_collision',
|
||||
cells: [
|
||||
{
|
||||
cellId: 'target-2',
|
||||
cellUrl: 'https://target-2.example.com',
|
||||
capacityRequests: 4_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
]
|
||||
}),
|
||||
/did not commit exactly/
|
||||
)
|
||||
})
|
||||
@@ -1,33 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
function accessToken() {
|
||||
const result = spawnSync('gcloud', ['auth', 'print-access-token'], {
|
||||
encoding: 'utf8', timeout: 30_000
|
||||
})
|
||||
const token = result.stdout.trim()
|
||||
if (result.status !== 0 || token.length < 20) {
|
||||
throw new Error('Google access token is unavailable')
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
export async function readCloudSqlBackends(environment, startedAt, endedAt) {
|
||||
const production = environment === 'production'
|
||||
if (!production && environment !== 'staging') throw new Error('Cloud SQL environment is invalid')
|
||||
const project = production ? 'onorca-cloud' : 'onorca-cloud-staging'
|
||||
const instance = production ? 'orca-cloud-auth-db' : 'orca-cloud-staging-auth-db'
|
||||
const url = new URL(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries`)
|
||||
url.searchParams.set('filter', `metric.type = "cloudsql.googleapis.com/database/postgresql/num_backends" AND resource.labels.database_id = "${project}:${instance}"`)
|
||||
url.searchParams.set('interval.startTime', startedAt)
|
||||
url.searchParams.set('interval.endTime', endedAt)
|
||||
url.searchParams.set('aggregation.alignmentPeriod', '60s')
|
||||
url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_MAX')
|
||||
url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_MAX')
|
||||
url.searchParams.set('view', 'FULL')
|
||||
const response = await fetch(url, {
|
||||
headers: { authorization: `Bearer ${accessToken()}` },
|
||||
redirect: 'error', signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Cloud SQL metric query returned ${response.status}`)
|
||||
return await response.json()
|
||||
}
|
||||
@@ -1,639 +0,0 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readCloudSqlBackends } from './relay-asia-cloud-sql-metrics.mjs'
|
||||
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
|
||||
|
||||
const REPOSITORY = RELAY_GITHUB_REPOSITORY
|
||||
const ADMISSION_WORKFLOW = relayWorkflowPath('operate-relay-asia-admission.yml')
|
||||
const STAGING_WORKFLOW = relayWorkflowPath('prove-relay-asia-staging.yml')
|
||||
const STAGING_CELL = 'staging-gce-c4'
|
||||
const C27 = 'production-gce-c27'
|
||||
const ASIA_REGION = 'asia-east2'
|
||||
const US_REGION = 'us-central1'
|
||||
// Each canary proves its own cell under production load; C28/C29 promotion consumes only C27's.
|
||||
const PRODUCTION_CANARIES = {
|
||||
[C27]: { kind: 'production-c27-canary', origin: 'https://c27.relay.onorca.dev', region: ASIA_REGION },
|
||||
'production-gce-c30': {
|
||||
kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev', region: ASIA_REGION
|
||||
},
|
||||
'production-gce-c31': {
|
||||
kind: 'production-c31-canary', origin: 'https://c31.relay.onorca.dev', region: ASIA_REGION
|
||||
},
|
||||
'production-gce-c32': {
|
||||
kind: 'production-c32-canary', origin: 'https://c32.relay.onorca.dev', region: US_REGION
|
||||
},
|
||||
'production-gce-c33': {
|
||||
kind: 'production-c33-canary', origin: 'https://c33.relay.onorca.dev', region: US_REGION
|
||||
},
|
||||
'production-gce-c34': {
|
||||
kind: 'production-c34-canary', origin: 'https://c34.relay.onorca.dev', region: ASIA_REGION
|
||||
},
|
||||
'production-gce-c35': {
|
||||
kind: 'production-c35-canary', origin: 'https://c35.relay.onorca.dev', region: ASIA_REGION
|
||||
}
|
||||
}
|
||||
const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/
|
||||
const SHA_PATTERN = /^[a-f0-9]{40}$/
|
||||
const MAX_LOG_EDGE_GAP_MS = 120_000
|
||||
const MAX_LOG_SAMPLE_GAP_MS = 120_000
|
||||
const CLOUD_SQL_LIMIT = 320
|
||||
const CANARY_MINIMUM_MS = 5 * 60_000
|
||||
const GENERATOR_CPU_PERCENT_LIMIT = 80
|
||||
const GENERATOR_EVENT_LOOP_P99_MS_LIMIT = 100
|
||||
const GENERATOR_RSS_GROWTH_MIB_LIMIT = 512
|
||||
const DATABASE_POOL_TRANSIENT_WAITERS_MAX = 4
|
||||
const DATABASE_POOL_TRANSIENT_WAIT_MS_MAX = 50
|
||||
|
||||
function object(value, label) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveInteger(value, label) {
|
||||
const number = Number(value)
|
||||
if (!Number.isSafeInteger(number) || number < 1) throw new Error(`${label} is invalid`)
|
||||
return number
|
||||
}
|
||||
|
||||
function instant(value, label) {
|
||||
const date = new Date(value)
|
||||
if (!Number.isFinite(date.valueOf())) throw new Error(`${label} is invalid`)
|
||||
return date
|
||||
}
|
||||
|
||||
function exactCells(actual, expected) {
|
||||
return Array.isArray(actual) &&
|
||||
JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort())
|
||||
}
|
||||
|
||||
function source(input, environment, workflow) {
|
||||
if (input.repository !== REPOSITORY) throw new Error('repository is invalid')
|
||||
if (!SHA_PATTERN.test(input.commitSha)) throw new Error('commit SHA is invalid')
|
||||
return {
|
||||
repository: input.repository,
|
||||
workflow,
|
||||
environment,
|
||||
runId: positiveInteger(input.runId, 'run ID'),
|
||||
runAttempt: positiveInteger(input.runAttempt, 'run attempt'),
|
||||
commitSha: input.commitSha
|
||||
}
|
||||
}
|
||||
|
||||
function baseEvidence(input, environment, cells, workflow = ADMISSION_WORKFLOW) {
|
||||
if (!DIGEST_PATTERN.test(input.imageDigest)) throw new Error('image digest is invalid')
|
||||
return {
|
||||
version: 1,
|
||||
source: source(input, environment, workflow),
|
||||
imageDigest: input.imageDigest,
|
||||
topology: {
|
||||
cellIds: cells,
|
||||
selectorGeneration: positiveInteger(input.selectorGeneration, 'selector generation')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildStagingEvidence(input) {
|
||||
const start = instant(input.startedAt, 'staging proof start')
|
||||
const end = instant(input.endedAt, 'staging proof end')
|
||||
const launch = loadReports(input.launchReport, 'launch load report')
|
||||
const expectedLoad = {
|
||||
controls: 20, splices: 20, slowReaders: 4, wedgedReaders: 1,
|
||||
minimumSeconds: 210
|
||||
}
|
||||
assertLoadReports(launch, expectedLoad)
|
||||
const metrics = runtimeMetrics(input.logs, start, end, STAGING_CELL)
|
||||
assertPassingRuntimeMetrics(metrics, 'staging proof')
|
||||
// Staging is quiet, so its one intentional sticky fallback must be the only one.
|
||||
if (
|
||||
number(metrics.regionFallbacks, 'regionFallbacks') !== 1 ||
|
||||
number(metrics.usRegionFallbacks, 'usRegionFallbacks') !== 1
|
||||
) throw new Error('staging proof regionFallbacks did not match the intentional probes')
|
||||
const minimumConcurrentSplices = expectedLoad.splices - expectedLoad.wedgedReaders
|
||||
if (
|
||||
metrics.targetControlsMax < expectedLoad.controls ||
|
||||
metrics.targetSplicesMax < minimumConcurrentSplices
|
||||
) {
|
||||
throw new Error('staging launch load did not reach C4 at the reviewed levels')
|
||||
}
|
||||
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
|
||||
if (metrics.cloudSqlBackendsMax >= CLOUD_SQL_LIMIT) {
|
||||
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
|
||||
}
|
||||
return {
|
||||
...baseEvidence(input, 'staging', [STAGING_CELL], STAGING_WORKFLOW),
|
||||
kind: 'staging-asia-readiness',
|
||||
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
|
||||
load: { launch: loadSummary(launch) },
|
||||
metrics
|
||||
}
|
||||
}
|
||||
|
||||
function loadReports(value, label) {
|
||||
if (!Array.isArray(value) || value.length < 2) throw new Error(`${label} must be sharded`)
|
||||
return value.map((report) => object(report, label))
|
||||
}
|
||||
|
||||
function total(reports, key) {
|
||||
return reports.reduce((sum, report) => sum + number(report[key], key), 0)
|
||||
}
|
||||
|
||||
function loadSummary(reports) {
|
||||
return {
|
||||
shards: reports.length,
|
||||
controls: total(reports, 'controls'),
|
||||
peakActive: total(reports, 'peakActive'),
|
||||
steadyMinimumActive: total(reports, 'steadyMinimumActive'),
|
||||
peakActiveSplices: total(reports, 'peakActiveSplices'),
|
||||
completedSplices: total(reports, 'completedSplices'),
|
||||
slowReaderSplicesCompleted: total(reports, 'slowReaderSplicesCompleted'),
|
||||
wedgedReaderSplicesClosed: total(reports, 'wedgedReaderSplicesClosed'),
|
||||
regionalFallbacksProved: total(reports, 'regionalFallbacksProved'),
|
||||
oldClientUsFirstProved: total(reports, 'oldClientUsFirstProved'),
|
||||
stickyAssignmentProved: total(reports, 'stickyAssignmentProved'),
|
||||
requestUnitInvitesOpened: total(reports, 'requestUnitInvitesOpened'),
|
||||
requestUnitPrincipalCounts: reports.map((report) => report.requestUnitPrincipalCount),
|
||||
requestUnitOverflowReasons:
|
||||
reports.map((report) => report.requestUnitOverflowReason).filter(Boolean),
|
||||
requestUnitCleanupProved: total(reports, 'requestUnitCleanupProved'),
|
||||
phaseBarrierPassed: reports.every((report) => report.phaseBarrierPassed === true),
|
||||
rebindProbesOpened: total(reports, 'rebindProbesOpened'),
|
||||
rebindOverflowReasons: reports.map((report) => report.rebindOverflowReason).filter(Boolean),
|
||||
readerQueueEvidence: reports.flatMap((report) => report.readerQueueEvidence),
|
||||
readerQueuedBytesPeak: Math.max(...reports.map((report) => report.readerQueuedBytesPeak)),
|
||||
generatorCpuPercentMax: Math.max(...reports.map((report) => report.generatorCpuPercent)),
|
||||
generatorEventLoopP99MsMax: Math.max(
|
||||
...reports.map((report) => report.generatorEventLoopP99Ms)
|
||||
),
|
||||
generatorRssGrowthMiBMax: Math.max(...reports.map((report) => report.generatorRssGrowthMiB)),
|
||||
configuredSteadySeconds: Math.min(...reports.map((report) => report.configuredSteadySeconds))
|
||||
}
|
||||
}
|
||||
|
||||
function assertLoadReports(reports, expected) {
|
||||
const shardCount = reports.length
|
||||
if (
|
||||
reports.some((report, index) =>
|
||||
report.event !== 'relay_load_complete' ||
|
||||
report.shardCount !== shardCount || report.shardIndex !== index ||
|
||||
report.relayAsiaLoadPrincipalCount !== 32 ||
|
||||
number(report.configuredSteadySeconds, 'staging steady seconds') < expected.minimumSeconds ||
|
||||
number(report.configuredSpliceHoldSeconds, 'staging splice hold seconds') <
|
||||
expected.minimumSeconds ||
|
||||
report.requiredLeaseHorizons !== 2 || report.phaseBarrierPassed !== true
|
||||
) ||
|
||||
total(reports, 'controls') !== expected.controls
|
||||
) {
|
||||
throw new Error('staging load profile does not match')
|
||||
}
|
||||
if (
|
||||
total(reports, 'peakActive') !== expected.controls ||
|
||||
total(reports, 'steadyMinimumActive') !== expected.controls
|
||||
) {
|
||||
throw new Error('staging load did not sustain the required controls')
|
||||
}
|
||||
for (const key of [
|
||||
'rampConnectionFailures', 'steadyConnectionFailures', 'transitionConnectionFailures',
|
||||
'unexpectedCloses', 'protocolErrors', 'refreshErrors', 'socketErrors', 'failedSplices'
|
||||
]) {
|
||||
if (total(reports, key) !== 0) throw new Error(`staging load ${key} must be zero`)
|
||||
}
|
||||
const peakActiveSplices = total(reports, 'peakActiveSplices')
|
||||
if (
|
||||
total(reports, 'configuredSplices') !== expected.splices ||
|
||||
total(reports, 'configuredSlowReaderSplices') !== expected.slowReaders ||
|
||||
total(reports, 'configuredWedgedReaderSplices') !== expected.wedgedReaders ||
|
||||
peakActiveSplices < expected.splices - expected.wedgedReaders ||
|
||||
peakActiveSplices > expected.splices ||
|
||||
total(reports, 'completedSplices') !== expected.splices - expected.wedgedReaders ||
|
||||
total(reports, 'slowReaderSplicesCompleted') !== expected.slowReaders ||
|
||||
total(reports, 'wedgedReaderSplicesClosed') !== expected.wedgedReaders
|
||||
) throw new Error('staging mixed load evidence does not match')
|
||||
if (
|
||||
total(reports, 'regionalFallbacksProved') !== 0 ||
|
||||
total(reports, 'oldClientUsFirstProved') !== 1 ||
|
||||
total(reports, 'stickyAssignmentProved') !== 1 ||
|
||||
total(reports, 'requestUnitInvitesOpened') !== 0 ||
|
||||
reports.some((report) => report.requestUnitPrincipalCount !== 0) ||
|
||||
total(reports, 'requestUnitCleanupProved') !== 0 ||
|
||||
reports.some((report) => report.requestUnitOverflowReason !== null) ||
|
||||
total(reports, 'rebindProbesOpened') !== 2 ||
|
||||
reports.some((report) => report.rebindOverflowReason !== null)
|
||||
) throw new Error('staging launch-path evidence does not match')
|
||||
const readerReports = reports.filter(
|
||||
(report) => report.configuredSlowReaderSplices + report.configuredWedgedReaderSplices > 0
|
||||
)
|
||||
if (expected.slowReaders > 0 && readerReports.length !== 1) {
|
||||
throw new Error('staging reader pressure must have one causal owner')
|
||||
}
|
||||
for (const report of reports) {
|
||||
const queue = report.readerQueueEvidence
|
||||
if (!Array.isArray(queue)) throw new Error('staging reader queue evidence is invalid')
|
||||
if (expected.slowReaders === 0 && queue.length !== 0) {
|
||||
throw new Error('control load unexpectedly contains reader queue evidence')
|
||||
}
|
||||
const ownsReaderPressure = readerReports.includes(report)
|
||||
if (expected.slowReaders > 0 && ownsReaderPressure && (
|
||||
queue.length !== 1 ||
|
||||
queue[0]?.origin !== 'https://c4.relay-staging.onorca.dev' ||
|
||||
number(queue[0]?.baselineBytes, 'reader queue baseline') >
|
||||
number(queue[0]?.peakBytes, 'reader queue peak') ||
|
||||
number(queue[0]?.increaseBytes, 'reader queue increase') <= 0 ||
|
||||
queue[0].peakBytes - queue[0].baselineBytes !== queue[0].increaseBytes ||
|
||||
report.readerQueuedBytesPeak !== queue[0].increaseBytes
|
||||
)) throw new Error('staging reader queue evidence is not causal')
|
||||
if (expected.slowReaders > 0 && !ownsReaderPressure && (
|
||||
queue.length !== 0 || report.readerQueuedBytesPeak !== 0
|
||||
)) throw new Error('non-owner shard contains reader queue evidence')
|
||||
}
|
||||
for (const report of reports) {
|
||||
if (
|
||||
number(report.generatorCpuPercent, 'generator CPU') >= GENERATOR_CPU_PERCENT_LIMIT ||
|
||||
number(report.generatorEventLoopP99Ms, 'generator event loop') >=
|
||||
GENERATOR_EVENT_LOOP_P99_MS_LIMIT ||
|
||||
number(report.generatorRssGrowthMiB, 'generator RSS growth') >=
|
||||
GENERATOR_RSS_GROWTH_MIB_LIMIT
|
||||
) throw new Error('staging load generator has insufficient headroom')
|
||||
const shutdown = object(report.shutdownEvidence, 'load shutdown evidence')
|
||||
if (
|
||||
shutdown.peerShutdowns !== report.controls || shutdown.activeControls !== 0 ||
|
||||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
|
||||
) throw new Error('staging load cleanup is incomplete')
|
||||
}
|
||||
}
|
||||
|
||||
function number(value, label) {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function sumMap(value, label) {
|
||||
const entries = Object.entries(object(value ?? {}, label))
|
||||
return entries.reduce((total, [key, count]) => {
|
||||
if (!key) throw new Error(`${label} is invalid`)
|
||||
return total + number(count, label)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function assertCoverage(timestamps, start, end, label) {
|
||||
if (timestamps.length === 0) throw new Error(`${label} has no samples`)
|
||||
const ordered = timestamps.map((value) => instant(value, `${label} timestamp`).valueOf())
|
||||
.sort((left, right) => left - right)
|
||||
if (
|
||||
ordered[0] > start.valueOf() + MAX_LOG_EDGE_GAP_MS ||
|
||||
ordered.at(-1) < end.valueOf() - MAX_LOG_EDGE_GAP_MS
|
||||
) throw new Error(`${label} does not cover the canary window`)
|
||||
if (ordered.some((timestamp, index) => index > 0 && timestamp - ordered[index - 1] > MAX_LOG_SAMPLE_GAP_MS)) {
|
||||
throw new Error(`${label} has a sampling gap`)
|
||||
}
|
||||
}
|
||||
|
||||
function pointValue(point) {
|
||||
const value = point?.value
|
||||
if (typeof value?.doubleValue === 'number') return value.doubleValue
|
||||
if (typeof value?.int64Value === 'string') return Number(value.int64Value)
|
||||
return NaN
|
||||
}
|
||||
|
||||
function cloudSqlMaximum(response, start, end) {
|
||||
const points = (object(response, 'Cloud SQL response').timeSeries ?? [])
|
||||
.flatMap((series) => series.points ?? [])
|
||||
assertCoverage(points.map((point) => point.interval?.endTime), start, end, 'Cloud SQL metrics')
|
||||
const values = points.map(pointValue)
|
||||
if (values.some((value) => !Number.isFinite(value) || value < 0)) {
|
||||
throw new Error('Cloud SQL backend metric is invalid')
|
||||
}
|
||||
return Math.max(...values)
|
||||
}
|
||||
|
||||
function productionCanary(cellId) {
|
||||
const canary = PRODUCTION_CANARIES[cellId]
|
||||
if (!canary) throw new Error('canary cell is not a reviewed production Asia canary')
|
||||
return { ...canary, label: cellId.split('-').at(-1).toUpperCase() }
|
||||
}
|
||||
|
||||
export function buildProductionCanaryEvidence(input) {
|
||||
const canary = productionCanary(input.cellId)
|
||||
const start = instant(input.startedAt, 'canary start')
|
||||
const end = instant(input.endedAt, 'canary end')
|
||||
if (end.valueOf() - start.valueOf() < CANARY_MINIMUM_MS) {
|
||||
throw new Error(`${canary.label} canary window is shorter than 5 minutes`)
|
||||
}
|
||||
const load = object(input.loadReport, `${canary.label} load report`)
|
||||
assertCanaryLoad(load, input.cellId)
|
||||
// Directors show a steady relay_cells lock and pool-wait baseline unrelated to the canary cell.
|
||||
const metrics = runtimeMetrics(input.logs, start, end, input.cellId, {
|
||||
gateDirectorDatabase: false, region: canary.region
|
||||
})
|
||||
assertPassingRuntimeMetrics(metrics, `${canary.label} canary`, canary.region)
|
||||
// Fallbacks are keyed by the host's target region. US-targeted ones come from unhinted or
|
||||
// US-preferring hosts an Asia cell cannot cause, so they are recorded but not gated. A US
|
||||
// cell gates neither: the US-targeted baseline is nonzero while the US fleet is full.
|
||||
if (
|
||||
canary.region === ASIA_REGION &&
|
||||
number(metrics.asiaRegionFallbacks, 'asiaRegionFallbacks') !== 0
|
||||
) {
|
||||
throw new Error(`${canary.label} canary asiaRegionFallbacks must be zero`)
|
||||
}
|
||||
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
|
||||
assertPassingCanary(metrics, input.cellId)
|
||||
return {
|
||||
...baseEvidence(input, 'production', [input.cellId]),
|
||||
kind: canary.kind,
|
||||
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
|
||||
load,
|
||||
metrics
|
||||
}
|
||||
}
|
||||
|
||||
function assertCanaryLoad(report, cellId) {
|
||||
const { label, origin } = productionCanary(cellId)
|
||||
if (
|
||||
report.event !== 'relay_load_complete' ||
|
||||
report.controls !== 1 || report.shardCount !== 1 || report.shardIndex !== 0 ||
|
||||
report.relayAsiaLoadPrincipalCount !== 1 ||
|
||||
number(report.configuredSteadySeconds, 'canary steady seconds') < 300 ||
|
||||
number(report.configuredSpliceHoldSeconds, 'canary splice hold seconds') < 60 ||
|
||||
number(report.requiredLeaseHorizons, 'canary lease horizons') < 2 ||
|
||||
report.peakActive !== 1 || report.steadyMinimumActive !== 1 ||
|
||||
report.configuredSplices !== 1 || report.peakActiveSplices !== 1 ||
|
||||
report.completedSplices !== 1 || report.failedSplices !== 0
|
||||
) throw new Error(`${label} control and splice canary did not match`)
|
||||
// Placement picks the least-loaded general cell in the canary's region, so the control lands on it.
|
||||
if (JSON.stringify(report.assignedCellOrigins) !== JSON.stringify([origin])) {
|
||||
throw new Error(`${label} canary load was not placed only on ${label}`)
|
||||
}
|
||||
for (const key of [
|
||||
'connectionFailures', 'unexpectedCloses', 'protocolErrors',
|
||||
'refreshErrors', 'socketErrors'
|
||||
]) {
|
||||
if (number(report[key], key) !== 0) throw new Error(`${label} canary ${key} must be zero`)
|
||||
}
|
||||
const shutdown = object(report.shutdownEvidence, `${label} load shutdown evidence`)
|
||||
if (
|
||||
shutdown.peerShutdowns !== 1 || shutdown.activeControls !== 0 ||
|
||||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
|
||||
) throw new Error(`${label} load cleanup is incomplete`)
|
||||
}
|
||||
|
||||
function runtimeMetrics(
|
||||
logs, start, end, targetCellId, { gateDirectorDatabase = true, region = ASIA_REGION } = {}
|
||||
) {
|
||||
const entries = logs.map((entry) => object(entry, 'runtime metric entry'))
|
||||
const directorEntries = entries.filter((entry) => entry.jsonPayload?.role === 'director')
|
||||
const cellEntries = entries.filter((entry) =>
|
||||
entry.jsonPayload?.role === 'cell' &&
|
||||
entry.jsonPayload?.cellId === targetCellId &&
|
||||
entry.jsonPayload?.region === region
|
||||
)
|
||||
assertCoverage(directorEntries.map((entry) => entry.timestamp), start, end, 'director metrics')
|
||||
assertCoverage(cellEntries.map((entry) => entry.timestamp), start, end, `${targetCellId} metrics`)
|
||||
const identifiedDirectors = Map.groupBy(
|
||||
directorEntries.filter((entry) => entry.resource?.labels?.instance_id),
|
||||
(entry) => entry.resource.labels.instance_id
|
||||
)
|
||||
for (const [instanceId, instanceEntries] of identifiedDirectors) {
|
||||
assertCoverage(instanceEntries.map((entry) => entry.timestamp), start, end, `director ${instanceId}`)
|
||||
}
|
||||
const directorPayloads = directorEntries.map((entry) => entry.jsonPayload)
|
||||
const cellPayloads = cellEntries.map((entry) => entry.jsonPayload)
|
||||
const placement = {
|
||||
asiaSelections: directorPayloads.reduce(
|
||||
(total, payload) => total + number(payload.selectedRegionsDelta?.['asia-east2'] ?? 0, 'Asia selections'), 0
|
||||
),
|
||||
regionFallbacks: directorPayloads.reduce(
|
||||
(total, payload) => total + sumMap(payload.regionFallbacksDelta, 'region fallbacks'), 0
|
||||
),
|
||||
asiaRegionFallbacks: directorPayloads.reduce(
|
||||
(total, payload) => total + number(
|
||||
payload.regionFallbacksDelta?.['asia-east2'] ?? 0,
|
||||
'Asia region fallbacks'
|
||||
), 0
|
||||
),
|
||||
usRegionFallbacks: directorPayloads.reduce(
|
||||
(total, payload) => total + number(
|
||||
payload.regionFallbacksDelta?.['us-central1'] ?? 0,
|
||||
'US region fallbacks'
|
||||
), 0
|
||||
),
|
||||
unavailableRegions: directorPayloads.reduce(
|
||||
(total, payload) => total + sumMap(payload.unavailableRegionsDelta, 'unavailable regions'), 0
|
||||
),
|
||||
// Only a US canary reads US selections, so Asia evidence keeps its exact field set.
|
||||
...(region === US_REGION ? {
|
||||
usSelections: directorPayloads.reduce(
|
||||
(total, payload) =>
|
||||
total + number(payload.selectedRegionsDelta?.[US_REGION] ?? 0, 'US selections'),
|
||||
0
|
||||
)
|
||||
} : {})
|
||||
}
|
||||
// Director before cell per metric keeps the old validation order.
|
||||
const split = (read) => ({ director: read(directorPayloads), cell: read(cellPayloads) })
|
||||
const sqlFailures = split((list) => list.reduce(
|
||||
(total, payload) => total + number(payload.sqlFailuresDelta, 'Relay SQL failures'), 0
|
||||
))
|
||||
const waitingMax = split((list) => Math.max(...list.map(
|
||||
(payload) => number(payload.databasePoolWaiting, 'database pool waiting')
|
||||
)))
|
||||
const waitersMax = split((list) => Math.max(...list.map(
|
||||
(payload) => number(payload.databasePoolWaitersMax, 'database pool waiters')
|
||||
)))
|
||||
const waitMsMax = split((list) => Math.max(...list.map(
|
||||
(payload) => number(payload.databasePoolWaitMsMax, 'database pool wait time')
|
||||
)))
|
||||
// Ungated director values stay reported so the artifact still shows them.
|
||||
const gated = gateDirectorDatabase
|
||||
? {
|
||||
relaySqlFailures: sqlFailures.director + sqlFailures.cell,
|
||||
databasePoolWaitingMax: Math.max(waitingMax.director, waitingMax.cell),
|
||||
databasePoolWaitersMax: Math.max(waitersMax.director, waitersMax.cell),
|
||||
databasePoolWaitMsMax: Math.max(waitMsMax.director, waitMsMax.cell)
|
||||
}
|
||||
: {
|
||||
relaySqlFailures: sqlFailures.cell,
|
||||
directorSqlFailures: sqlFailures.director,
|
||||
databasePoolWaitingMax: waitingMax.cell,
|
||||
databasePoolWaitersMax: waitersMax.cell,
|
||||
databasePoolWaitMsMax: waitMsMax.cell,
|
||||
directorDatabasePoolWaitingMax: waitingMax.director,
|
||||
directorDatabasePoolWaitersMax: waitersMax.director,
|
||||
directorDatabasePoolWaitMsMax: waitMsMax.director
|
||||
}
|
||||
return {
|
||||
...placement,
|
||||
...gated,
|
||||
targetControlsMax: Math.max(...cellPayloads.map(
|
||||
(payload) => number(payload.controls, 'target controls')
|
||||
)),
|
||||
targetSplicesMax: Math.max(...cellPayloads.map(
|
||||
(payload) => number(payload.splices, 'target splices')
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// Fleet-level proof that director metrics flowed; the placement check proves the cell itself.
|
||||
function assertPassingRuntimeMetrics(metrics, label, region = ASIA_REGION) {
|
||||
if (region === US_REGION) {
|
||||
if (number(metrics.usSelections, 'US selections') < 1) {
|
||||
throw new Error(`${label} observed no US selections`)
|
||||
}
|
||||
} else if (number(metrics.asiaSelections, 'Asia selections') < 1) {
|
||||
throw new Error(`${label} observed no Asia selections`)
|
||||
}
|
||||
for (const key of [
|
||||
'unavailableRegions', 'relaySqlFailures', 'databasePoolWaitingMax'
|
||||
]) {
|
||||
if (number(metrics[key], key) !== 0) throw new Error(`${label} ${key} must be zero`)
|
||||
}
|
||||
if (
|
||||
number(metrics.databasePoolWaitersMax, 'databasePoolWaitersMax') >
|
||||
DATABASE_POOL_TRANSIENT_WAITERS_MAX ||
|
||||
number(metrics.databasePoolWaitMsMax, 'databasePoolWaitMsMax') >
|
||||
DATABASE_POOL_TRANSIENT_WAIT_MS_MAX
|
||||
) throw new Error(`${label} transient database pool pressure exceeded its bound`)
|
||||
}
|
||||
|
||||
function assertPassingCanary(metrics, cellId) {
|
||||
const { label } = productionCanary(cellId)
|
||||
if (
|
||||
number(metrics.targetControlsMax, `${label} controls`) < 1 ||
|
||||
number(metrics.targetSplicesMax, `${label} splices`) < 1
|
||||
) throw new Error(`${label} canary traffic did not reach ${label}`)
|
||||
if (number(metrics.cloudSqlBackendsMax, 'Cloud SQL backends') >= CLOUD_SQL_LIMIT) {
|
||||
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
|
||||
}
|
||||
}
|
||||
|
||||
function assertProvenance(evidence, run, expected) {
|
||||
const evidenceSource = object(evidence.source, 'evidence source')
|
||||
if (
|
||||
run.id !== evidenceSource.runId ||
|
||||
run.run_attempt !== evidenceSource.runAttempt ||
|
||||
run.conclusion !== 'success' ||
|
||||
run.event !== 'workflow_dispatch' ||
|
||||
run.head_branch !== 'main' ||
|
||||
run.head_sha !== evidenceSource.commitSha ||
|
||||
run.repository?.full_name !== evidenceSource.repository ||
|
||||
run.path?.split('@')[0] !== expected.workflow ||
|
||||
evidenceSource.workflow !== expected.workflow ||
|
||||
evidenceSource.repository !== REPOSITORY ||
|
||||
expected.repository !== REPOSITORY ||
|
||||
evidenceSource.environment !== expected.environment ||
|
||||
evidenceSource.commitSha !== expected.commitSha
|
||||
) throw new Error('evidence workflow provenance does not match')
|
||||
}
|
||||
|
||||
export function verifyRolloutEvidence(evidence, run, expected) {
|
||||
object(evidence, 'evidence')
|
||||
object(run, 'workflow run')
|
||||
if (evidence.version !== 1 || evidence.kind !== expected.kind) {
|
||||
throw new Error('evidence kind is invalid')
|
||||
}
|
||||
assertProvenance(evidence, run, expected)
|
||||
if (evidence.imageDigest !== expected.imageDigest) throw new Error('evidence image digest does not match')
|
||||
if (!exactCells(evidence.topology?.cellIds, expected.cellIds)) {
|
||||
throw new Error('evidence topology does not match')
|
||||
}
|
||||
positiveInteger(evidence.topology?.selectorGeneration, 'evidence selector generation')
|
||||
if (
|
||||
expected.selectorGeneration !== undefined &&
|
||||
evidence.topology.selectorGeneration !== expected.selectorGeneration
|
||||
) throw new Error('evidence selector generation does not match')
|
||||
const now = instant(expected.now, 'verification time')
|
||||
const proofTime = instant(evidence.window?.endedAt, 'evidence time')
|
||||
const maxAgeMs = expected.kind === 'staging-asia-readiness' ? 24 * 60 * 60_000 : 6 * 60 * 60_000
|
||||
if (proofTime > now || now.valueOf() - proofTime.valueOf() > maxAgeMs) {
|
||||
throw new Error('rollout evidence is stale')
|
||||
}
|
||||
const canaryCell = Object.keys(PRODUCTION_CANARIES)
|
||||
.find((cellId) => PRODUCTION_CANARIES[cellId].kind === expected.kind)
|
||||
if (canaryCell) {
|
||||
if (!exactCells(expected.cellIds, [canaryCell])) throw new Error('evidence topology does not match')
|
||||
const start = instant(evidence.window?.startedAt, 'canary start')
|
||||
if (proofTime.valueOf() - start.valueOf() < CANARY_MINIMUM_MS) {
|
||||
throw new Error(`${productionCanary(canaryCell).label} canary window is shorter than 5 minutes`)
|
||||
}
|
||||
assertCanaryLoad(object(evidence.load, 'canary load'), canaryCell)
|
||||
assertPassingCanary(object(evidence.metrics, 'canary metrics'), canaryCell)
|
||||
}
|
||||
return evidence
|
||||
}
|
||||
|
||||
function argumentsMap(argv) {
|
||||
const values = new Map()
|
||||
for (let index = 1; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined || values.has(key.slice(2))) {
|
||||
throw new Error('invalid rollout evidence arguments')
|
||||
}
|
||||
values.set(key.slice(2), value)
|
||||
}
|
||||
return { command: argv[0], values }
|
||||
}
|
||||
|
||||
function required(values, key) {
|
||||
const value = values.get(key)
|
||||
if (!value) throw new Error(`missing --${key}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function commonInput(values) {
|
||||
return {
|
||||
repository: required(values, 'repository'), runId: required(values, 'run-id'),
|
||||
runAttempt: required(values, 'run-attempt'), commitSha: required(values, 'commit-sha'),
|
||||
imageDigest: required(values, 'image-digest'),
|
||||
selectorGeneration: required(values, 'selector-generation')
|
||||
}
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const { command, values } = argumentsMap(argv)
|
||||
const output = required(values, 'output')
|
||||
if (command === 'create-staging') {
|
||||
writeFileSync(output, `${JSON.stringify(buildStagingEvidence({
|
||||
...commonInput(values), startedAt: required(values, 'started-at'),
|
||||
endedAt: required(values, 'ended-at'),
|
||||
launchReport: JSON.parse(readFileSync(required(values, 'launch-report'), 'utf8')),
|
||||
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
|
||||
cloudSql: await readCloudSqlBackends(
|
||||
'staging', required(values, 'started-at'), required(values, 'ended-at')
|
||||
)
|
||||
}), null, 2)}\n`)
|
||||
return
|
||||
}
|
||||
if (command === 'create-canary') {
|
||||
const startedAt = required(values, 'started-at')
|
||||
const endedAt = required(values, 'ended-at')
|
||||
writeFileSync(output, `${JSON.stringify(buildProductionCanaryEvidence({
|
||||
...commonInput(values), cellId: required(values, 'cell-id'), startedAt, endedAt,
|
||||
loadReport: JSON.parse(readFileSync(required(values, 'load-report'), 'utf8')),
|
||||
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
|
||||
cloudSql: await readCloudSqlBackends('production', startedAt, endedAt)
|
||||
}), null, 2)}\n`)
|
||||
return
|
||||
}
|
||||
if (!['verify-staging', 'verify-c27'].includes(command)) throw new Error('invalid evidence command')
|
||||
const kind = command === 'verify-staging' ? 'staging-asia-readiness' : 'production-c27-canary'
|
||||
verifyRolloutEvidence(
|
||||
JSON.parse(readFileSync(required(values, 'evidence'), 'utf8')),
|
||||
JSON.parse(readFileSync(required(values, 'run-json'), 'utf8')),
|
||||
{
|
||||
kind, repository: REPOSITORY,
|
||||
workflow: kind === 'staging-asia-readiness' ? STAGING_WORKFLOW : ADMISSION_WORKFLOW,
|
||||
environment: kind === 'staging-asia-readiness' ? 'staging' : 'production',
|
||||
commitSha: required(values, 'commit-sha'), imageDigest: required(values, 'image-digest'),
|
||||
cellIds: [kind === 'staging-asia-readiness' ? STAGING_CELL : C27],
|
||||
selectorGeneration: values.has('selector-generation')
|
||||
? positiveInteger(values.get('selector-generation'), 'selector generation') : undefined,
|
||||
now: required(values, 'now')
|
||||
}
|
||||
)
|
||||
writeFileSync(output, 'verified\n')
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(2))
|
||||
@@ -1,592 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
|
||||
import {
|
||||
buildProductionCanaryEvidence,
|
||||
buildStagingEvidence,
|
||||
verifyRolloutEvidence
|
||||
} from './relay-asia-rollout-evidence.mjs'
|
||||
|
||||
const digest = `sha256:${'a'.repeat(64)}`
|
||||
const commitSha = 'b'.repeat(40)
|
||||
const repository = RELAY_GITHUB_REPOSITORY
|
||||
const start = new Date('2026-08-13T12:00:00.000Z')
|
||||
const end = new Date(start.valueOf() + 15 * 60_000)
|
||||
const canaryEnd = new Date(start.valueOf() + 5 * 60_000)
|
||||
|
||||
function sourceInput(overrides = {}) {
|
||||
return {
|
||||
repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest,
|
||||
selectorGeneration: 9, ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function metricLog(timestamp, role, cellId, overrides = {}) {
|
||||
return {
|
||||
timestamp: timestamp.toISOString(),
|
||||
resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} },
|
||||
jsonPayload: {
|
||||
role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1',
|
||||
controls: role === 'cell' ? 2_840 : 0,
|
||||
splices: role === 'cell' ? 120 : 0,
|
||||
selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {},
|
||||
regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0,
|
||||
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function completeLogs(cellId = 'production-gce-c27', windowEnd = end) {
|
||||
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1
|
||||
return Array.from({ length: samples }, (_, minute) => {
|
||||
const timestamp = new Date(start.valueOf() + minute * 60_000)
|
||||
return [
|
||||
metricLog(timestamp, 'director', 'production-director'),
|
||||
metricLog(timestamp, 'cell', cellId)
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function cloudSql(max = 319, windowEnd = end) {
|
||||
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000
|
||||
return {
|
||||
timeSeries: [{
|
||||
points: Array.from({ length: samples }, (_, minute) => ({
|
||||
interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() },
|
||||
value: { int64Value: String(minute === samples - 1 ? max : 300) }
|
||||
}))
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
function loadReport({
|
||||
controls, shardIndex, splices = 0, slow = 0, wedged = 0,
|
||||
launch = false
|
||||
}) {
|
||||
return {
|
||||
event: 'relay_load_complete', controls, shardCount: 4, shardIndex,
|
||||
configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2,
|
||||
configuredSpliceHoldSeconds: 210,
|
||||
configuredSplices: splices, configuredSlowReaderSplices: slow,
|
||||
configuredWedgedReaderSplices: wedged,
|
||||
peakActive: controls, steadyMinimumActive: controls,
|
||||
connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0,
|
||||
transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0,
|
||||
refreshErrors: 0, socketErrors: 0, failedSplices: 0,
|
||||
regionalFallbacksProved: 0,
|
||||
oldClientUsFirstProved: launch ? 1 : 0,
|
||||
stickyAssignmentProved: launch ? 1 : 0,
|
||||
requestUnitInvitesOpened: 0,
|
||||
requestUnitPrincipalCount: 0,
|
||||
relayAsiaLoadPrincipalCount: 32,
|
||||
requestUnitOverflowReason: null,
|
||||
requestUnitCleanupProved: 0,
|
||||
phaseBarrierPassed: true,
|
||||
rebindProbesOpened: launch ? 2 : 0,
|
||||
rebindOverflowReason: null,
|
||||
peakActiveSplices: splices, completedSplices: splices - wedged,
|
||||
slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged,
|
||||
readerQueuedBytesPeak: slow > 0 ? 1_024 : 0,
|
||||
generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10,
|
||||
readerQueueEvidence: slow > 0 ? [{
|
||||
origin: 'https://c4.relay-staging.onorca.dev',
|
||||
baselineBytes: 128,
|
||||
peakBytes: 1_152,
|
||||
increaseBytes: 1_024
|
||||
}] : [],
|
||||
shutdownEvidence: {
|
||||
peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stagingInput(overrides = {}) {
|
||||
const logs = completeLogs('staging-gce-c4')
|
||||
logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 }
|
||||
return {
|
||||
...sourceInput({ selectorGeneration: 4 }),
|
||||
startedAt: start.toISOString(), endedAt: end.toISOString(),
|
||||
launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({
|
||||
controls: 5, shardIndex, splices: 5, launch: shardIndex === 0,
|
||||
slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0
|
||||
})),
|
||||
logs, cloudSql: cloudSql(), ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function canaryInput(overrides = {}, cellId = 'production-gce-c27') {
|
||||
const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 })
|
||||
Object.assign(load, {
|
||||
shardCount: 1,
|
||||
configuredSteadySeconds: 300,
|
||||
configuredSpliceHoldSeconds: 60,
|
||||
relayAsiaLoadPrincipalCount: 1,
|
||||
assignedCellOrigins: [`https://${cellId.split('-').at(-1)}.relay.onorca.dev`]
|
||||
})
|
||||
return {
|
||||
...sourceInput(), cellId, startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(),
|
||||
loadReport: load,
|
||||
logs: completeLogs(cellId, canaryEnd), cloudSql: cloudSql(319, canaryEnd),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function workflowRun(evidence, overrides = {}) {
|
||||
return {
|
||||
id: evidence.source.runId, run_attempt: evidence.source.runAttempt,
|
||||
conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main',
|
||||
head_sha: evidence.source.commitSha,
|
||||
repository: { full_name: evidence.source.repository }, path: evidence.source.workflow,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function verifyExpected(kind, overrides = {}) {
|
||||
const staging = kind === 'staging-asia-readiness'
|
||||
return {
|
||||
kind, repository,
|
||||
workflow: staging
|
||||
? relayWorkflowPath('prove-relay-asia-staging.yml')
|
||||
: relayWorkflowPath('operate-relay-asia-admission.yml'),
|
||||
environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest,
|
||||
cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'],
|
||||
now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides
|
||||
}
|
||||
}
|
||||
|
||||
test('accepts the exact sharded staging load, telemetry, and provenance proof', () => {
|
||||
const evidence = buildStagingEvidence(stagingInput())
|
||||
assert.equal(evidence.load.launch.controls, 20)
|
||||
assert.equal(evidence.load.launch.peakActiveSplices, 20)
|
||||
assert.equal(evidence.load.launch.readerQueueEvidence.length, 1)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness')
|
||||
), evidence)
|
||||
})
|
||||
|
||||
test('ignores unrelated legacy cell metrics outside the Asia proof', () => {
|
||||
const input = stagingInput()
|
||||
const legacy = metricLog(start, 'cell', 'staging-gce-c1', {
|
||||
region: undefined,
|
||||
sqlFailuresDelta: 1
|
||||
})
|
||||
delete legacy.jsonPayload.databasePoolWaiting
|
||||
delete legacy.jsonPayload.databasePoolWaitersMax
|
||||
delete legacy.jsonPayload.databasePoolWaitMsMax
|
||||
input.logs.push(legacy)
|
||||
|
||||
assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0)
|
||||
})
|
||||
|
||||
test('accepts bounded transient pool waits without a sampled queue', () => {
|
||||
const input = stagingInput()
|
||||
input.logs[0].jsonPayload.databasePoolWaitersMax = 4
|
||||
input.logs[0].jsonPayload.databasePoolWaitMsMax = 50
|
||||
|
||||
const metrics = buildStagingEvidence(input).metrics
|
||||
assert.equal(metrics.databasePoolWaitingMax, 0)
|
||||
assert.equal(metrics.databasePoolWaitersMax, 4)
|
||||
assert.equal(metrics.databasePoolWaitMsMax, 50)
|
||||
})
|
||||
|
||||
test('requires exactly the intentional sticky-assignment fallback in staging', () => {
|
||||
const missing = stagingInput()
|
||||
missing.logs[0].jsonPayload.regionFallbacksDelta = {}
|
||||
assert.throws(() => buildStagingEvidence(missing), /intentional probes/)
|
||||
|
||||
const extra = stagingInput()
|
||||
extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
|
||||
assert.throws(() => buildStagingEvidence(extra), /intentional probes/)
|
||||
|
||||
const substituted = stagingInput()
|
||||
substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
|
||||
assert.throws(() => buildStagingEvidence(substituted), /intentional probes/)
|
||||
})
|
||||
|
||||
test('accepts the wedged splice outside the sustained non-wedged peak', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].peakActiveSplices = 4
|
||||
input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.splices = 19 })
|
||||
const evidence = buildStagingEvidence(input)
|
||||
assert.equal(evidence.load.launch.peakActiveSplices, 19)
|
||||
|
||||
input.launchReport[0].peakActiveSplices = 3
|
||||
assert.throws(() => buildStagingEvidence(input), /mixed load evidence/)
|
||||
})
|
||||
|
||||
test('rejects staging evidence with incomplete load or cleanup', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].steadyMinimumActive--
|
||||
assert.throws(() => buildStagingEvidence(input), /required controls/)
|
||||
const cleanup = stagingInput()
|
||||
cleanup.launchReport[0].shutdownEvidence.activeControls = 1
|
||||
assert.throws(() => buildStagingEvidence(cleanup), /cleanup/)
|
||||
const nonCausal = stagingInput()
|
||||
nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0
|
||||
assert.throws(() => buildStagingEvidence(nonCausal), /not causal/)
|
||||
const multipleOwners = stagingInput()
|
||||
multipleOwners.launchReport[0].configuredSlowReaderSplices--
|
||||
multipleOwners.launchReport[0].slowReaderSplicesCompleted--
|
||||
multipleOwners.launchReport[1] = loadReport({
|
||||
controls: 5, shardIndex: 1, splices: 5, slow: 1
|
||||
})
|
||||
assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/)
|
||||
const overloaded = stagingInput()
|
||||
overloaded.launchReport[0].generatorCpuPercent = 80
|
||||
assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/)
|
||||
const missingLaunchProof = stagingInput()
|
||||
missingLaunchProof.launchReport[0].rebindProbesOpened = 0
|
||||
assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/)
|
||||
const offTarget = stagingInput()
|
||||
offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.controls = 19 })
|
||||
assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/)
|
||||
})
|
||||
|
||||
test('rejects staging evidence with a shortened splice hold', () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].configuredSpliceHoldSeconds = 60
|
||||
assert.throws(() => buildStagingEvidence(input), /load profile does not match/)
|
||||
})
|
||||
|
||||
for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) {
|
||||
test(`rejects staging evidence with a ${label} splice hold`, () => {
|
||||
const input = stagingInput()
|
||||
input.launchReport[0].configuredSpliceHoldSeconds = value
|
||||
assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/)
|
||||
})
|
||||
}
|
||||
|
||||
for (const [label, mutate, message] of [
|
||||
['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/],
|
||||
['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/],
|
||||
['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/]
|
||||
]) {
|
||||
test(`rejects staging evidence without ${label} proof`, () => {
|
||||
const input = stagingInput()
|
||||
mutate(input)
|
||||
assert.throws(() => buildStagingEvidence(input), message)
|
||||
})
|
||||
}
|
||||
|
||||
test('rejects staging evidence from a non-canonical repository', () => {
|
||||
assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/)
|
||||
})
|
||||
|
||||
test('rejects mismatched staging provenance, digest, topology, or age', () => {
|
||||
const evidence = buildStagingEvidence(stagingInput())
|
||||
const expected = verifyExpected('staging-asia-readiness')
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, {
|
||||
head_sha: 'c'.repeat(40)
|
||||
}), expected), /provenance/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, commitSha: 'c'.repeat(40)
|
||||
}), /provenance/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, imageDigest: `sha256:${'c'.repeat(64)}`
|
||||
}), /image digest/)
|
||||
assert.throws(() => verifyRolloutEvidence({
|
||||
...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] }
|
||||
}, workflowRun(evidence), expected), /topology/)
|
||||
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
|
||||
...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString()
|
||||
}), /stale/)
|
||||
})
|
||||
|
||||
test('builds and verifies a passing continuous 5-minute C27 canary', () => {
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput())
|
||||
assert.equal(evidence.load.completedSplices, 1)
|
||||
assert.equal(evidence.metrics.asiaSelections, 6)
|
||||
assert.equal(evidence.metrics.cloudSqlBackendsMax, 319)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c27-canary', { selectorGeneration: 9 })
|
||||
), evidence)
|
||||
})
|
||||
|
||||
test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => {
|
||||
assert.throws(() => buildProductionCanaryEvidence(canaryInput({
|
||||
endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
|
||||
})), /shorter than 5 minutes/)
|
||||
const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) =>
|
||||
entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString()
|
||||
)
|
||||
assert.throws(() => buildProductionCanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/)
|
||||
assert.throws(() => buildProductionCanaryEvidence(canaryInput({
|
||||
logs: completeLogs('production-gce-c26', canaryEnd)
|
||||
})), /production-gce-c27 metrics has no samples/)
|
||||
})
|
||||
|
||||
test('rejects a C27 canary without a real control and splice', () => {
|
||||
const input = canaryInput()
|
||||
input.loadReport.completedSplices = 0
|
||||
assert.throws(() => buildProductionCanaryEvidence(input), /did not match/)
|
||||
const shortHold = canaryInput()
|
||||
shortHold.loadReport.configuredSpliceHoldSeconds = 59
|
||||
assert.throws(() => buildProductionCanaryEvidence(shortHold), /did not match/)
|
||||
})
|
||||
|
||||
for (const [label, mutation, message] of [
|
||||
['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/],
|
||||
['Asia-targeted region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /C27 canary asiaRegionFallbacks must be zero/],
|
||||
['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/],
|
||||
['C27 SQL failures', (input) => { input.logs[1].jsonPayload.sqlFailuresDelta = 1 }, /C27 canary relaySqlFailures must be zero/],
|
||||
['C27 pool waiting', (input) => { input.logs[1].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/],
|
||||
['C27 pool waiters', (input) => { input.logs[1].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/],
|
||||
['C27 pool wait time', (input) => { input.logs[1].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/],
|
||||
['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/],
|
||||
['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/],
|
||||
['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/]
|
||||
]) {
|
||||
test(`rejects C27 evidence with ${label}`, () => {
|
||||
const input = canaryInput()
|
||||
mutation(input)
|
||||
assert.throws(() => buildProductionCanaryEvidence(input), message)
|
||||
})
|
||||
}
|
||||
|
||||
test('rejects C27 evidence from a different selector generation', () => {
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput())
|
||||
assert.throws(() => verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c27-canary', { selectorGeneration: 10 })
|
||||
), /selector generation/)
|
||||
})
|
||||
|
||||
test('rejects a C27 canary whose control was placed on another cell', () => {
|
||||
for (const origins of [
|
||||
['https://c28.relay.onorca.dev'],
|
||||
['https://c27.relay.onorca.dev', 'https://c28.relay.onorca.dev'],
|
||||
[],
|
||||
undefined
|
||||
]) {
|
||||
const input = canaryInput()
|
||||
input.loadReport.assignedCellOrigins = origins
|
||||
assert.throws(() => buildProductionCanaryEvidence(input), /C27 canary load was not placed only on C27/)
|
||||
}
|
||||
})
|
||||
|
||||
const c30 = 'production-gce-c30'
|
||||
|
||||
test('builds and verifies a C30 canary from C30 runtime metrics and placement', () => {
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, c30))
|
||||
assert.equal(evidence.kind, 'production-c30-canary')
|
||||
assert.deepEqual(evidence.topology, { cellIds: [c30], selectorGeneration: 9 })
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
assert.throws(() => verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 10 })
|
||||
), /selector generation/)
|
||||
assert.throws(() => verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence), verifyExpected('production-c27-canary', { selectorGeneration: 9 })
|
||||
), /evidence kind is invalid/)
|
||||
})
|
||||
|
||||
test('builds a C31 canary that only C31 placement satisfies', () => {
|
||||
const c31 = 'production-gce-c31'
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, c31))
|
||||
assert.equal(evidence.kind, 'production-c31-canary')
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c31-canary', { cellIds: [c31], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
const onC30 = canaryInput({}, c31)
|
||||
onC30.loadReport.assignedCellOrigins = ['https://c30.relay.onorca.dev']
|
||||
assert.throws(() => buildProductionCanaryEvidence(onC30), /C31 canary load was not placed only on C31/)
|
||||
})
|
||||
|
||||
test('builds a C34 canary that only C34 placement satisfies', () => {
|
||||
const c34 = 'production-gce-c34'
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, c34))
|
||||
assert.equal(evidence.kind, 'production-c34-canary')
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c34-canary', { cellIds: [c34], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
const onC31 = canaryInput({}, c34)
|
||||
onC31.loadReport.assignedCellOrigins = ['https://c31.relay.onorca.dev']
|
||||
assert.throws(() => buildProductionCanaryEvidence(onC31), /C34 canary load was not placed only on C34/)
|
||||
})
|
||||
|
||||
test('builds a C35 canary that only C35 placement satisfies', () => {
|
||||
const c35 = 'production-gce-c35'
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, c35))
|
||||
assert.equal(evidence.kind, 'production-c35-canary')
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c35-canary', { cellIds: [c35], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
const onC34 = canaryInput({}, c35)
|
||||
onC34.loadReport.assignedCellOrigins = ['https://c34.relay.onorca.dev']
|
||||
assert.throws(() => buildProductionCanaryEvidence(onC34), /C35 canary load was not placed only on C35/)
|
||||
})
|
||||
|
||||
test('records but does not gate organic US-targeted fallbacks during a canary', () => {
|
||||
// Mirrors C31's 2026-10-01 canary: US-targeted fallbacks, none targeting Asia.
|
||||
const input = canaryInput({}, 'production-gce-c31')
|
||||
input.logs.filter((entry) => entry.jsonPayload.role === 'director').forEach((entry) => {
|
||||
entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24 }
|
||||
})
|
||||
const evidence = buildProductionCanaryEvidence(input)
|
||||
assert.equal(evidence.metrics.regionFallbacks, 144)
|
||||
assert.equal(evidence.metrics.usRegionFallbacks, 144)
|
||||
assert.equal(evidence.metrics.asiaRegionFallbacks, 0)
|
||||
input.logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 }
|
||||
assert.throws(() => buildProductionCanaryEvidence(input), /C31 canary asiaRegionFallbacks must be zero/)
|
||||
})
|
||||
|
||||
// A root-region cell logs region us-central1, and the directors place US hosts too.
|
||||
function usCanaryInput(cellId) {
|
||||
const input = canaryInput({}, cellId)
|
||||
for (const entry of input.logs) {
|
||||
if (entry.jsonPayload.role === 'cell') entry.jsonPayload.region = 'us-central1'
|
||||
else entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1, 'us-central1': 3 }
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
test('builds and verifies C32 and C33 canaries from their own us-central1 metrics', () => {
|
||||
for (const cellId of ['production-gce-c32', 'production-gce-c33']) {
|
||||
const label = cellId.split('-').at(-1).toUpperCase()
|
||||
const kind = `production-${label.toLowerCase()}-canary`
|
||||
const evidence = buildProductionCanaryEvidence(usCanaryInput(cellId))
|
||||
assert.equal(evidence.kind, kind)
|
||||
assert.equal(evidence.metrics.usSelections, 18)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence), verifyExpected(kind, { cellIds: [cellId], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
// The Asia-region filter would find none of a US cell's samples.
|
||||
assert.throws(
|
||||
() => buildProductionCanaryEvidence(canaryInput({}, cellId)),
|
||||
new RegExp(`${cellId} metrics`)
|
||||
)
|
||||
const elsewhere = usCanaryInput(cellId)
|
||||
elsewhere.loadReport.assignedCellOrigins = ['https://c26.relay.onorca.dev']
|
||||
assert.throws(
|
||||
() => buildProductionCanaryEvidence(elsewhere),
|
||||
new RegExp(`${label} canary load was not placed only on ${label}`)
|
||||
)
|
||||
const noUs = usCanaryInput(cellId)
|
||||
for (const entry of noUs.logs) {
|
||||
if (entry.jsonPayload.role === 'director') entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1 }
|
||||
}
|
||||
assert.throws(() => buildProductionCanaryEvidence(noUs), /observed no US selections/)
|
||||
}
|
||||
})
|
||||
|
||||
test('records but does not gate either region\'s fallbacks during a US canary', () => {
|
||||
const input = usCanaryInput('production-gce-c32')
|
||||
for (const entry of input.logs.filter(({ jsonPayload }) => jsonPayload.role === 'director')) {
|
||||
entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 }
|
||||
}
|
||||
const evidence = buildProductionCanaryEvidence(input)
|
||||
assert.equal(evidence.metrics.usRegionFallbacks, 144)
|
||||
assert.equal(evidence.metrics.asiaRegionFallbacks, 6)
|
||||
// Every other gate still binds a US canary.
|
||||
input.logs.find(({ jsonPayload }) => jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1
|
||||
assert.throws(() => buildProductionCanaryEvidence(input), /relaySqlFailures must be zero/)
|
||||
})
|
||||
|
||||
test('keeps the Asia canary evidence field set unchanged', () => {
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, 'production-gce-c31'))
|
||||
assert.equal('usSelections' in evidence.metrics, false)
|
||||
})
|
||||
|
||||
test('rejects a C30 canary that C30 did not serve', () => {
|
||||
const onLaunchCell = canaryInput({}, c30)
|
||||
onLaunchCell.loadReport.assignedCellOrigins = ['https://c27.relay.onorca.dev']
|
||||
assert.throws(() => buildProductionCanaryEvidence(onLaunchCell), /C30 canary load was not placed only on C30/)
|
||||
assert.throws(() => buildProductionCanaryEvidence(canaryInput({
|
||||
logs: completeLogs('production-gce-c27', canaryEnd)
|
||||
}, c30)), /production-gce-c30 metrics has no samples/)
|
||||
const idle = canaryInput({}, c30)
|
||||
idle.logs.filter((entry) => entry.jsonPayload.role === 'cell')
|
||||
.forEach((entry) => { entry.jsonPayload.splices = 0 })
|
||||
assert.throws(() => buildProductionCanaryEvidence(idle), /C30 canary traffic did not reach C30/)
|
||||
const poolPressure = canaryInput({}, c30)
|
||||
poolPressure.logs.at(-1).jsonPayload.databasePoolWaiting = 1
|
||||
assert.throws(() => buildProductionCanaryEvidence(poolPressure), /C30 canary databasePoolWaitingMax/)
|
||||
assert.throws(() => buildProductionCanaryEvidence(canaryInput({
|
||||
endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
|
||||
}, c30)), /C30 canary window is shorter than 5 minutes/)
|
||||
})
|
||||
|
||||
test('gates a C30 canary on C30 pool pressure and only reports the director baseline', () => {
|
||||
const directorPressure = canaryInput({}, c30)
|
||||
const directorLogs = directorPressure.logs.filter((entry) => entry.jsonPayload.role === 'director')
|
||||
Object.assign(directorLogs[0].jsonPayload, { databasePoolWaiting: 3 })
|
||||
Object.assign(directorLogs[1].jsonPayload, { databasePoolWaitersMax: 9, databasePoolWaitMsMax: 240 })
|
||||
const evidence = buildProductionCanaryEvidence(directorPressure)
|
||||
assert.deepEqual([
|
||||
evidence.metrics.databasePoolWaitingMax,
|
||||
evidence.metrics.databasePoolWaitersMax,
|
||||
evidence.metrics.databasePoolWaitMsMax
|
||||
], [0, 0, 0])
|
||||
assert.deepEqual([
|
||||
evidence.metrics.directorDatabasePoolWaitingMax,
|
||||
evidence.metrics.directorDatabasePoolWaitersMax,
|
||||
evidence.metrics.directorDatabasePoolWaitMsMax
|
||||
], [3, 9, 240])
|
||||
|
||||
for (const [field, value, message] of [
|
||||
['databasePoolWaiting', 1, /C30 canary databasePoolWaitingMax must be zero/],
|
||||
['databasePoolWaitersMax', 5, /C30 canary transient database pool pressure/],
|
||||
['databasePoolWaitMsMax', 51, /C30 canary transient database pool pressure/]
|
||||
]) {
|
||||
const cellPressure = canaryInput({}, c30)
|
||||
cellPressure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload[field] = value
|
||||
assert.throws(() => buildProductionCanaryEvidence(cellPressure), message)
|
||||
}
|
||||
})
|
||||
|
||||
test('gates a C30 canary on C30 SQL failures and only reports the director baseline', () => {
|
||||
const directorNoise = canaryInput({}, c30)
|
||||
directorNoise.logs.filter((entry) => entry.jsonPayload.role === 'director')
|
||||
.slice(0, 3).forEach((entry) => { entry.jsonPayload.sqlFailuresDelta = 2 })
|
||||
const evidence = buildProductionCanaryEvidence(directorNoise)
|
||||
assert.equal(evidence.metrics.relaySqlFailures, 0)
|
||||
assert.equal(evidence.metrics.directorSqlFailures, 6)
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
|
||||
const cellFailure = canaryInput({}, c30)
|
||||
cellFailure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1
|
||||
assert.throws(() => buildProductionCanaryEvidence(cellFailure), /C30 canary relaySqlFailures must be zero/)
|
||||
})
|
||||
|
||||
test('keeps staging gated on director database metrics without director-only fields', () => {
|
||||
const metrics = buildStagingEvidence(stagingInput()).metrics
|
||||
assert.equal(metrics.relaySqlFailures, 0)
|
||||
assert.equal(Object.keys(metrics).some((key) => key.startsWith('director')), false)
|
||||
for (const [field, value, message] of [
|
||||
['databasePoolWaiting', 1, /staging proof databasePoolWaitingMax must be zero/],
|
||||
['databasePoolWaitersMax', 5, /staging proof transient database pool pressure/],
|
||||
['databasePoolWaitMsMax', 51, /staging proof transient database pool pressure/]
|
||||
]) {
|
||||
const directorPressure = stagingInput()
|
||||
directorPressure.logs[0].jsonPayload[field] = value
|
||||
assert.throws(() => buildStagingEvidence(directorPressure), message)
|
||||
}
|
||||
|
||||
const directorFailure = stagingInput()
|
||||
directorFailure.logs[0].jsonPayload.sqlFailuresDelta = 1
|
||||
assert.throws(() => buildStagingEvidence(directorFailure), /staging proof relaySqlFailures must be zero/)
|
||||
})
|
||||
|
||||
test('accepts canaries only for the reviewed production Asia cells', () => {
|
||||
for (const cellId of ['production-gce-c28', 'production-gce-c29', 'staging-gce-c4', undefined]) {
|
||||
assert.throws(
|
||||
() => buildProductionCanaryEvidence({ ...canaryInput(), cellId }),
|
||||
/not a reviewed production Asia canary/
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -1,244 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import { relayWorkflowUrl } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readFileSync(
|
||||
relayWorkflowUrl('deploy-relay-asia-topology.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const iam = readFileSync(
|
||||
new URL('../../infra/terraform/relay-asia-topology-iam.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const cells = readFileSync(
|
||||
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const variables = readFileSync(
|
||||
new URL('../../infra/terraform/variables.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
test('uses only its exact workflow-bound topology identity', () => {
|
||||
assert.match(workflow, /production-cloud-sql-rollout/)
|
||||
assert.match(workflow, /relay-staging-mutation/)
|
||||
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER/)
|
||||
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT/)
|
||||
assert.doesNotMatch(workflow, /GCP_DEPLOY_SERVICE_ACCOUNT/)
|
||||
assert.match(
|
||||
iam,
|
||||
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_topology_workflow_file\}@refs\/heads\/main'/
|
||||
)
|
||||
assert.match(iam, /assertion\.ref == 'refs\/heads\/main'/)
|
||||
assert.match(iam, /assertion\.event_name == 'workflow_dispatch'/)
|
||||
assert.match(iam, /assertion\.environment == '\$\{var\.environment\}'/)
|
||||
})
|
||||
|
||||
test('accepts only the reviewed Asia topology waves', () => {
|
||||
const cases = /case "\$\{TARGET_ENVIRONMENT\}:\$\{TARGET_CELL_IDS\}" in\n([\s\S]*?)\n\s*esac/
|
||||
.exec(workflow)?.[1]
|
||||
assert.ok(cases)
|
||||
assert.deepEqual(
|
||||
[...cases.matchAll(/^\s*([a-z]+:[a-z0-9,-]+)\)(?: target_region=[a-z0-9-]+)? ;;$/gm)]
|
||||
.map((match) => match[1]),
|
||||
[
|
||||
'staging:staging-gce-c4',
|
||||
'production:production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
'production:production-gce-c30',
|
||||
'production:production-gce-c31',
|
||||
'production:production-gce-c32,production-gce-c33',
|
||||
'production:production-gce-c34',
|
||||
'production:production-gce-c35'
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
// Runs the workflow's own region and target blocks, so an Asia wave's targets stay byte-identical.
|
||||
function waveTargets(environment, cellIds) {
|
||||
const block = (first, last) => {
|
||||
const start = workflow.indexOf(first)
|
||||
const end = workflow.indexOf(last, start)
|
||||
assert.ok(start !== -1 && end !== -1, first)
|
||||
return workflow.slice(start, end + last.length).replace(/^ {10}/gm, '')
|
||||
}
|
||||
const region = block('target_region=asia-east2\n', '\n esac')
|
||||
const targets = block('file="${RUNNER_TEMP}/relay-asia-targets"', '\n done')
|
||||
const temp = mkdtempSync(join(tmpdir(), 'relay-topology-targets-'))
|
||||
try {
|
||||
const result = spawnSync('bash', ['-euo', 'pipefail', '-c',
|
||||
`${region}\nTARGET_REGION="\${target_region}"\n${targets}\n` +
|
||||
'echo "${TARGET_REGION}"; cat "${file}"'], {
|
||||
env: { ...process.env, TARGET_ENVIRONMENT: environment, TARGET_CELL_IDS: cellIds, RUNNER_TEMP: temp },
|
||||
encoding: 'utf8'
|
||||
})
|
||||
assert.equal(result.status, 0, result.stderr)
|
||||
const [waveRegion, ...lines] = result.stdout.trim().split('\n')
|
||||
return { region: waveRegion, targets: lines }
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
const cellTargets = (cellId) => ['instance_template', 'instance_group_manager', 'backend_service']
|
||||
.map((kind) => `-target=google_compute_${kind}.relay_gce_cell["${cellId}"]`)
|
||||
|
||||
test('targets the asia-east2 network for Asia waves and only the cell and URL map for US waves', () => {
|
||||
const asiaNetwork = ['subnetwork', 'router', 'router_nat']
|
||||
.map((kind) => `-target=google_compute_${kind}.relay_gce_additional["asia-east2"]`)
|
||||
const urlMap = '-target=google_compute_url_map.relay_gce[0]'
|
||||
for (const [environment, cellIds] of [
|
||||
['staging', 'staging-gce-c4'],
|
||||
['production', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['production', 'production-gce-c30'],
|
||||
['production', 'production-gce-c31']
|
||||
]) {
|
||||
assert.deepEqual(waveTargets(environment, cellIds), {
|
||||
region: 'asia-east2',
|
||||
targets: [...asiaNetwork, urlMap, ...cellIds.split(',').flatMap(cellTargets)]
|
||||
}, cellIds)
|
||||
}
|
||||
const usWave = 'production-gce-c32,production-gce-c33'
|
||||
assert.deepEqual(waveTargets('production', usWave), {
|
||||
region: 'us-central1',
|
||||
targets: [urlMap, ...usWave.split(',').flatMap(cellTargets)]
|
||||
})
|
||||
})
|
||||
|
||||
// The live-image overlay refuses a declared non-target cell with no template, so the two US cells
|
||||
// declared together must also plan together; a lone C32 or C33 plan is not a reviewed wave.
|
||||
test('plans the two declared US cells as one wave', () => {
|
||||
const cases = /case "\$\{TARGET_ENVIRONMENT\}:\$\{TARGET_CELL_IDS\}" in\n([\s\S]*?)\n\s*esac/
|
||||
.exec(workflow)?.[1]
|
||||
assert.doesNotMatch(cases, /production:production-gce-c3[23]\)/)
|
||||
})
|
||||
|
||||
test('plans only additive Asia topology and applies the saved plan', () => {
|
||||
assert.doesNotMatch(workflow, /manage_artifact_dns/)
|
||||
for (const target of [
|
||||
'relay_gce_additional',
|
||||
'google_compute_instance_template.relay_gce_cell',
|
||||
'google_compute_instance_group_manager.relay_gce_cell',
|
||||
'google_compute_backend_service.relay_gce_cell',
|
||||
'google_compute_url_map.relay_gce'
|
||||
]) assert.match(workflow, new RegExp(target.replaceAll('.', '\\.')))
|
||||
assert.match(workflow, /apply -input=false -auto-approve "\$\{\{ steps\.plan\.outputs\.plan \}\}"/)
|
||||
assert.match(workflow, /prepare-relay-asia-topology-input\.mjs/)
|
||||
assert.doesNotMatch(workflow, /steps\.variables\.outputs\.file/)
|
||||
assert.match(workflow, /\.variables\.relay_gce_cells\.value/)
|
||||
assert.match(
|
||||
workflow,
|
||||
/\.variables\.relay_gce_additional_region_subnetwork_cidrs\.value/
|
||||
)
|
||||
// Console evaluates every output against state and fails while a declared cell has no MIG.
|
||||
assert.doesNotMatch(workflow, /terraform[^\n]*console/)
|
||||
assert.equal(
|
||||
(workflow.match(/-var-file="\$\{TF_VARS\}" -var-file="\$\{\{ steps\.live-images\.outputs\.file \}\}"/g) ?? []).length,
|
||||
2
|
||||
)
|
||||
assert.equal((workflow.match(/-var-file=/g) ?? []).length, 5)
|
||||
assert.doesNotMatch(workflow, /terraform[^\n]*apply[^\n]*-target/)
|
||||
assert.doesNotMatch(workflow, /google_(?:sql|cloudflare|dns|certificate_manager)/)
|
||||
})
|
||||
|
||||
test('validates before apply and proves convergence afterward', () => {
|
||||
assert.equal((workflow.match(/validate-relay-asia-topology-plan\.mjs/g) ?? []).length, 2)
|
||||
assert.match(workflow, /APPLY_RELAY_ASIA_TOPOLOGY/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.changes'/)
|
||||
assert.match(workflow, /Register the exact new cells atomically as migration-only/)
|
||||
})
|
||||
|
||||
test('checks the connection budget and production live ceiling before planning', () => {
|
||||
assert.match(workflow, /relay-cloud-sql-connection-budget\.mjs/)
|
||||
assert.match(workflow, /gcloud sql instances describe "\$\{CLOUD_SQL_INSTANCE\}"/)
|
||||
assert.match(workflow, /select\(\.name == "max_connections"\)/)
|
||||
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/)
|
||||
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/)
|
||||
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS: '500'/)
|
||||
assert.match(workflow, /live_max="\$\{VERIFIED_DEFAULT_MAX_CONNECTIONS\}"/)
|
||||
assert.doesNotMatch(workflow, /live_max=\d/)
|
||||
assert.match(workflow, /live_source=verified-shape-default/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/)
|
||||
assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/)
|
||||
assert.match(workflow, /test "\$\{live_max\}" = "\$\{checked_max\}"/)
|
||||
assert.ok(
|
||||
workflow.indexOf('relay-cloud-sql-connection-budget.mjs') <
|
||||
workflow.indexOf('terraform -chdir=infra/terraform plan')
|
||||
)
|
||||
})
|
||||
|
||||
test('binds computed Asia references to the matching Terraform cell resources', () => {
|
||||
assert.match(cells, /instance_template = google_compute_instance_template\.relay_gce_cell\[each\.key\]\.self_link/)
|
||||
assert.match(cells, /group\s+= google_compute_instance_group_manager\.relay_gce_cell\[each\.key\]\.instance_group/)
|
||||
assert.match(cells, /default_service = google_compute_backend_service\.relay_gce_cell\[cell\.key\]\.id/)
|
||||
assert.match(cells, /subnetwork = local\.relay_gce_subnetworks\[each\.value\.region\]/)
|
||||
})
|
||||
|
||||
test('keeps cross-variable region constraints in Terraform 1.5 check blocks', () => {
|
||||
assert.doesNotMatch(variables, /region != var\.region/)
|
||||
assert.doesNotMatch(variables, /cell\.region == var\.region/)
|
||||
assert.match(cells, /check "relay_gce_fixed_one_topology"[\s\S]*?region != var\.region/)
|
||||
assert.match(cells, /cell\.region == var\.region[\s\S]*?configured subnetwork/)
|
||||
})
|
||||
|
||||
test('the custom role cannot delete topology or mutate SQL and DNS', () => {
|
||||
assert.doesNotMatch(iam, /compute\.[A-Za-z]+\.delete/)
|
||||
assert.doesNotMatch(
|
||||
iam,
|
||||
/roles\/viewer|cloudsql\.instances\.(?:update|delete)|dns\.|certificatemanager|cloudflare/i
|
||||
)
|
||||
assert.match(iam, /resource "google_project_iam_custom_role" "github_relay_asia_topology_read"/)
|
||||
assert.match(iam, /"cloudsql\.instances\.get"/)
|
||||
assert.match(iam, /"run\.revisions\.get"/)
|
||||
assert.match(iam, /"run\.services\.get"/)
|
||||
assert.match(iam, /"serviceusage\.services\.list"/)
|
||||
assert.match(iam, /"compute\.networks\.updatePolicy"/)
|
||||
assert.match(iam, /"compute\.healthChecks\.useReadOnly"/)
|
||||
assert.match(iam, /"compute\.instanceGroups\.create"/)
|
||||
assert.match(iam, /"compute\.instances\.use"/)
|
||||
assert.match(iam, /roles\/storage\.objectAdmin/)
|
||||
assert.match(iam, /default\.tfstate/)
|
||||
assert.match(iam, /default\.tflock/)
|
||||
assert.match(
|
||||
iam,
|
||||
/resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list"[\s\S]*?permissions = \["storage\.objects\.list"\]/
|
||||
)
|
||||
assert.match(
|
||||
iam,
|
||||
/resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list"[\s\S]*?role\s+= google_project_iam_custom_role\.github_relay_asia_topology_state_list\[0\]\.id/
|
||||
)
|
||||
})
|
||||
|
||||
test('plans every non-target cell at its served image, read from state templates only', () => {
|
||||
const step = /- id: live-images\n[\s\S]*?\n\n/.exec(workflow)?.[0]
|
||||
assert.ok(step)
|
||||
assert.match(step, /terraform -chdir=infra\/terraform show -json \| jq -ce '\[/)
|
||||
assert.match(step, /\.type == "google_compute_instance_template" and \.name == "relay_gce_cell"/)
|
||||
assert.match(step, /\{ index, metadata_startup_script: \.values\.metadata_startup_script \}/)
|
||||
assert.match(step, /relay-live-cell-image-overlay\.mjs/)
|
||||
assert.match(step, /--cell-ids "\$\{TARGET_CELL_IDS\}"/)
|
||||
// The committed map comes from a read-only plan over the same targets, never from console.
|
||||
assert.match(step, /mapfile -t targets < "\$\{\{ steps\.targets\.outputs\.file \}\}"/)
|
||||
assert.match(
|
||||
step,
|
||||
/terraform -chdir=infra\/terraform plan -input=false -refresh=false -lock=false \\\n\s+-var-file="\$\{TF_VARS\}" "\$\{targets\[@\]\}" -out="\$\{committed_plan\}" > \/dev\/null/
|
||||
)
|
||||
assert.match(step, /show -json "\$\{committed_plan\}" \\\n\s+\| jq -ce '\.variables\.relay_gce_cells\.value \| objects' > "\$\{cells\}"/)
|
||||
assert.equal((step.match(/terraform -chdir=infra\/terraform (?:plan|apply)/g) ?? []).length, 1)
|
||||
assert.ok(workflow.indexOf('- id: targets') < workflow.indexOf('- id: live-images'))
|
||||
assert.ok(workflow.indexOf('- id: live-images') < workflow.indexOf('- name: Create and validate the saved topology plan'))
|
||||
})
|
||||
|
||||
test('the deployments output tolerates a cell declared before its topology apply', () => {
|
||||
const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8')
|
||||
const start = outputs.indexOf('output "relay_gce_cell_deployments" {')
|
||||
const block = outputs.slice(start, outputs.indexOf('\n}\n', start))
|
||||
const lookups = [...block.matchAll(/^\s+\w+\s+=\s+(.*\.relay_gce_cell\[cell_id\].*)$/gm)].map((match) => match[1])
|
||||
assert.equal(lookups.length, 6)
|
||||
for (const lookup of lookups) {
|
||||
assert.match(lookup, /^try\(google_compute_\w+\.relay_gce_cell\[cell_id\]\.\w+, null\)$/)
|
||||
}
|
||||
})
|
||||
@@ -1,73 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { delimiter, join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import { readRelayWorkflow, relayTreePath } from './relay-repository.mjs'
|
||||
|
||||
const workflow = readRelayWorkflow('monitor-relay-clock-skew.yml')
|
||||
const productionTfvars = readFileSync(
|
||||
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
// The measuring step's `run: |` block, unindented as the runner receives it.
|
||||
function measureScript() {
|
||||
const lines = workflow.split('\n')
|
||||
const step = lines.findIndex((line) => line.includes('name: Measure Date-header skew'))
|
||||
const start = lines.findIndex((line, index) => index > step && line.trim() === 'run: |') + 1
|
||||
assert.ok(step !== -1 && start > 0, 'measuring step not found')
|
||||
const end = lines.findIndex(
|
||||
(line, index) => index >= start && line.trim() !== '' && !line.startsWith(' '.repeat(10))
|
||||
)
|
||||
return lines.slice(start, end === -1 ? undefined : end).map((line) => line.slice(10)).join('\n')
|
||||
}
|
||||
|
||||
// Runs the step in a checkout holding `tfvars`, with every cell answering like a fenced one so
|
||||
// each probe is named without a clock to judge.
|
||||
function runMonitor(tfvars) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'relay-clock-skew-'))
|
||||
try {
|
||||
const bin = join(root, 'bin')
|
||||
mkdirSync(bin)
|
||||
writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\nprintf "HTTP/2 404 \\r\\n\\r\\n"\n')
|
||||
chmodSync(join(bin, 'curl'), 0o755)
|
||||
const environments = join(root, 'cloud', 'infra', 'terraform', 'environments')
|
||||
mkdirSync(environments, { recursive: true })
|
||||
if (tfvars !== undefined) writeFileSync(join(environments, 'production.tfvars'), tfvars)
|
||||
return spawnSync('bash', ['-e', '-c', measureScript()], {
|
||||
cwd: join(root, 'cloud'),
|
||||
env: { ...process.env, PATH: `${bin}${delimiter}${process.env.PATH}` },
|
||||
encoding: 'utf8'
|
||||
})
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
test('checks out the tfvars the measuring step reads', () => {
|
||||
const tfvars = relayTreePath('infra/terraform/environments/production.tfvars')
|
||||
assert.ok(workflow.includes(`sparse-checkout: ${tfvars}\n`))
|
||||
assert.ok(workflow.includes('sparse-checkout-cone-mode: false\n'))
|
||||
})
|
||||
|
||||
test('probes every cell production.tfvars declares', () => {
|
||||
const declared = [
|
||||
...productionTfvars.matchAll(/"production-gce-c\d+"\s*=\s*\{([\s\S]*?)\n {2}\}/g)
|
||||
].map(([, body]) => /\bhostname\s*=\s*"([^"]+)"/.exec(body)?.[1])
|
||||
assert.ok(declared.length > 0)
|
||||
const result = runMonitor(productionTfvars)
|
||||
assert.deepEqual(
|
||||
[...result.stdout.matchAll(/^(\S+): not serving/gm)].map(([, cell]) => cell),
|
||||
declared
|
||||
)
|
||||
})
|
||||
|
||||
test('fails instead of probing nothing when no cell can be read', () => {
|
||||
for (const tfvars of [undefined, 'relay_gce_cells = {}\n']) {
|
||||
const result = runMonitor(tfvars)
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(result.stdout, /::error::read no relay cells/)
|
||||
}
|
||||
})
|
||||
@@ -1,148 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const DEFAULT_PATHS = {
|
||||
productionTfvars: new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
|
||||
terraformVariables: new URL('../../infra/terraform/variables.tf', import.meta.url),
|
||||
relayConfig: new URL('../../apps/relay/src/config.ts', import.meta.url)
|
||||
}
|
||||
|
||||
// Auth and API live outside the Relay tree, so their consumption is published as a contract
|
||||
// rather than parsed from their source. production-cloud-sql-app-consumers.test.mjs binds it back.
|
||||
const APP_CONSUMERS_CONTRACT = new URL(
|
||||
'../contracts/production-cloud-sql-app-consumers.json',
|
||||
import.meta.url
|
||||
)
|
||||
|
||||
const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections']
|
||||
|
||||
export function readProductionCloudSqlAppConsumers(contract) {
|
||||
const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8'))
|
||||
for (const field of APP_CONSUMER_FIELDS) {
|
||||
const value = parsed[field]
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`could not read ${field} from the app consumer contract`)
|
||||
}
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function requiredInteger(source, pattern, label) {
|
||||
const value = Number(source.match(pattern)?.[1])
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`could not read ${label}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function productionCells(source, defaultPoolMax) {
|
||||
const fencedMatch = source.match(/relay_gce_fenced_cells\s*=\s*\[([^\]]*)\]/)
|
||||
if (!fencedMatch) throw new Error('could not read fenced Relay cells')
|
||||
const fenced = new Set([...fencedMatch[1].matchAll(/"([^"]+)"/g)].map((match) => match[1]))
|
||||
const cells = [...source.matchAll(/"(production-gce-[^"]+)"\s*=\s*\{([\s\S]*?)\n\s*\}/g)].map(
|
||||
([, id, body]) => ({
|
||||
id,
|
||||
fenced: fenced.has(id),
|
||||
region: body.match(/\bregion\s*=\s*"([^"]+)"/)?.[1] ?? 'us-central1',
|
||||
poolMax: body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)
|
||||
? Number(body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)[1])
|
||||
: defaultPoolMax
|
||||
})
|
||||
)
|
||||
if (cells.length === 0) throw new Error('could not read production Relay cells')
|
||||
return cells
|
||||
}
|
||||
|
||||
export function calculateRelayCloudSqlConnectionBudget(inputs) {
|
||||
const consumers = {
|
||||
cells: inputs.cellPoolTotal + inputs.asiaCellCount * inputs.asiaPoolMax,
|
||||
directors: inputs.directorInstances * inputs.directorPoolMax,
|
||||
auth: inputs.authInstances * inputs.authPoolMax,
|
||||
api: inputs.apiInstances * inputs.apiPoolMax
|
||||
}
|
||||
const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0)
|
||||
const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax
|
||||
const candidateOverlap = {
|
||||
relayDirectorCandidate: retainedDirectorRollback * 2,
|
||||
apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax,
|
||||
authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax,
|
||||
relayCells: retainedDirectorRollback
|
||||
}
|
||||
const rolloutOverlap = Math.max(...Object.values(candidateOverlap))
|
||||
const operatingMaximum = configuredMaximum + rolloutOverlap + inputs.maintenanceAdminAllowance
|
||||
const usableCeiling = inputs.maxConnections - inputs.explicitReserve
|
||||
const budgetedTotal = operatingMaximum + inputs.explicitReserve
|
||||
return {
|
||||
maxConnections: inputs.maxConnections,
|
||||
consumers,
|
||||
asia: { cells: inputs.asiaCellCount, poolMax: inputs.asiaPoolMax },
|
||||
configuredMaximum,
|
||||
rolloutOverlap: {
|
||||
...candidateOverlap,
|
||||
retainedDirectorRollback,
|
||||
maximum: rolloutOverlap,
|
||||
reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps'
|
||||
},
|
||||
maintenanceAdminAllowance: inputs.maintenanceAdminAllowance,
|
||||
maintenanceAdminAllowanceReason: 'covers bounded work outside configured services',
|
||||
explicitReserve: inputs.explicitReserve,
|
||||
explicitReserveReason: 'remains unavailable to configured services and planned rollouts',
|
||||
usableCeiling,
|
||||
operatingMaximum,
|
||||
remainingWithinUsableCeiling: usableCeiling - operatingMaximum,
|
||||
budgetedTotal,
|
||||
unallocated: inputs.maxConnections - budgetedTotal,
|
||||
withinBudget: operatingMaximum <= usableCeiling && budgetedTotal < inputs.maxConnections
|
||||
}
|
||||
}
|
||||
|
||||
export function readRelayCloudSqlConnectionBudget({
|
||||
sources,
|
||||
appConsumers,
|
||||
proposedAsiaCellCount = 3,
|
||||
asiaPoolMax = 10,
|
||||
maxConnections,
|
||||
maintenanceAdminAllowance = 5,
|
||||
explicitReserve = 10
|
||||
} = {}) {
|
||||
const read = (name) => sources?.[name] ?? readFileSync(DEFAULT_PATHS[name], 'utf8')
|
||||
const apps = readProductionCloudSqlAppConsumers(appConsumers)
|
||||
const productionTfvars = read('productionTfvars')
|
||||
const terraformVariables = read('terraformVariables')
|
||||
const relayConfig = read('relayConfig')
|
||||
const cells = productionCells(
|
||||
productionTfvars,
|
||||
requiredInteger(relayConfig, /RELAY_DATABASE_POOL_MAX\s*=\s*(\d+)/, 'Relay pool maximum')
|
||||
)
|
||||
const poweredCells = cells.filter(({ fenced }) => !fenced)
|
||||
const configuredAsiaCells = poweredCells.filter(({ region }) => region === 'asia-east2')
|
||||
const nonAsiaCells = poweredCells.filter(({ region }) => region !== 'asia-east2')
|
||||
const cellPoolTotal = nonAsiaCells.reduce((total, cell) => total + cell.poolMax, 0)
|
||||
const asiaCellCount = configuredAsiaCells.length || proposedAsiaCellCount
|
||||
const configuredAsiaPoolMax = configuredAsiaCells[0]?.poolMax ?? asiaPoolMax
|
||||
if (configuredAsiaCells.some(({ poolMax }) => poolMax !== configuredAsiaPoolMax)) {
|
||||
throw new Error('Asia Relay cells must use one checked pool maximum')
|
||||
}
|
||||
return calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal,
|
||||
asiaCellCount,
|
||||
asiaPoolMax: configuredAsiaPoolMax,
|
||||
directorInstances: requiredInteger(productionTfvars, /relay_max_instances\s*=\s*(\d+)/, 'director instances'),
|
||||
directorPoolMax: requiredInteger(
|
||||
terraformVariables,
|
||||
/variable\s+"relay_director_database_pool_max"[\s\S]*?default\s*=\s*(\d+)/,
|
||||
'director pool maximum'
|
||||
),
|
||||
authInstances: apps.authInstances,
|
||||
authPoolMax: apps.authPoolMax,
|
||||
apiInstances: apps.apiInstances,
|
||||
apiPoolMax: apps.apiPoolMax,
|
||||
maxConnections: maxConnections ?? apps.maxConnections,
|
||||
maintenanceAdminAllowance,
|
||||
explicitReserve
|
||||
})
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
console.log(JSON.stringify({ event: 'relay_cloud_sql_connection_budget', ...report }, null, 2))
|
||||
if (!report.withinBudget) process.exitCode = 1
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
calculateRelayCloudSqlConnectionBudget,
|
||||
readRelayCloudSqlConnectionBudget
|
||||
} from './relay-cloud-sql-connection-budget.mjs'
|
||||
|
||||
test('production shared consumers keep allowance and reserve below the ceiling', () => {
|
||||
// cells: 22 pools at 10 (220, C32/C33 included) + the seven asia-east2 pools at 16 (112).
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
|
||||
assert.deepEqual(report.consumers, { cells: 332, directors: 15, auth: 20, api: 50 })
|
||||
assert.deepEqual(report.asia, { cells: 7, poolMax: 16 })
|
||||
assert.equal(report.configuredMaximum, 417)
|
||||
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
|
||||
assert.equal(report.rolloutOverlap.apiCandidate, 65)
|
||||
assert.equal(report.rolloutOverlap.authCandidate, 35)
|
||||
assert.equal(report.rolloutOverlap.relayCells, 15)
|
||||
assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15)
|
||||
assert.equal(report.rolloutOverlap.maximum, 65)
|
||||
assert.equal(report.maintenanceAdminAllowance, 5)
|
||||
assert.equal(report.explicitReserve, 10)
|
||||
assert.equal(report.usableCeiling, 490)
|
||||
assert.equal(report.operatingMaximum, 487)
|
||||
assert.equal(report.remainingWithinUsableCeiling, 3)
|
||||
assert.equal(report.budgetedTotal, 497)
|
||||
assert.equal(report.unallocated, 3)
|
||||
assert.equal(report.withinBudget, true)
|
||||
})
|
||||
|
||||
test('fails closed when pool growth consumes the explicit reserve', () => {
|
||||
const report = calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal: 200,
|
||||
asiaCellCount: 3,
|
||||
asiaPoolMax: 20,
|
||||
directorInstances: 5,
|
||||
directorPoolMax: 3,
|
||||
authInstances: 2,
|
||||
authPoolMax: 10,
|
||||
apiInstances: 20,
|
||||
apiPoolMax: 5,
|
||||
maxConnections: 400,
|
||||
maintenanceAdminAllowance: 5,
|
||||
explicitReserve: 10
|
||||
})
|
||||
|
||||
assert.equal(report.operatingMaximum, 515)
|
||||
assert.equal(report.withinBudget, false)
|
||||
})
|
||||
|
||||
test('excludes fenced cell pools and reads per-cell pool overrides', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget({
|
||||
proposedAsiaCellCount: 1,
|
||||
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
|
||||
sources: {
|
||||
productionTfvars: `
|
||||
relay_max_instances = 1
|
||||
relay_gce_fenced_cells = ["production-gce-c1"]
|
||||
relay_gce_cells = {
|
||||
"production-gce-c1" = { database_pool_max = 99
|
||||
}
|
||||
"production-gce-c2" = { database_pool_max = 4
|
||||
}
|
||||
}
|
||||
`,
|
||||
terraformVariables: [
|
||||
'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
'variable "push_max_instances" { default = 1 }',
|
||||
'variable "push_database_pool_max" { default = 2 }'
|
||||
].join('\n'),
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 100,
|
||||
maintenanceAdminAllowance: 1,
|
||||
explicitReserve: 1
|
||||
})
|
||||
|
||||
assert.equal(report.consumers.cells, 14)
|
||||
assert.equal(report.operatingMaximum, 46)
|
||||
assert.equal(report.budgetedTotal, 47)
|
||||
})
|
||||
|
||||
test('refuses an Asia cell whose pool differs from its siblings', () => {
|
||||
const budget = (c30PoolMax) => readRelayCloudSqlConnectionBudget({
|
||||
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 500 },
|
||||
sources: {
|
||||
productionTfvars: `
|
||||
relay_max_instances = 1
|
||||
relay_gce_fenced_cells = []
|
||||
relay_gce_cells = {
|
||||
${['c27', 'c28', 'c29', 'c30'].map((hostname) => ` "production-gce-${hostname}" = {
|
||||
region = "asia-east2"
|
||||
database_pool_max = ${hostname === 'c30' ? c30PoolMax : 16}
|
||||
}`).join('\n')}
|
||||
}
|
||||
`,
|
||||
terraformVariables: [
|
||||
'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
'variable "push_max_instances" { default = 1 }',
|
||||
'variable "push_database_pool_max" { default = 2 }'
|
||||
].join('\n'),
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 500
|
||||
})
|
||||
assert.deepEqual(budget(16).asia, { cells: 4, poolMax: 16 })
|
||||
assert.throws(() => budget(10), /Asia Relay cells must use one checked pool maximum/)
|
||||
})
|
||||
|
||||
test('dedicated push scaling does not consume shared capacity', () => {
|
||||
const report = readRelayCloudSqlConnectionBudget({
|
||||
proposedAsiaCellCount: 1,
|
||||
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
|
||||
sources: {
|
||||
productionTfvars: `
|
||||
relay_max_instances = 1
|
||||
push_max_instances = 3
|
||||
relay_gce_fenced_cells = []
|
||||
relay_gce_cells = {
|
||||
"production-gce-c2" = { database_pool_max = 4
|
||||
}
|
||||
}
|
||||
`,
|
||||
terraformVariables: [
|
||||
'variable "relay_director_database_pool_max" { default = 3 }',
|
||||
'variable "push_max_instances" { default = 1 }',
|
||||
'variable "push_database_pool_max" { default = 2 }'
|
||||
].join('\n'),
|
||||
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
|
||||
},
|
||||
maxConnections: 100,
|
||||
maintenanceAdminAllowance: 1,
|
||||
explicitReserve: 1
|
||||
})
|
||||
|
||||
assert.equal(report.consumers.push, undefined)
|
||||
assert.equal(report.rolloutOverlap.pushCandidate, undefined)
|
||||
assert.equal(report.operatingMaximum, 46)
|
||||
})
|
||||
|
||||
test('requires strict headroom below the physical ceiling', () => {
|
||||
const report = calculateRelayCloudSqlConnectionBudget({
|
||||
cellPoolTotal: 20,
|
||||
asiaCellCount: 0,
|
||||
asiaPoolMax: 10,
|
||||
directorInstances: 1,
|
||||
directorPoolMax: 3,
|
||||
authInstances: 1,
|
||||
authPoolMax: 10,
|
||||
apiInstances: 1,
|
||||
apiPoolMax: 5,
|
||||
maxConnections: 50,
|
||||
maintenanceAdminAllowance: 9,
|
||||
explicitReserve: 3
|
||||
})
|
||||
|
||||
assert.equal(report.budgetedTotal, 63)
|
||||
assert.equal(report.withinBudget, false)
|
||||
})
|
||||
|
||||
test('pages Relay channels when Cloud SQL backends consume headroom', () => {
|
||||
const terraform = readFileSync(
|
||||
new URL('../../infra/terraform/relay-observability.tf', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const policy = terraform.match(
|
||||
/resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" \{([\s\S]*?)\n\}/
|
||||
)?.[1]
|
||||
|
||||
assert.ok(policy)
|
||||
assert.match(policy, /notification_channels\s*=\s*var\.relay_alert_notification_channels/)
|
||||
})
|
||||
@@ -1,37 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Why: the metric keys on the event name and fields the switch-file reader writes. A rename on
|
||||
// either side silences the alert without failing anything.
|
||||
|
||||
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
|
||||
const terraform = read('../../infra/terraform/relay-observability.tf')
|
||||
const channel = read('../../apps/relay/src/relay-control-flag-channel.ts')
|
||||
const cellFlags = read('../../apps/relay/src/cell-flags.ts')
|
||||
|
||||
const block = (kind, name) => {
|
||||
const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
|
||||
assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
|
||||
return body
|
||||
}
|
||||
|
||||
test('the metric counts the event and fields the reader writes', () => {
|
||||
const unreadableLine = /event: 'orca_relay_control_flags_unreadable',\s*object: input\.objectName,\s*failure,/
|
||||
assert.match(channel, unreadableLine)
|
||||
assert.match(cellFlags, /return `cells\/\$\{cellId\}\.json`/)
|
||||
const metric = block('google_logging_metric', 'relay_control_flags_unreadable')
|
||||
assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_control_flags_unreadable\\"'))
|
||||
assert.ok(metric.includes('REGEXP_EXTRACT(jsonPayload.object, \\"cells/([^.]+)\\")'))
|
||||
assert.ok(metric.includes('failure = "EXTRACT(jsonPayload.failure)"'))
|
||||
})
|
||||
|
||||
test('the alert fires on one line per cell and stays in the console', () => {
|
||||
const alert = block('google_monitoring_alert_policy', 'relay_control_flags_unreadable')
|
||||
// The reader logs once per change of cause, so a stuck cell writes one line; a higher bar misses it.
|
||||
assert.match(alert, /threshold_value = 0\n/)
|
||||
assert.match(alert, /notification_channels = \[\]/)
|
||||
assert.ok(alert.includes('metric.type=\\"logging.googleapis.com/user/orca_relay_control_flags_unreadable\\"'))
|
||||
assert.ok(alert.includes('group_by_fields = ["metric.label.\\"cell_id\\""]'))
|
||||
})
|
||||
@@ -1,76 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Why: these alerts key on event names and field paths the relay writes (the rejection fence and
|
||||
// the lease shadow). A rename on either side silences the alert without failing anything.
|
||||
|
||||
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
|
||||
const terraform = read('../../infra/terraform/relay-observability.tf')
|
||||
|
||||
const block = (kind, name) => {
|
||||
const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
|
||||
assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
|
||||
return body
|
||||
}
|
||||
|
||||
const incidentMetric = (key) => {
|
||||
const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1]
|
||||
assert.ok(body, `relay_incident_metrics.${key} not found`)
|
||||
return body
|
||||
}
|
||||
|
||||
test('the fence metrics count the exact events on directors and cells', () => {
|
||||
for (const [key, event] of [
|
||||
['database_rejection_fenced', 'orca_relay_database_rejection_fenced'],
|
||||
['process_fatal', 'orca_relay_process_fatal']
|
||||
]) {
|
||||
const metric = incidentMetric(key)
|
||||
assert.ok(metric.includes(`jsonPayload.event=\\"${event}\\"`), key)
|
||||
assert.ok(metric.includes('resource.type=\\"cloud_run_revision\\"'), key)
|
||||
assert.ok(metric.includes('resource.type=\\"gce_instance\\"'), key)
|
||||
}
|
||||
})
|
||||
|
||||
test('the fenced alert is one fleet-wide series, and the fatal alert leaves cells to the exit alert', () => {
|
||||
const fenced = block('google_monitoring_alert_policy', 'relay_database_rejection_fenced')
|
||||
assert.ok(
|
||||
fenced.includes(
|
||||
'query = "sum(increase(logging_googleapis_com:user_orca_relay_database_rejection_fenced[5m])) > 0"'
|
||||
)
|
||||
)
|
||||
const fatal = block('google_monitoring_alert_policy', 'relay_process_fatal')
|
||||
assert.ok(fatal.includes('resource.type=\\"cloud_run_revision\\" AND metric.type=\\"logging.googleapis.com/user/orca_relay_process_fatal\\"'))
|
||||
assert.doesNotMatch(fatal, /gce_instance/)
|
||||
// The cell side of a fatal rejection still pages, once, through the exit alert.
|
||||
block('google_monitoring_alert_policy', 'relay_cell_process_exit')
|
||||
})
|
||||
|
||||
test('the bad-signature alert stays in the console', () => {
|
||||
const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature')
|
||||
assert.match(alert, /notification_channels = \[\]/)
|
||||
})
|
||||
|
||||
test('the lease shadow metrics extract every class the cell writes', () => {
|
||||
const classes = [
|
||||
'agree',
|
||||
'disagree',
|
||||
'absent',
|
||||
'expired',
|
||||
'badSignature',
|
||||
'wrongHost',
|
||||
'wrongCell',
|
||||
'epochBehind',
|
||||
'epochAhead'
|
||||
]
|
||||
for (const field of classes) {
|
||||
assert.match(terraform, new RegExp(`field = "classes\\.${field}"`), field)
|
||||
}
|
||||
assert.match(terraform, /field = "dbReadMs\.p99"/)
|
||||
assert.match(terraform, /field = "dbRefused"/)
|
||||
const metric = block('google_logging_metric', 'relay_assignment_lease_shadow')
|
||||
assert.ok(metric.includes('jsonPayload.event=\\"orca_relay_assignment_lease_shadow\\"'))
|
||||
const alert = block('google_monitoring_alert_policy', 'relay_assignment_lease_bad_signature')
|
||||
assert.ok(alert.includes('user_orca_relay_assignment_lease_shadow_bad_signature_sum'))
|
||||
})
|
||||
@@ -1,106 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { parseArguments as parseBlueGreen } from './deploy-relay-blue-green.mjs'
|
||||
import { parseArguments as parseGceCandidate } from './deploy-relay-gce-candidate.mjs'
|
||||
import { RELAY_WORKFLOW_DIRECTORY, readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
// Every workflow's real argv, literals as written: a flag combination the parser refuses
|
||||
// (staging's lone --admin-audience once was one) fails here instead of on a release.
|
||||
|
||||
const PROJECT = 'orca-test-project'
|
||||
const DIGEST = `sha256:${'a'.repeat(64)}`
|
||||
|
||||
// Shell-expanded values only; anything a workflow writes literally is parsed as written.
|
||||
const SAMPLES = {
|
||||
project: PROJECT,
|
||||
image: `relay@${DIGEST}`,
|
||||
'predecessor-image-digest': DIGEST,
|
||||
'director-cells-json': JSON.stringify([
|
||||
{ id: 'cell-a', url: 'https://cell-a.example.test', region: 'us-central1', capacityRequests: 10, initiallyEnabled: true }
|
||||
]),
|
||||
'regional-placement-secret-version': '3',
|
||||
'min-instances': '1',
|
||||
'max-instances': '5',
|
||||
'prune-revisions': 'false',
|
||||
'bootstrap-runtime-identity': 'false',
|
||||
'expected-rehome-generation': '4',
|
||||
'region-correction-cohort-percent': '10',
|
||||
'rehome-audience': 'https://relay.example.test/v1/admin/host-drain',
|
||||
'director-origin': 'https://relay.example.test',
|
||||
'admin-audience': 'https://relay.example.test/v1/admin/drain',
|
||||
'reserve-placement': 'preserve',
|
||||
'shadow-seat-feed-cells': 'preserve',
|
||||
mode: 'audit'
|
||||
}
|
||||
|
||||
function sample(flag) {
|
||||
if (flag in SAMPLES) return SAMPLES[flag]
|
||||
if (flag.endsWith('service-account')) return `${flag.slice(0, 20).replace(/-$/, '')}@${PROJECT}.iam.gserviceaccount.com`
|
||||
return `sample-${flag}`
|
||||
}
|
||||
|
||||
// Flags a workflow passes through a bash array (`"${name[@]}"`), from every `name=(...)` it
|
||||
// assigns; parsing them all at once is the widest set that workflow can send.
|
||||
function arrayArguments(source, name) {
|
||||
const flags = []
|
||||
for (const match of source.matchAll(new RegExp(`\\b${name}=\\(([^)]*)\\)`, 'g'))) {
|
||||
for (const entry of match[1].matchAll(/(--[a-z0-9-]+)\s+("[^"]*"|\S+)/g)) {
|
||||
flags.push([entry[1], entry[2].replace(/^"(.*)"$/, '$1')])
|
||||
}
|
||||
}
|
||||
if (flags.length === 0) throw new Error(`no flags assigned to ${name}`)
|
||||
return flags
|
||||
}
|
||||
|
||||
function invocations(source, script) {
|
||||
const lines = source.split('\n')
|
||||
const found = []
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
if (!lines[index].includes(`dev/scripts/${script}`)) continue
|
||||
const argv = []
|
||||
let line = lines[index]
|
||||
while (line.trimEnd().endsWith('\\')) {
|
||||
line = lines[++index]
|
||||
const array = /^\s*"\$\{([a-z_]+)\[@\]\}"\s*\\?\s*$/.exec(line)
|
||||
if (array) {
|
||||
for (const [flag, value] of arrayArguments(source, array[1])) {
|
||||
argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const match = /^\s*(--[a-z0-9-]+)\s+(.+?)\s*\\?\s*$/.exec(line)
|
||||
if (!match) throw new Error(`unparsed ${script} argument line: ${line.trim()}`)
|
||||
const [, flag, raw] = match
|
||||
// A trailing `)"` closes the command substitution the invocation sits in.
|
||||
const value = raw.replace(/\)"$/, '').replace(/^"(.*)"$/, '$1')
|
||||
argv.push(flag, value.includes('$') ? sample(flag.slice(2)) : value)
|
||||
}
|
||||
found.push(argv)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
const SCRIPTS = [
|
||||
['deploy-relay-blue-green.mjs', parseBlueGreen],
|
||||
['deploy-relay-gce-candidate.mjs', parseGceCandidate]
|
||||
]
|
||||
|
||||
for (const [script, parse] of SCRIPTS) {
|
||||
test(`every workflow's ${script} argv parses`, () => {
|
||||
let count = 0
|
||||
for (const file of readdirSync(RELAY_WORKFLOW_DIRECTORY).filter((name) => name.endsWith('.yml'))) {
|
||||
for (const argv of invocations(readFileSync(new URL(file, RELAY_WORKFLOW_DIRECTORY), 'utf8'), script)) {
|
||||
count++
|
||||
assert.doesNotThrow(() => parse(argv), `${file}: ${argv.join(' ')}`)
|
||||
}
|
||||
}
|
||||
assert.ok(count > 0, `no workflow runs ${script}`)
|
||||
})
|
||||
}
|
||||
|
||||
test('the production and staging director workflows are among the parsed', () => {
|
||||
for (const name of ['deploy-relay-production-director.yml', 'deploy-relay-staging.yml']) {
|
||||
assert.equal(invocations(readRelayWorkflow(name), 'deploy-relay-blue-green.mjs').length, 1, name)
|
||||
}
|
||||
})
|
||||
@@ -1,335 +0,0 @@
|
||||
// Pure pieces of the director deploy driver: the exact inputs each existing workflow receives,
|
||||
// and the parsers that read a run's result back. No process, network, or clock access here.
|
||||
|
||||
import {
|
||||
RELAY_GITHUB_REPOSITORY,
|
||||
RELAY_WORKFLOW_FILE_PREFIX,
|
||||
relayWorkflowFile
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
export const REPOSITORY = RELAY_GITHUB_REPOSITORY
|
||||
export const WORKFLOW_REF = 'main'
|
||||
export const PROJECT = 'onorca-cloud'
|
||||
export const REGION = 'us-central1'
|
||||
export const DIRECTOR_SERVICE = 'orca-cloud-relay'
|
||||
export const ROLLBACK_TAG = 'selector-rollback'
|
||||
export const IMAGE_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
|
||||
// Any reviewed registration wave is accepted by admission inspect; the launch wave never changes.
|
||||
const ADMISSION_INSPECT_CELLS = 'production-gce-c27,production-gce-c28,production-gce-c29'
|
||||
|
||||
const workflow = (name, title) => ({ file: relayWorkflowFile(name), name: title })
|
||||
export const WORKFLOWS = {
|
||||
publish: workflow('publish-relay-production.yml', 'Publish Relay Production Image'),
|
||||
director: workflow('deploy-relay-production-director.yml', 'Deploy Relay Production Director'),
|
||||
rehome: workflow('operate-relay-production-rehome.yml', 'Operate Relay Production Rehome'),
|
||||
admission: workflow('operate-relay-asia-admission.yml', 'Operate Relay Asia Admission'),
|
||||
monitor: workflow('monitor-relay-production.yml', 'Monitor Relay Production')
|
||||
}
|
||||
|
||||
// Read-only or unrelated to the production rollout lane, so they never block a deploy.
|
||||
const NON_BLOCKING_WORKFLOWS = new Set(
|
||||
['verify.yml', 'monitor-relay-clock-skew.yml'].map(relayWorkflowFile)
|
||||
)
|
||||
|
||||
const DIGEST = /^sha256:[a-f0-9]{64}$/
|
||||
const COMMIT = /^[a-f0-9]{40}$/
|
||||
const PRODUCTION_CELL = /^production-gce-c[1-9][0-9]?$/
|
||||
const MEMBERSHIP_KEYS = ['existingOnly', 'migrationOnly', 'general']
|
||||
|
||||
export function requireDigest(value, label) {
|
||||
if (typeof value !== 'string' || !DIGEST.test(value))
|
||||
throw new Error(`${label} is not an immutable sha256 digest`)
|
||||
return value
|
||||
}
|
||||
|
||||
export function requireCommit(value, label) {
|
||||
if (typeof value !== 'string' || !COMMIT.test(value))
|
||||
throw new Error(`${label} is not a full commit SHA`)
|
||||
return value
|
||||
}
|
||||
|
||||
function requireGeneration(value, label) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} is invalid`)
|
||||
return value
|
||||
}
|
||||
|
||||
export function blocksDeploy(path) {
|
||||
// A `@ref` suffix never appears on run paths today; stripping it keeps the check fail-closed.
|
||||
const file = String(path ?? '')
|
||||
.split('@')[0]
|
||||
.split('/')
|
||||
.at(-1)
|
||||
return (
|
||||
file.startsWith(RELAY_WORKFLOW_FILE_PREFIX) &&
|
||||
file.endsWith('.yml') &&
|
||||
!NON_BLOCKING_WORKFLOWS.has(file)
|
||||
)
|
||||
}
|
||||
|
||||
export function membershipInput(cells) {
|
||||
return cells.length === 0 ? 'none' : [...cells].sort().join(',')
|
||||
}
|
||||
|
||||
export function parseSelector(value, label) {
|
||||
const selector = {
|
||||
generation: requireGeneration(value?.generation, `${label} generation`),
|
||||
membership: {}
|
||||
}
|
||||
for (const key of MEMBERSHIP_KEYS) {
|
||||
const cells = value?.membership?.[key]
|
||||
if (!Array.isArray(cells) || cells.some((cell) => !PRODUCTION_CELL.test(cell))) {
|
||||
throw new Error(`${label} ${key} membership is invalid`)
|
||||
}
|
||||
selector.membership[key] = [...cells].sort()
|
||||
}
|
||||
const all = MEMBERSHIP_KEYS.flatMap((key) => selector.membership[key])
|
||||
if (new Set(all).size !== all.length) throw new Error(`${label} membership has duplicates`)
|
||||
return selector
|
||||
}
|
||||
|
||||
function selectorInputs(selector) {
|
||||
return {
|
||||
'expected-selector-generation': String(selector.generation),
|
||||
'expected-existing-only-cells': membershipInput(selector.membership.existingOnly),
|
||||
'expected-migration-only-cells': membershipInput(selector.membership.migrationOnly),
|
||||
'expected-general-cells': membershipInput(selector.membership.general)
|
||||
}
|
||||
}
|
||||
|
||||
export function parseControl(value, label) {
|
||||
const control = value ?? {}
|
||||
const integers = [
|
||||
'generation',
|
||||
'notBefore',
|
||||
'ratePerMinute',
|
||||
'preferenceMaxAgeMs',
|
||||
'drainGraceMs'
|
||||
]
|
||||
if (
|
||||
typeof control.enabled !== 'boolean' ||
|
||||
integers.some((key) => !Number.isSafeInteger(control[key]))
|
||||
) {
|
||||
throw new Error(`${label} is not a complete regional rehome control`)
|
||||
}
|
||||
if (control.hostCooldownMs !== undefined && !Number.isSafeInteger(control.hostCooldownMs)) {
|
||||
throw new Error(`${label} host cooldown is invalid`)
|
||||
}
|
||||
return Object.fromEntries(
|
||||
[...integers, 'enabled', 'hostCooldownMs'].map((key) => [key, control[key]])
|
||||
)
|
||||
}
|
||||
|
||||
const INTEGER_INPUT = /^(0|[1-9][0-9]*)$/
|
||||
|
||||
/**
|
||||
* The last check before `gh workflow run`. Builders also render dry-run placeholders such as
|
||||
* `<published digest>`; this guarantees none of them, or a malformed digest, is ever dispatched.
|
||||
*/
|
||||
export function validateDispatchInputs(inputs) {
|
||||
for (const [key, value] of Object.entries(inputs)) {
|
||||
if (typeof value !== 'string' || value.startsWith('<'))
|
||||
throw new Error(`input ${key} is not resolved`)
|
||||
if (key.endsWith('digest') && !DIGEST.test(value))
|
||||
throw new Error(`input ${key} is not a sha256 digest`)
|
||||
if (
|
||||
(key.includes('generation') ||
|
||||
['not-before', 'monitor-run-id', 'monitor-run-attempt'].includes(key)) &&
|
||||
!INTEGER_INPUT.test(value)
|
||||
) {
|
||||
throw new Error(`input ${key} is not an integer`)
|
||||
}
|
||||
}
|
||||
return inputs
|
||||
}
|
||||
|
||||
export function admissionInspectInputs(servingDigest) {
|
||||
return {
|
||||
environment: 'production',
|
||||
mode: 'inspect',
|
||||
'cell-ids': ADMISSION_INSPECT_CELLS,
|
||||
'image-digest': servingDigest
|
||||
}
|
||||
}
|
||||
|
||||
function rehomeInputs(mode, { director, selector, controlGeneration }) {
|
||||
return {
|
||||
mode,
|
||||
'director-image-digest': director.servingDigest,
|
||||
'rollback-image-digest': director.rollbackDigest,
|
||||
...selectorInputs(selector),
|
||||
'expected-control-generation': String(controlGeneration)
|
||||
}
|
||||
}
|
||||
|
||||
export function rehomeInspectInputs(context) {
|
||||
return rehomeInputs('inspect', context)
|
||||
}
|
||||
|
||||
// Pause keeps every durable field as inspected; only `enabled` and the generation change.
|
||||
// Every `confirmation` is the phrase the operator typed, never filled in here.
|
||||
export function rehomePauseInputs({ director, selector, control, confirmation }) {
|
||||
return {
|
||||
...rehomeInputs('pause', { director, selector, controlGeneration: control.generation }),
|
||||
'not-before': String(control.notBefore),
|
||||
'rate-per-minute': String(control.ratePerMinute),
|
||||
'preference-max-age-ms': String(control.preferenceMaxAgeMs),
|
||||
'host-cooldown-ms': String(control.hostCooldownMs ?? 604_800_000),
|
||||
'drain-grace-ms': String(control.drainGraceMs),
|
||||
confirmation
|
||||
}
|
||||
}
|
||||
|
||||
export function rehomeEnableInputs({
|
||||
director,
|
||||
selector,
|
||||
control,
|
||||
controlGeneration,
|
||||
notBefore,
|
||||
monitor,
|
||||
confirmation
|
||||
}) {
|
||||
return {
|
||||
...rehomeInputs('enable', { director, selector, controlGeneration }),
|
||||
'not-before': String(notBefore),
|
||||
// The enable job accepts exactly 10 per minute.
|
||||
'rate-per-minute': '10',
|
||||
'preference-max-age-ms': String(control.preferenceMaxAgeMs),
|
||||
'host-cooldown-ms': String(control.hostCooldownMs),
|
||||
'drain-grace-ms': String(control.drainGraceMs),
|
||||
'monitor-run-id': String(monitor.runId),
|
||||
'monitor-run-attempt': String(monitor.attempt),
|
||||
confirmation
|
||||
}
|
||||
}
|
||||
|
||||
export function publishInputs() {
|
||||
return { mode: 'publish' }
|
||||
}
|
||||
|
||||
export function directorDeployInputs({ imageDigest, predecessorDigest, rehomeGeneration }) {
|
||||
return {
|
||||
'image-digest': imageDigest,
|
||||
'regional-placement-mode': 'preserve',
|
||||
'region-correction-cohort-percent': 'preserve',
|
||||
'reserve-placement': 'preserve',
|
||||
'shadow-seat-feed-cells': 'preserve',
|
||||
'prune-incompatible-revisions': 'false',
|
||||
'expected-rehome-generation': String(rehomeGeneration),
|
||||
'bootstrap-runtime-identity': 'false',
|
||||
// Required by the form even without the bootstrap; it is only format-checked then.
|
||||
'predecessor-image-digest': predecessorDigest
|
||||
}
|
||||
}
|
||||
|
||||
export function configureInputs({
|
||||
cells,
|
||||
cellImageDigest,
|
||||
directorDigest,
|
||||
selectorGeneration,
|
||||
confirmation
|
||||
}) {
|
||||
return {
|
||||
environment: 'production',
|
||||
mode: 'configure',
|
||||
'cell-ids': cells.join(','),
|
||||
'selector-generation': String(selectorGeneration),
|
||||
'image-digest': cellImageDigest,
|
||||
'director-image-digest': directorDigest,
|
||||
confirmation
|
||||
}
|
||||
}
|
||||
|
||||
export function monitorDryRunInputs(selector) {
|
||||
return {
|
||||
mode: 'dry-run',
|
||||
...selectorInputs(selector),
|
||||
'migration-policy': 'strict',
|
||||
'recovery-source-cell-id': 'none',
|
||||
'capacity-cell-id': 'none'
|
||||
}
|
||||
}
|
||||
|
||||
export function parseConfigureWave(value) {
|
||||
const separator = String(value).lastIndexOf('=')
|
||||
const cells = String(value)
|
||||
.slice(0, separator)
|
||||
.split(',')
|
||||
.map((cell) => cell.trim())
|
||||
.filter(Boolean)
|
||||
const cellImageDigest = String(value).slice(separator + 1)
|
||||
if (
|
||||
separator < 1 ||
|
||||
cells.length === 0 ||
|
||||
new Set(cells).size !== cells.length ||
|
||||
cells.some((cell) => !PRODUCTION_CELL.test(cell))
|
||||
) {
|
||||
throw new Error('--configure must be <cell>[,<cell>...]=sha256:<cell image digest>')
|
||||
}
|
||||
return { cells, cellImageDigest: requireDigest(cellImageDigest, '--configure cell image digest') }
|
||||
}
|
||||
|
||||
/** The single 100% revision and the selector-rollback revision of `gcloud run services describe`. */
|
||||
export function directorRevisions(service) {
|
||||
const traffic = Array.isArray(service?.status?.traffic) ? service.status.traffic : []
|
||||
const serving = traffic.filter((entry) => (entry.percent ?? 0) > 0)
|
||||
const rollback = traffic.filter((entry) => entry.tag === ROLLBACK_TAG)
|
||||
if (serving.length !== 1 || serving[0].percent !== 100 || !serving[0].revisionName) {
|
||||
throw new Error('director does not serve one revision at 100%')
|
||||
}
|
||||
if (rollback.length !== 1 || !rollback[0].revisionName) {
|
||||
throw new Error(`director has no single ${ROLLBACK_TAG} revision`)
|
||||
}
|
||||
return { servingRevision: serving[0].revisionName, rollbackRevision: rollback[0].revisionName }
|
||||
}
|
||||
|
||||
export function revisionDigest(revision, label) {
|
||||
const image = revision?.spec?.containers?.[0]?.image
|
||||
const [repository, digest] = String(image ?? '').split('@')
|
||||
if (repository !== IMAGE_REPOSITORY)
|
||||
throw new Error(`${label} does not run the production relay image`)
|
||||
return requireDigest(digest, `${label} image`)
|
||||
}
|
||||
|
||||
/** The relay push line, `sha-<commit>: digest: sha256:… size: …`, must name the registry digest. */
|
||||
export function logConfirmsPublishedDigest(log, commit, digest) {
|
||||
return String(log)
|
||||
.split('\n')
|
||||
.some((line) => line.includes(`sha-${commit}: digest: ${digest} size:`))
|
||||
}
|
||||
|
||||
/** The last `relay_regional_rehome_control` JSON line a rehome run printed, of any mode. */
|
||||
export function rehomeResultFromLog(log) {
|
||||
let found
|
||||
for (const line of String(log).split('\n')) {
|
||||
const start = line.indexOf('{"event":"relay_regional_rehome_control"')
|
||||
if (start < 0) continue
|
||||
try {
|
||||
found = JSON.parse(line.slice(start).trim())
|
||||
} catch {
|
||||
// A truncated or echoed line is not the result.
|
||||
}
|
||||
}
|
||||
if (!found) throw new Error('the rehome run printed no control result')
|
||||
return {
|
||||
mode: found.mode,
|
||||
recovered: found.recovered,
|
||||
control: parseControl(found.control, `rehome ${found.mode} control`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The generation a run paused rehome at, or undefined. Only two lines prove a run paused it: a
|
||||
* `pause`, and a failed enable's `recover-enable` that itself disabled rehome (`recovered: true`).
|
||||
* `recovered: false` means rehome was already disabled, possibly by a director safety pause.
|
||||
*/
|
||||
export function pausedGeneration(result) {
|
||||
const paused =
|
||||
!result.control.enabled &&
|
||||
(result.mode === 'pause' || (result.mode === 'recover-enable' && result.recovered === true))
|
||||
return paused ? result.control.generation : undefined
|
||||
}
|
||||
|
||||
export function admissionInspectResult(result) {
|
||||
if (result?.mode !== 'inspect') throw new Error('admission result is not an inspect')
|
||||
return parseSelector(result, 'admission selector')
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
RELAY_REPOSITORY_ROOT,
|
||||
relayTreePath,
|
||||
relayWorkflowPath
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
const SHA = /^[a-f0-9]{40}$/
|
||||
|
||||
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
|
||||
// production; identical content across two commits is what makes the older commit's verdict binding.
|
||||
export const TRUSTED_EVIDENCE_CODE_PATHS = [
|
||||
// Produces and seals the 15-minute dry-run evidence.
|
||||
relayWorkflowPath('monitor-relay-production.yml'),
|
||||
relayWorkflowPath('monitor-relay-production-job.yml'),
|
||||
// Rehome enable downloads it, verifies its authority, and mutates production on it.
|
||||
relayWorkflowPath('operate-relay-production-rehome.yml'),
|
||||
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
|
||||
// Same-cap seals and spends canary authority, which a later batch accepts only from this code.
|
||||
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
|
||||
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
|
||||
// Sealing, verification, the wave/canary authority, and the path constants below.
|
||||
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
|
||||
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
|
||||
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
|
||||
relayTreePath('dev/scripts/relay-repository.mjs'),
|
||||
// Every other script those jobs run against live production.
|
||||
relayTreePath('dev/scripts/check-relay-same-cap-headroom.mjs'),
|
||||
relayTreePath('dev/scripts/infra.mjs'),
|
||||
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
|
||||
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
|
||||
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
|
||||
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
|
||||
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
|
||||
relayTreePath('dev/scripts/relay-admin-transient-retry.mjs'),
|
||||
relayTreePath('dev/scripts/relay-reserve-cell-refusal.mjs'),
|
||||
// The exit metric filter the same-cap pre-drain sample's crash rule reads.
|
||||
relayTreePath('infra/terraform/relay-observability.tf'),
|
||||
// The monitor, the live preflight recheck, and the same-cap pre-drain sample, plus anything that
|
||||
// changes their behaviour.
|
||||
relayTreePath('apps/relay-ops'),
|
||||
relayTreePath('package.json'),
|
||||
relayTreePath('pnpm-lock.yaml'),
|
||||
relayTreePath('pnpm-workspace.yaml'),
|
||||
// The Cloud SQL rollout lease every mutation job takes and releases.
|
||||
'.github/actions/cloud-sql-rollout-lease'
|
||||
]
|
||||
|
||||
function git(root, args) {
|
||||
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
|
||||
if (result.error) throw new Error('relay evidence provenance cannot run git')
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts evidence sealed at a different commit only when the current commit descends from it and
|
||||
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
|
||||
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
|
||||
*/
|
||||
export function requireSameEvidenceCode({
|
||||
sealedSha,
|
||||
currentSha,
|
||||
label,
|
||||
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
|
||||
}) {
|
||||
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
|
||||
throw new Error(`${label} commit is invalid`)
|
||||
}
|
||||
if (sealedSha === currentSha) return
|
||||
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
|
||||
throw new Error(`${label} commit cannot be compared without a git checkout`)
|
||||
}
|
||||
for (const sha of [sealedSha, currentSha]) {
|
||||
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
|
||||
throw new Error(
|
||||
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
|
||||
)
|
||||
}
|
||||
}
|
||||
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
|
||||
if (ancestry.status === 1) {
|
||||
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
|
||||
}
|
||||
if (ancestry.status !== 0) {
|
||||
throw new Error(`${label} commit ancestry could not be determined`)
|
||||
}
|
||||
const diff = git(repositoryRoot, [
|
||||
'diff',
|
||||
'--name-only',
|
||||
sealedSha,
|
||||
currentSha,
|
||||
'--',
|
||||
...TRUSTED_EVIDENCE_CODE_PATHS
|
||||
])
|
||||
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
|
||||
const changed = diff.stdout.split('\n').filter(Boolean)
|
||||
if (changed.length > 0) {
|
||||
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,84 +0,0 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Committed images lag what same-cap rolls serve, and a URL map target pulls every cell's
|
||||
// template into the plan, so a non-target cell must be planned at the image it serves.
|
||||
const IMAGE_PATTERN = /^[a-z0-9.-]+\/[a-z0-9-]+\/[a-z0-9-]+\/relay@sha256:[a-f0-9]{64}$/
|
||||
|
||||
function liveRelayImage(script, committedImage, cellId) {
|
||||
const repository = committedImage.split('@')[0]
|
||||
const pulled = [...script.matchAll(/^docker pull '([^']+)'$/gm)]
|
||||
.map((match) => match[1])
|
||||
.filter((image) => image.split('@')[0] === repository)
|
||||
const digest = /^\s*printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'$/m.exec(script)?.[1]
|
||||
if (pulled.length !== 1 || !IMAGE_PATTERN.test(pulled[0]) || pulled[0].split('@')[1] !== digest) {
|
||||
throw new Error(`${cellId} live template has no single pinned Relay image`)
|
||||
}
|
||||
return pulled[0]
|
||||
}
|
||||
|
||||
export function overlayRelayLiveCellImages({ committedCells, liveTemplates, targetCellIds }) {
|
||||
if (!committedCells || Array.isArray(committedCells) || typeof committedCells !== 'object') {
|
||||
throw new Error('committed Relay cells must be an object')
|
||||
}
|
||||
if (!Array.isArray(liveTemplates)) throw new Error('live templates must be an array')
|
||||
const targets = new Set(targetCellIds)
|
||||
for (const cellId of targets) {
|
||||
if (!committedCells[cellId]) throw new Error(`${cellId} is not a committed Relay cell`)
|
||||
}
|
||||
const scripts = new Map()
|
||||
for (const template of liveTemplates) {
|
||||
if (typeof template?.index !== 'string' || typeof template.metadata_startup_script !== 'string') {
|
||||
throw new Error('live template entry is malformed')
|
||||
}
|
||||
if (scripts.has(template.index)) throw new Error(`${template.index} has more than one live template`)
|
||||
scripts.set(template.index, template.metadata_startup_script)
|
||||
}
|
||||
const cells = {}
|
||||
for (const [cellId, cell] of Object.entries(committedCells)) {
|
||||
if (targets.has(cellId)) {
|
||||
cells[cellId] = cell
|
||||
continue
|
||||
}
|
||||
const script = scripts.get(cellId)
|
||||
// A declared cell with no live template would be created here, outside the reviewed wave.
|
||||
if (script === undefined) throw new Error(`${cellId} is not a target and has no live template`)
|
||||
cells[cellId] = { ...cell, image: liveRelayImage(script, cell.image, cellId) }
|
||||
}
|
||||
return { relay_gce_cells: cells }
|
||||
}
|
||||
|
||||
function argumentsFrom(argv) {
|
||||
const values = {}
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const key = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
|
||||
values[key.slice(2)] = value
|
||||
}
|
||||
for (const key of ['cells-json', 'live-templates-json', 'cell-ids', 'output']) {
|
||||
if (!values[key]) throw new Error(`missing --${key}`)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
function readJsonFile(path, label) {
|
||||
const text = readFileSync(path, 'utf8')
|
||||
if (!text.trim()) throw new Error(`${label} is empty`)
|
||||
return JSON.parse(text)
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const values = argumentsFrom(process.argv.slice(2))
|
||||
const committedCells = readJsonFile(values['cells-json'], 'committed Relay cells')
|
||||
const overlay = overlayRelayLiveCellImages({
|
||||
committedCells,
|
||||
liveTemplates: readJsonFile(values['live-templates-json'], 'live Relay templates'),
|
||||
targetCellIds: values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean)
|
||||
})
|
||||
writeFileSync(values.output, `${JSON.stringify(overlay)}\n`)
|
||||
const drifted = Object.keys(committedCells).filter(
|
||||
(cellId) => overlay.relay_gce_cells[cellId].image !== committedCells[cellId].image
|
||||
)
|
||||
console.log(JSON.stringify({ cells: Object.keys(committedCells).length, drifted }))
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { overlayRelayLiveCellImages } from './relay-live-cell-image-overlay.mjs'
|
||||
|
||||
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
|
||||
const committed = `${repository}@sha256:${'a'.repeat(64)}`
|
||||
const served = `${repository}@sha256:${'b'.repeat(64)}`
|
||||
const template = readFileSync(
|
||||
new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
// Renders only the two lines the overlay reads, from the real template, so a template edit fails here.
|
||||
function startupScript(image) {
|
||||
return template
|
||||
.replaceAll('${trimprefix(regex("@sha256:[a-f0-9]{64}$", relay_image), "@")}', image.split('@')[1])
|
||||
.replaceAll('${relay_image}', image)
|
||||
.replaceAll('${cloud_sql_proxy_image}', `gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:${'c'.repeat(64)}`)
|
||||
}
|
||||
|
||||
const cell = (image) => ({ hostname: 'x', region: 'asia-east2', zone: 'asia-east2-a', image, connection_hard_cap: 3000 })
|
||||
const committedCells = {
|
||||
'production-gce-c1': cell(committed),
|
||||
'production-gce-c27': cell(committed),
|
||||
'production-gce-c30': cell(committed)
|
||||
}
|
||||
const live = [
|
||||
{ index: 'production-gce-c1', metadata_startup_script: startupScript(committed) },
|
||||
{ index: 'production-gce-c27', metadata_startup_script: startupScript(served) }
|
||||
]
|
||||
|
||||
test('plans every non-target cell at its served image while the target has no template yet', () => {
|
||||
const overlay = overlayRelayLiveCellImages({
|
||||
committedCells, liveTemplates: live, targetCellIds: ['production-gce-c30']
|
||||
})
|
||||
assert.deepEqual(overlay, {
|
||||
relay_gce_cells: {
|
||||
'production-gce-c1': cell(committed),
|
||||
'production-gce-c27': cell(served),
|
||||
'production-gce-c30': cell(committed)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('leaves a target cell at its committed image even once it has a live template', () => {
|
||||
const overlay = overlayRelayLiveCellImages({
|
||||
committedCells,
|
||||
liveTemplates: [...live, { index: 'production-gce-c30', metadata_startup_script: startupScript(served) }],
|
||||
targetCellIds: ['production-gce-c30']
|
||||
})
|
||||
assert.equal(overlay.relay_gce_cells['production-gce-c30'].image, committed)
|
||||
})
|
||||
|
||||
test('refuses a non-target cell that has no live template', () => {
|
||||
assert.throws(() => overlayRelayLiveCellImages({
|
||||
committedCells, liveTemplates: live.slice(1), targetCellIds: ['production-gce-c30']
|
||||
}), /production-gce-c1 is not a target and has no live template/)
|
||||
})
|
||||
|
||||
test('refuses a live template without one pinned Relay image', () => {
|
||||
const digestMismatch = startupScript(served)
|
||||
.replace(`%s\\n' 'sha256:${'b'.repeat(64)}'`, `%s\\n' 'sha256:${'d'.repeat(64)}'`)
|
||||
for (const script of [
|
||||
digestMismatch,
|
||||
startupScript(served).replace(`docker pull '${served}'`, ''),
|
||||
`${startupScript(served)}\ndocker pull '${committed}'`,
|
||||
startupScript(`${repository}:latest`)
|
||||
]) {
|
||||
assert.throws(() => overlayRelayLiveCellImages({
|
||||
committedCells,
|
||||
liveTemplates: [live[0], { index: 'production-gce-c27', metadata_startup_script: script }],
|
||||
targetCellIds: ['production-gce-c30']
|
||||
}), /production-gce-c27 live template has no single pinned Relay image/)
|
||||
}
|
||||
})
|
||||
|
||||
test('refuses duplicate live templates and an undeclared target', () => {
|
||||
assert.throws(() => overlayRelayLiveCellImages({
|
||||
committedCells, liveTemplates: [...live, live[1]], targetCellIds: ['production-gce-c30']
|
||||
}), /more than one live template/)
|
||||
assert.throws(() => overlayRelayLiveCellImages({
|
||||
committedCells, liveTemplates: live, targetCellIds: ['production-gce-c31']
|
||||
}), /production-gce-c31 is not a committed Relay cell/)
|
||||
})
|
||||
|
||||
test('builds the overlay from plan variables when the target cell has no template in state yet', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'relay-live-cell-image-overlay-'))
|
||||
try {
|
||||
const cells = join(dir, 'cells.json')
|
||||
const templates = join(dir, 'templates.json')
|
||||
const output = join(dir, 'overlay.tfvars.json')
|
||||
// Plan variables carry the map as written: optional attributes the tfvars omit stay absent.
|
||||
const { connection_hard_cap: _omitted, ...c30 } = cell(committed)
|
||||
writeFileSync(cells, JSON.stringify({ ...committedCells, 'production-gce-c30': c30 }))
|
||||
writeFileSync(templates, JSON.stringify(live))
|
||||
assert.ok(!live.some(({ index }) => index === 'production-gce-c30'))
|
||||
const run = (cellsPath) => spawnSync(process.execPath, [
|
||||
fileURLToPath(new URL('./relay-live-cell-image-overlay.mjs', import.meta.url)),
|
||||
'--cells-json', cellsPath, '--live-templates-json', templates,
|
||||
'--cell-ids', 'production-gce-c30', '--output', output
|
||||
], { encoding: 'utf8' })
|
||||
const result = run(cells)
|
||||
assert.equal(result.status, 0, result.stderr)
|
||||
assert.deepEqual(JSON.parse(result.stdout), { cells: 3, drifted: ['production-gce-c27'] })
|
||||
assert.deepEqual(JSON.parse(readFileSync(output, 'utf8')).relay_gce_cells['production-gce-c30'], c30)
|
||||
const empty = join(dir, 'empty.json')
|
||||
writeFileSync(empty, '')
|
||||
const failed = run(empty)
|
||||
assert.notEqual(failed.status, 0)
|
||||
assert.match(failed.stderr, /committed Relay cells is empty/)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -1,45 +0,0 @@
|
||||
// A refused home cell is not fleet capacity, so it gets its own bucket rather
|
||||
// than inflating the capacity count a run is read for.
|
||||
const ASSIGNMENT_REJECTION_BUCKETS = {
|
||||
relay_capacity_exhausted: 'assignment_capacity_exhausted',
|
||||
relay_connection_headroom_exhausted: 'assignment_capacity_exhausted',
|
||||
relay_home_cell_unavailable: 'assignment_home_cell_unavailable'
|
||||
}
|
||||
|
||||
export function relayLoadFailureReason(error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
|
||||
if (tokenExchange) return `token_http_${tokenExchange[1]}`
|
||||
const assignment =
|
||||
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_[a-z_]+))?$/.exec(message)
|
||||
if (assignment?.[1] === '503' && assignment[2]) {
|
||||
return ASSIGNMENT_REJECTION_BUCKETS[assignment[2]] ?? `assignment_http_${assignment[1]}`
|
||||
}
|
||||
if (assignment) return `assignment_http_${assignment[1]}`
|
||||
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
|
||||
if (closed) return `control_close_${closed[1]}`
|
||||
if (message === 'control open timeout') return 'control_open_timeout'
|
||||
if (message === 'control response timeout') return 'control_response_timeout'
|
||||
if (message === 'relay token exchange timeout') return 'token_timeout'
|
||||
if (message === 'relay assignment timeout') return 'assignment_timeout'
|
||||
if (message === 'WebSocket was closed before the connection was established') {
|
||||
return 'socket_closed_before_open'
|
||||
}
|
||||
if (message === 'relay token exchange omitted token') return 'token_response_invalid'
|
||||
if (message === 'relay assignment response invalid') return 'assignment_response_invalid'
|
||||
if (message === 'expected host challenge') return 'host_challenge_invalid'
|
||||
if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed'
|
||||
if (message === 'expected host hello acknowledgement') return 'host_ack_invalid'
|
||||
const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message)
|
||||
if (socketResponse) return `socket_http_${socketResponse[1]}`
|
||||
if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) {
|
||||
return 'socket_transport'
|
||||
}
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
export function discardFailedLoadSocket(socket) {
|
||||
if (!socket) return
|
||||
socket.on('error', () => undefined)
|
||||
socket.terminate()
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
discardFailedLoadSocket,
|
||||
relayLoadFailureReason
|
||||
} from './relay-load-connection-failure.mjs'
|
||||
|
||||
test('classifies only bounded aggregate connection failure reasons', () => {
|
||||
assert.equal(relayLoadFailureReason(new Error('relay token exchange failed: 503')), 'token_http_503')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay assignment failed: 503')), 'assignment_http_503')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(
|
||||
new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
|
||||
),
|
||||
'assignment_capacity_exhausted'
|
||||
)
|
||||
for (const status of [400, 429, 500]) {
|
||||
assert.equal(
|
||||
relayLoadFailureReason(
|
||||
new Error(`${`relay assignment failed: ${status}`} relay_capacity_exhausted`)
|
||||
),
|
||||
`assignment_http_${status}`
|
||||
)
|
||||
}
|
||||
assert.equal(relayLoadFailureReason(new Error('control closed: 4404 wrong cell')), 'control_close_4404')
|
||||
assert.equal(relayLoadFailureReason(new Error('control open timeout')), 'control_open_timeout')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay token exchange timeout')), 'token_timeout')
|
||||
assert.equal(relayLoadFailureReason(new Error('relay assignment timeout')), 'assignment_timeout')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('relay token exchange omitted token')),
|
||||
'token_response_invalid'
|
||||
)
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('relay assignment response invalid')),
|
||||
'assignment_response_invalid'
|
||||
)
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('Unexpected server response: 503 Service Unavailable')),
|
||||
'socket_http_503'
|
||||
)
|
||||
assert.equal(relayLoadFailureReason(new Error('connect ECONNRESET 127.0.0.1')), 'socket_transport')
|
||||
assert.equal(relayLoadFailureReason(new Error('expected host challenge')), 'host_challenge_invalid')
|
||||
assert.equal(
|
||||
relayLoadFailureReason(new Error('host proof challenge did not decrypt')),
|
||||
'host_challenge_decrypt_failed'
|
||||
)
|
||||
assert.equal(relayLoadFailureReason(new Error('expected host hello acknowledgement')), 'host_ack_invalid')
|
||||
assert.equal(relayLoadFailureReason(new Error('host-sensitive detail')), 'unknown')
|
||||
})
|
||||
|
||||
test('absorbs the setup error emitted while discarding a failed socket', () => {
|
||||
const socket = new EventEmitter()
|
||||
socket.terminate = () => socket.emit('error', new Error('closed before open'))
|
||||
|
||||
assert.doesNotThrow(() => discardFailedLoadSocket(socket))
|
||||
})
|
||||
@@ -1,918 +0,0 @@
|
||||
import { createHash, createHmac } from 'node:crypto'
|
||||
import { createRequire } from 'node:module'
|
||||
import { controlPhase } from './relay-load-model.mjs'
|
||||
import { discardFailedLoadSocket } from './relay-load-connection-failure.mjs'
|
||||
|
||||
const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url))
|
||||
const nacl = requireFromRelay('tweetnacl')
|
||||
const WebSocket = requireFromRelay('ws')
|
||||
const { SignJWT } = await import(requireFromRelay.resolve('jose'))
|
||||
const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import(
|
||||
requireFromRelay.resolve('@orca-cloud/relay-contract')
|
||||
)
|
||||
|
||||
export const CONTROL_SILENCE_LIMIT_MS = 75_000
|
||||
const CONTROL_SILENCE_CHECK_INTERVAL_MS = 15_000
|
||||
|
||||
const REPORTABLE_ASSIGNMENT_ERRORS = [
|
||||
'relay_capacity_exhausted',
|
||||
'relay_connection_headroom_exhausted',
|
||||
'relay_home_cell_unavailable'
|
||||
]
|
||||
|
||||
export function waitForOpen(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs)
|
||||
const finish = (error) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('open', onOpen)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve()
|
||||
}
|
||||
const onOpen = () => finish()
|
||||
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('open', onOpen)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
export function nextJson(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data) => {
|
||||
try {
|
||||
finish(undefined, JSON.parse(data.toString()))
|
||||
} catch (error) {
|
||||
finish(error)
|
||||
}
|
||||
}
|
||||
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
export function nextFrame(socket, timeoutMs = 10_000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data, binary) => finish(undefined, { bytes: Buffer.from(data), binary })
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function receiveBinaryStream(socket, expectedBytes, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let receivedBytes = 0
|
||||
const hash = createHash('sha256')
|
||||
const timer = setTimeout(() => finish(new Error('relay stream timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onMessage = (data, binary) => {
|
||||
if (!binary) return finish(new Error('relay changed stream opcode'))
|
||||
const bytes = Buffer.from(data)
|
||||
receivedBytes += bytes.byteLength
|
||||
hash.update(bytes)
|
||||
if (receivedBytes > expectedBytes) return finish(new Error('relay expanded reader stream'))
|
||||
if (receivedBytes === expectedBytes) {
|
||||
finish(undefined, { bytes: receivedBytes, digest: hash.digest('hex') })
|
||||
}
|
||||
}
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
socket.on('message', onMessage)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
function closeInfo(socket, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => finish(new Error('reader close timeout')), timeoutMs)
|
||||
const finish = (error, value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
if (error) reject(error)
|
||||
else resolve(value)
|
||||
}
|
||||
const onClose = (code, reason) => finish(undefined, { code, reason: reason.toString() })
|
||||
const onError = (error) => finish(error)
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
export function relayLoadWedgedCloseAccepted(closeCodes) {
|
||||
return closeCodes.length === 2 && closeCodes[1] === 4429 &&
|
||||
(closeCodes[0] === 4429 || closeCodes[0] === 1006)
|
||||
}
|
||||
|
||||
function waitForClose(socket, timeoutMs = 10_000) {
|
||||
if (!socket || socket.readyState === socket.CLOSED) return Promise.resolve()
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
socket.off('close', onClose)
|
||||
discardFailedLoadSocket(socket)
|
||||
reject(new Error('control close timeout'))
|
||||
}, timeoutMs)
|
||||
const onClose = () => {
|
||||
clearTimeout(timer)
|
||||
resolve()
|
||||
}
|
||||
socket.once('close', onClose)
|
||||
})
|
||||
}
|
||||
|
||||
async function cancelResponse(response) {
|
||||
try {
|
||||
await response.body?.cancel()
|
||||
} catch {
|
||||
// Preserve the bounded failure classification.
|
||||
}
|
||||
}
|
||||
|
||||
function proofForChallenge(challenge, hostSecretKey) {
|
||||
const plaintext = nacl.box.open(
|
||||
Buffer.from(challenge.ciphertextB64, 'base64'),
|
||||
Buffer.from(challenge.nonceB64, 'base64'),
|
||||
Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'),
|
||||
hostSecretKey
|
||||
)
|
||||
if (!plaintext) throw new Error('host proof challenge did not decrypt')
|
||||
const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`)
|
||||
const transcriptLength = new DataView(
|
||||
plaintext.buffer,
|
||||
plaintext.byteOffset + domain.length,
|
||||
4
|
||||
).getUint32(0, false)
|
||||
const transcriptStart = domain.length + 4
|
||||
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
|
||||
const secret = plaintext.slice(transcriptStart + transcriptLength)
|
||||
return createHmac('sha256', secret).update(buildHostProofMacInput(transcript)).digest('base64')
|
||||
}
|
||||
|
||||
export class RelayLoadControlPeer {
|
||||
constructor(index, options, observe) {
|
||||
if (options.directorOrigin && options.targetOrigin) {
|
||||
throw new Error('provide either directorOrigin or targetOrigin, not both')
|
||||
}
|
||||
this.index = index
|
||||
this.options = options
|
||||
this.observe = observe
|
||||
// A twin shares a host's keys, so it can seat the same relay host on a second cell.
|
||||
this.keys = options.keys ?? nacl.box.keyPair()
|
||||
this.relayHostId = createHash('sha256')
|
||||
.update(this.keys.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
this.phase = controlPhase(index, options.seed)
|
||||
this.socket = null
|
||||
this.generation = undefined
|
||||
this.controlResumeSecret = undefined
|
||||
this.lastAssignment = undefined
|
||||
this.refreshTimer = null
|
||||
this.stopped = false
|
||||
this.connecting = false
|
||||
this.inFlight = new Set()
|
||||
this.shutdownPromise = null
|
||||
this.abortController = new AbortController()
|
||||
this.drainExpected = false
|
||||
this.controlWaiters = new Set()
|
||||
this.spliceSockets = new Set()
|
||||
this.spliceSequence = 0
|
||||
}
|
||||
|
||||
// `assignment`, when given, is used instead of asking the director.
|
||||
connect(assignment) {
|
||||
if (
|
||||
this.stopped ||
|
||||
this.connecting ||
|
||||
(this.socket !== null && this.socket.readyState === this.socket.OPEN)
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
this.connecting = true
|
||||
const operation = this.connectOnce(assignment)
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => {
|
||||
this.connecting = false
|
||||
this.inFlight.delete(operation)
|
||||
}
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
assignedCellUrl() {
|
||||
return this.lastAssignment?.cellUrl
|
||||
}
|
||||
|
||||
async connectOnce(givenAssignment) {
|
||||
let socket = null
|
||||
try {
|
||||
const relayToken = await this.relayToken()
|
||||
if (this.stopped) return
|
||||
const assignment = givenAssignment ?? (await this.assignment(relayToken))
|
||||
if (this.stopped) return
|
||||
this.lastAssignment = assignment
|
||||
socket = this.createSocket(assignment, relayToken)
|
||||
this.socket = socket
|
||||
this.drainExpected = false
|
||||
await waitForOpen(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
|
||||
appVersion: 'relay-load',
|
||||
...(this.generation === undefined ? {} : { previousGeneration: this.generation }),
|
||||
...(this.controlResumeSecret === undefined
|
||||
? {}
|
||||
: { controlResumeSecret: this.controlResumeSecret })
|
||||
})
|
||||
)
|
||||
const challenge = await nextJson(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-challenge-ack',
|
||||
challengeId: challenge.challengeId,
|
||||
proofB64: proofForChallenge(challenge, this.keys.secretKey)
|
||||
})
|
||||
)
|
||||
const ack = await nextJson(socket)
|
||||
if (this.stopped || this.socket !== socket) return
|
||||
if (ack.type !== 'host-hello-ack') throw new Error('expected host hello acknowledgement')
|
||||
this.generation = ack.generation
|
||||
this.controlResumeSecret = ack.controlResumeSecret
|
||||
socket.on('message', (data) => this.onMessage(socket, data))
|
||||
this.watchSilence(socket)
|
||||
socket.once('close', (code) => this.onClose(socket, code))
|
||||
socket.once('error', (error) => this.observe('socketError', { index: this.index, error }))
|
||||
this.observe('connected', { index: this.index })
|
||||
this.scheduleRefresh(this.phase.refreshOffsetMs)
|
||||
} catch (error) {
|
||||
discardFailedLoadSocket(socket)
|
||||
if (this.socket === socket) this.socket = null
|
||||
if (this.stopped) return
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
createSocket(assignment, relayToken) {
|
||||
return new WebSocket(`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, {
|
||||
headers: { authorization: `Bearer ${relayToken}` },
|
||||
perMessageDeflate: false
|
||||
})
|
||||
}
|
||||
|
||||
shutdown() {
|
||||
if (this.shutdownPromise) return this.shutdownPromise
|
||||
this.stopped = true
|
||||
this.abortController.abort()
|
||||
if (this.refreshTimer) clearTimeout(this.refreshTimer)
|
||||
this.refreshTimer = null
|
||||
this.shutdownPromise = this.shutdownOnce()
|
||||
return this.shutdownPromise
|
||||
}
|
||||
|
||||
async shutdownOnce() {
|
||||
const socket = this.socket
|
||||
const closed = waitForClose(socket)
|
||||
for (const spliceSocket of this.spliceSockets) {
|
||||
if (
|
||||
spliceSocket.readyState !== spliceSocket.CLOSED &&
|
||||
spliceSocket.readyState !== spliceSocket.CLOSING
|
||||
) {
|
||||
spliceSocket.close(1000, 'load complete')
|
||||
}
|
||||
}
|
||||
this.rejectControlWaiters(new Error('control stopped'))
|
||||
if (socket && socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'load complete')
|
||||
}
|
||||
const settled = async () => {
|
||||
while (this.inFlight.size > 0) {
|
||||
await Promise.allSettled([...this.inFlight])
|
||||
}
|
||||
}
|
||||
await Promise.all([closed, settled()])
|
||||
this.observe('shutdown', {
|
||||
index: this.index,
|
||||
activeControls: this.socket?.readyState === this.socket?.OPEN ? 1 : 0,
|
||||
activeSpliceSockets: this.spliceSockets.size,
|
||||
inFlightOperations: this.inFlight.size,
|
||||
refreshTimerActive: this.refreshTimer !== null
|
||||
})
|
||||
}
|
||||
|
||||
openSplice(options = {}) {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
const operation = this.openSpliceOnce(options)
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
openInviteOffer() {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
const operation = this.openInviteOfferOnce()
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
async openInviteOfferOnce() {
|
||||
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
|
||||
throw new Error('active control required for invite offer')
|
||||
}
|
||||
const sequence = this.spliceSequence++
|
||||
const reqId = `load-offer-${this.index}-${sequence}`
|
||||
const response = this.waitForControlMessage(
|
||||
(message) =>
|
||||
message.reqId === reqId &&
|
||||
(message.type === 'invite-created' || message.type === 'control-error')
|
||||
)
|
||||
this.socket.send(JSON.stringify({
|
||||
type: 'invite-create',
|
||||
reqId,
|
||||
relayDeviceId: `load-offer-device-${this.index}-${sequence}`
|
||||
}))
|
||||
const result = await response
|
||||
if (result.type === 'control-error') throw new Error(`invite offer failed: ${result.code}`)
|
||||
if (
|
||||
typeof result.inviteToken !== 'string' ||
|
||||
!Number.isSafeInteger(result.expiresAt) ||
|
||||
result.expiresAt <= Date.now()
|
||||
) throw new Error('relay invite offer response invalid')
|
||||
}
|
||||
|
||||
async openSpliceOnce({
|
||||
payloadBytes = 64,
|
||||
readerMode = 'normal',
|
||||
readerHoldMs = 0,
|
||||
streamBytes = payloadBytes,
|
||||
frameBytes = payloadBytes,
|
||||
observeReaderPressure = async () => undefined,
|
||||
readerDelay = async (ms) => await new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
slowReaderHoldMs = 0,
|
||||
holdMs = 0
|
||||
} = {}) {
|
||||
if (
|
||||
!this.socket ||
|
||||
this.socket.readyState !== this.socket.OPEN ||
|
||||
this.generation === undefined ||
|
||||
this.lastAssignment === undefined
|
||||
) {
|
||||
throw new Error('active control required for splice')
|
||||
}
|
||||
if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 1) {
|
||||
throw new Error('splice payload bytes must be positive')
|
||||
}
|
||||
const sequence = this.spliceSequence++
|
||||
const reqId = `load-invite-${this.index}-${sequence}`
|
||||
const relayDeviceId = `load-device-${this.index}-${sequence}`
|
||||
let phone
|
||||
let data
|
||||
let opened = false
|
||||
try {
|
||||
const invitePromise = this.waitForControlMessage(
|
||||
(message) => message.type === 'invite-created' && message.reqId === reqId
|
||||
)
|
||||
this.socket.send(JSON.stringify({ type: 'invite-create', reqId, relayDeviceId }))
|
||||
const invite = await invitePromise
|
||||
if (typeof invite.inviteToken !== 'string') throw new Error('relay invite response invalid')
|
||||
|
||||
phone = this.createClientSocket(this.lastAssignment)
|
||||
this.trackSpliceSocket(phone)
|
||||
await waitForOpen(phone)
|
||||
const connectionPromise = this.waitForControlMessage(
|
||||
(message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId
|
||||
)
|
||||
phone.send(
|
||||
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
|
||||
)
|
||||
const connection = await connectionPromise
|
||||
if (typeof connection.connId !== 'string' || typeof connection.connTicket !== 'string') {
|
||||
throw new Error('relay connection response invalid')
|
||||
}
|
||||
|
||||
data = this.createHostDataSocket(this.lastAssignment, connection.connId)
|
||||
this.trackSpliceSocket(data)
|
||||
await waitForOpen(data)
|
||||
const phoneHello = nextJson(phone)
|
||||
data.send(
|
||||
JSON.stringify({
|
||||
type: 'host-data-auth',
|
||||
v: 1,
|
||||
connTicket: connection.connTicket,
|
||||
generation: this.generation
|
||||
})
|
||||
)
|
||||
if ((await phoneHello).ok !== true) throw new Error('relay rejected load splice')
|
||||
|
||||
if (slowReaderHoldMs > 0 && readerMode === 'normal') {
|
||||
readerMode = 'slow'
|
||||
readerHoldMs = slowReaderHoldMs
|
||||
streamBytes = payloadBytes
|
||||
frameBytes = payloadBytes
|
||||
}
|
||||
if (!['normal', 'slow', 'wedged'].includes(readerMode)) {
|
||||
throw new Error('reader mode is invalid')
|
||||
}
|
||||
const pausedSocket = readerMode === 'normal' ? undefined : phone._socket
|
||||
if (readerMode !== 'normal' && !pausedSocket) throw new Error('reader transport unavailable')
|
||||
if (readerMode === 'wedged') {
|
||||
const closes = [closeInfo(phone, readerHoldMs + 10_000), closeInfo(data, readerHoldMs + 10_000)]
|
||||
pausedSocket.pause()
|
||||
const readerPausedAt = Date.now()
|
||||
const readerPressure = observeReaderPressure({
|
||||
cellOrigin: this.lastAssignment.cellUrl,
|
||||
readerMode,
|
||||
streamBytes
|
||||
})
|
||||
const [sent] = await Promise.all([
|
||||
this.sendReaderStream(data, sequence, streamBytes, frameBytes, readerDelay),
|
||||
readerPressure
|
||||
])
|
||||
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
|
||||
pausedSocket.resume()
|
||||
const closeEvidence = await Promise.all(closes)
|
||||
const closeCodes = closeEvidence.map(({ code }) => code)
|
||||
if (!relayLoadWedgedCloseAccepted(closeCodes)) {
|
||||
throw new Error(`wedged reader close codes: ${closeCodes.join(',')}`)
|
||||
}
|
||||
opened = true
|
||||
this.observe('spliceOpened', { index: this.index, readerMode })
|
||||
this.observe('spliceWedged', { index: this.index, code: 4429, streamBytes: sent.bytes })
|
||||
return
|
||||
}
|
||||
|
||||
const expectedStreamBytes = readerMode === 'slow' ? streamBytes : payloadBytes
|
||||
const expectedFrameBytes = readerMode === 'slow' ? frameBytes : payloadBytes
|
||||
const phoneStream = receiveBinaryStream(
|
||||
phone,
|
||||
expectedStreamBytes,
|
||||
readerMode === 'slow' ? readerHoldMs + 10_000 : 10_000
|
||||
)
|
||||
const readerPausedAt = pausedSocket ? Date.now() : 0
|
||||
if (pausedSocket) pausedSocket.pause()
|
||||
const readerPressure = readerMode === 'slow'
|
||||
? observeReaderPressure({
|
||||
cellOrigin: this.lastAssignment.cellUrl,
|
||||
readerMode,
|
||||
streamBytes: expectedStreamBytes
|
||||
})
|
||||
: Promise.resolve()
|
||||
const [sent] = await Promise.all([
|
||||
this.sendReaderStream(
|
||||
data,
|
||||
sequence,
|
||||
expectedStreamBytes,
|
||||
expectedFrameBytes,
|
||||
readerDelay
|
||||
),
|
||||
readerPressure
|
||||
])
|
||||
if (readerMode === 'slow') {
|
||||
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
|
||||
pausedSocket.resume()
|
||||
}
|
||||
const receivedByPhone = await phoneStream
|
||||
if (receivedByPhone.bytes !== sent.bytes || receivedByPhone.digest !== sent.digest) {
|
||||
throw new Error('relay changed host-to-client splice payload')
|
||||
}
|
||||
|
||||
const textPayload = `orca-relay-load:${this.index}:${sequence}:${payloadBytes}`
|
||||
const dataFrame = nextFrame(data)
|
||||
phone.send(textPayload)
|
||||
const receivedByHost = await dataFrame
|
||||
if (receivedByHost.binary || receivedByHost.bytes.toString() !== textPayload) {
|
||||
throw new Error('relay changed client-to-host splice payload')
|
||||
}
|
||||
opened = true
|
||||
this.observe('spliceOpened', { index: this.index, readerMode })
|
||||
if (holdMs > 0 && !(await this.waitForSpliceHold(holdMs, [phone, data]))) return
|
||||
this.observe('spliceCompleted', { index: this.index, readerMode })
|
||||
} catch (error) {
|
||||
if (!this.stopped) this.observe('spliceFailed', { index: this.index, error })
|
||||
throw error
|
||||
} finally {
|
||||
await Promise.all([this.closeSpliceSocket(phone), this.closeSpliceSocket(data)])
|
||||
if (opened) this.observe('spliceClosed', { index: this.index })
|
||||
}
|
||||
}
|
||||
|
||||
waitForSpliceHold(holdMs, sockets) {
|
||||
if (this.stopped) return Promise.resolve(false)
|
||||
return new Promise((resolve, reject) => {
|
||||
const finish = (error, completed = false) => {
|
||||
clearTimeout(timer)
|
||||
this.abortController.signal.removeEventListener('abort', onAbort)
|
||||
for (const socket of sockets) {
|
||||
socket.off('close', onClose)
|
||||
socket.off('error', onError)
|
||||
}
|
||||
if (error) reject(error)
|
||||
else resolve(completed)
|
||||
}
|
||||
const onAbort = () => finish(undefined, false)
|
||||
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
|
||||
const onError = (error) => finish(error)
|
||||
const timer = setTimeout(() => finish(undefined, true), holdMs)
|
||||
this.abortController.signal.addEventListener('abort', onAbort, { once: true })
|
||||
for (const socket of sockets) {
|
||||
socket.once('close', onClose)
|
||||
socket.once('error', onError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
createClientSocket(assignment) {
|
||||
return new WebSocket(
|
||||
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/connect/${this.relayHostId}`,
|
||||
{ perMessageDeflate: false }
|
||||
)
|
||||
}
|
||||
|
||||
createHostDataSocket(assignment, connId) {
|
||||
return new WebSocket(
|
||||
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/data/${connId}`,
|
||||
{ perMessageDeflate: false }
|
||||
)
|
||||
}
|
||||
|
||||
splicePayload(sequence, payloadBytes) {
|
||||
const seed = createHash('sha256')
|
||||
.update(`orca-relay-load:${this.index}:${sequence}`)
|
||||
.digest()
|
||||
return Buffer.allocUnsafe(payloadBytes).map((_, index) => seed[index % seed.length])
|
||||
}
|
||||
|
||||
async sendReaderStream(socket, sequence, streamBytes, frameBytes, delay) {
|
||||
const hash = createHash('sha256')
|
||||
let sentBytes = 0
|
||||
let frameIndex = 0
|
||||
while (sentBytes < streamBytes) {
|
||||
const bytes = Math.min(frameBytes, streamBytes - sentBytes)
|
||||
const payload = this.splicePayload(sequence + frameIndex, bytes)
|
||||
await this.sendReaderFrame(socket, payload)
|
||||
hash.update(payload)
|
||||
sentBytes += bytes
|
||||
frameIndex++
|
||||
while (socket.bufferedAmount > frameBytes) await delay(10)
|
||||
}
|
||||
return { bytes: sentBytes, digest: hash.digest('hex') }
|
||||
}
|
||||
|
||||
sendReaderFrame(socket, payload) {
|
||||
if (socket.send.length < 2) {
|
||||
socket.send(payload)
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('reader send timeout')), 10_000)
|
||||
socket.send(payload, (error) => {
|
||||
clearTimeout(timer)
|
||||
if (error) reject(error)
|
||||
else resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
trackSpliceSocket(socket) {
|
||||
this.spliceSockets.add(socket)
|
||||
socket.once('close', () => this.spliceSockets.delete(socket))
|
||||
}
|
||||
|
||||
async closeSpliceSocket(socket) {
|
||||
if (!socket) return
|
||||
const closed = waitForClose(socket).catch(() => undefined)
|
||||
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'splice complete')
|
||||
}
|
||||
await closed
|
||||
this.spliceSockets.delete(socket)
|
||||
}
|
||||
|
||||
async openRebindProbe() {
|
||||
if (
|
||||
!this.socket ||
|
||||
this.socket.readyState !== this.socket.OPEN ||
|
||||
this.generation === undefined ||
|
||||
this.controlResumeSecret === undefined ||
|
||||
this.lastAssignment === undefined
|
||||
) {
|
||||
throw new Error('active control required for rebind probe')
|
||||
}
|
||||
const relayToken = await this.relayToken()
|
||||
const socket = new WebSocket(
|
||||
`${this.lastAssignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`,
|
||||
{
|
||||
headers: { authorization: `Bearer ${relayToken}` },
|
||||
perMessageDeflate: false
|
||||
}
|
||||
)
|
||||
try {
|
||||
await waitForOpen(socket)
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
assignmentEpoch: this.lastAssignment.assignmentEpoch,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
|
||||
appVersion: 'relay-load-rebind-proof',
|
||||
previousGeneration: this.generation,
|
||||
controlResumeSecret: this.controlResumeSecret
|
||||
})
|
||||
)
|
||||
const challenge = await nextJson(socket)
|
||||
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
|
||||
socket.on('error', () => undefined)
|
||||
const closed = new Promise((resolve) => socket.once('close', resolve))
|
||||
return {
|
||||
close: async () => {
|
||||
const closeCompleted = waitForClose(socket)
|
||||
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
|
||||
socket.close(1000, 'rebind boundary proved')
|
||||
}
|
||||
await closeCompleted
|
||||
},
|
||||
closed,
|
||||
isOpen: () => socket.readyState === socket.OPEN
|
||||
}
|
||||
} catch (error) {
|
||||
discardFailedLoadSocket(socket)
|
||||
await waitForClose(socket).catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async relayToken() {
|
||||
const accessToken = this.options.accessTokenProvider
|
||||
? await this.options.accessTokenProvider()
|
||||
: this.options.accessToken
|
||||
if (accessToken) {
|
||||
const body = await this.requestJson(
|
||||
`${this.options.authOrigin}/v1/desktop/auth/relay-token`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${accessToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
relayHostId: this.relayHostId,
|
||||
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64')
|
||||
})
|
||||
},
|
||||
'relay token exchange timeout',
|
||||
(status) => `relay token exchange failed: ${status}`
|
||||
)
|
||||
if (typeof body.relayToken !== 'string') throw new Error('relay token exchange omitted token')
|
||||
if (!this.stopped) this.observe('token', { index: this.index })
|
||||
return body.relayToken
|
||||
}
|
||||
const token = await new SignJWT({
|
||||
prof: `load-profile-${this.index}`,
|
||||
org: 'relay-load',
|
||||
purpose: 'host-control',
|
||||
relayHostId: this.relayHostId
|
||||
})
|
||||
.setProtectedHeader({ alg: 'ES256', kid: this.options.signingKeyId })
|
||||
.setIssuer(this.options.authOrigin)
|
||||
.setAudience('orca-relay')
|
||||
.setSubject(`load-user-${this.index}`)
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('5m')
|
||||
.sign(this.options.signingKey)
|
||||
if (!this.stopped) this.observe('token', { index: this.index })
|
||||
return token
|
||||
}
|
||||
|
||||
async requestAssignment(preferredRegion) {
|
||||
return await this.assignment(await this.relayToken(), preferredRegion)
|
||||
}
|
||||
|
||||
async assignment(relayToken, preferredRegion = this.options.preferredRegion) {
|
||||
if (!this.options.directorOrigin) {
|
||||
if (!this.options.targetOrigin) throw new Error('relay target origin missing')
|
||||
return { cellUrl: this.options.targetOrigin, assignmentEpoch: 1 }
|
||||
}
|
||||
const body = await this.requestJson(
|
||||
`${this.options.directorOrigin}/v1/assign`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${relayToken}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
relayHostId: this.relayHostId,
|
||||
...(preferredRegion ? { preferredRegion } : {}),
|
||||
// Desktops declare a reconnect after an established control; the director's sticky lane needs it.
|
||||
...(this.generation === undefined ? {} : { reconnect: true })
|
||||
})
|
||||
},
|
||||
'relay assignment timeout',
|
||||
(status, errorCode) =>
|
||||
`relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`,
|
||||
REPORTABLE_ASSIGNMENT_ERRORS
|
||||
)
|
||||
if (
|
||||
typeof body.cellUrl !== 'string' ||
|
||||
!Number.isSafeInteger(body.assignmentEpoch) ||
|
||||
body.assignmentEpoch < 1
|
||||
) {
|
||||
throw new Error('relay assignment response invalid')
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async requestJson(url, init, timeoutMessage, httpErrorMessage, allowedErrorCodes = []) {
|
||||
const controller = new AbortController()
|
||||
const onShutdown = () => controller.abort()
|
||||
if (this.abortController.signal.aborted) controller.abort()
|
||||
else this.abortController.signal.addEventListener('abort', onShutdown, { once: true })
|
||||
let timedOut = false
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true
|
||||
controller.abort()
|
||||
}, this.options.requestTimeoutMs ?? 10_000)
|
||||
try {
|
||||
const response = await fetch(url, { ...init, signal: controller.signal })
|
||||
if (!response.ok) {
|
||||
let bodyConsumed = false
|
||||
let errorCode
|
||||
if (allowedErrorCodes.length > 0) {
|
||||
try {
|
||||
const body = await response.json()
|
||||
bodyConsumed = true
|
||||
if (allowedErrorCodes.includes(body?.error)) errorCode = body.error
|
||||
} catch {
|
||||
// Preserve the bounded status-only classification.
|
||||
}
|
||||
}
|
||||
if (!bodyConsumed) await cancelResponse(response)
|
||||
throw new Error(httpErrorMessage(response.status, errorCode))
|
||||
}
|
||||
return await response.json()
|
||||
} catch (error) {
|
||||
if (timedOut) throw new Error(timeoutMessage, { cause: error })
|
||||
throw error
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
this.abortController.signal.removeEventListener('abort', onShutdown)
|
||||
}
|
||||
}
|
||||
|
||||
onMessage(socket, data) {
|
||||
let message
|
||||
try {
|
||||
message = JSON.parse(data.toString())
|
||||
} catch {
|
||||
this.observe('protocolError', { index: this.index })
|
||||
return
|
||||
}
|
||||
for (const waiter of this.controlWaiters) {
|
||||
if (waiter.matches(message)) {
|
||||
this.controlWaiters.delete(waiter)
|
||||
clearTimeout(waiter.timer)
|
||||
waiter.resolve(message)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (message.type === 'ping') {
|
||||
socket.send(JSON.stringify({ type: 'pong', t: message.t }))
|
||||
this.observe('ping', { index: this.index })
|
||||
} else if (message.type === 'drain') {
|
||||
this.drainExpected = true
|
||||
this.observe('drain', { index: this.index })
|
||||
}
|
||||
}
|
||||
|
||||
// Desktops close a control after 75 s with no inbound frame (the cell pings every 15 s); the
|
||||
// load balancer keeps a dead backend's client socket open, so without this a host never re-assigns.
|
||||
watchSilence(socket) {
|
||||
let lastInboundAt = Date.now()
|
||||
const noteInbound = () => {
|
||||
lastInboundAt = Date.now()
|
||||
}
|
||||
socket.on('message', noteInbound)
|
||||
socket.on('ping', noteInbound)
|
||||
socket.on('pong', noteInbound)
|
||||
const timer = setInterval(() => {
|
||||
if (Date.now() - lastInboundAt > CONTROL_SILENCE_LIMIT_MS) socket.terminate()
|
||||
}, CONTROL_SILENCE_CHECK_INTERVAL_MS)
|
||||
timer.unref?.()
|
||||
socket.once('close', () => clearInterval(timer))
|
||||
}
|
||||
|
||||
onClose(socket, code) {
|
||||
if (this.socket !== socket) return
|
||||
this.socket = null
|
||||
if (this.refreshTimer) clearTimeout(this.refreshTimer)
|
||||
this.refreshTimer = null
|
||||
this.rejectControlWaiters(new Error(`control closed: ${code}`))
|
||||
this.observe('closed', {
|
||||
index: this.index,
|
||||
code,
|
||||
stopped: this.stopped,
|
||||
expectedDrain: this.drainExpected
|
||||
})
|
||||
this.drainExpected = false
|
||||
}
|
||||
|
||||
waitForControlMessage(matches, timeoutMs = 10_000) {
|
||||
if (this.stopped) return Promise.reject(new Error('control stopped'))
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = {
|
||||
matches,
|
||||
resolve,
|
||||
reject,
|
||||
timer: setTimeout(() => {
|
||||
this.controlWaiters.delete(waiter)
|
||||
reject(new Error('control response timeout'))
|
||||
}, timeoutMs)
|
||||
}
|
||||
this.controlWaiters.add(waiter)
|
||||
})
|
||||
}
|
||||
|
||||
rejectControlWaiters(error) {
|
||||
for (const waiter of this.controlWaiters) {
|
||||
clearTimeout(waiter.timer)
|
||||
waiter.reject(error)
|
||||
}
|
||||
this.controlWaiters.clear()
|
||||
}
|
||||
|
||||
scheduleRefresh(delayMs) {
|
||||
if (this.stopped) return
|
||||
const socket = this.socket
|
||||
const reschedule = () => {
|
||||
if (this.socket === socket) this.scheduleRefresh(this.phase.refreshIntervalMs)
|
||||
}
|
||||
this.refreshTimer = setTimeout(() => {
|
||||
void this.refresh().then(reschedule, reschedule)
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
refresh() {
|
||||
if (this.stopped) return Promise.resolve()
|
||||
const operation = this.refreshOnce()
|
||||
this.inFlight.add(operation)
|
||||
const finish = () => this.inFlight.delete(operation)
|
||||
operation.then(finish, finish)
|
||||
return operation
|
||||
}
|
||||
|
||||
async refreshOnce() {
|
||||
const socket = this.socket
|
||||
if (this.stopped || !socket || socket.readyState !== socket.OPEN) return
|
||||
try {
|
||||
const relayJwt = await this.relayToken()
|
||||
if (this.stopped || this.socket !== socket || socket.readyState !== socket.OPEN) return
|
||||
socket.send(JSON.stringify({ type: 'auth-refresh', relayJwt }))
|
||||
this.observe('refresh', { index: this.index })
|
||||
} catch (error) {
|
||||
if (!this.stopped) this.observe('refreshError', { index: this.index, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,147 +0,0 @@
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
function integer(value) {
|
||||
return Number.isSafeInteger(value) && value >= 0 ? value : undefined
|
||||
}
|
||||
|
||||
export function assertRelayLoadDirectorCapacityToken(config, now = Date.now, timeoutMs = 0) {
|
||||
if (!config.adminToken || config.adminToken.length > 8_192) {
|
||||
throw new Error('director capacity identity token is unavailable')
|
||||
}
|
||||
const origin = new URL(config.directorOrigin)
|
||||
if (origin.protocol !== 'https:' || origin.origin !== config.directorOrigin) {
|
||||
throw new Error('director capacity origin must be canonical HTTPS')
|
||||
}
|
||||
let claims
|
||||
try {
|
||||
const parts = config.adminToken.split('.')
|
||||
if (parts.length !== 3) throw new Error('invalid token shape')
|
||||
claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'))
|
||||
} catch {
|
||||
throw new Error('director capacity identity token is invalid')
|
||||
}
|
||||
const expectedAudience = new URL('/v1/admin/drain', origin).toString()
|
||||
const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud]
|
||||
const expiresAt = integer(claims.exp)
|
||||
if (
|
||||
!audiences.includes(expectedAudience) ||
|
||||
typeof claims.email !== 'string' ||
|
||||
claims.email.length === 0 ||
|
||||
claims.email_verified !== true ||
|
||||
expiresAt === undefined ||
|
||||
expiresAt * 1_000 <= now() + timeoutMs
|
||||
) {
|
||||
throw new Error('director capacity identity token is not bound to this proof')
|
||||
}
|
||||
}
|
||||
|
||||
function matchingHeartbeat(status, config) {
|
||||
const capacity = status?.connectionCapacity
|
||||
const runtime = status?.runtime
|
||||
const heartbeatAt = integer(runtime?.lastHeartbeatAt)
|
||||
const matches =
|
||||
status?.cellId === config.cellId &&
|
||||
status?.admissionState === 'general' &&
|
||||
runtime?.ready === true &&
|
||||
runtime?.heartbeatFresh === true &&
|
||||
capacity?.heartbeatFresh === true &&
|
||||
integer(capacity?.hardCap) === config.hardCap &&
|
||||
integer(capacity?.unobservedBound) === config.unobservedBound &&
|
||||
integer(capacity?.normalAdmissionPause) === config.requiredConnections &&
|
||||
integer(capacity?.observedConnections) === config.requiredConnections &&
|
||||
integer(capacity?.enforcedConnectionUnits) === config.requiredConnections &&
|
||||
integer(capacity?.inFlightConnections) === 0 &&
|
||||
integer(capacity?.reservedConnectionUnits) === 0 &&
|
||||
integer(capacity?.pendingControlReservations) === 0 &&
|
||||
heartbeatAt !== undefined
|
||||
return matches ? heartbeatAt : undefined
|
||||
}
|
||||
|
||||
async function cellStatus(fetchImpl, config) {
|
||||
const response = await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.adminToken}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ v: 1, cellId: config.cellId }),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new Error('director capacity identity was rejected')
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.arrayBuffer().catch(() => undefined)
|
||||
return undefined
|
||||
}
|
||||
const result = await response.json().catch(() => undefined)
|
||||
if (!result?.status) throw new Error('director capacity status is invalid')
|
||||
return result.status
|
||||
}
|
||||
|
||||
export async function waitForRelayLoadDirectorCapacity(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? 120_000
|
||||
const pollMs = overrides.pollMs ?? 1_000
|
||||
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
|
||||
const deadline = now() + timeoutMs
|
||||
let baselineHeartbeatAt = config.baselineHeartbeatAt
|
||||
let previousHeartbeatAt
|
||||
let matchingSamples = 0
|
||||
const requiredSamples = config.requiredSamples ?? 2
|
||||
for (;;) {
|
||||
const status = await cellStatus(fetchImpl, config)
|
||||
const currentHeartbeatAt = integer(status?.runtime?.lastHeartbeatAt)
|
||||
if (baselineHeartbeatAt === undefined && currentHeartbeatAt !== undefined) {
|
||||
baselineHeartbeatAt = currentHeartbeatAt
|
||||
}
|
||||
const heartbeatAt = status ? matchingHeartbeat(status, config) : undefined
|
||||
if (heartbeatAt !== undefined && heartbeatAt > baselineHeartbeatAt) {
|
||||
if (previousHeartbeatAt === undefined || heartbeatAt > previousHeartbeatAt) {
|
||||
previousHeartbeatAt = heartbeatAt
|
||||
matchingSamples++
|
||||
if (matchingSamples === requiredSamples) return { heartbeatAt }
|
||||
}
|
||||
} else {
|
||||
previousHeartbeatAt = undefined
|
||||
matchingSamples = 0
|
||||
}
|
||||
if (now() >= deadline) throw new Error('director capacity did not converge after recovery')
|
||||
await delay(pollMs)
|
||||
}
|
||||
}
|
||||
|
||||
function matchingRequestUnits(status, config) {
|
||||
return (
|
||||
status?.cellId === config.cellId &&
|
||||
status?.admissionState === 'general' &&
|
||||
status?.capacityRequests === config.capacityRequests &&
|
||||
status?.reservedRequests === config.expectedRequestUnits &&
|
||||
status?.activityRequestUnits === config.expectedRequestUnits &&
|
||||
status?.activityLeases === config.expectedActivityLeases &&
|
||||
status?.runtime?.observedRequests === config.expectedRequestUnits &&
|
||||
status?.runtime?.ready === true &&
|
||||
status?.runtime?.heartbeatFresh === true
|
||||
)
|
||||
}
|
||||
|
||||
export async function waitForRelayLoadRequestUnits(config, overrides = {}) {
|
||||
const fetchImpl = overrides.fetch ?? fetch
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs ?? 120_000
|
||||
const pollMs = overrides.pollMs ?? 1_000
|
||||
const requiredSamples = overrides.requiredSamples ?? 2
|
||||
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
|
||||
const deadline = now() + timeoutMs
|
||||
let matches = 0
|
||||
for (;;) {
|
||||
const status = await cellStatus(fetchImpl, config)
|
||||
matches = matchingRequestUnits(status, config) ? matches + 1 : 0
|
||||
if (matches === requiredSamples) return
|
||||
if (now() >= deadline) throw new Error('Relay request-unit accounting did not converge')
|
||||
await delay(pollMs)
|
||||
}
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
assertRelayLoadDirectorCapacityToken,
|
||||
waitForRelayLoadDirectorCapacity,
|
||||
waitForRelayLoadRequestUnits
|
||||
} from './relay-load-director-capacity-gate.mjs'
|
||||
|
||||
function token(claims) {
|
||||
const encode = (value) => Buffer.from(JSON.stringify(value)).toString('base64url')
|
||||
return `${encode({ alg: 'none' })}.${encode(claims)}.signature`
|
||||
}
|
||||
|
||||
const config = {
|
||||
directorOrigin: 'https://relay-staging.example.com',
|
||||
adminToken: token({
|
||||
aud: 'https://relay-staging.example.com/v1/admin/drain',
|
||||
email: 'capacity@example.com',
|
||||
email_verified: true,
|
||||
exp: 4_000_000_000
|
||||
}),
|
||||
cellId: 'staging-gce-c3',
|
||||
hardCap: 1_000,
|
||||
unobservedBound: 60,
|
||||
requiredConnections: 840
|
||||
}
|
||||
|
||||
function status(lastHeartbeatAt, overrides = {}) {
|
||||
return {
|
||||
cellId: config.cellId,
|
||||
admissionState: 'general',
|
||||
runtime: { ready: true, heartbeatFresh: true, lastHeartbeatAt, observedRequests: 0 },
|
||||
connectionCapacity: {
|
||||
hardCap: 1_000,
|
||||
unobservedBound: 60,
|
||||
normalAdmissionPause: 840,
|
||||
observedConnections: 840,
|
||||
enforcedConnectionUnits: 840,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
pendingControlReservations: 0,
|
||||
heartbeatFresh: true,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function response(value, responseStatus = 200) {
|
||||
return {
|
||||
ok: responseStatus >= 200 && responseStatus < 300,
|
||||
status: responseStatus,
|
||||
json: async () => value,
|
||||
arrayBuffer: async () => new ArrayBuffer(0)
|
||||
}
|
||||
}
|
||||
|
||||
test('requires two advancing exact director capacity heartbeats', async () => {
|
||||
const heartbeats = [101, 116, 131]
|
||||
let calls = 0
|
||||
const result = await waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: status(heartbeats[calls++]) }),
|
||||
delay: async () => undefined
|
||||
})
|
||||
assert.equal(calls, 3)
|
||||
assert.deepEqual(result, { heartbeatAt: 131 })
|
||||
})
|
||||
|
||||
test('resets after a newer heartbeat undercounts recovered controls', async () => {
|
||||
const samples = [
|
||||
status(101),
|
||||
status(116),
|
||||
status(131, { observedConnections: 839 }),
|
||||
status(146),
|
||||
status(161)
|
||||
]
|
||||
let calls = 0
|
||||
await waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: samples[calls++] }),
|
||||
delay: async () => undefined
|
||||
})
|
||||
assert.equal(calls, 5)
|
||||
})
|
||||
|
||||
test('fails closed when exact advancing telemetry never converges', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => response({ status: status(101) }),
|
||||
delay: async (milliseconds) => {
|
||||
elapsed += milliseconds
|
||||
},
|
||||
now: () => elapsed,
|
||||
timeoutMs: 2_000,
|
||||
pollMs: 1_000
|
||||
}),
|
||||
/did not converge/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects an unauthorized capacity identity without retrying', async () => {
|
||||
let calls = 0
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(config, {
|
||||
fetch: async () => {
|
||||
calls++
|
||||
return response({}, 401)
|
||||
},
|
||||
delay: async () => undefined
|
||||
}),
|
||||
/identity was rejected/
|
||||
)
|
||||
assert.equal(calls, 1)
|
||||
})
|
||||
|
||||
test('binds the admin token to the canonical director audience', async () => {
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(
|
||||
{
|
||||
...config,
|
||||
directorOrigin: 'https://relay-staging.example.com/path'
|
||||
},
|
||||
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
|
||||
),
|
||||
/canonical HTTPS/
|
||||
)
|
||||
await assert.rejects(
|
||||
waitForRelayLoadDirectorCapacity(
|
||||
{
|
||||
...config,
|
||||
adminToken: token({
|
||||
aud: 'https://other.example.com/v1/admin/drain',
|
||||
email: 'capacity@example.com',
|
||||
email_verified: true,
|
||||
exp: 4_000_000_000
|
||||
})
|
||||
},
|
||||
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
|
||||
),
|
||||
/not bound/
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects a missing or malformed admin token during startup preflight', () => {
|
||||
assert.throws(
|
||||
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: undefined }, () => 0),
|
||||
/unavailable/
|
||||
)
|
||||
assert.throws(
|
||||
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: 'not-a-jwt' }, () => 0),
|
||||
/invalid/
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
assertRelayLoadDirectorCapacityToken(
|
||||
{
|
||||
...config,
|
||||
adminToken: token({
|
||||
aud: 'https://relay-staging.example.com/v1/admin/drain',
|
||||
exp: 4_000_000_000
|
||||
})
|
||||
},
|
||||
() => 0
|
||||
),
|
||||
/not bound/
|
||||
)
|
||||
})
|
||||
|
||||
test('supports one newer exact post-probe heartbeat', async () => {
|
||||
const heartbeats = [146, 161]
|
||||
let calls = 0
|
||||
const result = await waitForRelayLoadDirectorCapacity(
|
||||
{ ...config, requiredSamples: 1 },
|
||||
{
|
||||
fetch: async () => response({ status: status(heartbeats[calls++]) }),
|
||||
delay: async () => undefined,
|
||||
now: () => 0
|
||||
}
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
assert.deepEqual(result, { heartbeatAt: 161 })
|
||||
})
|
||||
|
||||
test('requires consecutive exact request-unit accounting samples', async () => {
|
||||
const requestConfig = {
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 6_000,
|
||||
expectedActivityLeases: 6_000
|
||||
}
|
||||
const samples = [5_999, 6_000, 6_000]
|
||||
let calls = 0
|
||||
await waitForRelayLoadRequestUnits(requestConfig, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: {
|
||||
...status(100).runtime,
|
||||
observedRequests: samples[calls]
|
||||
},
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: samples[calls],
|
||||
activityRequestUnits: samples[calls],
|
||||
activityLeases: samples[calls++]
|
||||
}
|
||||
}),
|
||||
delay: async () => undefined,
|
||||
now: () => 0
|
||||
})
|
||||
assert.equal(calls, 3)
|
||||
})
|
||||
|
||||
test('requires the cell runtime to observe every request unit', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(waitForRelayLoadRequestUnits({
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 6_000,
|
||||
expectedActivityLeases: 6_000,
|
||||
timeoutMs: 1_000
|
||||
}, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: { ...status(100).runtime, observedRequests: 5_999 },
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: 6_000,
|
||||
activityRequestUnits: 6_000,
|
||||
activityLeases: 6_000
|
||||
}
|
||||
}),
|
||||
delay: async (milliseconds) => { elapsed += milliseconds },
|
||||
now: () => elapsed,
|
||||
pollMs: 1_000
|
||||
}), /did not converge/)
|
||||
})
|
||||
|
||||
test('fails closed when request-unit accounting does not clean up', async () => {
|
||||
let elapsed = 0
|
||||
await assert.rejects(waitForRelayLoadRequestUnits({
|
||||
...config,
|
||||
capacityRequests: 6_000,
|
||||
expectedRequestUnits: 0,
|
||||
expectedActivityLeases: 0,
|
||||
timeoutMs: 2_000
|
||||
}, {
|
||||
fetch: async () => response({
|
||||
status: {
|
||||
...status(100),
|
||||
runtime: { ...status(100).runtime, observedRequests: 1 },
|
||||
capacityRequests: 6_000,
|
||||
reservedRequests: 1,
|
||||
activityRequestUnits: 1,
|
||||
activityLeases: 1
|
||||
}
|
||||
}),
|
||||
delay: async (milliseconds) => { elapsed += milliseconds },
|
||||
now: () => elapsed,
|
||||
pollMs: 1_000
|
||||
}), /did not converge/)
|
||||
})
|
||||
@@ -1,66 +0,0 @@
|
||||
// Step 5's equal-epoch case: two fresh assigns for one host that reach two director instances
|
||||
// can each book a cell at the same epoch. The host connects with the first answer, and a twin
|
||||
// with the same keys holds the other seat half-open, as a desktop that joined both would.
|
||||
export async function proveRelayLoadDuplicateAssign({ pairs, holdMs, delay, failureReason }) {
|
||||
const result = {
|
||||
probes: pairs.length,
|
||||
bothAnswered: 0,
|
||||
splitCells: 0,
|
||||
sameEpochSplits: 0,
|
||||
joinedBoth: 0,
|
||||
refusalsByReason: {},
|
||||
connectFailuresByReason: {},
|
||||
// Not secrets: synthetic load hosts, listed so the ledger rows can be checked afterwards.
|
||||
splits: []
|
||||
}
|
||||
const count = (bucket, error) => {
|
||||
const reason = failureReason(error)
|
||||
bucket[reason] = (bucket[reason] ?? 0) + 1
|
||||
}
|
||||
const twins = []
|
||||
await Promise.all(
|
||||
pairs.map(async ({ primary, twin }) => {
|
||||
const answers = await Promise.allSettled([
|
||||
primary.requestAssignment(),
|
||||
primary.requestAssignment()
|
||||
])
|
||||
for (const answer of answers) {
|
||||
if (answer.status === 'rejected') count(result.refusalsByReason, answer.reason)
|
||||
}
|
||||
const [first, second] = answers
|
||||
.filter((answer) => answer.status === 'fulfilled')
|
||||
.map((answer) => answer.value)
|
||||
if (!first) return
|
||||
if (second) result.bothAnswered++
|
||||
const split = second !== undefined && second.cellUrl !== first.cellUrl
|
||||
if (split) {
|
||||
result.splitCells++
|
||||
if (second.assignmentEpoch === first.assignmentEpoch) result.sameEpochSplits++
|
||||
result.splits.push({
|
||||
relayHostId: primary.relayHostId,
|
||||
first: { cellUrl: first.cellUrl, assignmentEpoch: first.assignmentEpoch },
|
||||
second: { cellUrl: second.cellUrl, assignmentEpoch: second.assignmentEpoch }
|
||||
})
|
||||
}
|
||||
try {
|
||||
await primary.connect(first)
|
||||
} catch (error) {
|
||||
count(result.connectFailuresByReason, error)
|
||||
return
|
||||
}
|
||||
if (!split) return
|
||||
try {
|
||||
await twin.connect(second)
|
||||
result.joinedBoth++
|
||||
twins.push(twin)
|
||||
} catch (error) {
|
||||
count(result.connectFailuresByReason, error)
|
||||
}
|
||||
})
|
||||
)
|
||||
if (twins.length > 0) {
|
||||
await delay(holdMs)
|
||||
await Promise.all(twins.map((twin) => twin.shutdown()))
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { proveRelayLoadDuplicateAssign } from './relay-load-duplicate-assign.mjs'
|
||||
|
||||
const c2 = 'https://c2.relay-staging.onorca.dev'
|
||||
const c3 = 'https://c3.relay-staging.onorca.dev'
|
||||
|
||||
function pair(relayHostId, answers, { twinFails = false } = {}) {
|
||||
const queue = [...answers]
|
||||
const log = { primary: [], twin: [], twinShutdowns: 0 }
|
||||
const primary = {
|
||||
relayHostId,
|
||||
requestAssignment: async () => {
|
||||
const answer = queue.shift()
|
||||
if (answer instanceof Error) throw answer
|
||||
return answer
|
||||
},
|
||||
connect: async (assignment) => {
|
||||
log.primary.push(assignment)
|
||||
}
|
||||
}
|
||||
const twin = {
|
||||
connect: async (assignment) => {
|
||||
if (twinFails) throw new Error('control closed: 4409 wrong cell')
|
||||
log.twin.push(assignment)
|
||||
},
|
||||
shutdown: async () => {
|
||||
log.twinShutdowns++
|
||||
}
|
||||
}
|
||||
return { pair: { primary, twin }, log }
|
||||
}
|
||||
|
||||
const failureReason = (error) => (error.message.startsWith('control closed: 4409') ? 'control_close_4409' : 'other')
|
||||
|
||||
test('counts same-epoch splits, joins the second cell with a twin, then lets it go', async () => {
|
||||
const split = pair('aaaaaaaaaaaaaaaa', [
|
||||
{ cellUrl: c2, assignmentEpoch: 5 },
|
||||
{ cellUrl: c3, assignmentEpoch: 5 }
|
||||
])
|
||||
const same = pair('bbbbbbbbbbbbbbbb', [
|
||||
{ cellUrl: c2, assignmentEpoch: 3 },
|
||||
{ cellUrl: c2, assignmentEpoch: 3 }
|
||||
])
|
||||
const delays = []
|
||||
const result = await proveRelayLoadDuplicateAssign({
|
||||
pairs: [split.pair, same.pair],
|
||||
holdMs: 20_000,
|
||||
delay: async (ms) => {
|
||||
delays.push(ms)
|
||||
},
|
||||
failureReason
|
||||
})
|
||||
|
||||
assert.equal(result.probes, 2)
|
||||
assert.equal(result.bothAnswered, 2)
|
||||
assert.equal(result.splitCells, 1)
|
||||
assert.equal(result.sameEpochSplits, 1)
|
||||
assert.equal(result.joinedBoth, 1)
|
||||
assert.deepEqual(result.splits, [
|
||||
{
|
||||
relayHostId: 'aaaaaaaaaaaaaaaa',
|
||||
first: { cellUrl: c2, assignmentEpoch: 5 },
|
||||
second: { cellUrl: c3, assignmentEpoch: 5 }
|
||||
}
|
||||
])
|
||||
assert.deepEqual(split.log.primary, [{ cellUrl: c2, assignmentEpoch: 5 }])
|
||||
assert.deepEqual(split.log.twin, [{ cellUrl: c3, assignmentEpoch: 5 }])
|
||||
assert.equal(split.log.twinShutdowns, 1)
|
||||
assert.deepEqual(same.log.twin, [])
|
||||
assert.deepEqual(delays, [20_000])
|
||||
})
|
||||
|
||||
test('a split at different epochs is not a same-epoch split', async () => {
|
||||
const split = pair('cccccccccccccccc', [
|
||||
{ cellUrl: c2, assignmentEpoch: 5 },
|
||||
{ cellUrl: c3, assignmentEpoch: 6 }
|
||||
])
|
||||
const result = await proveRelayLoadDuplicateAssign({
|
||||
pairs: [split.pair],
|
||||
holdMs: 1_000,
|
||||
delay: async () => undefined,
|
||||
failureReason
|
||||
})
|
||||
assert.equal(result.splitCells, 1)
|
||||
assert.equal(result.sameEpochSplits, 0)
|
||||
})
|
||||
|
||||
test('a refused second assign still connects the host once and holds nothing', async () => {
|
||||
const refused = pair('dddddddddddddddd', [
|
||||
{ cellUrl: c2, assignmentEpoch: 2 },
|
||||
new Error('relay assignment failed: 503')
|
||||
])
|
||||
const delays = []
|
||||
const result = await proveRelayLoadDuplicateAssign({
|
||||
pairs: [refused.pair],
|
||||
holdMs: 20_000,
|
||||
delay: async (ms) => {
|
||||
delays.push(ms)
|
||||
},
|
||||
failureReason
|
||||
})
|
||||
assert.equal(result.bothAnswered, 0)
|
||||
assert.deepEqual(result.refusalsByReason, { other: 1 })
|
||||
assert.deepEqual(refused.log.primary, [{ cellUrl: c2, assignmentEpoch: 2 }])
|
||||
assert.deepEqual(delays, [])
|
||||
})
|
||||
|
||||
test('a twin the second cell refuses is counted, not joined', async () => {
|
||||
const split = pair(
|
||||
'eeeeeeeeeeeeeeee',
|
||||
[
|
||||
{ cellUrl: c2, assignmentEpoch: 5 },
|
||||
{ cellUrl: c3, assignmentEpoch: 5 }
|
||||
],
|
||||
{ twinFails: true }
|
||||
)
|
||||
const result = await proveRelayLoadDuplicateAssign({
|
||||
pairs: [split.pair],
|
||||
holdMs: 20_000,
|
||||
delay: async () => undefined,
|
||||
failureReason
|
||||
})
|
||||
assert.equal(result.joinedBoth, 0)
|
||||
assert.deepEqual(result.connectFailuresByReason, { control_close_4409: 1 })
|
||||
assert.equal(split.log.twinShutdowns, 0)
|
||||
})
|
||||
@@ -1,76 +0,0 @@
|
||||
const HEARTBEAT_INTERVAL_MS = 15_000
|
||||
const REFRESH_MIN_MS = 180_000
|
||||
const REFRESH_MAX_MS = 240_000
|
||||
|
||||
function mix32(value) {
|
||||
let mixed = value >>> 0
|
||||
mixed = Math.imul(mixed ^ (mixed >>> 16), 0x21f0aaad)
|
||||
mixed = Math.imul(mixed ^ (mixed >>> 15), 0x735a2d97)
|
||||
return (mixed ^ (mixed >>> 15)) >>> 0
|
||||
}
|
||||
|
||||
function fraction(seed, index, stream) {
|
||||
return mix32(seed ^ Math.imul(index + 1, 0x9e3779b1) ^ stream) / 0x1_0000_0000
|
||||
}
|
||||
|
||||
export function controlPhase(controlIndex, seed = 0x4f524341) {
|
||||
const refreshIntervalMs = Math.round(
|
||||
REFRESH_MIN_MS + fraction(seed, controlIndex, 2) * (REFRESH_MAX_MS - REFRESH_MIN_MS)
|
||||
)
|
||||
return {
|
||||
heartbeatOffsetMs: Math.floor(fraction(seed, controlIndex, 1) * HEARTBEAT_INTERVAL_MS),
|
||||
refreshIntervalMs,
|
||||
refreshOffsetMs: Math.floor(fraction(seed, controlIndex, 3) * refreshIntervalMs),
|
||||
reconnectJitterMs: Math.floor(fraction(seed, controlIndex, 4) * 30_000)
|
||||
}
|
||||
}
|
||||
|
||||
export function modeledRelayLoad(controlCount, durationMs = 15 * 60_000, seed) {
|
||||
if (!Number.isInteger(controlCount) || controlCount < 1) throw new Error('controlCount must be positive')
|
||||
const bins = Array.from({ length: Math.ceil(durationMs / 1000) }, () => ({ pings: 0, refreshes: 0 }))
|
||||
for (let controlIndex = 0; controlIndex < controlCount; controlIndex++) {
|
||||
const phase = controlPhase(controlIndex, seed)
|
||||
for (let at = phase.heartbeatOffsetMs; at < durationMs; at += HEARTBEAT_INTERVAL_MS) {
|
||||
bins[Math.floor(at / 1000)].pings++
|
||||
}
|
||||
for (let at = phase.refreshOffsetMs; at < durationMs; at += phase.refreshIntervalMs) {
|
||||
bins[Math.floor(at / 1000)].refreshes++
|
||||
}
|
||||
}
|
||||
const totals = bins.reduce(
|
||||
(result, bin) => ({
|
||||
pings: result.pings + bin.pings,
|
||||
refreshes: result.refreshes + bin.refreshes
|
||||
}),
|
||||
{ pings: 0, refreshes: 0 }
|
||||
)
|
||||
const durationSeconds = durationMs / 1000
|
||||
return {
|
||||
controlCount,
|
||||
durationMs,
|
||||
expectedPingRate: controlCount / (HEARTBEAT_INTERVAL_MS / 1000),
|
||||
expectedRefreshRate: controlCount / ((REFRESH_MIN_MS + REFRESH_MAX_MS) / 2 / 1000),
|
||||
observedPingRate: totals.pings / durationSeconds,
|
||||
observedRefreshRate: totals.refreshes / durationSeconds,
|
||||
maxPingBurst: Math.max(...bins.map(({ pings }) => pings)),
|
||||
maxRefreshBurst: Math.max(...bins.map(({ refreshes }) => refreshes))
|
||||
}
|
||||
}
|
||||
|
||||
export function assertSpreadModel(model) {
|
||||
const pingTolerance = model.expectedPingRate * 0.03 + 1
|
||||
const refreshTolerance = model.expectedRefreshRate * 0.08 + 1
|
||||
if (Math.abs(model.observedPingRate - model.expectedPingRate) > pingTolerance) {
|
||||
throw new Error('modeled heartbeat rate diverged from the 15-second contract')
|
||||
}
|
||||
if (Math.abs(model.observedRefreshRate - model.expectedRefreshRate) > refreshTolerance) {
|
||||
throw new Error('modeled token refresh rate diverged from the 180-240 second contract')
|
||||
}
|
||||
if (model.maxPingBurst > model.expectedPingRate * 1.3 + 5) {
|
||||
throw new Error('heartbeat phase spreading produced a reconnect cliff')
|
||||
}
|
||||
// One-second bins have Poisson-sized tails even with uniform phase spreading.
|
||||
if (model.maxRefreshBurst > model.expectedRefreshRate * 1.75 + 5) {
|
||||
throw new Error('refresh phase spreading produced an auth herd')
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { assertSpreadModel, controlPhase, modeledRelayLoad } from './relay-load-model.mjs'
|
||||
|
||||
test('phases are deterministic, bounded, and separated by stream', () => {
|
||||
assert.deepEqual(controlPhase(42), controlPhase(42))
|
||||
assert.notDeepEqual(controlPhase(42), controlPhase(43))
|
||||
const phase = controlPhase(42)
|
||||
assert.ok(phase.heartbeatOffsetMs >= 0 && phase.heartbeatOffsetMs < 15_000)
|
||||
assert.ok(phase.refreshIntervalMs >= 180_000 && phase.refreshIntervalMs <= 240_000)
|
||||
assert.ok(phase.refreshOffsetMs >= 0 && phase.refreshOffsetMs < phase.refreshIntervalMs)
|
||||
assert.ok(phase.reconnectJitterMs >= 0 && phase.reconnectJitterMs < 30_000)
|
||||
})
|
||||
|
||||
for (const [controls, expectedPings, expectedRefreshes] of [
|
||||
[4_000, 267, 19],
|
||||
[10_000, 667, 48]
|
||||
]) {
|
||||
test(`${controls} modeled controls spread heartbeat and token refresh load`, () => {
|
||||
const model = modeledRelayLoad(controls)
|
||||
assert.equal(Math.round(model.expectedPingRate), expectedPings)
|
||||
assert.equal(Math.round(model.expectedRefreshRate), expectedRefreshes)
|
||||
assert.doesNotThrow(() => assertSpreadModel(model))
|
||||
})
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { access, mkdir, open } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
export async function waitForRelayLoadPhaseBarrier(config, overrides = {}) {
|
||||
const delay = overrides.delay ?? delayDefault
|
||||
const now = overrides.now ?? Date.now
|
||||
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs
|
||||
if (
|
||||
typeof config.directory !== 'string' || config.directory.length === 0 ||
|
||||
!Number.isSafeInteger(config.shardCount) || config.shardCount < 2 ||
|
||||
!Number.isSafeInteger(config.shardIndex) || config.shardIndex < 0 ||
|
||||
config.shardIndex >= config.shardCount ||
|
||||
!Number.isSafeInteger(timeoutMs) || timeoutMs < 1
|
||||
) throw new Error('invalid Relay load phase barrier')
|
||||
|
||||
await mkdir(config.directory, { recursive: true })
|
||||
const marker = join(config.directory, `${config.shardIndex}.ready`)
|
||||
const handle = await open(marker, 'wx')
|
||||
await handle.close()
|
||||
const deadline = now() + timeoutMs
|
||||
for (;;) {
|
||||
const ready = await Promise.all(
|
||||
Array.from({ length: config.shardCount }, async (_, index) => {
|
||||
try {
|
||||
await access(join(config.directory, `${index}.ready`))
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
})
|
||||
)
|
||||
if (ready.every(Boolean)) return
|
||||
if (now() >= deadline) throw new Error('Relay load phase barrier timed out')
|
||||
await delay(100)
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
|
||||
|
||||
const loadHarness = await readFile(new URL('./load-relay-controls.mjs', import.meta.url), 'utf8')
|
||||
|
||||
test('releases every shard only after all readiness markers exist', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
|
||||
try {
|
||||
let firstResolved = false
|
||||
const first = waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1_000
|
||||
}).then(() => { firstResolved = true })
|
||||
await new Promise((resolve) => setTimeout(resolve, 20))
|
||||
assert.equal(firstResolved, false)
|
||||
await Promise.all([
|
||||
first,
|
||||
waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 1, timeoutMs: 1_000
|
||||
})
|
||||
])
|
||||
assert.equal(firstResolved, true)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('fails closed on a duplicate shard or incomplete barrier', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
|
||||
try {
|
||||
const nowValues = [0, 2]
|
||||
await assert.rejects(
|
||||
waitForRelayLoadPhaseBarrier(
|
||||
{ directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 },
|
||||
{ now: () => nowValues.shift() ?? 2, delay: async () => undefined }
|
||||
),
|
||||
/timed out/
|
||||
)
|
||||
await assert.rejects(
|
||||
waitForRelayLoadPhaseBarrier({
|
||||
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1
|
||||
}),
|
||||
/EEXIST/
|
||||
)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('synchronizes splice ramps after every shard finishes reader baselines', () => {
|
||||
assert.match(
|
||||
loadHarness,
|
||||
/createRelayLoadReaderEvidence[\s\S]*?phaseBarrierDir\}-splices[\s\S]*?splicePromises/
|
||||
)
|
||||
})
|
||||
|
||||
test('budgets both shard barriers and the splice ramp in token lifetime', () => {
|
||||
assert.match(
|
||||
loadHarness,
|
||||
/phaseBarrierDir \? 2 \* config\.phaseBarrierTimeoutMs : 0[\s\S]*?config\.spliceRampMs/
|
||||
)
|
||||
})
|
||||
@@ -1,29 +0,0 @@
|
||||
export async function proveRelayLoadPlacementBoundary({ peer, failureReason }) {
|
||||
let connected = false
|
||||
try {
|
||||
await peer.connect()
|
||||
connected = true
|
||||
} catch (error) {
|
||||
const reason = failureReason(error)
|
||||
if (reason !== 'assignment_capacity_exhausted') {
|
||||
throw new Error(`placement overflow was not rejected: ${reason}`)
|
||||
}
|
||||
return reason
|
||||
} finally {
|
||||
await peer.shutdown()
|
||||
}
|
||||
if (connected) throw new Error('placement overflow unexpectedly connected')
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRegionalFallback({ peer, blockedOrigin }) {
|
||||
try {
|
||||
await peer.connect()
|
||||
const assignedOrigin = peer.assignedCellUrl()
|
||||
if (!assignedOrigin || assignedOrigin === blockedOrigin) {
|
||||
throw new Error('regional fallback did not leave the full preferred cell')
|
||||
}
|
||||
return true
|
||||
} finally {
|
||||
await peer.shutdown()
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
proveRelayLoadPlacementBoundary,
|
||||
proveRelayLoadRegionalFallback
|
||||
} from './relay-load-placement-boundary.mjs'
|
||||
|
||||
function peer(connect) {
|
||||
return { connect, shutdown: async () => undefined }
|
||||
}
|
||||
|
||||
test('requires the next fresh placement to receive HTTP 503', async () => {
|
||||
assert.equal(
|
||||
await proveRelayLoadPlacementBoundary({
|
||||
peer: peer(async () => {
|
||||
throw new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
|
||||
}),
|
||||
failureReason: (error) =>
|
||||
error.message === 'relay assignment failed: 503 relay_connection_headroom_exhausted'
|
||||
? 'assignment_capacity_exhausted'
|
||||
: 'unknown'
|
||||
}),
|
||||
'assignment_capacity_exhausted'
|
||||
)
|
||||
await assert.rejects(
|
||||
proveRelayLoadPlacementBoundary({
|
||||
peer: peer(async () => undefined),
|
||||
failureReason: () => 'unknown'
|
||||
}),
|
||||
/placement overflow unexpectedly connected/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires a preferred-region fallback to leave the full cell', async () => {
|
||||
let shutdowns = 0
|
||||
assert.equal(await proveRelayLoadRegionalFallback({
|
||||
peer: {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => 'https://c3.relay-staging.onorca.dev',
|
||||
shutdown: async () => { shutdowns++ }
|
||||
},
|
||||
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
|
||||
}), true)
|
||||
assert.equal(shutdowns, 1)
|
||||
await assert.rejects(proveRelayLoadRegionalFallback({
|
||||
peer: {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => 'https://c4.relay-staging.onorca.dev',
|
||||
shutdown: async () => undefined
|
||||
},
|
||||
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
|
||||
}), /did not leave/)
|
||||
})
|
||||
@@ -1,440 +0,0 @@
|
||||
export const RELAY_CONTROL_LEASE_HORIZON_SECONDS = 105
|
||||
export const RELAY_LOAD_SPLICE_HIGH_WATER_BYTES = 256 * 1024
|
||||
export const RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS = 10_000
|
||||
export const RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES = 16
|
||||
export const RELAY_LOAD_MAX_AGGREGATE_READER_BYTES = 64 * 1024 * 1024
|
||||
|
||||
const DEFAULT_SLOW_READER_STREAM_BYTES = 1024 * 1024
|
||||
const DEFAULT_WEDGED_READER_STREAM_BYTES = 8 * 1024 * 1024
|
||||
const DEFAULT_READER_FRAME_BYTES = 64 * 1024
|
||||
|
||||
export function parseRelayLoadArguments(argv) {
|
||||
const values = new Map()
|
||||
const flags = new Set()
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const argument = argv[index]
|
||||
if (argument === '--') continue
|
||||
if (
|
||||
[
|
||||
'--allow-partial',
|
||||
'--allow-planned-transition-retries',
|
||||
'--skip-rebind-overflow-check'
|
||||
].includes(argument)
|
||||
) {
|
||||
flags.add(argument)
|
||||
continue
|
||||
}
|
||||
if (!argument.startsWith('--') || index + 1 >= argv.length) {
|
||||
throw new Error(`invalid argument: ${argument}`)
|
||||
}
|
||||
values.set(argument, argv[++index])
|
||||
}
|
||||
const integer = (name, fallback) => {
|
||||
const parsed = Number(values.get(name) ?? fallback)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a nonnegative integer`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
const targetOrigin = values.get('--target-origin')?.replace(/\/$/, '')
|
||||
const directorOrigin = values.get('--director-origin')?.replace(/\/$/, '')
|
||||
const authOrigin = values.get('--auth-origin')?.replace(/\/$/, '')
|
||||
if ((!targetOrigin && !directorOrigin) || (targetOrigin && directorOrigin) || !authOrigin) {
|
||||
throw new Error('provide --auth-origin and exactly one target or director origin')
|
||||
}
|
||||
const controls = integer('--controls', 100)
|
||||
const maxRampConnectionFailures = integer('--max-ramp-connection-failures', 0)
|
||||
const maxUnexpectedCloses = integer('--max-unexpected-closes', 0)
|
||||
const rebindProbes = integer('--rebind-probes', 0)
|
||||
const placementOverflowProbes = integer('--placement-overflow-probes', 0)
|
||||
const regionalFallbackProbes = integer('--regional-fallback-probes', 0)
|
||||
const regionBehaviorProbes = integer('--region-behavior-probes', 0)
|
||||
const duplicateAssignProbes = integer('--duplicate-assign-probes', 0)
|
||||
const duplicateAssignHoldMs = integer('--duplicate-assign-hold-ms', 20_000)
|
||||
const requestUnitInvites = integer('--request-unit-invites', 0)
|
||||
const requestUnitInvitesPerSecond = integer('--request-unit-invites-per-second', 0)
|
||||
const requestUnitPrincipalCount = integer('--request-unit-principals', 0)
|
||||
const relayAsiaLoadPrincipalCount = integer('--relay-asia-load-principals', 0)
|
||||
const requestUnitOverflowProbes = integer('--request-unit-overflow-probes', 0)
|
||||
const requestUnitCapacity = values.has('--request-unit-capacity')
|
||||
? integer('--request-unit-capacity', 0)
|
||||
: undefined
|
||||
const requestUnitCleanupTimeoutMs = integer(
|
||||
'--request-unit-cleanup-timeout-seconds',
|
||||
0
|
||||
) * 1000
|
||||
const rebindHoldMs = integer('--rebind-hold-ms', 4_000)
|
||||
const rebindDelayMs = integer('--rebind-delay-seconds', 0) * 1000
|
||||
const capacityCellId = values.get('--capacity-cell-id')
|
||||
const capacityCellOrigin = values.get('--capacity-cell-origin')?.replace(/\/$/, '')
|
||||
const capacityHardCap = values.has('--capacity-hard-cap')
|
||||
? integer('--capacity-hard-cap', 0)
|
||||
: undefined
|
||||
const capacityUnobservedBound = values.has('--capacity-unobserved-bound')
|
||||
? integer('--capacity-unobserved-bound', 0)
|
||||
: undefined
|
||||
const shardCount = integer('--shard-count', 1)
|
||||
const shardIndex = integer('--shard-index', 0)
|
||||
const durationMs = integer('--duration-seconds', 900) * 1000
|
||||
const spliceHoldMs = integer(
|
||||
'--splice-hold-seconds',
|
||||
values.get('--duration-seconds') ?? 900
|
||||
) * 1000
|
||||
const splices = integer('--splices', 0)
|
||||
const resumePhones = integer('--resume-phones', 0)
|
||||
const resumeIntervalMs = integer('--resume-interval-seconds', 20) * 1000
|
||||
const resumeHoldMs = integer('--resume-hold-seconds', 10) * 1000
|
||||
const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000
|
||||
const slowReaderSplices = integer('--slow-reader-splices', 0)
|
||||
const wedgedReaderSplices = integer('--wedged-reader-splices', 0)
|
||||
const splicePayloadBytes = integer('--splice-payload-bytes', 1_024)
|
||||
const slowReaderStreamBytes = integer(
|
||||
'--slow-reader-stream-bytes',
|
||||
DEFAULT_SLOW_READER_STREAM_BYTES
|
||||
)
|
||||
const wedgedReaderStreamBytes = integer(
|
||||
'--wedged-reader-stream-bytes',
|
||||
DEFAULT_WEDGED_READER_STREAM_BYTES
|
||||
)
|
||||
const readerFrameBytes = integer('--reader-frame-bytes', DEFAULT_READER_FRAME_BYTES)
|
||||
const slowReaderHoldMs = integer('--slow-reader-hold-ms', 2_000)
|
||||
const wedgedReaderHoldMs = integer('--wedged-reader-hold-ms', 12_000)
|
||||
const maxGeneratorRssGrowthMiB = integer('--max-generator-rss-growth-mib', 512)
|
||||
const requiredLeaseHorizons = integer('--required-lease-horizons', 0)
|
||||
const aggregateControls = values.has('--aggregate-controls')
|
||||
? integer('--aggregate-controls', 0)
|
||||
: controls
|
||||
const aggregateSplices = values.has('--aggregate-splices')
|
||||
? integer('--aggregate-splices', 0)
|
||||
: splices
|
||||
const aggregateRequestUnitInvites = values.has('--aggregate-request-unit-invites')
|
||||
? integer('--aggregate-request-unit-invites', 0)
|
||||
: requestUnitInvites
|
||||
const phaseBarrierDir = values.get('--phase-barrier-dir')
|
||||
const phaseBarrierTimeoutMs = integer('--phase-barrier-timeout-seconds', 180) * 1000
|
||||
if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000')
|
||||
if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls')
|
||||
if (splices > controls) throw new Error('--splices cannot exceed --controls')
|
||||
if (splices + resumePhones > controls) {
|
||||
throw new Error('--splices plus --resume-phones cannot exceed --controls')
|
||||
}
|
||||
if (resumePhones > 0 && resumeIntervalMs < 1_000) {
|
||||
throw new Error('--resume-interval-seconds must be at least 1')
|
||||
}
|
||||
if (shardCount > 1 && capacityHardCap !== undefined) {
|
||||
if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) {
|
||||
throw new Error('capacity-bound sharding requires explicit aggregate controls and splices')
|
||||
}
|
||||
if (aggregateControls !== controls * shardCount || aggregateSplices !== splices * shardCount) {
|
||||
throw new Error('aggregate controls and splices must match every equal-sized shard')
|
||||
}
|
||||
} else if (aggregateControls !== controls || aggregateSplices !== splices) {
|
||||
throw new Error('aggregate controls and splices require matching sharded local counts')
|
||||
}
|
||||
if (
|
||||
capacityHardCap !== undefined &&
|
||||
aggregateControls + 2 * aggregateSplices > capacityHardCap - 100
|
||||
) {
|
||||
throw new Error('controls plus splice connection units exceed ordinary cell admission')
|
||||
}
|
||||
if (slowReaderSplices + wedgedReaderSplices > splices) {
|
||||
throw new Error('reader splice counts cannot exceed --splices')
|
||||
}
|
||||
if (splices > 0 && (spliceHoldMs < 1_000 || spliceHoldMs > durationMs)) {
|
||||
throw new Error('--splice-hold-seconds must be between 1 and the steady duration')
|
||||
}
|
||||
if (slowReaderSplices + wedgedReaderSplices > 0 && !directorOrigin) {
|
||||
throw new Error('reader evidence requires --director-origin')
|
||||
}
|
||||
if (splicePayloadBytes < 1 || splicePayloadBytes > 1_048_576) {
|
||||
throw new Error('--splice-payload-bytes must be between 1 and 1048576')
|
||||
}
|
||||
if (
|
||||
slowReaderSplices > 0 &&
|
||||
slowReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
|
||||
) {
|
||||
throw new Error('--slow-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
|
||||
}
|
||||
if (
|
||||
wedgedReaderSplices > 0 &&
|
||||
wedgedReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
|
||||
) {
|
||||
throw new Error('--wedged-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
|
||||
}
|
||||
const localReaderSplices = slowReaderSplices + wedgedReaderSplices
|
||||
const localReaderBytes = slowReaderSplices * slowReaderStreamBytes +
|
||||
wedgedReaderSplices * wedgedReaderStreamBytes
|
||||
const aggregateReaderSplices = values.has('--aggregate-reader-splices')
|
||||
? integer('--aggregate-reader-splices', 0)
|
||||
: localReaderSplices * shardCount
|
||||
const aggregateReaderBytes = values.has('--aggregate-reader-bytes')
|
||||
? integer('--aggregate-reader-bytes', 0)
|
||||
: localReaderBytes * shardCount
|
||||
if (
|
||||
aggregateReaderSplices < localReaderSplices ||
|
||||
aggregateReaderBytes < localReaderBytes ||
|
||||
(shardCount === 1 &&
|
||||
(aggregateReaderSplices !== localReaderSplices || aggregateReaderBytes !== localReaderBytes))
|
||||
) {
|
||||
throw new Error('aggregate reader bounds do not cover the local shard')
|
||||
}
|
||||
if (aggregateReaderSplices > RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES) {
|
||||
throw new Error('aggregate reader splice count exceeds the reviewed bound')
|
||||
}
|
||||
if (aggregateReaderBytes > RELAY_LOAD_MAX_AGGREGATE_READER_BYTES) {
|
||||
throw new Error('aggregate reader stream bytes exceed the reviewed bound')
|
||||
}
|
||||
if (readerFrameBytes < 1 || readerFrameBytes > 1_048_576) {
|
||||
throw new Error('--reader-frame-bytes must be between 1 and 1048576')
|
||||
}
|
||||
if (slowReaderSplices > 0 && slowReaderHoldMs >= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
|
||||
throw new Error('--slow-reader-hold-ms must stay below the wedged timeout')
|
||||
}
|
||||
if (wedgedReaderSplices > 0 && wedgedReaderHoldMs <= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
|
||||
throw new Error('--wedged-reader-hold-ms must exceed the wedged timeout')
|
||||
}
|
||||
if (wedgedReaderHoldMs > 30_000) {
|
||||
throw new Error('--wedged-reader-hold-ms cannot exceed 30000')
|
||||
}
|
||||
if (maxGeneratorRssGrowthMiB < 1) {
|
||||
throw new Error('--max-generator-rss-growth-mib must be positive')
|
||||
}
|
||||
if (placementOverflowProbes > 1) {
|
||||
throw new Error('--placement-overflow-probes must be zero or one')
|
||||
}
|
||||
if (regionalFallbackProbes > 1) {
|
||||
throw new Error('--regional-fallback-probes must be zero or one')
|
||||
}
|
||||
if (regionBehaviorProbes > 1 || requestUnitOverflowProbes > 1) {
|
||||
throw new Error('regional behavior and request-unit overflow probes must be zero or one')
|
||||
}
|
||||
if (placementOverflowProbes > 0 && shardCount > 1) {
|
||||
throw new Error('placement overflow proof requires one coordinated generator')
|
||||
}
|
||||
if (
|
||||
placementOverflowProbes > 0 &&
|
||||
(!directorOrigin ||
|
||||
!capacityCellId ||
|
||||
capacityHardCap === undefined ||
|
||||
capacityUnobservedBound === undefined)
|
||||
) {
|
||||
throw new Error('placement overflow requires exact capacity cell, hard cap, and bound')
|
||||
}
|
||||
if (regionalFallbackProbes > 0 && (
|
||||
!directorOrigin || !capacityCellId || !capacityCellOrigin ||
|
||||
capacityHardCap === undefined || capacityUnobservedBound === undefined ||
|
||||
(shardCount > 1 && shardIndex !== 0)
|
||||
)) throw new Error('regional fallback requires the coordinating capacity shard')
|
||||
if (regionBehaviorProbes > 0 && (
|
||||
!directorOrigin || !capacityCellOrigin || preferredRegionValue(values) !== 'asia-east2' ||
|
||||
(shardCount > 1 && shardIndex !== 0)
|
||||
)) throw new Error('regional behavior proof requires the coordinating Asia shard')
|
||||
if (duplicateAssignProbes > 1_000) {
|
||||
throw new Error('--duplicate-assign-probes cannot exceed 1000')
|
||||
}
|
||||
if (duplicateAssignProbes > 0 && !directorOrigin) {
|
||||
throw new Error('duplicate assign probes require --director-origin')
|
||||
}
|
||||
if (duplicateAssignHoldMs < 1_000 || duplicateAssignHoldMs > 120_000) {
|
||||
throw new Error('--duplicate-assign-hold-ms must be between 1000 and 120000')
|
||||
}
|
||||
if (phaseBarrierDir && shardCount < 2) {
|
||||
throw new Error('phase barrier requires multiple shards')
|
||||
}
|
||||
if (phaseBarrierTimeoutMs < 1_000) {
|
||||
throw new Error('phase barrier timeout must be at least one second')
|
||||
}
|
||||
if (requestUnitInvites > 0) {
|
||||
if (
|
||||
!directorOrigin || !capacityCellId || requestUnitCapacity === undefined ||
|
||||
requestUnitCapacity < 1 || requestUnitInvitesPerSecond < 1 ||
|
||||
requestUnitInvitesPerSecond > 20 || requestUnitPrincipalCount < 1 ||
|
||||
requestUnitPrincipalCount > 32 || requestUnitInvites > requestUnitPrincipalCount * 30 ||
|
||||
aggregateSplices !== 0 || !phaseBarrierDir ||
|
||||
aggregateRequestUnitInvites !== requestUnitInvites * shardCount ||
|
||||
aggregateControls + aggregateRequestUnitInvites !== requestUnitCapacity
|
||||
) throw new Error('request-unit proof does not reach the exact reviewed capacity')
|
||||
} else if (
|
||||
requestUnitCapacity !== undefined || requestUnitInvitesPerSecond !== 0 ||
|
||||
requestUnitPrincipalCount !== 0 ||
|
||||
requestUnitOverflowProbes > 0 ||
|
||||
requestUnitCleanupTimeoutMs > 0 || aggregateRequestUnitInvites !== 0
|
||||
) throw new Error('request-unit proof options require invite offers')
|
||||
if (
|
||||
requestUnitOverflowProbes > 0 &&
|
||||
(shardIndex !== 0 || requestUnitCleanupTimeoutMs < 600_000)
|
||||
) throw new Error('request-unit overflow requires the cleanup-owning coordinator')
|
||||
if (requestUnitCleanupTimeoutMs > 0 && requestUnitOverflowProbes !== 1) {
|
||||
throw new Error('request-unit cleanup requires the overflow proof')
|
||||
}
|
||||
// A US cell's canary uses the same synthetic principals, aimed at the root region instead.
|
||||
if (
|
||||
relayAsiaLoadPrincipalCount > 32 ||
|
||||
(relayAsiaLoadPrincipalCount > 0 &&
|
||||
(!directorOrigin || !['asia-east2', 'us-central1'].includes(preferredRegionValue(values))))
|
||||
) throw new Error('Relay Asia load principals require a regional director proof')
|
||||
if (capacityCellOrigin) {
|
||||
const origin = new URL(capacityCellOrigin)
|
||||
if (origin.protocol !== 'https:' || origin.origin !== capacityCellOrigin) {
|
||||
throw new Error('--capacity-cell-origin must be canonical HTTPS')
|
||||
}
|
||||
}
|
||||
if (shardCount < 1 || shardIndex >= shardCount) throw new Error('invalid shard index/count')
|
||||
const minimumDurationMs = requiredLeaseHorizons * RELAY_CONTROL_LEASE_HORIZON_SECONDS * 1000
|
||||
if (durationMs < minimumDurationMs) {
|
||||
throw new Error(`--duration-seconds must cover ${requiredLeaseHorizons} lease horizons`)
|
||||
}
|
||||
return {
|
||||
targetOrigin,
|
||||
directorOrigin,
|
||||
authOrigin,
|
||||
preferredRegion: preferredRegionValue(values),
|
||||
controls,
|
||||
maxRampConnectionFailures,
|
||||
maxUnexpectedCloses,
|
||||
rebindProbes,
|
||||
placementOverflowProbes,
|
||||
regionalFallbackProbes,
|
||||
regionBehaviorProbes,
|
||||
duplicateAssignProbes,
|
||||
duplicateAssignHoldMs,
|
||||
requestUnitInvites,
|
||||
requestUnitInvitesPerSecond,
|
||||
requestUnitPrincipalCount,
|
||||
relayAsiaLoadPrincipalCount,
|
||||
requestUnitOverflowProbes,
|
||||
requestUnitCapacity,
|
||||
requestUnitCleanupTimeoutMs,
|
||||
rebindHoldMs,
|
||||
rebindDelayMs,
|
||||
capacityCellId,
|
||||
capacityCellOrigin,
|
||||
capacityHardCap,
|
||||
capacityUnobservedBound,
|
||||
durationMs,
|
||||
spliceHoldMs,
|
||||
rampMs: integer('--ramp-seconds', 60) * 1000,
|
||||
rampStartDelayMs: integer('--ramp-start-delay-ms', 0),
|
||||
reconnectMaxMs: integer('--reconnect-max-seconds', 30) * 1000,
|
||||
// 0 keeps the old behaviour: the first splice failure fails the run.
|
||||
spliceRetryMaxMs: integer('--splice-retry-max-seconds', 0) * 1000,
|
||||
shardCount,
|
||||
shardIndex,
|
||||
aggregateControls,
|
||||
aggregateSplices,
|
||||
aggregateRequestUnitInvites,
|
||||
phaseBarrierDir,
|
||||
phaseBarrierTimeoutMs,
|
||||
aggregateReaderSplices,
|
||||
aggregateReaderBytes,
|
||||
signingKeyFile: values.get('--signing-key-file'),
|
||||
splices,
|
||||
resumePhones,
|
||||
resumeIntervalMs,
|
||||
resumeHoldMs,
|
||||
spliceRampMs,
|
||||
splicePayloadBytes,
|
||||
slowReaderSplices,
|
||||
wedgedReaderSplices,
|
||||
slowReaderStreamBytes,
|
||||
wedgedReaderStreamBytes,
|
||||
readerFrameBytes,
|
||||
slowReaderHoldMs,
|
||||
wedgedReaderHoldMs,
|
||||
maxGeneratorRssGrowthMiB,
|
||||
requiredLeaseHorizons,
|
||||
allowPartial: flags.has('--allow-partial'),
|
||||
allowPlannedTransitionRetries: flags.has('--allow-planned-transition-retries'),
|
||||
requireRebindOverflow: !flags.has('--skip-rebind-overflow-check')
|
||||
}
|
||||
}
|
||||
|
||||
function preferredRegionValue(values) {
|
||||
return values.get('--preferred-region')
|
||||
}
|
||||
|
||||
export function relayLoadSpliceIndexes(config) {
|
||||
return Array.from(
|
||||
{ length: config.splices },
|
||||
(_, localIndex) => localIndex * config.shardCount + config.shardIndex
|
||||
)
|
||||
}
|
||||
|
||||
// Resume phones ride the controls after the splice controls, so the two never share a peer.
|
||||
export function relayLoadResumePhoneIndexes(config) {
|
||||
return Array.from(
|
||||
{ length: config.resumePhones },
|
||||
(_, localIndex) => (config.splices + localIndex) * config.shardCount + config.shardIndex
|
||||
)
|
||||
}
|
||||
|
||||
export function relayLoadSpliceStartDelayMs(config, localIndex) {
|
||||
if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) {
|
||||
throw new Error('invalid local splice index')
|
||||
}
|
||||
const totalSplices = config.splices * config.shardCount
|
||||
if (totalSplices <= 1) return 0
|
||||
const globalOrdinal = localIndex * config.shardCount + config.shardIndex
|
||||
return Math.floor(globalOrdinal * config.spliceRampMs / (totalSplices - 1))
|
||||
}
|
||||
|
||||
export function relayLoadPrincipalIndex(peerIndex, shardCount, principalCount) {
|
||||
if (
|
||||
!Number.isSafeInteger(peerIndex) || peerIndex < 0 ||
|
||||
!Number.isSafeInteger(shardCount) || shardCount < 1 ||
|
||||
!Number.isSafeInteger(principalCount) || principalCount < 1
|
||||
) throw new Error('invalid Relay load principal mapping')
|
||||
return Math.floor(peerIndex / shardCount) % principalCount
|
||||
}
|
||||
|
||||
export function relayLoadSpliceProfile(config, spliceIndex) {
|
||||
if (spliceIndex < config.wedgedReaderSplices) {
|
||||
return {
|
||||
readerMode: 'wedged',
|
||||
readerHoldMs: config.wedgedReaderHoldMs,
|
||||
streamBytes: config.wedgedReaderStreamBytes,
|
||||
frameBytes: config.readerFrameBytes
|
||||
}
|
||||
}
|
||||
if (spliceIndex < config.wedgedReaderSplices + config.slowReaderSplices) {
|
||||
return {
|
||||
readerMode: 'slow',
|
||||
readerHoldMs: config.slowReaderHoldMs,
|
||||
streamBytes: config.slowReaderStreamBytes,
|
||||
frameBytes: config.readerFrameBytes
|
||||
}
|
||||
}
|
||||
return {
|
||||
readerMode: 'normal',
|
||||
readerHoldMs: 0,
|
||||
streamBytes: config.splicePayloadBytes,
|
||||
frameBytes: config.splicePayloadBytes
|
||||
}
|
||||
}
|
||||
|
||||
export function relayLoadReaderEvidenceError(result, config) {
|
||||
const readerSplices = config.slowReaderSplices + config.wedgedReaderSplices
|
||||
if (readerSplices > 0 && result.generatorRssGrowthMiB > config.maxGeneratorRssGrowthMiB) {
|
||||
return 'load generator exceeded its RSS growth budget'
|
||||
}
|
||||
if (result.slowReaderSplicesCompleted !== config.slowReaderSplices) {
|
||||
return 'slow-reader streams did not all complete'
|
||||
}
|
||||
if (result.wedgedReaderSplicesClosed !== config.wedgedReaderSplices) {
|
||||
return 'wedged-reader streams did not all close at the relay limit'
|
||||
}
|
||||
if (
|
||||
readerSplices > 0 &&
|
||||
(result.readerQueueEvidence.length === 0 ||
|
||||
result.readerQueueEvidence.some(({ increaseBytes }) => increaseBytes < 1))
|
||||
) {
|
||||
return 'reader streams produced no causal Relay queued-byte evidence'
|
||||
}
|
||||
if (
|
||||
config.wedgedReaderSplices > 0 &&
|
||||
result.readerClosesByCode['4429'] !== config.wedgedReaderSplices
|
||||
) {
|
||||
return 'wedged-reader streams did not close with 4429'
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -1,427 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
parseRelayLoadArguments,
|
||||
relayLoadPrincipalIndex,
|
||||
relayLoadReaderEvidenceError,
|
||||
relayLoadResumePhoneIndexes,
|
||||
relayLoadSpliceIndexes,
|
||||
relayLoadSpliceProfile,
|
||||
relayLoadSpliceStartDelayMs
|
||||
} from './relay-load-profile.mjs'
|
||||
|
||||
const required = ['--auth-origin', 'https://auth.test', '--director-origin', 'https://relay.test']
|
||||
|
||||
test('accepts the 2840-control two-lease-horizon Asia proof profile', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls',
|
||||
'2840',
|
||||
'--duration-seconds',
|
||||
'210',
|
||||
'--required-lease-horizons',
|
||||
'2',
|
||||
'--preferred-region',
|
||||
'asia-east2',
|
||||
'--relay-asia-load-principals',
|
||||
'32',
|
||||
'--capacity-hard-cap',
|
||||
'3000'
|
||||
])
|
||||
|
||||
assert.equal(config.controls, 2840)
|
||||
assert.equal(config.durationMs, 210_000)
|
||||
assert.equal(config.requiredLeaseHorizons, 2)
|
||||
assert.equal(config.preferredRegion, 'asia-east2')
|
||||
assert.equal(config.relayAsiaLoadPrincipalCount, 32)
|
||||
assert.equal(config.splices, 0)
|
||||
})
|
||||
|
||||
test('binds synthetic Relay principals to a bounded Asia director proof', () => {
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--relay-asia-load-principals', '1'
|
||||
]), /require a regional director proof/)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'asia-east2',
|
||||
'--relay-asia-load-principals', '33'
|
||||
]), /require a regional director proof/)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'europe-west1', '--relay-asia-load-principals', '1'
|
||||
]), /require a regional director proof/)
|
||||
// A US cell's canary aims the one synthetic principal at us-central1.
|
||||
assert.equal(parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'us-central1', '--relay-asia-load-principals', '1'
|
||||
]).preferredRegion, 'us-central1')
|
||||
})
|
||||
|
||||
test('rejects a mixed profile beyond the ordinary 2900-unit boundary', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '2840',
|
||||
'--splices', '31',
|
||||
'--capacity-hard-cap', '3000'
|
||||
]),
|
||||
/exceed ordinary cell admission/
|
||||
)
|
||||
expectMixedProfile(parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '2600',
|
||||
'--splices', '120',
|
||||
'--capacity-hard-cap', '3000'
|
||||
]))
|
||||
})
|
||||
|
||||
test('requires reviewed aggregate totals for capacity-bound shards', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0'
|
||||
]),
|
||||
/requires explicit aggregate controls and splices/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0'
|
||||
]),
|
||||
/must match every equal-sized shard/
|
||||
)
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0'
|
||||
])
|
||||
assert.equal(config.aggregateControls, 2840)
|
||||
assert.equal(config.aggregateSplices, 0)
|
||||
})
|
||||
|
||||
test('allows one coordinating shard to prove regional fallback', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--capacity-unobserved-bound', '60', '--rebind-probes', '160'
|
||||
])
|
||||
assert.equal(config.regionalFallbackProbes, 1)
|
||||
assert.equal(config.capacityCellOrigin, 'https://c4.relay-staging.onorca.dev')
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '1',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--capacity-unobserved-bound', '60'
|
||||
]), /coordinating capacity shard/)
|
||||
})
|
||||
|
||||
test('accepts the exact sharded request-unit and region behavior proof', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--preferred-region', 'asia-east2',
|
||||
'--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4',
|
||||
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
|
||||
'--request-unit-invites', '790', '--request-unit-invites-per-second', '2',
|
||||
'--request-unit-principals', '32',
|
||||
'--aggregate-request-unit-invites', '3160',
|
||||
'--request-unit-capacity', '6000', '--request-unit-overflow-probes', '1',
|
||||
'--request-unit-cleanup-timeout-seconds', '630',
|
||||
'--region-behavior-probes', '1', '--phase-barrier-dir', '/tmp/load-barrier'
|
||||
])
|
||||
assert.equal(config.aggregateRequestUnitInvites, 3_160)
|
||||
assert.equal(config.requestUnitCapacity, 6_000)
|
||||
assert.equal(config.requestUnitPrincipalCount, 32)
|
||||
assert.equal(config.requestUnitCleanupTimeoutMs, 630_000)
|
||||
assert.equal(config.regionBehaviorProbes, 1)
|
||||
assert.equal(config.phaseBarrierDir, '/tmp/load-barrier')
|
||||
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '789',
|
||||
'--request-unit-invites-per-second', '2',
|
||||
'--request-unit-principals', '32',
|
||||
'--aggregate-request-unit-invites', '3156', '--request-unit-capacity', '6000',
|
||||
'--phase-barrier-dir', '/tmp/load-barrier'
|
||||
]), /does not reach the exact reviewed capacity/)
|
||||
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--controls', '710', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2840', '--aggregate-splices', '0',
|
||||
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '790',
|
||||
'--request-unit-invites-per-second', '2', '--request-unit-principals', '26',
|
||||
'--aggregate-request-unit-invites', '3160', '--request-unit-capacity', '6000',
|
||||
'--phase-barrier-dir', '/tmp/load-barrier'
|
||||
]), /does not reach the exact reviewed capacity/)
|
||||
})
|
||||
|
||||
function expectMixedProfile(config) {
|
||||
assert.equal(config.controls + 2 * config.splices, 2840)
|
||||
}
|
||||
|
||||
test('rejects a run shorter than its required lease horizons', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--duration-seconds',
|
||||
'209',
|
||||
'--required-lease-horizons',
|
||||
'2'
|
||||
]),
|
||||
/must cover 2 lease horizons/
|
||||
)
|
||||
})
|
||||
|
||||
test('bounds an explicit splice hold within the steady window', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '1',
|
||||
'--splices', '1',
|
||||
'--duration-seconds', '300',
|
||||
'--splice-hold-seconds', '60'
|
||||
])
|
||||
assert.equal(config.durationMs, 300_000)
|
||||
assert.equal(config.spliceHoldMs, 60_000)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '1',
|
||||
'--splices', '1',
|
||||
'--duration-seconds', '300',
|
||||
'--splice-hold-seconds', '301'
|
||||
]), /between 1 and the steady duration/)
|
||||
})
|
||||
|
||||
test('maps splice ownership deterministically within a shard', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls',
|
||||
'4',
|
||||
'--splices',
|
||||
'3',
|
||||
'--shard-count',
|
||||
'4',
|
||||
'--shard-index',
|
||||
'2'
|
||||
])
|
||||
|
||||
assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10])
|
||||
})
|
||||
|
||||
test('puts resume phones on the controls after the splice controls', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--controls', '4', '--splices', '1', '--resume-phones', '2',
|
||||
'--shard-count', '4', '--shard-index', '2'
|
||||
])
|
||||
|
||||
assert.deepEqual(relayLoadSpliceIndexes(config), [2])
|
||||
assert.deepEqual(relayLoadResumePhoneIndexes(config), [6, 10])
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '1', '--resume-phones', '2']),
|
||||
/--splices plus --resume-phones cannot exceed --controls/
|
||||
)
|
||||
})
|
||||
|
||||
test('staggered shards form one deterministic splice ramp', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '4',
|
||||
'--splices', '3',
|
||||
'--splice-ramp-seconds', '11',
|
||||
'--shard-count', '4',
|
||||
'--shard-index', '2'
|
||||
])
|
||||
|
||||
assert.equal(config.spliceRampMs, 11_000)
|
||||
assert.deepEqual(
|
||||
[0, 1, 2].map((index) => relayLoadSpliceStartDelayMs(config, index)),
|
||||
[2_000, 6_000, 10_000]
|
||||
)
|
||||
assert.throws(() => relayLoadSpliceStartDelayMs(config, 3), /invalid local splice index/)
|
||||
})
|
||||
|
||||
test('distributes each shard invite wave below the account rate limit', () => {
|
||||
const identities = Array.from(
|
||||
{ length: 710 },
|
||||
(_, localIndex) => relayLoadPrincipalIndex(localIndex * 4 + 2, 4, 32)
|
||||
)
|
||||
const offers = Array.from({ length: 790 }, (_, index) => identities[index % identities.length])
|
||||
const counts = new Map()
|
||||
for (const principal of offers) counts.set(principal, (counts.get(principal) ?? 0) + 1)
|
||||
assert.equal(counts.size, 32)
|
||||
assert.equal(Math.max(...counts.values()), 26)
|
||||
})
|
||||
|
||||
test('requires exactly one assignment mode', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--target-origin',
|
||||
'https://cell.test'
|
||||
]),
|
||||
/exactly one target or director origin/
|
||||
)
|
||||
})
|
||||
|
||||
test('requires separate recoverable and wedged reader profiles', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '4',
|
||||
'--splices', '3',
|
||||
'--slow-reader-splices', '1',
|
||||
'--wedged-reader-splices', '1',
|
||||
'--slow-reader-stream-bytes', '524288',
|
||||
'--wedged-reader-stream-bytes', '1048576',
|
||||
'--slow-reader-hold-ms', '9000',
|
||||
'--wedged-reader-hold-ms', '11000'
|
||||
])
|
||||
|
||||
assert.deepEqual(relayLoadSpliceProfile(config, 0), {
|
||||
readerMode: 'wedged', readerHoldMs: 11_000, streamBytes: 1_048_576, frameBytes: 65_536
|
||||
})
|
||||
assert.deepEqual(relayLoadSpliceProfile(config, 1), {
|
||||
readerMode: 'slow', readerHoldMs: 9_000, streamBytes: 524_288, frameBytes: 65_536
|
||||
})
|
||||
assert.equal(relayLoadSpliceProfile(config, 2).readerMode, 'normal')
|
||||
})
|
||||
|
||||
test('bounds reader stream, timeout, and splice load per shard', () => {
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '3']),
|
||||
/splices cannot exceed/
|
||||
)
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRelayLoadArguments([
|
||||
...required,
|
||||
'--splices',
|
||||
'1',
|
||||
'--slow-reader-splices',
|
||||
'2'
|
||||
]),
|
||||
/reader splice counts cannot exceed/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--slow-reader-splices', '1',
|
||||
'--slow-reader-stream-bytes', '262144'
|
||||
]),
|
||||
/must exceed the 256 KiB/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--wedged-reader-splices', '1',
|
||||
'--wedged-reader-stream-bytes', '262144'
|
||||
]),
|
||||
/must exceed the 256 KiB/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--slow-reader-splices', '1',
|
||||
'--slow-reader-hold-ms', '10000'
|
||||
]),
|
||||
/must stay below the wedged timeout/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--splices', '1', '--wedged-reader-splices', '1',
|
||||
'--wedged-reader-hold-ms', '10000'
|
||||
]),
|
||||
/must exceed the wedged timeout/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '17', '--splices', '17', '--slow-reader-splices', '17'
|
||||
]),
|
||||
/reader splice count exceeds/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRelayLoadArguments([
|
||||
...required, '--controls', '9', '--splices', '9', '--wedged-reader-splices', '9'
|
||||
]),
|
||||
/reader stream bytes exceed/
|
||||
)
|
||||
})
|
||||
|
||||
test('supports one bounded reader-owning shard without multiplying its pressure', () => {
|
||||
const owner = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '0',
|
||||
'--aggregate-controls', '2600', '--aggregate-splices', '120',
|
||||
'--slow-reader-splices', '10', '--wedged-reader-splices', '1',
|
||||
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
|
||||
])
|
||||
const peer = parseRelayLoadArguments([
|
||||
...required,
|
||||
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
|
||||
'--shard-count', '4', '--shard-index', '1',
|
||||
'--aggregate-controls', '2600', '--aggregate-splices', '120',
|
||||
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
|
||||
])
|
||||
|
||||
assert.equal(owner.aggregateReaderSplices, 11)
|
||||
assert.equal(owner.aggregateReaderBytes, 18 * 1024 * 1024)
|
||||
assert.equal(peer.slowReaderSplices + peer.wedgedReaderSplices, 0)
|
||||
assert.equal(peer.aggregateReaderSplices, 11)
|
||||
})
|
||||
|
||||
test('requires queue, memory, and expected close evidence', () => {
|
||||
const config = parseRelayLoadArguments([
|
||||
...required, '--splices', '2', '--slow-reader-splices', '1',
|
||||
'--wedged-reader-splices', '1', '--max-generator-rss-growth-mib', '100'
|
||||
])
|
||||
const passing = {
|
||||
generatorRssGrowthMiB: 50,
|
||||
slowReaderSplicesCompleted: 1,
|
||||
wedgedReaderSplicesClosed: 1,
|
||||
readerQueueEvidence: [
|
||||
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 65_544, increaseBytes: 65_536 }
|
||||
],
|
||||
readerClosesByCode: { '4429': 1 }
|
||||
}
|
||||
|
||||
assert.equal(relayLoadReaderEvidenceError(passing, config), undefined)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({
|
||||
...passing,
|
||||
readerQueueEvidence: [
|
||||
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 8, increaseBytes: 0 }
|
||||
]
|
||||
}, config),
|
||||
/no causal Relay queued-byte evidence/
|
||||
)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({ ...passing, generatorRssGrowthMiB: 101 }, config),
|
||||
/RSS growth budget/
|
||||
)
|
||||
assert.match(
|
||||
relayLoadReaderEvidenceError({ ...passing, readerClosesByCode: { '4429': 0 } }, config),
|
||||
/did not close with 4429/
|
||||
)
|
||||
})
|
||||
|
||||
test('bounds duplicate-assign probes to a director run', () => {
|
||||
const config = parseRelayLoadArguments([...required, '--duplicate-assign-probes', '100'])
|
||||
assert.equal(config.duplicateAssignProbes, 100)
|
||||
assert.equal(config.duplicateAssignHoldMs, 20_000)
|
||||
assert.equal(parseRelayLoadArguments(required).duplicateAssignProbes, 0)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
'--auth-origin', 'https://auth.test', '--target-origin', 'https://cell.test',
|
||||
'--duplicate-assign-probes', '1'
|
||||
]), /require --director-origin/)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--duplicate-assign-probes', '1001'
|
||||
]), /cannot exceed 1000/)
|
||||
assert.throws(() => parseRelayLoadArguments([
|
||||
...required, '--duplicate-assign-probes', '1', '--duplicate-assign-hold-ms', '500'
|
||||
]), /between 1000 and 120000/)
|
||||
})
|
||||
@@ -1,51 +0,0 @@
|
||||
const DEFAULT_TIMEOUT_MS = 8_000
|
||||
const DEFAULT_POLL_MS = 100
|
||||
|
||||
export async function createRelayLoadReaderEvidence(origins, dependencies) {
|
||||
const distinctOrigins = [...new Set(origins)].sort()
|
||||
const baselines = new Map(await Promise.all(distinctOrigins.map(async (origin) => [
|
||||
origin,
|
||||
await dependencies.readQueuedBytes(origin)
|
||||
])))
|
||||
const peaks = new Map(baselines)
|
||||
const pending = new Map()
|
||||
const now = dependencies.now ?? Date.now
|
||||
const delay = dependencies.delay
|
||||
const timeoutMs = dependencies.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
||||
const pollMs = dependencies.pollMs ?? DEFAULT_POLL_MS
|
||||
|
||||
const observe = async ({ cellOrigin }) => {
|
||||
if (!baselines.has(cellOrigin)) throw new Error('reader origin lacks a run baseline')
|
||||
if (peaks.get(cellOrigin) > baselines.get(cellOrigin)) return
|
||||
const current = pending.get(cellOrigin)
|
||||
if (current) return await current
|
||||
const proof = (async () => {
|
||||
const baseline = baselines.get(cellOrigin)
|
||||
const deadline = now() + timeoutMs
|
||||
for (;;) {
|
||||
const queuedBytes = await dependencies.readQueuedBytes(cellOrigin)
|
||||
peaks.set(cellOrigin, Math.max(peaks.get(cellOrigin), queuedBytes))
|
||||
if (queuedBytes > baseline) return
|
||||
if (now() >= deadline) {
|
||||
throw new Error('reader stream produced no causal Relay queued-byte increase')
|
||||
}
|
||||
await delay(pollMs)
|
||||
}
|
||||
})()
|
||||
pending.set(cellOrigin, proof)
|
||||
try {
|
||||
await proof
|
||||
} finally {
|
||||
pending.delete(cellOrigin)
|
||||
}
|
||||
}
|
||||
|
||||
const snapshot = () => distinctOrigins.map((origin) => ({
|
||||
origin,
|
||||
baselineBytes: baselines.get(origin),
|
||||
peakBytes: peaks.get(origin),
|
||||
increaseBytes: peaks.get(origin) - baselines.get(origin)
|
||||
}))
|
||||
|
||||
return { observe, snapshot }
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
|
||||
|
||||
test('requires a queue increase above the pre-injection baseline for every origin', async () => {
|
||||
const samples = new Map([
|
||||
['https://a.test', [7, 7, 11]],
|
||||
['https://b.test', [0, 3]]
|
||||
])
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence([...samples.keys()], {
|
||||
readQueuedBytes: async (origin) => samples.get(origin).shift(),
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await Promise.all([
|
||||
evidence.observe({ cellOrigin: 'https://a.test' }),
|
||||
evidence.observe({ cellOrigin: 'https://b.test' })
|
||||
])
|
||||
|
||||
assert.deepEqual(evidence.snapshot(), [
|
||||
{ origin: 'https://a.test', baselineBytes: 7, peakBytes: 11, increaseBytes: 4 },
|
||||
{ origin: 'https://b.test', baselineBytes: 0, peakBytes: 3, increaseBytes: 3 }
|
||||
])
|
||||
})
|
||||
|
||||
test('shares one causal proof across concurrent readers on the same cell', async () => {
|
||||
const samples = [4, 4, 9]
|
||||
let reads = 0
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => { reads++; return samples.shift() },
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await Promise.all([
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' }),
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
])
|
||||
|
||||
assert.equal(reads, 3)
|
||||
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
|
||||
})
|
||||
|
||||
test('reuses a completed causal proof for later readers on the same cell', async () => {
|
||||
const samples = [4, 9]
|
||||
let reads = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => { reads++; return samples.shift() },
|
||||
delay: async () => undefined
|
||||
})
|
||||
|
||||
await evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
await evidence.observe({ cellOrigin: 'https://cell.test' })
|
||||
|
||||
assert.equal(reads, 2)
|
||||
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
|
||||
})
|
||||
|
||||
test('rejects a pre-existing nonzero queue that never increases', async () => {
|
||||
let now = 0
|
||||
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
|
||||
readQueuedBytes: async () => 9,
|
||||
delay: async (ms) => { now += ms },
|
||||
now: () => now,
|
||||
timeoutMs: 200,
|
||||
pollMs: 100
|
||||
})
|
||||
|
||||
await assert.rejects(
|
||||
evidence.observe({ cellOrigin: 'https://cell.test' }),
|
||||
/no causal Relay queued-byte increase/
|
||||
)
|
||||
})
|
||||
@@ -1,59 +0,0 @@
|
||||
export async function waitForRelayLoadRebindGate({
|
||||
delay,
|
||||
delayMs,
|
||||
activeCount,
|
||||
requiredCount
|
||||
}) {
|
||||
await delay(delayMs)
|
||||
const active = activeCount()
|
||||
if (active !== requiredCount) {
|
||||
throw new Error(`rebind boundary requires ${requiredCount} active controls, found ${active}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount,
|
||||
holdMs,
|
||||
delay,
|
||||
failureReason,
|
||||
requireOverflow = true
|
||||
}) {
|
||||
if (probeCount === 0) return { opened: 0, overflowReason: null }
|
||||
if (peers.length < probeCount) throw new Error('insufficient active controls for rebind proof')
|
||||
|
||||
const probes = []
|
||||
try {
|
||||
const opened = await Promise.allSettled(
|
||||
peers.slice(0, probeCount).map((peer) => peer.openRebindProbe())
|
||||
)
|
||||
for (const result of opened) {
|
||||
if (result.status === 'fulfilled') probes.push(result.value)
|
||||
}
|
||||
const rejected = opened.find((result) => result.status === 'rejected')
|
||||
if (rejected) throw rejected.reason
|
||||
|
||||
let overflowReason = null
|
||||
if (requireOverflow) {
|
||||
try {
|
||||
const overflow = await peers[0].openRebindProbe()
|
||||
await overflow.close()
|
||||
} catch (error) {
|
||||
overflowReason = failureReason(error)
|
||||
}
|
||||
if (overflowReason !== 'socket_http_503') {
|
||||
throw new Error(`rebind overflow was not rejected at the hard cap: ${overflowReason}`)
|
||||
}
|
||||
}
|
||||
const closedIndex = await Promise.race([
|
||||
delay(holdMs).then(() => -1),
|
||||
...probes.map((probe, index) => probe.closed.then(() => index))
|
||||
])
|
||||
if (closedIndex >= 0 || probes.some((probe) => !probe.isOpen())) {
|
||||
throw new Error('rebind probe closed before the hold completed')
|
||||
}
|
||||
return { opened: probes.length, overflowReason }
|
||||
} finally {
|
||||
await Promise.all(probes.map((probe) => probe.close()))
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
proveRelayLoadRebindBoundary,
|
||||
waitForRelayLoadRebindGate
|
||||
} from './relay-load-rebind-boundary.mjs'
|
||||
|
||||
function peer(open) {
|
||||
return { openRebindProbe: open }
|
||||
}
|
||||
|
||||
function probe(onClose = () => undefined) {
|
||||
let open = true
|
||||
let resolveClosed
|
||||
const closed = new Promise((resolve) => {
|
||||
resolveClosed = resolve
|
||||
})
|
||||
return {
|
||||
close: () => {
|
||||
if (!open) return
|
||||
open = false
|
||||
onClose()
|
||||
resolveClosed()
|
||||
},
|
||||
closed,
|
||||
isOpen: () => open
|
||||
}
|
||||
}
|
||||
|
||||
test('holds the requested rebind overlap and requires a hard-cap rejection', async () => {
|
||||
let openCalls = 0
|
||||
let closes = 0
|
||||
let heldFor = null
|
||||
const peers = [
|
||||
peer(async () => {
|
||||
openCalls++
|
||||
if (openCalls === 3) throw new Error('Unexpected server response: 503')
|
||||
return probe(() => closes++)
|
||||
}),
|
||||
peer(async () => {
|
||||
openCalls++
|
||||
return probe(() => closes++)
|
||||
})
|
||||
]
|
||||
const result = await proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount: 2,
|
||||
holdMs: 4_000,
|
||||
delay: async (milliseconds) => {
|
||||
heldFor = milliseconds
|
||||
},
|
||||
failureReason: (error) =>
|
||||
error.message.includes('503') ? 'socket_http_503' : 'unknown'
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { opened: 2, overflowReason: 'socket_http_503' })
|
||||
assert.equal(heldFor, 4_000)
|
||||
assert.equal(closes, 2)
|
||||
})
|
||||
|
||||
test('closes successful probes when a boundary probe fails', async () => {
|
||||
let closes = 0
|
||||
const peers = [
|
||||
peer(async () => probe(() => closes++)),
|
||||
peer(async () => {
|
||||
throw new Error('probe failed')
|
||||
})
|
||||
]
|
||||
|
||||
await assert.rejects(
|
||||
proveRelayLoadRebindBoundary({
|
||||
peers,
|
||||
probeCount: 2,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown'
|
||||
}),
|
||||
/probe failed/
|
||||
)
|
||||
assert.equal(closes, 1)
|
||||
})
|
||||
|
||||
test('waits for every replacement socket to finish closing', async () => {
|
||||
let finishClose
|
||||
const closeFinished = new Promise((resolve) => {
|
||||
finishClose = resolve
|
||||
})
|
||||
const closingProbe = probe()
|
||||
closingProbe.close = () => closeFinished
|
||||
let completed = false
|
||||
const boundary = proveRelayLoadRebindBoundary({
|
||||
peers: [peer(async () => closingProbe)],
|
||||
probeCount: 1,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
}).then(() => {
|
||||
completed = true
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
assert.equal(completed, false)
|
||||
finishClose()
|
||||
await boundary
|
||||
assert.equal(completed, true)
|
||||
})
|
||||
|
||||
test('can prove reserved replacement headroom below the physical cap', async () => {
|
||||
let closes = 0
|
||||
const result = await proveRelayLoadRebindBoundary({
|
||||
peers: [peer(async () => probe(() => closes++))],
|
||||
probeCount: 1,
|
||||
holdMs: 0,
|
||||
delay: async () => undefined,
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
})
|
||||
assert.deepEqual(result, { opened: 1, overflowReason: null })
|
||||
assert.equal(closes, 1)
|
||||
})
|
||||
|
||||
test('fails when a replacement closes before the hold completes', async () => {
|
||||
let heldProbe
|
||||
await assert.rejects(
|
||||
proveRelayLoadRebindBoundary({
|
||||
peers: [
|
||||
peer(async () => {
|
||||
heldProbe = probe()
|
||||
return heldProbe
|
||||
})
|
||||
],
|
||||
probeCount: 1,
|
||||
holdMs: 4_000,
|
||||
delay: async () => {
|
||||
heldProbe.close()
|
||||
},
|
||||
failureReason: () => 'unknown',
|
||||
requireOverflow: false
|
||||
}),
|
||||
/closed before the hold completed/
|
||||
)
|
||||
})
|
||||
|
||||
test('delays the boundary until every ordinary control has recovered', async () => {
|
||||
let active = 899
|
||||
await assert.rejects(
|
||||
waitForRelayLoadRebindGate({
|
||||
delay: async () => undefined,
|
||||
delayMs: 0,
|
||||
activeCount: () => active,
|
||||
requiredCount: 900
|
||||
}),
|
||||
/requires 900 active controls/
|
||||
)
|
||||
await waitForRelayLoadRebindGate({
|
||||
delay: async () => {
|
||||
active = 900
|
||||
},
|
||||
delayMs: 1,
|
||||
activeCount: () => active,
|
||||
requiredCount: 900
|
||||
})
|
||||
})
|
||||
@@ -1,35 +0,0 @@
|
||||
const ASSIGNMENT_RETRY_DELAY_MS = 5_100
|
||||
|
||||
const waitPastAssignmentRateLimit = (schedule) =>
|
||||
new Promise((resolve) => schedule(resolve, ASSIGNMENT_RETRY_DELAY_MS))
|
||||
|
||||
export async function proveRelayLoadRegionBehavior({
|
||||
oldClientPeer,
|
||||
stickyPeer,
|
||||
asiaOrigin,
|
||||
scheduleAssignmentRetry = setTimeout
|
||||
}) {
|
||||
try {
|
||||
await oldClientPeer.connect()
|
||||
if (!oldClientPeer.assignedCellUrl() || oldClientPeer.assignedCellUrl() === asiaOrigin) {
|
||||
throw new Error('unhinted client did not use the US-first path')
|
||||
}
|
||||
} finally {
|
||||
await oldClientPeer.shutdown()
|
||||
}
|
||||
|
||||
try {
|
||||
await stickyPeer.connect()
|
||||
if (stickyPeer.assignedCellUrl() !== asiaOrigin) {
|
||||
throw new Error('preferred Asia client did not reach the Asia cell')
|
||||
}
|
||||
await waitPastAssignmentRateLimit(scheduleAssignmentRetry)
|
||||
const reassigned = await stickyPeer.requestAssignment('us-central1')
|
||||
if (reassigned.cellUrl !== asiaOrigin) {
|
||||
throw new Error('valid sticky assignment moved after preference changed')
|
||||
}
|
||||
} finally {
|
||||
await stickyPeer.shutdown()
|
||||
}
|
||||
return { oldClientUsFirst: true, stickyAssignmentPreserved: true }
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
|
||||
|
||||
function peer(origin, reassigned = origin) {
|
||||
let shutdowns = 0
|
||||
return {
|
||||
connect: async () => undefined,
|
||||
assignedCellUrl: () => origin,
|
||||
requestAssignment: async () => ({ cellUrl: reassigned }),
|
||||
shutdown: async () => { shutdowns++ },
|
||||
shutdowns: () => shutdowns
|
||||
}
|
||||
}
|
||||
|
||||
test('proves unhinted US-first placement and sticky Asia preservation', async () => {
|
||||
const oldClientPeer = peer('https://c3.relay-staging.onorca.dev')
|
||||
const stickyPeer = peer('https://c4.relay-staging.onorca.dev')
|
||||
let retryDelayMs = 0
|
||||
assert.deepEqual(await proveRelayLoadRegionBehavior({
|
||||
oldClientPeer,
|
||||
stickyPeer,
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve, delayMs) => {
|
||||
retryDelayMs = delayMs
|
||||
resolve()
|
||||
}
|
||||
}), { oldClientUsFirst: true, stickyAssignmentPreserved: true })
|
||||
assert.equal(retryDelayMs, 5_100)
|
||||
assert.equal(oldClientPeer.shutdowns(), 1)
|
||||
assert.equal(stickyPeer.shutdowns(), 1)
|
||||
})
|
||||
|
||||
test('rejects Asia placement for an unhinted client or a moved sticky assignment', async () => {
|
||||
await assert.rejects(proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: peer('https://c4.relay-staging.onorca.dev'),
|
||||
stickyPeer: peer('https://c4.relay-staging.onorca.dev'),
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve) => resolve()
|
||||
}), /US-first/)
|
||||
await assert.rejects(proveRelayLoadRegionBehavior({
|
||||
oldClientPeer: peer('https://c3.relay-staging.onorca.dev'),
|
||||
stickyPeer: peer('https://c4.relay-staging.onorca.dev', 'https://c3.relay-staging.onorca.dev'),
|
||||
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
|
||||
scheduleAssignmentRetry: (resolve) => resolve()
|
||||
}), /sticky assignment moved/)
|
||||
})
|
||||
@@ -1,43 +0,0 @@
|
||||
import { setTimeout as delayDefault } from 'node:timers/promises'
|
||||
|
||||
export async function openRelayLoadInviteOffers({
|
||||
peers,
|
||||
count,
|
||||
ratePerSecond,
|
||||
concurrency = 8,
|
||||
delay = delayDefault,
|
||||
now = Date.now
|
||||
}) {
|
||||
if (
|
||||
!Array.isArray(peers) || peers.length === 0 ||
|
||||
!Number.isSafeInteger(count) || count < 0 ||
|
||||
!Number.isSafeInteger(ratePerSecond) || ratePerSecond < 1 || ratePerSecond > 20 ||
|
||||
!Number.isSafeInteger(concurrency) || concurrency < 1
|
||||
) throw new Error('invalid Relay invite-offer load')
|
||||
let next = 0
|
||||
let nextStartAt = now()
|
||||
const workers = Array.from({ length: Math.min(concurrency, count) }, async () => {
|
||||
for (;;) {
|
||||
const index = next++
|
||||
if (index >= count) return
|
||||
const scheduledAt = Math.max(nextStartAt, now())
|
||||
nextStartAt = scheduledAt + 1_000 / ratePerSecond
|
||||
await delay(Math.max(0, scheduledAt - now()))
|
||||
await peers[index % peers.length].openInviteOffer()
|
||||
}
|
||||
})
|
||||
await Promise.all(workers)
|
||||
return count
|
||||
}
|
||||
|
||||
export async function proveRelayLoadRequestUnitBoundary(peer) {
|
||||
try {
|
||||
await peer.openInviteOffer()
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'invite offer failed: relay_capacity_exhausted') {
|
||||
return 'relay_capacity_exhausted'
|
||||
}
|
||||
throw new Error('request-unit overflow was not rejected safely', { cause: error })
|
||||
}
|
||||
throw new Error('request-unit overflow unexpectedly succeeded')
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import {
|
||||
openRelayLoadInviteOffers,
|
||||
proveRelayLoadRequestUnitBoundary
|
||||
} from './relay-load-request-unit-boundary.mjs'
|
||||
|
||||
test('distributes the exact invite count with bounded concurrency', async () => {
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const delays = []
|
||||
const calls = [0, 0, 0]
|
||||
const peers = calls.map((_, index) => ({
|
||||
async openInviteOffer() {
|
||||
calls[index]++
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await Promise.resolve()
|
||||
active--
|
||||
}
|
||||
}))
|
||||
assert.equal(await openRelayLoadInviteOffers({
|
||||
peers,
|
||||
count: 8,
|
||||
ratePerSecond: 2,
|
||||
concurrency: 2,
|
||||
delay: async (milliseconds) => { delays.push(milliseconds) },
|
||||
now: () => 0
|
||||
}), 8)
|
||||
assert.deepEqual(calls, [3, 3, 2])
|
||||
assert.ok(peak <= 2)
|
||||
assert.equal(delays.length, 8)
|
||||
assert.ok(Math.max(...delays) >= 3_500)
|
||||
})
|
||||
|
||||
test('accepts only the exact request-unit exhaustion error', async () => {
|
||||
assert.equal(await proveRelayLoadRequestUnitBoundary({
|
||||
openInviteOffer: async () => {
|
||||
throw new Error('invite offer failed: relay_capacity_exhausted')
|
||||
}
|
||||
}), 'relay_capacity_exhausted')
|
||||
await assert.rejects(
|
||||
proveRelayLoadRequestUnitBoundary({
|
||||
openInviteOffer: async () => { throw new Error('control response timeout') }
|
||||
}),
|
||||
/not rejected safely/
|
||||
)
|
||||
await assert.rejects(
|
||||
proveRelayLoadRequestUnitBoundary({ openInviteOffer: async () => undefined }),
|
||||
/unexpectedly succeeded/
|
||||
)
|
||||
})
|
||||
@@ -1,220 +0,0 @@
|
||||
import { createHash, randomBytes } from 'node:crypto'
|
||||
import { nextFrame, nextJson, waitForOpen } from './relay-load-control-peer.mjs'
|
||||
|
||||
// A paired phone's reconnects: one device install over an invite splice, then real device
|
||||
// resumes (accept, then the desktop's device-resume-confirm), timed from the phone's side.
|
||||
|
||||
const ECHO_INTERVAL_MS = 5_000
|
||||
|
||||
const resumeTokenHash = (token) => createHash('sha256').update(token).digest('base64url')
|
||||
|
||||
function controlAnswer(peer, reqId, type) {
|
||||
return peer.waitForControlMessage(
|
||||
(message) =>
|
||||
message.reqId === reqId && (message.type === type || message.type === 'control-error')
|
||||
)
|
||||
}
|
||||
|
||||
function closeCode(socket, timeoutMs = 2_000) {
|
||||
if (socket.readyState === socket.CLOSED) return Promise.resolve(undefined)
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => resolve(undefined), timeoutMs)
|
||||
socket.once('close', (code) => {
|
||||
clearTimeout(timer)
|
||||
resolve(code)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// The phone dials, the desktop's control answers conn-open, and the desktop attaches its data leg.
|
||||
async function dialPhone(peer, credential, relayDeviceId) {
|
||||
if (!peer.socket || peer.socket.readyState !== peer.socket.OPEN || peer.generation === undefined) {
|
||||
throw Object.assign(new Error('active control required for a phone'), { reason: 'no-control' })
|
||||
}
|
||||
const assignment = peer.lastAssignment
|
||||
const phone = peer.createClientSocket(assignment)
|
||||
peer.trackSpliceSocket(phone)
|
||||
let data
|
||||
try {
|
||||
const setData = (socket) => {
|
||||
data = socket
|
||||
}
|
||||
return await attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData })
|
||||
} catch (error) {
|
||||
// A close before any hello names its code, e.g. 4503 while the database is down.
|
||||
const closed = /closed: (\d+)/.exec(String(error.message))
|
||||
throw Object.assign(error, {
|
||||
reason: error.reason ?? (closed ? `hello-${closed[1]}` : undefined),
|
||||
phone,
|
||||
data
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) {
|
||||
await waitForOpen(phone)
|
||||
const connectionPromise = peer.waitForControlMessage(
|
||||
(message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId
|
||||
)
|
||||
connectionPromise.catch(() => undefined)
|
||||
const hello = nextJson(phone)
|
||||
const startedAt = performance.now()
|
||||
phone.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
|
||||
// A refused phone gets a failed hello and no conn-open.
|
||||
const first = await Promise.race([
|
||||
connectionPromise.then((connection) => ({ connection })),
|
||||
hello.then((answer) => ({ answer }))
|
||||
])
|
||||
if (first.answer) {
|
||||
const code = first.answer.code ?? (await closeCode(phone))
|
||||
throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` })
|
||||
}
|
||||
const { connection } = first
|
||||
const data = peer.createHostDataSocket(assignment, connection.connId)
|
||||
setData(data)
|
||||
peer.trackSpliceSocket(data)
|
||||
await waitForOpen(data)
|
||||
data.send(
|
||||
JSON.stringify({
|
||||
type: 'host-data-auth',
|
||||
v: 1,
|
||||
connTicket: connection.connTicket,
|
||||
generation: peer.generation
|
||||
})
|
||||
)
|
||||
const answer = await hello
|
||||
const acceptMs = performance.now() - startedAt
|
||||
if (answer.ok !== true) {
|
||||
const code = answer.code ?? (await closeCode(phone))
|
||||
throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` })
|
||||
}
|
||||
return { phone, data, connId: connection.connId, acceptMs }
|
||||
}
|
||||
|
||||
async function echo(phone, data, payload) {
|
||||
const toHost = nextFrame(data)
|
||||
phone.send(payload)
|
||||
if ((await toHost).bytes.toString() !== payload) throw new Error('relay changed phone payload')
|
||||
const toPhone = nextFrame(phone)
|
||||
data.send(Buffer.from(payload))
|
||||
if ((await toPhone).bytes.toString() !== payload) throw new Error('relay changed host payload')
|
||||
}
|
||||
|
||||
function rttMs(socket, timeoutMs = 5_000) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = performance.now()
|
||||
const timer = setTimeout(() => finish(undefined), timeoutMs)
|
||||
const finish = (value) => {
|
||||
clearTimeout(timer)
|
||||
socket.off('pong', onPong)
|
||||
resolve(value)
|
||||
}
|
||||
const onPong = () => finish(performance.now() - startedAt)
|
||||
socket.on('pong', onPong)
|
||||
socket.ping()
|
||||
})
|
||||
}
|
||||
|
||||
export async function installRelayLoadResumeDevice(peer, relayDeviceId) {
|
||||
const inviteReqId = `load-resume-invite-${peer.index}-${peer.spliceSequence++}`
|
||||
const invite = controlAnswer(peer, inviteReqId, 'invite-created')
|
||||
peer.socket.send(JSON.stringify({ type: 'invite-create', reqId: inviteReqId, relayDeviceId }))
|
||||
const created = await invite
|
||||
if (created.type !== 'invite-created') throw new Error(`invite failed: ${created.code}`)
|
||||
const splice = await dialPhone(peer, created.inviteToken, relayDeviceId)
|
||||
try {
|
||||
const token = randomBytes(32).toString('base64url')
|
||||
const reqId = `load-resume-install-${peer.index}-${peer.spliceSequence++}`
|
||||
const installed = controlAnswer(peer, reqId, 'device-credential-installed')
|
||||
peer.socket.send(
|
||||
JSON.stringify({
|
||||
type: 'device-credential-install',
|
||||
v: 1,
|
||||
reqId,
|
||||
relayDeviceId,
|
||||
newResumeTokenHash: resumeTokenHash(token),
|
||||
authorization: { mode: 'relay-basis', basisConnId: splice.connId }
|
||||
})
|
||||
)
|
||||
const result = await installed
|
||||
if (result.type !== 'device-credential-installed') {
|
||||
throw new Error(`device install failed: ${result.code}`)
|
||||
}
|
||||
return token
|
||||
} finally {
|
||||
await Promise.all([peer.closeSpliceSocket(splice.phone), peer.closeSpliceSocket(splice.data)])
|
||||
}
|
||||
}
|
||||
|
||||
// One resume: accept, confirm, then hold the splice with periodic echoes. Never throws.
|
||||
export async function resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { holdMs, stopping, delay }) {
|
||||
const attempt = { at: Date.now(), cellUrl: peer.lastAssignment?.cellUrl }
|
||||
let splice
|
||||
try {
|
||||
splice = await dialPhone(peer, token, relayDeviceId)
|
||||
attempt.acceptMs = Math.round(splice.acceptMs)
|
||||
const reqId = `load-resume-confirm-${peer.index}-${peer.spliceSequence++}`
|
||||
const confirmed = controlAnswer(peer, reqId, 'device-resume-confirmed')
|
||||
const confirmStartedAt = performance.now()
|
||||
peer.socket.send(
|
||||
JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: splice.connId })
|
||||
)
|
||||
const result = await confirmed
|
||||
attempt.confirmMs = Math.round(performance.now() - confirmStartedAt)
|
||||
if (result.type !== 'device-resume-confirmed') {
|
||||
attempt.confirmFailure = String(result.code)
|
||||
} else {
|
||||
attempt.renewed = result.renewed
|
||||
}
|
||||
const rtt = await rttMs(splice.phone)
|
||||
if (rtt !== undefined) attempt.rttMs = Math.round(rtt)
|
||||
const holdUntil = Date.now() + holdMs
|
||||
attempt.echoes = 0
|
||||
while (!stopping() && Date.now() < holdUntil) {
|
||||
await echo(splice.phone, splice.data, `orca-relay-resume:${peer.index}:${attempt.echoes}`)
|
||||
attempt.echoes++
|
||||
await delay(Math.min(ECHO_INTERVAL_MS, Math.max(0, holdUntil - Date.now())))
|
||||
}
|
||||
attempt.ok = attempt.confirmFailure === undefined
|
||||
} catch (error) {
|
||||
attempt.ok = false
|
||||
attempt.failure = error.reason ?? (splice ? 'held-splice' : 'accept')
|
||||
attempt.error = String(error.message).slice(0, 160)
|
||||
if (splice) attempt.heldCloseCode = await closeCode(splice.phone, 100)
|
||||
splice ??= { phone: error.phone, data: error.data }
|
||||
} finally {
|
||||
await Promise.all([peer.closeSpliceSocket(splice?.phone), peer.closeSpliceSocket(splice?.data)])
|
||||
}
|
||||
return attempt
|
||||
}
|
||||
|
||||
const percentile = (values, fraction) => {
|
||||
if (values.length === 0) return undefined
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))]
|
||||
}
|
||||
|
||||
export function relayLoadResumeSummary(attempts) {
|
||||
const accepts = attempts.flatMap((attempt) =>
|
||||
attempt.acceptMs === undefined ? [] : [attempt.acceptMs]
|
||||
)
|
||||
const confirms = attempts.flatMap((attempt) =>
|
||||
attempt.confirmMs === undefined || attempt.confirmFailure ? [] : [attempt.confirmMs]
|
||||
)
|
||||
const rtts = attempts.flatMap((attempt) => (attempt.rttMs === undefined ? [] : [attempt.rttMs]))
|
||||
const failures = {}
|
||||
for (const attempt of attempts) {
|
||||
const reason = attempt.confirmFailure ? `confirm-${attempt.confirmFailure}` : attempt.failure
|
||||
if (reason) failures[reason] = (failures[reason] ?? 0) + 1
|
||||
}
|
||||
return {
|
||||
attempts: attempts.length,
|
||||
ok: attempts.filter((attempt) => attempt.ok).length,
|
||||
acceptMsP50: percentile(accepts, 0.5),
|
||||
acceptMsP95: percentile(accepts, 0.95),
|
||||
confirmMsP50: percentile(confirms, 0.5),
|
||||
confirmMsP95: percentile(confirms, 0.95),
|
||||
rttMsP50: percentile(rtts, 0.5),
|
||||
failures
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { relayLoadResumeSummary } from './relay-load-resume-phone.mjs'
|
||||
|
||||
test('summarizes resume latency from completed steps and failures by reason', () => {
|
||||
const summary = relayLoadResumeSummary([
|
||||
{ ok: true, acceptMs: 100, confirmMs: 20, rttMs: 10 },
|
||||
{ ok: true, acceptMs: 300, confirmMs: 40, rttMs: 10 },
|
||||
{ ok: false, acceptMs: 200, confirmMs: 900, confirmFailure: 'database_unavailable' },
|
||||
{ ok: false, failure: 'hello-4503' },
|
||||
{ ok: false, failure: 'hello-4503' }
|
||||
])
|
||||
|
||||
assert.deepEqual(summary, {
|
||||
attempts: 5,
|
||||
ok: 2,
|
||||
acceptMsP50: 200,
|
||||
acceptMsP95: 300,
|
||||
confirmMsP50: 40,
|
||||
confirmMsP95: 40,
|
||||
rttMsP50: 10,
|
||||
failures: { 'confirm-database_unavailable': 1, 'hello-4503': 2 }
|
||||
})
|
||||
})
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user