diff --git a/cloud/dev/scripts/drive-relay-director-deploy.mjs b/cloud/dev/scripts/drive-relay-director-deploy.mjs new file mode 100644 index 00000000000..1f783cd179c --- /dev/null +++ b/cloud/dev/scripts/drive-relay-director-deploy.mjs @@ -0,0 +1,987 @@ +// Operator-local driver for a production relay director deploy. It only dispatches the existing +// audited workflows and reads their results back; every safety check stays in those workflows. +// +// It keeps no state between runs. Every decision comes from live state read at the start: the +// serving director, its configured cells, and the rehome control. The one fact live state cannot +// show, that a pause is this driver's own, is the run that made it (`--pause-run`), checked +// against that run's log and the live generation. On any stop it prints the command that +// finishes the deploy from wherever it got to. +import { spawnSync } from 'node:child_process' +import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { homedir, tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { createInterface } from 'node:readline/promises' +import { pathToFileURL } from 'node:url' +import { verifyDryRunAuthority } from './relay-monitor-evidence.mjs' +import { + DIRECTOR_SERVICE, + IMAGE_REPOSITORY, + PROJECT, + REGION, + REPOSITORY, + WORKFLOW_REF, + WORKFLOWS, + admissionInspectInputs, + admissionInspectResult, + blocksDeploy, + configureInputs, + directorDeployInputs, + directorRevisions, + logConfirmsPublishedDigest, + monitorDryRunInputs, + parseConfigureWave, + pausedGeneration, + publishInputs, + rehomeEnableInputs, + rehomeInspectInputs, + rehomePauseInputs, + rehomeResultFromLog, + requireCommit, + requireDigest, + revisionDigest, + validateDispatchInputs +} from './relay-director-deploy-plan.mjs' + +const ACTIVE_RUN_STATUSES = ['queued', 'in_progress', 'waiting', 'requested', 'pending'] +// The enable job verifies the monitor within 5 minutes of completion after ~2.5 minutes of setup. +export const MONITOR_MAX_AGE_AT_ENABLE_MS = 150_000 +// The manual procedure watched the new director for 5+ minutes before configuring cells. +export const SOAK_MS = 5 * 60_000 +// Traffic moves about a minute before the deploy run completes (ops-log 05:00:33Z vs 05:01:34Z). +const TRAFFIC_SWITCH_LEAD_MS = 60_000 +// Request logs land up to a minute late; reading at the window's end would undercount it. +const LOG_INGESTION_LAG_MS = 60_000 +const DIRECTOR_5XX_FILTER = [ + 'resource.type="cloud_run_revision"', + `resource.labels.service_name="${DIRECTOR_SERVICE}"`, + `logName="projects/${PROJECT}/logs/run.googleapis.com%2Frequests"`, + 'httpRequest.status>=500' +].join(' AND ') +const LOG_COUNT_LIMIT = 5_000 +const LOG_ATTEMPTS = 6 +const LOG_INTERVAL_MS = 10_000 +const REHOME_HISTORY_RUNS = 5 +const RUN_ID = /^[1-9][0-9]*$/ + +export class DriverStop extends Error {} + +export function parseDriverArguments(argv, home = homedir()) { + const config = { + dryRun: false, + leaveRehomePaused: false, + configure: [], + logDirectory: join(home, '.orca', 'relay-director-deploy') + } + const runId = (value, key) => { + if (!RUN_ID.test(value)) throw new Error(`${key} must be a run ID`) + return Number(value) + } + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index] + if (key === '--dry-run' || key === '--leave-rehome-paused') { + config[key === '--dry-run' ? 'dryRun' : 'leaveRehomePaused'] = true + continue + } + const value = argv[index + 1] + if (value === undefined || value.startsWith('--')) throw new Error(`${key} needs a value`) + index += 1 + if (key === '--commit') config.commit = requireCommit(value, '--commit') + else if (key === '--configure') config.configure.push(parseConfigureWave(value)) + else if (key === '--publish-run') config.publishRun = runId(value, key) + else if (key === '--pause-run') config.pauseRun = runId(value, key) + else if (key === '--rehome-generation') { + if (!/^(0|[1-9][0-9]*)$/.test(value)) throw new Error('--rehome-generation is invalid') + config.rehomeGeneration = Number(value) + } else if (key === '--log-directory') config.logDirectory = resolve(value) + else throw new Error(`unsupported argument ${key}`) + } + if (!config.commit) throw new Error('missing --commit <40-character main commit to publish>') + if (config.pauseRun && config.leaveRehomePaused) { + throw new Error('--pause-run and --leave-rehome-paused contradict each other') + } + return config +} + +const timestamp = (ms) => new Date(ms).toISOString() +const runUrl = (runId) => `https://github.com/${REPOSITORY}/actions/runs/${runId}` +// Argument lists whose values never contain spaces. +const words = (text) => text.split(' ') + +export function createDriver(config, deps) { + let logPath + const typedPhrases = new Map() + // Everything learned this run; nothing outlives the process. + const known = { director: undefined, selector: undefined, control: undefined } + let rollbackPoint + let published = config.publishRun ? { runId: config.publishRun } : undefined + let owned // { generation, runId }: the pause this driver made, the only rehome state it owns + // A pause or enable this run cannot vouch for: `changing` while it may still apply on its own, + // `unconfirmed` once it finished without a usable result. { kind, name, runId?, url? } + let uncertain + let login + const ownRunIds = new Set() + let enabled = false + + function log(message) { + const line = `${timestamp(deps.now())} ${message}` + deps.print(line) + if (logPath) appendFileSync(logPath, `${line}\n`) + } + + function command(program, args, input) { + const result = deps.run(program, args, input) + if (result.status !== 0) { + const detail = String(result.stderr ?? '') + .trim() + .split('\n') + .slice(-3) + .join(' | ') + throw new Error(`${program} ${args.slice(0, 4).join(' ')} failed: ${detail}`) + } + return String(result.stdout ?? '') + } + + const gh = (args, input) => command('gh', args, input) + const ghJson = (args) => JSON.parse(gh(args)) + const gcloudJson = (args) => + JSON.parse(command('gcloud', [...args, '--project', PROJECT, '--format=json'])) + + function readDirector() { + const service = gcloudJson( + words(`run services describe ${DIRECTOR_SERVICE} --region ${REGION}`) + ) + const { servingRevision, rollbackRevision } = directorRevisions(service) + const describe = (revision) => + gcloudJson(words(`run revisions describe ${revision} --region ${REGION}`)) + const serving = describe(servingRevision) + const cellsJson = serving.spec.containers[0].env?.find( + (variable) => variable.name === 'ORCA_RELAY_CELLS_JSON' + )?.value + return { + servingRevision, + servingDigest: revisionDigest(serving, `revision ${servingRevision}`), + servingCreatedAt: serving.metadata?.creationTimestamp, + cells: new Set(JSON.parse(cellsJson ?? '[]').map((cell) => cell.id)), + rollbackRevision, + rollbackDigest: revisionDigest(describe(rollbackRevision), `revision ${rollbackRevision}`) + } + } + + function describeDirector(director) { + return `serving ${director.servingRevision} ${director.servingDigest}; rollback ${director.rollbackRevision} ${director.rollbackDigest}` + } + + // Paginated per status: the repository-wide first page can hide an in-flight relay run. + function activeRuns() { + const active = [] + for (const status of ACTIVE_RUN_STATUSES) { + const lines = gh( + words( + `api --paginate -X GET repos/${REPOSITORY}/actions/runs -f status=${status} -f per_page=100 --jq .workflow_runs[]|{id,path,name,status}` + ) + ) + for (const run of lines.split('\n').filter(Boolean).map(JSON.parse)) { + if (blocksDeploy(run.path) && !ownRunIds.has(run.id)) + active.push(`${run.name} ${runUrl(run.id)} (${run.status})`) + } + } + return active + } + + function requireQuietLane() { + const active = activeRuns() + if (active.length > 0) { + throw new DriverStop( + `relay workflows are in flight; wait for them:\n ${active.join('\n ')}` + ) + } + } + + function viewRun(runId) { + return ghJson( + words( + `run view ${runId} -R ${REPOSITORY} --json status,conclusion,attempt,headSha,headBranch,event,workflowName` + ) + ) + } + + async function waitForRun(run) { + for (;;) { + deps.stream( + 'gh', + words(`run watch ${run.runId} -R ${REPOSITORY} --exit-status --interval 10`) + ) + const view = viewRun(run.runId) + if (view.status === 'completed') { + log(`${run.name}: ${view.conclusion} ${run.url}`) + return { ...run, conclusion: view.conclusion, attempt: view.attempt } + } + await deps.sleep(LOG_INTERVAL_MS) + } + } + + // Dispatches and watches one workflow run. The run ID comes only from the URL `gh workflow run` + // prints, so another dispatch by the same account can never be mistaken for this one. + async function dispatch({ name, workflow, inputs: build, changesRehome = false }) { + const inputs = validateDispatchInputs(build(view())) + // Lets a signal that arrived during synchronous work stop the driver before it dispatches. + await new Promise((resolveYield) => setImmediate(resolveYield)) + requireQuietLane() + log(`dispatch ${name}: ${workflow.file} ${JSON.stringify(inputs)}`) + if (changesRehome) uncertain = { kind: 'changing', name } + const output = gh( + words(`workflow run ${workflow.file} -R ${REPOSITORY} --ref ${WORKFLOW_REF} --json`), + JSON.stringify(inputs) + ) + const printed = output.match(/\/actions\/runs\/([0-9]+)/) + if (!printed) { + throw new DriverStop( + `gh printed no run URL for ${workflow.file}; upgrade gh. The run may exist: find it before re-running` + ) + } + const runId = Number(printed[1]) + const url = runUrl(runId) + ownRunIds.add(runId) + if (changesRehome) Object.assign(uncertain, { runId, url }) + log(`${name}: run ${url}`) + const dispatched = viewRun(runId) + if ( + dispatched.workflowName !== workflow.name || + dispatched.event !== 'workflow_dispatch' || + dispatched.headBranch !== WORKFLOW_REF + ) { + throw new DriverStop( + `run ${runUrl(runId)} is not a ${workflow.file} dispatch on ${WORKFLOW_REF}` + ) + } + // `uncertain` is cleared by the step only once it has read the run's result. + return await waitForRun({ name, runId, url, headSha: dispatched.headSha }) + } + + function requireSuccess(run) { + if (run.conclusion !== 'success') { + throw new DriverStop(`${run.name} run ${run.url} concluded ${run.conclusion}`) + } + } + + async function runLog(runId) { + for (let attempt = 1; ; attempt += 1) { + try { + const text = gh(words(`run view ${runId} -R ${REPOSITORY} --log`)) + if (text.trim()) return text + } catch (error) { + if (attempt >= LOG_ATTEMPTS) + throw new Error(`log for ${runUrl(runId)} is unavailable: ${error.message}`) + } + if (attempt >= LOG_ATTEMPTS) throw new Error(`log for ${runUrl(runId)} is empty`) + await deps.sleep(LOG_INTERVAL_MS) + } + } + + async function rehomeResult(runId) { + try { + return rehomeResultFromLog(await runLog(runId)) + } catch { + return undefined + } + } + + async function withArtifact(runId, artifact, read) { + const directory = mkdtempSync(join(tmpdir(), 'relay-director-deploy-')) + try { + gh(words(`run download ${runId} -R ${REPOSITORY} -n ${artifact} -D ${directory}`)) + return await read(directory) + } finally { + rmSync(directory, { recursive: true, force: true }) + } + } + + async function typed(phrase, meaning = '') { + if (typedPhrases.has(phrase)) return phrase + const answer = (await deps.prompt(`Type ${phrase} to continue${meaning}: `)).trim() + if (answer !== phrase) + throw new DriverStop(`expected ${phrase}; nothing further was dispatched`) + typedPhrases.set(phrase, answer) + log(`operator typed ${phrase}`) + return answer + } + + // The digest the publish run pushed: the registry's digest for the commit's tag, which the push + // line in the run's own log must name too. + async function publishedDigest(runId) { + const view = viewRun(runId) + if (view.workflowName !== WORKFLOWS.publish.name || view.conclusion !== 'success') { + throw new DriverStop(`${runUrl(runId)} is not a successful ${WORKFLOWS.publish.file} run`) + } + if (view.headSha !== config.commit) { + throw new DriverStop( + `publish ${runUrl(runId)} built ${view.headSha}, not the reviewed ${config.commit}; do not deploy it` + ) + } + const tag = `${IMAGE_REPOSITORY}:sha-${config.commit}` + const digest = command( + 'gcloud', + words( + `artifacts docker images describe ${tag} --project ${PROJECT} --format=value(image_summary.digest)` + ) + ).trim() + requireDigest(digest, 'registry digest of the published tag') + if (!logConfirmsPublishedDigest(await runLog(runId), config.commit, digest)) { + throw new DriverStop( + `registry digest ${digest} is not the digest ${runUrl(runId)} pushed; the tag moved` + ) + } + return digest + } + + async function lastKnownRehomeGeneration() { + const runs = ghJson( + words( + `run list -R ${REPOSITORY} --workflow ${WORKFLOWS.rehome.file} --status completed --limit ${REHOME_HISTORY_RUNS} --json databaseId` + ) + ) + for (const run of runs) { + const result = await rehomeResult(run.databaseId) + if (result) { + log( + `rehome generation candidate ${result.control.generation} from ${runUrl(run.databaseId)}` + ) + return result.control.generation + } + } + throw new DriverStop( + 'no recent rehome run printed the control generation; pass --rehome-generation' + ) + } + + // Values every input builder reads. A dry run shows what is not known yet as ``, + // which validateDispatchInputs refuses, so a placeholder can never be dispatched. + function view() { + const unknown = (label) => `<${label}>` + const placeholder = (label) => new Proxy({}, { get: () => unknown(label) }) + const fromAdmission = [unknown('from admission inspect')] + const control = known.control + return { + director: known.director, + selector: known.selector ?? { + generation: fromAdmission[0], + membership: new Proxy({}, { get: () => fromAdmission }) + }, + control: control ?? placeholder('inspected'), + pausedGeneration: + owned?.generation ?? + (control?.enabled === false + ? control.generation + : unknown('rehome generation after pause')), + published: published?.digest ?? unknown('published digest'), + rollbackDigest: rollbackPoint?.digest ?? known.director?.servingDigest, + monitor: placeholder('monitor run'), + afterDeploy: placeholder('digest read from gcloud after the deploy'), + notBefore: config.dryRun ? unknown('now, epoch ms') : Math.floor(deps.now() / 1000) * 1000, + confirmation: (phrase) => + typedPhrases.has(phrase) ? phrase : unknown(`operator types ${phrase}`) + } + } + + const deployPending = () => known.director.servingDigest !== published?.digest + const pendingWaves = () => + config.configure.filter((wave) => wave.cells.some((cell) => !known.director.cells.has(cell))) + const work = () => deployPending() || pendingWaves().length > 0 + + function count5xx(fromMs, toMs) { + const filter = `${DIRECTOR_5XX_FILTER} AND timestamp>="${timestamp(fromMs)}" AND timestamp<"${timestamp(toMs)}"` + return command('gcloud', [ + 'logging', + 'read', + filter, + ...words(`--project ${PROJECT} --limit ${LOG_COUNT_LIMIT} --format=value(timestamp)`) + ]) + .split('\n') + .filter(Boolean).length + } + + // Director 5xx over SOAK_MS on the new image against the same span before its revision existed. + // The window starts at the traffic switch when this run deployed, otherwise now, so a re-run + // after a tripped soak judges fresh traffic. + async function soak(deployedAt) { + const created = Date.parse(known.director.servingCreatedAt) + const start = + deployedAt === undefined ? deps.now() : Math.max(deployedAt - TRAFFIC_SWITCH_LEAD_MS, created) + const end = start + SOAK_MS + const readAt = end + LOG_INGESTION_LAG_MS + if (deps.now() < readAt) { + log( + `soak: watching the new director until ${timestamp(end)}, reading at ${timestamp(readAt)}` + ) + await deps.sleep(readAt - deps.now()) + } + const before = count5xx(created - SOAK_MS, created) + const after = count5xx(start, end) + log( + `soak: director 5xx ${after} in ${SOAK_MS / 60_000} min on the new image, ${before} before it` + ) + if (after >= LOG_COUNT_LIMIT || after > 2 * before + 25) { + throw new DriverStop( + `director 5xx rose from ${before} to ${after} after the deploy; investigate before configuring cells` + ) + } + } + + // The same audited check the enable job runs, on the same sealed files. + async function monitorCompletedAt(run) { + const incidentId = `relay-${run.runId}-dry-run` + return await withArtifact( + run.runId, + `relay-monitor-dry-run-${run.runId}-${run.attempt}`, + async (directory) => { + try { + const argv = Object.entries({ + directory, + 'incident-id': incidentId, + 'run-id': run.runId, + 'run-attempt': run.attempt, + 'commit-sha': run.headSha, + mode: 'dry-run', + 'required-migration-policy': 'strict' + }).flatMap(([key, value]) => [`--${key}`, String(value)]) + const { state } = await verifyDryRunAuthority(argv, deps.now) + log(`monitor GREEN, completed ${state.completedAt}`) + return state.completedAt + } catch (error) { + // The freeze reason lives only in the sealed state, never in the run log. + const sealed = (() => { + try { + return JSON.parse(readFileSync(join(directory, `${incidentId}.state.json`), 'utf8')) + } catch { + return {} + } + })() + const failures = (sealed.failures ?? []).map((failure) => + [ + failure.source, + failure.code, + failure.signal, + failure.observed, + failure.threshold + ].join(' ') + ) + throw new DriverStop( + [ + `monitor ${run.url} is not usable: ${error.message}`, + `frozenAt ${sealed.frozenAt}`, + ...failures + ].join('\n ') + ) + } + } + ) + } + + async function preflight() { + log('preflight: relay workflow lane, target commit, serving director') + if (config.dryRun) { + const active = activeRuns() + if (active.length > 0) + log( + `WARNING relay workflows are in flight; a real run would stop:\n ${active.join('\n ')}` + ) + } else { + requireQuietLane() + } + if (published) { + published.digest = await publishedDigest(published.runId) + } else { + const main = gh(['api', `repos/${REPOSITORY}/commits/${WORKFLOW_REF}`, '--jq', '.sha']).trim() + if (main !== config.commit) { + throw new DriverStop( + `${WORKFLOW_REF} is at ${main}, not the reviewed ${config.commit}; review the difference and run with --commit ${main}` + ) + } + } + known.director = readDirector() + if (known.director.servingDigest !== published?.digest) { + rollbackPoint = { + revision: known.director.servingRevision, + digest: known.director.servingDigest + } + } + log(describeDirector(known.director)) + let claim + if (config.pauseRun) { + // Only this operator's own rehome-control run can prove a pause belongs to this driver. + const claimed = ghJson( + words( + `api repos/${REPOSITORY}/actions/runs/${config.pauseRun} --jq {path,actor:.triggering_actor.login}` + ) + ) + if ( + !claimed.path?.split('@')[0].endsWith(`/${WORKFLOWS.rehome.file}`) || + claimed.actor !== login + ) { + throw new DriverStop( + `${runUrl(config.pauseRun)} is not a ${WORKFLOWS.rehome.file} run by ${login}, so it cannot prove a pause is this driver's` + ) + } + const result = await rehomeResult(config.pauseRun) + claim = result && pausedGeneration(result) + if (claim === undefined) { + throw new DriverStop( + `${runUrl(config.pauseRun)} printed no pause of its own, so it cannot prove a pause is this driver's` + ) + } + } + // A director safety pause moves the generation without a run; the inspect then fails closed. + const generation = config.rehomeGeneration ?? (await lastKnownRehomeGeneration()) + if (config.dryRun) { + known.control = undefined + return generation + } + const [admissionStep, inspectStep] = readSteps(generation) + const admission = await dispatch(admissionStep) + requireSuccess(admission) + known.selector = await withArtifact( + admission.runId, + `relay-asia-admission-result-${admission.runId}-${admission.attempt}`, + (directory) => + admissionInspectResult(JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8'))) + ) + const inspect = await dispatch(inspectStep) + if (inspect.conclusion !== 'success') { + throw new DriverStop( + `rehome inspect at generation ${generation} failed (${inspect.url}): the generation, selector or a digest moved. A director safety pause bumps the generation. Read the log, then pass --rehome-generation` + ) + } + known.control = (await rehomeResult(inspect.runId))?.control + if (!known.control) throw new DriverStop(`rehome inspect ${inspect.url} printed no control`) + const control = known.control + log( + `rehome: generation ${control.generation}, ${control.enabled ? 'ENABLED' : 'disabled'}; selector generation ${known.selector.generation}` + ) + if (claim !== undefined) { + if (!control.enabled && control.generation === claim) + owned = { generation: claim, runId: config.pauseRun } + else if (control.enabled && control.generation === claim + 1) enabled = true + else + throw new DriverStop( + `rehome is generation ${control.generation} ${control.enabled ? 'enabled' : 'paused'}, not the pause ${runUrl(config.pauseRun)} made at ${claim}. Something else changed it, such as a director safety pause; this driver will not touch it` + ) + } else if (control.enabled && config.leaveRehomePaused) { + throw new DriverStop( + 'rehome is enabled; --leave-rehome-paused accepts only a paused switch. Drop it' + ) + } else if (!control.enabled && !config.leaveRehomePaused) { + throw new DriverStop( + `rehome is paused at generation ${control.generation}, and this run did not pause it. If an earlier run of this driver did, re-run with the --pause-run its log printed. Otherwise pass --leave-rehome-paused to deploy and leave it paused` + ) + } + if (control.hostCooldownMs === undefined && (control.enabled || owned)) { + throw new DriverStop( + 'the director reports no per-host rehome cooldown, so enable would refuse; not pausing' + ) + } + return generation + } + + // The two read-only inspects every real run starts with; never resumed, always run fresh. + function readSteps(generation) { + return [ + { + name: 'preflight-admission', + workflow: WORKFLOWS.admission, + inputs: (v) => admissionInspectInputs(v.director.servingDigest) + }, + { + name: 'preflight-rehome', + workflow: WORKFLOWS.rehome, + inputs: (v) => rehomeInspectInputs({ ...v, controlGeneration: generation }) + } + ] + } + + // The one ordered plan. `when` reads live state, so a re-run skips what is already done; a dry run + // prints every step whose need it cannot know yet. + function plan() { + let deployedAt + let verified + let monitor + return [ + { + name: 'publish', + workflow: WORKFLOWS.publish, + inputs: publishInputs, + when: () => !published, + run: async (step) => { + const run = await dispatch(step) + requireSuccess(run) + published = { runId: run.runId } + published.digest = await publishedDigest(run.runId) + log(`published ${IMAGE_REPOSITORY}@${published.digest}`) + } + }, + { + name: 'pause', + workflow: WORKFLOWS.rehome, + changesRehome: true, + inputs: (v) => + rehomePauseInputs({ ...v, confirmation: v.confirmation('PAUSE_REGIONAL_REHOMING') }), + when: () => known.control?.enabled === true && !owned && work(), + run: async (step) => { + await typed('PAUSE_REGIONAL_REHOMING') + const run = await dispatch(step) + const result = await rehomeResult(run.runId) + const generation = result && pausedGeneration(result) + if (generation !== known.control.generation + 1) { + uncertain.kind = 'unconfirmed' + throw new DriverStop( + `${run.url} (${run.conclusion}) printed no pause at generation ${known.control.generation + 1}` + ) + } + owned = { generation, runId: run.runId } + uncertain = undefined + log( + `REHOME PAUSED at generation ${generation}. To finish from here after any stop: ${rerunCommand()}` + ) + } + }, + { + name: 'deploy', + workflow: WORKFLOWS.director, + inputs: (v) => + directorDeployInputs({ + imageDigest: v.published, + predecessorDigest: v.rollbackDigest, + rehomeGeneration: v.pausedGeneration + }), + when: () => !published || deployPending(), + run: async (step) => { + requireSuccess(await dispatch(step)) + deployedAt = deps.now() + known.director = readDirector() + if (deployPending()) + throw new DriverStop( + `deploy: ${describeDirector(known.director)}, not ${published.digest}` + ) + } + }, + ...config.configure.slice(0, 1).map(() => ({ + name: 'soak', + description: `wait ${SOAK_MS / 60_000} min, then compare director 5xx`, + // A wave already configured means an earlier run passed the soak on this image. + when: () => pendingWaves().length === config.configure.length, + run: () => soak(deployedAt) + })), + ...config.configure.map((wave) => ({ + name: `configure:${wave.cells.join(',')}`, + workflow: WORKFLOWS.admission, + inputs: (v) => + configureInputs({ + ...wave, + directorDigest: v.published, + selectorGeneration: v.selector.generation, + confirmation: v.confirmation('CONFIGURE_ASIA_DIRECTOR') + }), + when: () => pendingWaves().includes(wave), + run: async (step) => { + await typed('CONFIGURE_ASIA_DIRECTOR') + requireSuccess(await dispatch(step)) + known.director = readDirector() + if (deployPending()) + throw new DriverStop( + `${step.name}: ${describeDirector(known.director)}, not ${published.digest}` + ) + } + })), + // Inspect binds the exact serving and rollback digests, so a wrong one fails here, read-only, + // before 15 minutes of monitor evidence is spent on it. + { + name: 'verify-identities', + workflow: WORKFLOWS.rehome, + inputs: (v) => + rehomeInspectInputs({ + director: verified ?? v.afterDeploy, + selector: v.selector, + controlGeneration: v.pausedGeneration + }), + when: () => Boolean(owned), + run: async (step) => { + verified = readDirector() + const run = await dispatch(step) + const control = (await rehomeResult(run.runId))?.control + if ( + run.conclusion !== 'success' || + control?.enabled !== false || + control.generation !== owned.generation + ) { + throw new DriverStop( + `verify-identities ${run.url} (${run.conclusion}) did not find rehome paused at ${owned.generation} with ${describeDirector(verified)}` + ) + } + } + }, + { + name: 'monitor', + workflow: WORKFLOWS.monitor, + inputs: (v) => monitorDryRunInputs(v.selector), + when: () => Boolean(owned), + run: async (step) => { + // The operator arms the enable before the 15-minute watch; it still dispatches only on green. + await typed( + 'ENABLE_REGIONAL_REHOMING', + ` (this arms an automatic enable, sent about 17 min from now and only if the monitor is green and its evidence is at most ${MONITOR_MAX_AGE_AT_ENABLE_MS / 1000} s old)` + ) + const run = await dispatch(step) + requireSuccess(run) + monitor = { + runId: run.runId, + attempt: run.attempt, + completedAt: await monitorCompletedAt(run) + } + } + }, + { + name: 'enable', + workflow: WORKFLOWS.rehome, + changesRehome: true, + inputs: (v) => + rehomeEnableInputs({ + ...v, + director: verified ?? v.afterDeploy, + controlGeneration: v.pausedGeneration, + monitor: monitor ?? v.monitor, + confirmation: v.confirmation('ENABLE_REGIONAL_REHOMING') + }), + when: () => Boolean(owned), + run: async (step) => { + const ageMs = deps.now() - Date.parse(monitor.completedAt) + if (ageMs > MONITOR_MAX_AGE_AT_ENABLE_MS) { + throw new DriverStop( + `monitor evidence is ${Math.round(ageMs / 1000)} s old, past the ${MONITOR_MAX_AGE_AT_ENABLE_MS / 1000} s budget; re-run for a fresh monitor` + ) + } + const now = readDirector() + if ( + now.servingDigest !== verified.servingDigest || + now.rollbackDigest !== verified.rollbackDigest + ) { + throw new DriverStop( + `the director changed after its digests were verified: ${describeDirector(now)}` + ) + } + if (known.control.ratePerMinute !== 10) + log( + `enable starts at the job's fixed 10 hosts/min (was ${known.control.ratePerMinute})` + ) + const run = await dispatch(step) + const result = await rehomeResult(run.runId) + if (result) known.control = result.control + // A failed run is never an enable, whatever it printed last. + if ( + run.conclusion === 'success' && + result?.mode === 'enable' && + result.control.enabled && + result.control.generation === owned.generation + 1 + ) { + owned = undefined + uncertain = undefined + enabled = true + log(`REHOME RE-ENABLED at generation ${result.control.generation}`) + return + } + if (result?.mode !== 'recover-enable') { + uncertain.kind = 'unconfirmed' + throw new DriverStop(`${run.url} (${run.conclusion}) printed no enable result`) + } + // Recovery that disabled rehome itself is this driver's pause too. Recovery that found it + // already disabled at another generation found someone else's pause, such as a director + // safety pause: this driver gives up ownership and will never lift it. + uncertain = undefined + const recovered = pausedGeneration(result) + if (recovered !== undefined) owned = { generation: recovered, runId: run.runId } + else if (result.control.generation !== owned.generation) owned = undefined + throw new DriverStop( + `enable ${run.url} failed; the job's recovery left rehome paused at generation ${result.control.generation}` + ) + } + } + ] + } + + function rerunCommand(pauseRun = owned?.runId) { + return [ + 'node dev/scripts/drive-relay-director-deploy.mjs', + `--commit ${config.commit}`, + ...(published?.digest ? [`--publish-run ${published.runId}`] : []), + ...(pauseRun ? [`--pause-run ${pauseRun}`] : []), + ...(config.leaveRehomePaused ? ['--leave-rehome-paused'] : []), + ...config.configure.map( + (wave) => `--configure ${wave.cells.join(',')}=${wave.cellImageDigest}` + ) + ].join(' ') + } + + function stopReport(reason) { + const lines = [`STOPPED: ${reason}`, '', 'State now:'] + const workflowPage = `https://github.com/${REPOSITORY}/actions/workflows/${WORKFLOWS.rehome.file}` + const where = uncertain?.url ?? `(gh printed no URL; find it at ${workflowPage})` + if (uncertain?.name === 'pause') { + lines.push( + uncertain.kind === 'changing' + ? `- *** REHOME IS CHANGING: the pause run ${where} was dispatched and applies on its own, if it has not already. ***` + : `- *** PAUSE UNCONFIRMED: ${where} printed no pause. Rehome MAY BE PAUSED. ***`, + ' Inspect rehome before walking away.' + ) + if (uncertain.runId) { + lines.push( + ` If it paused rehome at generation ${known.control.generation + 1}, finish with: cd cloud && ${rerunCommand(uncertain.runId)}` + ) + } + } else if (uncertain?.name === 'enable') { + lines.push( + uncertain.kind === 'changing' + ? `- *** REHOME IS CHANGING: the enable run ${where} applies on its own: it enables rehome, or disables it again if it fails. ***` + : `- *** ENABLE UNCONFIRMED: ${where} did not confirm the enable. Rehome may be enabled, or paused. ***`, + ` If it enabled rehome, or failed before applying, finish with: cd cloud && ${rerunCommand(owned.runId)}` + ) + if (uncertain.runId) { + lines.push( + ` If its recovery paused rehome again, finish with: cd cloud && ${rerunCommand(uncertain.runId)}` + ) + } + } else if (owned) { + lines.push( + `- *** REHOME IS PAUSED by this driver at generation ${owned.generation} (${runUrl(owned.runId)}). It stays paused until the re-run below enables it. ***` + ) + } else if (known.control) { + const state = + enabled || known.control.enabled + ? 'enabled' + : 'PAUSED, not by this driver; it will not be re-enabled here' + lines.push(`- rehome: generation ${known.control.generation} as last read, ${state}`) + } else { + lines.push('- rehome: not read; this run did not change it') + } + try { + lines.push(`- director now: ${describeDirector(readDirector())}`) + } catch (error) { + lines.push(`- director: could not re-read (${error.message})`) + } + if (published?.digest) + lines.push(`- published: ${published.digest} (${runUrl(published.runId)})`) + if (rollbackPoint) { + lines.push( + `- rollback point: ${rollbackPoint.revision} ${rollbackPoint.digest}. To undo the deploy, while rehome is paused:` + ) + lines.push( + ` ${ghCommand(WORKFLOWS.director, directorDeployInputs({ imageDigest: rollbackPoint.digest, predecessorDigest: rollbackPoint.digest, rehomeGeneration: owned?.generation ?? '' }))}` + ) + } + if (!config.dryRun && !uncertain) { + lines.push( + '', + `Re-run to finish from here (it re-reads everything and skips what is done):`, + ` cd cloud && ${rerunCommand()}` + ) + } + return lines.join('\n') + } + + function ghCommand(workflow, inputs) { + const fields = Object.entries(inputs).map(([key, value]) => `-f ${key}=${value}`) + return `gh workflow run ${workflow.file} -R ${REPOSITORY} --ref ${WORKFLOW_REF} ${fields.join(' ')}` + } + + async function run() { + const stamp = timestamp(deps.now()).replace(/[:.]/g, '-') + mkdirSync(config.logDirectory, { recursive: true, mode: 0o700 }) + logPath = join( + config.logDirectory, + `${stamp}-${config.commit.slice(0, 12)}${config.dryRun ? '-dry-run' : ''}.log` + ) + log(`${config.dryRun ? 'dry run' : 'start'}: ${rerunCommand()}`) + try { + login = gh(['api', 'user', '--jq', '.login']).trim() + const generation = await preflight() + const steps = plan() + for (const step of [...(config.dryRun ? readSteps(generation) : []), ...steps]) { + const needed = config.dryRun || step.when() ? '' : ' (done or not needed)' + const detail = step.workflow + ? `${step.workflow.file} ${JSON.stringify(step.inputs({ ...view(), control: { ...view().control, generation } }))}` + : step.description + log(`plan ${step.name}${needed}: ${detail}`) + } + if (config.dryRun) { + log('dry run: nothing dispatched') + return { dryRun: true } + } + if (steps.some((step) => step.when())) await typed(`DEPLOY ${config.commit.slice(0, 12)}`) + for (const step of steps) if (step.when()) await step.run(step) + const rehome = + enabled || known.control.enabled ? 'enabled' : 'left paused (--leave-rehome-paused)' + log( + `DONE: ${describeDirector(known.director)}; rehome ${rehome}${rollbackPoint ? `; rollback point ${rollbackPoint.revision} ${rollbackPoint.digest}` : ''}` + ) + return { done: true, logPath } + } catch (error) { + for (const line of stopReport(error.message).split('\n')) log(line) + throw Object.assign(error instanceof DriverStop ? error : new DriverStop(error.message), { + reported: true, + logPath + }) + } + } + + // Ctrl-C, SIGTERM or a closed terminal still leaves the operator the live state and the re-run. + function interrupt(signal) { + if (logPath) for (const line of stopReport(`interrupted by ${signal}`).split('\n')) log(line) + } + + return { run, interrupt } +} + +function defaultDependencies() { + return { + run: (program, args, input) => + spawnSync(program, args, { + encoding: 'utf8', + input, + maxBuffer: 256 * 1024 * 1024, + stdio: ['pipe', 'pipe', 'pipe'] + }), + stream: (program, args) => + spawnSync(program, args, { stdio: ['ignore', 'inherit', 'inherit'] }).status, + now: () => Date.now(), + sleep: (ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)), + print: (line) => process.stdout.write(`${line}\n`), + prompt: async (question) => { + const reader = createInterface({ input: process.stdin, output: process.stdout }) + try { + return await reader.question(question) + } finally { + reader.close() + } + } + } +} + +export async function main(argv = process.argv.slice(2), deps = defaultDependencies()) { + const driver = createDriver(parseDriverArguments(argv), deps) + for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], + ['SIGHUP', 129] + ]) { + process.once(signal, () => { + driver.interrupt(signal) + process.exit(code) + }) + } + return await driver.run() +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + if (!error.reported) + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/cloud/dev/scripts/drive-relay-director-deploy.test.mjs b/cloud/dev/scripts/drive-relay-director-deploy.test.mjs new file mode 100644 index 00000000000..460bdf9b0a6 --- /dev/null +++ b/cloud/dev/scripts/drive-relay-director-deploy.test.mjs @@ -0,0 +1,1036 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { + MONITOR_MAX_AGE_AT_ENABLE_MS, + createDriver, + parseDriverArguments +} from './drive-relay-director-deploy.mjs' +import { + IMAGE_REPOSITORY, + REPOSITORY, + WORKFLOWS, + blocksDeploy, + parseConfigureWave, + pausedGeneration, + rehomeResultFromLog, + validateDispatchInputs +} from './relay-director-deploy-plan.mjs' +import { RELAY_WORKFLOW_FILE_PREFIX, relayWorkflowPath } from './relay-repository.mjs' + +const COMMIT = 'a'.repeat(40) +const OLD = `sha256:${'1'.repeat(64)}` +const NEW = `sha256:${'2'.repeat(64)}` +const CELL = `sha256:${'3'.repeat(64)}` +const START = Date.parse('2026-10-05T04:50:00Z') +const MEMBERSHIP = { + existingOnly: ['production-gce-c1'], + migrationOnly: ['production-gce-c34'], + general: ['production-gce-c27', 'production-gce-c7'] +} +const CONTROL = { + generation: 39, + enabled: true, + observationStartedAt: 1_786_687_676_179, + notBefore: 1_790_934_023_000, + ratePerMinute: 10, + preferenceMaxAgeMs: 86_400_000, + hostCooldownMs: 604_800_000, + drainGraceMs: 3_600_000 +} + +function controlLine(mode, control, extra = {}) { + const result = { event: 'relay_regional_rehome_control', mode, ...extra, control } + return `operate / control\tUNKNOWN STEP\t2026-10-05T04:51:27Z ${JSON.stringify(result)}` +} + +function monitorFiles(run, now, overrides = {}) { + const incidentId = `relay-${run.id}-dry-run` + const at = (offset) => new Date(now - offset).toISOString() + const state = JSON.stringify({ + schemaVersion: 4, + incidentId, + environment: 'production', + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + durationMinutes: 15, + intervalMs: 60_000, + sampleCount: 16, + frozenAt: null, + failures: [], + startedAt: at(16 * 60_000), + windowStartedAt: at(15 * 60_000), + lastSampleAt: at(0), + completedAt: at(0), + ...overrides + }) + const manifest = { + schemaVersion: 1, + incidentId, + runId: String(run.id), + runAttempt: 1, + commitSha: run.headSha, + mode: 'dry-run', + files: { [`${incidentId}.state.json`]: createHash('sha256').update(state).digest('hex') } + } + return { + [`${incidentId}.state.json`]: state, + 'evidence-manifest.json': JSON.stringify(manifest) + } +} + +// A model of GitHub Actions and Cloud Run: each dispatch does what the real workflow would do to +// `world`, including the side effects a failing run leaves behind. +function fakeWorld(overrides = {}) { + const world = { + now: START, + main: COMMIT, + nextRunId: 37_000_000_000, + runs: [], + active: [], + printUrl: true, + revision: 700, + serving: { revision: 'orca-cloud-relay-00700-qor', digest: OLD, createdAt: START - 86_400_000 }, + rollback: { revision: 'orca-cloud-relay-00699-das', digest: OLD }, + cells: ['production-gce-c1'], + registry: {}, + selector: { generation: 345, attemptId: 'x', membership: MEMBERSHIP }, + control: CONTROL, + publishedDigest: NEW, + pushLogDigest: undefined, + fail: {}, + monitorState: {}, + director5xx: () => 10, + prompts: [], + answer: (question) => question.match(/^Type (\S+(?: [0-9a-f]{12})?) to continue/)[1], + printed: [], + ...overrides + } + // The last rehome run before this deploy, where the driver finds the current generation. + world.runs.push({ + id: 1, + file: WORKFLOWS.rehome.file, + conclusion: 'success', + log: controlLine('enable', world.control) + }) + world.dispatches = () => world.runs.filter((run) => run.dispatched) + return world +} + +function promote(world, digest) { + world.revision += 5 + world.rollback = { revision: `orca-cloud-relay-00${world.revision - 1}-rbk`, digest } + world.serving = { + revision: `orca-cloud-relay-00${world.revision}-new`, + digest, + createdAt: world.now + } +} + +const nextGeneration = (world, enabled, step = 1) => { + world.control = { ...world.control, generation: world.control.generation + step, enabled } +} + +function simulate(world, run) { + const { inputs } = run + const failure = world.fail[run.key] + delete world.fail[run.key] + run.conclusion = failure ? 'failure' : 'success' + if (failure === 'before-apply') return + if (run.file === WORKFLOWS.admission.file && inputs.mode === 'inspect') { + const result = { + v: 1, + mode: 'inspect', + generation: world.selector.generation, + membership: world.selector.membership + } + run.artifacts[`relay-asia-admission-result-${run.id}-1`] = { + 'result.json': JSON.stringify(result) + } + } else if (run.file === WORKFLOWS.admission.file) { + assert.equal(inputs.confirmation, 'CONFIGURE_ASIA_DIRECTOR') + promote(world, inputs['director-image-digest']) + world.cells = [...world.cells, ...inputs['cell-ids'].split(',')] + } else if (run.file === WORKFLOWS.rehome.file) { + const identities = + inputs['director-image-digest'] === world.serving.digest && + inputs['rollback-image-digest'] === world.rollback.digest + const matches = + Number(inputs['expected-control-generation']) === world.control.generation && + Number(inputs['expected-selector-generation']) === world.selector.generation + if (inputs.mode === 'pause' && matches && world.control.enabled) { + assert.equal(inputs.confirmation, 'PAUSE_REGIONAL_REHOMING') + // The real job applies the pause before anything else can fail. + nextGeneration(world, false) + run.log = controlLine('pause', world.control) + } else if (inputs.mode === 'enable') { + assert.equal(inputs.confirmation, 'ENABLE_REGIONAL_REHOMING') + assert.equal( + world.runs.find((other) => other.id === Number(inputs['monitor-run-id']))?.file, + WORKFLOWS.monitor.file + ) + if (matches && identities && !failure) { + nextGeneration(world, true) + run.log = controlLine('enable', world.control) + return + } + run.conclusion = 'failure' + if (failure === 'applied-silent') { + // Applied, then a later step failed and the recovery printed nothing. + nextGeneration(world, true) + run.log = controlLine('enable', world.control) + return + } + // Applied, then the job's own recovery disabled it again; or a director safety pause got + // there first and recovery found it disabled. + if (failure === 'applied-then-recovered') nextGeneration(world, false, 2) + if (failure === 'safety-pause') nextGeneration(world, false, 3) + run.log = controlLine('recover-enable', world.control, { + recovered: failure === 'applied-then-recovered' + }) + } else if (!matches || (inputs.mode === 'inspect' && !identities)) { + run.conclusion = 'failure' + } else { + run.log = controlLine(inputs.mode, world.control) + } + } else if (run.file === WORKFLOWS.publish.file) { + world.registry[`${IMAGE_REPOSITORY}:sha-${run.headSha}`] = world.publishedDigest + run.log = `publish\tBuild\tsha-${run.headSha}: digest: ${world.pushLogDigest ?? world.publishedDigest} size: 3241` + } else if (run.file === WORKFLOWS.director.file) { + if ( + world.control.enabled || + Number(inputs['expected-rehome-generation']) !== world.control.generation + ) { + run.conclusion = 'failure' + return + } + promote(world, inputs['image-digest']) + // Blue/green takes minutes; traffic moves about one before the run completes. + world.now += 4 * 60_000 + } else if (run.file === WORKFLOWS.monitor.file) { + world.now += 16 * 60_000 + run.artifacts[`relay-monitor-dry-run-${run.id}-1`] = monitorFiles( + run, + world.now, + world.monitorState + ) + } +} + +function flag(args, name) { + const index = args.indexOf(name) + return index < 0 ? undefined : args[index + 1] +} + +function gh(world, args, input) { + const ok = (value) => ({ + status: 0, + stdout: typeof value === 'string' ? value : JSON.stringify(value), + stderr: '' + }) + if (args[0] === 'api' && args[1] === 'user') return ok('operator\n') + if (args[0] === 'api' && args.includes('--paginate')) { + const status = flag(args, '-f').slice('status='.length) + return ok( + world.active + .filter((run) => run.status === status) + .map((run) => `${JSON.stringify(run)}\n`) + .join('') + ) + } + if (args[0] === 'api' && args[1].includes('/actions/runs/')) { + const claimed = world.runs.find( + (candidate) => candidate.id === Number(args[1].split('/').at(-1)) + ) + return ok({ + path: relayWorkflowPath(claimed.file.slice(RELAY_WORKFLOW_FILE_PREFIX.length)), + actor: claimed.actor ?? 'operator' + }) + } + if (args[0] === 'api') return ok(`${world.main}\n`) + if (args[0] === 'workflow') { + const workflow = Object.values(WORKFLOWS).find((candidate) => candidate.file === args[2]) + const inputs = JSON.parse(input) + const run = { + id: world.nextRunId++, + file: workflow.file, + name: workflow.name, + inputs, + key: `${workflow.file}:${inputs.mode ?? 'deploy'}`, + dispatched: true, + headSha: world.main, + artifacts: {}, + log: '' + } + world.runs.push(run) + simulate(world, run) + return ok( + world.printUrl ? `https://github.com/${REPOSITORY}/actions/runs/${run.id}\n` : 'Created\n' + ) + } + const run = world.runs.find((candidate) => candidate.id === Number(args[2])) + if (args[1] === 'list') { + const runs = world.runs + .filter((candidate) => candidate.file === flag(args, '--workflow')) + .reverse() + return ok(runs.map((candidate) => ({ databaseId: candidate.id }))) + } + if (args[1] === 'view' && args.includes('--log')) { + return world.unreadableLogs?.(run) ? { status: 1, stdout: '', stderr: 'HTTP 502' } : ok(run.log) + } + if (args[1] === 'view') { + return ok({ + status: 'completed', + conclusion: run.conclusion, + attempt: 1, + headSha: run.headSha, + headBranch: 'main', + event: 'workflow_dispatch', + workflowName: run.name + }) + } + if (args[1] === 'download') { + const files = run.artifacts[flag(args, '-n')] + if (!files) return { status: 1, stdout: '', stderr: 'no artifact' } + for (const [name, content] of Object.entries(files)) + writeFileSync(join(flag(args, '-D'), name), content) + return ok('') + } + throw new Error(`unexpected gh ${args.join(' ')}`) +} + +function gcloud(world, args) { + const ok = (value) => ({ + status: 0, + stdout: typeof value === 'string' ? value : JSON.stringify(value), + stderr: '' + }) + if (args[1] === 'services') { + return ok({ + status: { + traffic: [ + { revisionName: world.serving.revision, percent: 100 }, + { revisionName: world.rollback.revision, percent: 0, tag: 'selector-rollback' } + ] + } + }) + } + if (args[1] === 'revisions') { + const revision = [world.serving, world.rollback].find( + (candidate) => candidate.revision === args[3] + ) + const cells = JSON.stringify(world.cells.map((id) => ({ id }))) + return ok({ + metadata: { creationTimestamp: new Date(revision.createdAt ?? START).toISOString() }, + spec: { + containers: [ + { + image: `${IMAGE_REPOSITORY}@${revision.digest}`, + env: [{ name: 'ORCA_RELAY_CELLS_JSON', value: cells }] + } + ] + } + }) + } + if (args[0] === 'artifacts') return ok(`${world.registry[args[4]] ?? ''}\n`) + if (args[0] === 'logging') { + const [, from, to] = args[2].match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) + world.reads = [ + ...(world.reads ?? []), + { from: Date.parse(from), to: Date.parse(to), readAt: world.now } + ] + return ok('t\n'.repeat(world.director5xx(Date.parse(from), Date.parse(to)))) + } + throw new Error(`unexpected gcloud ${args.join(' ')}`) +} + +function dependencies(world) { + return { + run: (program, args, input) => + program === 'gh' ? gh(world, args, input) : gcloud(world, args), + stream: () => 0, + now: () => world.now, + sleep: async (ms) => { + world.now += ms + }, + print: (line) => { + // Every command the driver prints must be one its own parser accepts. + for (const [, argv] of line.matchAll(/drive-relay-director-deploy\.mjs ([^*]+?)\s*$/g)) { + assert.doesNotThrow(() => parseDriverArguments(argv.split(' ')), line) + } + world.printed.push(line) + }, + prompt: async (question) => { + world.questions = [...(world.questions ?? []), question] + const answer = world.answer(question) + world.prompts.push(question.match(/^Type (\S+(?: [0-9a-f]{12})?) to continue/)[1]) + return answer + } + } +} + +const logDirectory = () => mkdtempSync(join(tmpdir(), 'relay-deploy-test-')) + +function drive(world, argv, deps = dependencies(world)) { + return createDriver( + parseDriverArguments([...argv, '--log-directory', logDirectory()]), + deps + ).run() +} + +const start = (world, extra = [], deps) => drive(world, ['--commit', COMMIT, ...extra], deps) + +// Runs the last printed command, or the last one on a line containing `marker`. +function rerun(world, marker = '') { + const command = world.printed + .filter((line) => line.includes(marker) && line.includes('drive-relay-director-deploy.mjs ')) + .at(-1) + const argv = command + .slice(command.indexOf('.mjs ') + 5) + .trim() + .split(' ') + world.printed.length = 0 + return drive(world, argv) +} + +const stopped = (promise) => + promise.then( + () => assert.fail('expected the driver to stop'), + (error) => error + ) + +function keys(world) { + return world + .dispatches() + .map((run) => run.key.slice(RELAY_WORKFLOW_FILE_PREFIX.length).replace('.yml', '')) +} + +const dispatched = (world, key) => world.dispatches().filter((run) => run.key === key) +const REHOME = (mode) => `${WORKFLOWS.rehome.file}:${mode}` +const DEPLOY = `${WORKFLOWS.director.file}:deploy` +const PUBLISH = `${WORKFLOWS.publish.file}:publish` +const MONITOR = `${WORKFLOWS.monitor.file}:dry-run` +const report = (world) => world.printed.join('\n') +const live = (world) => [world.control.generation, world.control.enabled] + +test('publishes before pausing, types every phrase, and enables on the digests now serving', async () => { + const world = fakeWorld() + assert.equal((await start(world)).done, true) + assert.deepEqual(keys(world), [ + 'operate-relay-asia-admission:inspect', + 'operate-relay-production-rehome:inspect', + 'publish-relay-production:publish', + 'operate-relay-production-rehome:pause', + 'deploy-relay-production-director:deploy', + 'operate-relay-production-rehome:inspect', + 'monitor-relay-production:dry-run', + 'operate-relay-production-rehome:enable' + ]) + assert.deepEqual(world.prompts, [ + 'DEPLOY aaaaaaaaaaaa', + 'PAUSE_REGIONAL_REHOMING', + 'ENABLE_REGIONAL_REHOMING' + ]) + assert.ok( + world.questions.some((question) => + question.includes( + 'arms an automatic enable, sent about 17 min from now and only if the monitor is green and its evidence is at most 150 s old' + ) + ) + ) + const [deploy] = dispatched(world, DEPLOY) + assert.deepEqual( + [ + deploy.inputs['image-digest'], + deploy.inputs['predecessor-image-digest'], + deploy.inputs['expected-rehome-generation'] + ], + [NEW, OLD, '40'] + ) + const [enable] = dispatched(world, REHOME('enable')) + const [monitor] = dispatched(world, MONITOR) + assert.deepEqual( + [ + enable.inputs['director-image-digest'], + enable.inputs['rollback-image-digest'], + enable.inputs['expected-control-generation'], + enable.inputs['monitor-run-id'] + ], + [NEW, NEW, '40', String(monitor.id)] + ) + assert.equal(monitor.inputs['expected-general-cells'], 'production-gce-c27,production-gce-c7') + assert.deepEqual(live(world), [41, true]) +}) + +test('a wrong phrase stops before the first mutation', async () => { + const world = fakeWorld({ answer: () => 'yes' }) + assert.match((await stopped(start(world))).message, /expected DEPLOY aaaaaaaaaaaa/) + assert.deepEqual(keys(world), [ + 'operate-relay-asia-admission:inspect', + 'operate-relay-production-rehome:inspect' + ]) +}) + +test('F2: rehome found paused is never adopted; --leave-rehome-paused deploys and leaves it paused', async () => { + const world = fakeWorld({ control: { ...CONTROL, generation: 52, enabled: false } }) + assert.match( + (await stopped(start(world))).message, + /did not pause it.*--pause-run.*--leave-rehome-paused/s + ) + assert.equal(dispatched(world, PUBLISH).length, 0) + await start(world, ['--leave-rehome-paused']) + assert.equal( + dispatched(world, REHOME('pause')).length + dispatched(world, REHOME('enable')).length, + 0 + ) + assert.equal(dispatched(world, DEPLOY)[0].inputs['expected-rehome-generation'], '52') + assert.deepEqual(live(world), [52, false]) +}) + +test('F2: main moving is caught before any rehome change, and after the pause it changes nothing', async () => { + const moved = fakeWorld({ main: 'b'.repeat(40) }) + assert.match((await stopped(start(moved))).message, /not the reviewed/) + assert.equal(moved.dispatches().length, 0) + + const world = fakeWorld() + const deps = dependencies(world) + const run = deps.run + deps.run = (program, args, input) => { + const result = run(program, args, input) + if (args[0] === 'workflow' && JSON.parse(input).mode === 'pause') world.main = 'b'.repeat(40) + return result + } + assert.equal((await start(world, [], deps)).done, true) + assert.deepEqual(live(world), [41, true]) +}) + +test('a publish whose log disagrees with the registry stops before rehome is touched', async () => { + const world = fakeWorld({ pushLogDigest: CELL }) + assert.match((await stopped(start(world))).message, /tag moved/) + assert.equal(dispatched(world, REHOME('pause')).length, 0) + assert.match(report(world), /rehome: generation 39 as last read, enabled/) +}) + +test('dry run dispatches nothing and prints every step', async () => { + const world = fakeWorld() + const directory = logDirectory() + await createDriver( + parseDriverArguments([ + '--commit', + COMMIT, + '--dry-run', + '--configure', + `production-gce-c34=${CELL}`, + '--log-directory', + directory + ]), + dependencies(world) + ).run() + assert.equal(world.dispatches().length, 0) + assert.equal(world.prompts.length, 0) + const plan = world.printed.filter((line) => line.includes(' plan ')) + assert.equal(plan.length, 10) + assert.ok(plan.some((line) => line.includes('"expected-control-generation":"39"'))) + assert.ok( + plan.some((line) => line.includes('"confirmation":""')) + ) + assert.deepEqual( + readdirSync(directory).map((name) => name.endsWith('-dry-run.log')), + [true] + ) +}) + +test('an in-flight relay workflow stops the preflight; no printed run URL stops the dispatch', async () => { + const busy = fakeWorld({ + active: [ + { + id: 9, + path: relayWorkflowPath('deploy-relay-production-same-cap.yml'), + name: 'Same cap', + status: 'waiting' + } + ] + }) + assert.match((await stopped(start(busy))).message, /in flight/) + assert.equal(busy.dispatches().length, 0) + const silent = fakeWorld({ printUrl: false }) + assert.match((await stopped(start(silent))).message, /printed no run URL/) +}) + +test('F1: an interrupt while the pause is in flight says rehome is changing, and the printed command finishes', async () => { + const world = fakeWorld() + const deps = dependencies(world) + let driver + deps.stream = () => { + if (world.dispatches().at(-1)?.inputs.mode === 'pause' && !world.interrupted) { + world.interrupted = true + driver.interrupt('SIGINT') + throw new Error('killed') + } + return 0 + } + driver = createDriver( + parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), + deps + ) + await driver.run().catch(() => {}) + assert.match( + report(world), + /STOPPED: interrupted by SIGINT[\s\S]*REHOME IS CHANGING: the pause run .* applies on its own/ + ) + assert.match(report(world), /finish with: cd cloud && .* --publish-run \d+ --pause-run \d+/) + assert.equal(await rerun(world).then((result) => result.done), true) + assert.equal(dispatched(world, REHOME('pause')).length, 1) + assert.equal(dispatched(world, PUBLISH).length, 1) + assert.deepEqual(live(world), [41, true]) +}) + +test('F1: an interrupt while the enable is in flight never says rehome is paused', async () => { + const world = fakeWorld() + const deps = dependencies(world) + let driver + deps.stream = () => { + if (world.dispatches().at(-1)?.inputs.mode === 'enable') driver.interrupt('SIGTERM') + return 0 + } + driver = createDriver( + parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), + deps + ) + await driver.run() + const text = report(world) + assert.match( + text, + /REHOME IS CHANGING: the enable run .* it enables rehome, or disables it again if it fails/ + ) + assert.doesNotMatch(text.slice(text.indexOf('interrupted')), /REHOME IS PAUSED/) +}) + +test('a pause run that printed nothing is PAUSE UNCONFIRMED, and a later safety pause is never lifted', async () => { + const world = fakeWorld({ fail: { [REHOME('pause')]: 'before-apply' } }) + await stopped(start(world)) + assert.match(report(world), /PAUSE UNCONFIRMED: .* printed no pause. Rehome MAY BE PAUSED/) + // A director safety pause lands; the operator follows the report's command. + nextGeneration(world, false) + assert.match((await stopped(rerun(world))).message, /printed no pause of its own/) + assert.equal(dispatched(world, REHOME('enable')).length, 0) +}) + +test('a failed deploy leaves its pause owned, and the printed command finishes without pausing or publishing again', async () => { + const world = fakeWorld({ fail: { [DEPLOY]: 'before-apply' } }) + await stopped(start(world)) + assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) + assert.match(report(world), /rollback point: orca-cloud-relay-00700-qor/) + await rerun(world) + assert.equal(dispatched(world, REHOME('pause')).length, 1) + assert.equal(dispatched(world, PUBLISH).length, 1) + assert.equal(dispatched(world, DEPLOY).length, 2) + assert.deepEqual(live(world), [41, true]) +}) + +test('a frozen monitor stops with its failures; the re-run runs a fresh one', async () => { + const failures = [ + { source: 'auth', code: 'threshold_equal', signal: 'health', observed: 0, threshold: 1 } + ] + const world = fakeWorld({ monitorState: { frozenAt: '2026-10-05T05:30:00Z', failures } }) + assert.match( + (await stopped(start(world))).message, + /incomplete or stale[\s\S]*auth threshold_equal health 0 1/ + ) + world.monitorState = {} + await rerun(world) + assert.equal(dispatched(world, MONITOR).length, 2) + assert.equal(dispatched(world, DEPLOY).length, 1) + assert.deepEqual(live(world), [41, true]) +}) + +test('stale monitor evidence is not spent on an enable', async () => { + const world = fakeWorld() + const deps = dependencies(world) + const prompt = deps.prompt + deps.prompt = async (question) => { + const answer = await prompt(question) + if (answer === 'ENABLE_REGIONAL_REHOMING') { + const completed = world.now + 16 * 60_000 - MONITOR_MAX_AGE_AT_ENABLE_MS - 1000 + const at = (offset) => new Date(completed - offset).toISOString() + world.monitorState = { + completedAt: at(0), + lastSampleAt: at(0), + windowStartedAt: at(15 * 60_000), + startedAt: at(16 * 60_000) + } + } + return answer + } + assert.match((await stopped(start(world, [], deps))).message, /past the 150 s budget/) + assert.equal(dispatched(world, REHOME('enable')).length, 0) +}) + +test('the ops-log 05:41Z case: an enable that failed before applying is finished by the printed command', async () => { + const world = fakeWorld({ fail: { [REHOME('enable')]: 'not-applied' } }) + await stopped(start(world)) + assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) + await rerun(world) + assert.deepEqual(live(world), [41, true]) +}) + +test('an enable whose own recovery paused rehome again hands ownership to that run', async () => { + const world = fakeWorld({ fail: { [REHOME('enable')]: 'applied-then-recovered' } }) + await stopped(start(world)) + const [enable] = dispatched(world, REHOME('enable')) + assert.match( + report(world), + new RegExp(`REHOME IS PAUSED by this driver at generation 42 \\(.*${enable.id}\\)`) + ) + await rerun(world) + assert.deepEqual(live(world), [43, true]) +}) + +test('F3: a director safety pause found by the enable recovery is never adopted or lifted', async () => { + const world = fakeWorld({ fail: { [REHOME('enable')]: 'safety-pause' } }) + await stopped(start(world)) + assert.match(report(world), /rehome: generation 43 as last read, PAUSED, not by this driver/) + const stop = report(world).slice(report(world).indexOf('STOPPED')) + assert.doesNotMatch(stop, /--pause-run/) + assert.match((await stopped(rerun(world))).message, /did not pause it/) + const [pause] = dispatched(world, REHOME('pause')) + const claim = await stopped( + drive(world, [ + '--commit', + COMMIT, + '--publish-run', + String(dispatched(world, PUBLISH)[0].id), + '--pause-run', + String(pause.id) + ]) + ) + assert.match(claim.message, /not the pause .* made at 40/) + assert.equal(dispatched(world, REHOME('enable')).length, 1) + assert.deepEqual(live(world), [43, false]) +}) + +test('P1: a green enable with an unreadable log is ENABLE UNCONFIRMED, and the printed command settles it', async () => { + const world = fakeWorld({ unreadableLogs: (run) => run.inputs?.mode === 'enable' }) + await stopped(start(world)) + assert.match( + report(world), + /ENABLE UNCONFIRMED: .* did not confirm the enable. Rehome may be enabled, or paused/ + ) + world.unreadableLogs = undefined + assert.equal((await rerun(world, 'If it enabled rehome')).done, true) + assert.equal(dispatched(world, REHOME('enable')).length, 1) + assert.deepEqual(live(world), [41, true]) +}) + +test('configure waits out a soak anchored at the traffic switch, then takes a typed phrase', async () => { + const world = fakeWorld() + await start(world, ['--configure', `production-gce-c34=${CELL}`]) + const [configure] = dispatched(world, `${WORKFLOWS.admission.file}:configure`) + assert.deepEqual( + [ + configure.inputs['cell-ids'], + configure.inputs['image-digest'], + configure.inputs['director-image-digest'], + configure.inputs['selector-generation'] + ], + ['production-gce-c34', CELL, NEW, '345'] + ) + assert.ok(world.prompts.includes('CONFIGURE_ASIA_DIRECTOR')) + const [before, after] = world.reads + assert.equal(after.to - after.from, 5 * 60_000) + // The deploy ran 4 minutes; traffic moved a minute before it completed. + assert.equal(after.from, START + 3 * 60_000) + assert.equal(before.to, START) + assert.ok(after.readAt >= after.to + 60_000) +}) + +test('F5: a tripped soak is judged again on fresh traffic by the printed command', async () => { + const world = fakeWorld({ director5xx: (from) => (from >= START ? 200 : 10) }) + assert.match( + (await stopped(start(world, ['--configure', `production-gce-c34=${CELL}`]))).message, + /5xx rose from 10 to 200/ + ) + assert.match(report(world), /REHOME IS PAUSED by this driver at generation 40/) + world.director5xx = () => 10 + world.now += 30 * 60_000 + const rerunAt = world.now + await rerun(world) + assert.equal(dispatched(world, `${WORKFLOWS.admission.file}:configure`).length, 1) + assert.ok(world.reads.at(-1).from >= rerunAt) + assert.deepEqual(live(world), [41, true]) +}) + +test('P8 and G1: the printed command never reports DONE without reading rehome', async () => { + const world = fakeWorld() + await start(world) + const before = world.dispatches().length + world.prompts.length = 0 + const publishRun = String(dispatched(world, PUBLISH)[0].id) + assert.equal((await drive(world, ['--commit', COMMIT, '--publish-run', publishRun])).done, true) + assert.deepEqual(keys(world).slice(before), [ + 'operate-relay-asia-admission:inspect', + 'operate-relay-production-rehome:inspect' + ]) + assert.equal(world.prompts.length, 0) + assert.match(report(world), /DONE: .* rehome enabled/) + + // G1: the deploy is done but the driver's pause still holds; dropping --pause-run must not hide it. + const paused = fakeWorld({ monitorState: { frozenAt: '2026-10-05T05:30:00Z' } }) + await stopped(start(paused)) + const run = String(dispatched(paused, PUBLISH)[0].id) + assert.match( + (await stopped(drive(paused, ['--commit', COMMIT, '--publish-run', run]))).message, + /did not pause it/ + ) + assert.deepEqual(live(paused), [40, false]) +}) + +test("G2: --pause-run accepts only this operator's own rehome-control run", async () => { + const world = fakeWorld({ control: { ...CONTROL, generation: 40, enabled: false } }) + // Another operator paused rehome by hand. + world.runs.push({ + id: 900, + file: WORKFLOWS.rehome.file, + actor: 'someone-else', + conclusion: 'success', + log: controlLine('pause', world.control) + }) + assert.match( + (await stopped(start(world, ['--pause-run', '900']))).message, + /is not a .* run by operator/ + ) + // A run of another workflow cannot prove a pause either. + world.runs.push({ + id: 901, + file: WORKFLOWS.monitor.file, + conclusion: 'success', + log: controlLine('pause', world.control) + }) + assert.match( + (await stopped(start(world, ['--pause-run', '901']))).message, + /is not a .* run by operator/ + ) + assert.equal(world.dispatches().length, 0) + assert.deepEqual(live(world), [40, false]) +}) + +test('G3: a failed enable run is never reported RE-ENABLED, whatever it printed last', async () => { + const world = fakeWorld({ fail: { [REHOME('enable')]: 'applied-silent' } }) + await stopped(start(world)) + assert.doesNotMatch(report(world), /RE-ENABLED|DONE/) + assert.match(report(world), /ENABLE UNCONFIRMED: .* did not confirm the enable/) + // The printed command re-reads live state: rehome is at the pause's generation + 1, enabled. + const pauseRun = String(dispatched(world, REHOME('pause'))[0].id) + const publishRun = String(dispatched(world, PUBLISH)[0].id) + assert.equal( + (await drive(world, ['--commit', COMMIT, '--publish-run', publishRun, '--pause-run', pauseRun])) + .done, + true + ) + assert.equal(dispatched(world, REHOME('enable')).length, 1) +}) + +test('Q1 and C1: after a hard kill mid-pause, the re-run names the pause its log recorded and finishes with it', async () => { + const world = fakeWorld() + const deps = dependencies(world) + const directory = logDirectory() + let logAtKill + deps.stream = () => { + if (world.dispatches().at(-1)?.key !== REHOME('pause')) return 0 + // SIGKILL writes no stop report: only what was logged before the watch survives. + logAtKill = readFileSync(join(directory, readdirSync(directory)[0]), 'utf8') + throw new Error('SIGKILL') + } + await stopped( + createDriver( + parseDriverArguments(['--commit', COMMIT, '--log-directory', directory]), + deps + ).run() + ) + world.printed.length = 0 + const [pause] = dispatched(world, REHOME('pause')) + assert.match(logAtKill, new RegExp(`runs/${pause.id}`)) + const publishRun = String(dispatched(world, PUBLISH)[0].id) + assert.match( + (await stopped(drive(world, ['--commit', COMMIT, '--publish-run', publishRun]))).message, + /did not pause it.*--pause-run its log printed/ + ) + assert.deepEqual(live(world), [40, false]) + const argv = ['--commit', COMMIT, '--publish-run', publishRun, '--pause-run', String(pause.id)] + assert.equal((await drive(world, argv)).done, true) + assert.deepEqual(live(world), [41, true]) +}) + +test('Q5: a failed enable followed by an unexplained disable is never adopted', async () => { + const world = fakeWorld() + const deps = dependencies(world) + const run = deps.run + deps.run = (program, args, input) => { + const result = run(program, args, input) + if (args[0] === 'workflow' && JSON.parse(input).mode === 'enable') { + world.runs.at(-1).conclusion = 'failure' + world.control = { ...world.control, generation: 42, enabled: false } + } + return result + } + await stopped(start(world, [], deps)) + assert.match(report(world), /ENABLE UNCONFIRMED/) + assert.doesNotMatch(report(world), /RE-ENABLED|DONE/) + const commands = ['failed before applying', 'paused rehome again'].map((marker) => { + const line = world.printed.findLast((printed) => printed.includes(marker)) + return line.slice(line.indexOf('.mjs ') + 5).split(' ') + }) + for (const argv of commands) { + assert.match((await stopped(drive(world, argv))).message, /--rehome-generation|cannot prove/) + const pinned = await stopped(drive(world, [...argv, '--rehome-generation', '42'])) + assert.match(pinned.message, /not the pause|cannot prove/) + } + assert.equal(dispatched(world, REHOME('enable')).length, 1) + assert.deepEqual(live(world), [42, false]) +}) + +test('C1: a pause whose run cannot be viewed, or printed no URL, is reported as REHOME IS CHANGING', async () => { + const world = fakeWorld() + const deps = dependencies(world) + const run = deps.run + deps.run = (program, args, input) => { + const pause = dispatched(world, REHOME('pause'))[0] + if (pause && args[1] === 'view' && args[2] === String(pause.id) && args.includes('--json')) { + return { status: 1, stdout: '', stderr: 'HTTP 502' } + } + return run(program, args, input) + } + await stopped(start(world, [], deps)) + const [pause] = dispatched(world, REHOME('pause')) + assert.match(report(world), new RegExp(`REHOME IS CHANGING: the pause run .*${pause.id}`)) + assert.match(report(world), new RegExp(`finish with: cd cloud && .*--pause-run ${pause.id}`)) + assert.doesNotMatch(report(world), /Re-run to finish/) + + const silent = fakeWorld() + const silentDeps = dependencies(silent) + const silentRun = silentDeps.run + silentDeps.run = (program, args, input) => { + const result = silentRun(program, args, input) + return args[0] === 'workflow' && JSON.parse(input).mode === 'pause' + ? { ...result, stdout: 'Created\n' } + : result + } + await stopped(start(silent, [], silentDeps)) + assert.match( + report(silent), + /REHOME IS CHANGING: the pause run \(gh printed no URL; find it at https:/ + ) + assert.doesNotMatch(report(silent), /Re-run to finish/) +}) + +test('C2: --leave-rehome-paused refuses an enabled switch instead of pausing it', async () => { + const world = fakeWorld() + assert.match( + (await stopped(start(world, ['--leave-rehome-paused']))).message, + /accepts only a paused switch/ + ) + assert.equal(dispatched(world, REHOME('pause')).length + dispatched(world, PUBLISH).length, 0) +}) + +test('G4: an interrupt during the enable prints both commands that can finish', async () => { + const world = fakeWorld() + const deps = dependencies(world) + let driver + deps.stream = () => { + if (world.dispatches().at(-1)?.inputs.mode === 'enable') driver.interrupt('SIGINT') + return 0 + } + driver = createDriver( + parseDriverArguments(['--commit', COMMIT, '--log-directory', logDirectory()]), + deps + ) + await driver.run() + const [pause] = dispatched(world, REHOME('pause')) + const [enable] = dispatched(world, REHOME('enable')) + assert.match( + report(world), + new RegExp( + `If it enabled rehome, or failed before applying, finish with: cd cloud && .*--pause-run ${pause.id}` + ) + ) + assert.match( + report(world), + new RegExp( + `If its recovery paused rehome again, finish with: cd cloud && .*--pause-run ${enable.id}` + ) + ) +}) + +test("G5: the driver's own run, still listed as in progress, does not block its next step", async () => { + const world = fakeWorld() + const deps = dependencies(world) + const run = deps.run + deps.run = (program, args, input) => { + const result = run(program, args, input) + if (args[0] === 'workflow' && JSON.parse(input).mode === 'dry-run') { + const monitor = world.runs.at(-1) + world.active.push({ + id: monitor.id, + path: relayWorkflowPath('monitor-relay-production.yml'), + name: 'Monitor', + status: 'in_progress' + }) + } + return result + } + assert.equal((await start(world, [], deps)).done, true) + assert.deepEqual(live(world), [41, true]) +}) + +test('argument parsing and plan helpers fail closed', () => { + assert.throws(() => parseDriverArguments([]), /missing --commit/) + assert.throws(() => parseDriverArguments(['--commit', 'abc']), /full commit SHA/) + assert.throws( + () => parseDriverArguments(['--commit', COMMIT, '--pause-run', 'x']), + /must be a run ID/ + ) + assert.throws( + () => parseDriverArguments(['--commit', COMMIT, '--pause-run', '5', '--leave-rehome-paused']), + /contradict/ + ) + assert.throws( + () => parseDriverArguments(['--commit', COMMIT, '--configure', 'production-gce-c34']), + /--configure must be/ + ) + assert.deepEqual(parseConfigureWave(`production-gce-c27,production-gce-c28=${CELL}`).cells, [ + 'production-gce-c27', + 'production-gce-c28' + ]) + assert.throws( + () => validateDispatchInputs({ confirmation: '' }), + /not resolved/ + ) + assert.throws( + () => validateDispatchInputs({ 'image-digest': 'sha256:abc' }), + /not a sha256 digest/ + ) + assert.ok(blocksDeploy(relayWorkflowPath('push-deploy.yml'))) + assert.ok(blocksDeploy(`${relayWorkflowPath('push-deploy.yml')}@refs/heads/main`)) + assert.ok(!blocksDeploy(relayWorkflowPath('monitor-relay-clock-skew.yml'))) + const disabled = { ...CONTROL, enabled: false } + assert.equal(pausedGeneration(rehomeResultFromLog(controlLine('pause', disabled))), 39) + assert.equal( + pausedGeneration( + rehomeResultFromLog(controlLine('recover-enable', disabled, { recovered: true })) + ), + 39 + ) + assert.equal( + pausedGeneration( + rehomeResultFromLog(controlLine('recover-enable', disabled, { recovered: false })) + ), + undefined + ) + assert.equal(pausedGeneration(rehomeResultFromLog(controlLine('inspect', disabled))), undefined) + assert.throws(() => rehomeResultFromLog('nothing'), /printed no control/) +}) diff --git a/cloud/dev/scripts/relay-director-deploy-plan.mjs b/cloud/dev/scripts/relay-director-deploy-plan.mjs new file mode 100644 index 00000000000..aaeb82bddc9 --- /dev/null +++ b/cloud/dev/scripts/relay-director-deploy-plan.mjs @@ -0,0 +1,333 @@ +// Pure pieces of the director deploy driver: the exact inputs each existing workflow receives, +// and the parsers that read a run's result back. No process, network, or clock access here. + +import { + RELAY_GITHUB_REPOSITORY, + RELAY_WORKFLOW_FILE_PREFIX, + relayWorkflowFile +} from './relay-repository.mjs' + +export const REPOSITORY = RELAY_GITHUB_REPOSITORY +export const WORKFLOW_REF = 'main' +export const PROJECT = 'onorca-cloud' +export const REGION = 'us-central1' +export const DIRECTOR_SERVICE = 'orca-cloud-relay' +export const ROLLBACK_TAG = 'selector-rollback' +export const IMAGE_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay' +// Any reviewed registration wave is accepted by admission inspect; the launch wave never changes. +const ADMISSION_INSPECT_CELLS = 'production-gce-c27,production-gce-c28,production-gce-c29' + +const workflow = (name, title) => ({ file: relayWorkflowFile(name), name: title }) +export const WORKFLOWS = { + publish: workflow('publish-relay-production.yml', 'Publish Relay Production Image'), + director: workflow('deploy-relay-production-director.yml', 'Deploy Relay Production Director'), + rehome: workflow('operate-relay-production-rehome.yml', 'Operate Relay Production Rehome'), + admission: workflow('operate-relay-asia-admission.yml', 'Operate Relay Asia Admission'), + monitor: workflow('monitor-relay-production.yml', 'Monitor Relay Production') +} + +// Read-only or unrelated to the production rollout lane, so they never block a deploy. +const NON_BLOCKING_WORKFLOWS = new Set( + ['verify.yml', 'monitor-relay-clock-skew.yml'].map(relayWorkflowFile) +) + +const DIGEST = /^sha256:[a-f0-9]{64}$/ +const COMMIT = /^[a-f0-9]{40}$/ +const PRODUCTION_CELL = /^production-gce-c[1-9][0-9]?$/ +const MEMBERSHIP_KEYS = ['existingOnly', 'migrationOnly', 'general'] + +export function requireDigest(value, label) { + if (typeof value !== 'string' || !DIGEST.test(value)) + throw new Error(`${label} is not an immutable sha256 digest`) + return value +} + +export function requireCommit(value, label) { + if (typeof value !== 'string' || !COMMIT.test(value)) + throw new Error(`${label} is not a full commit SHA`) + return value +} + +function requireGeneration(value, label) { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} is invalid`) + return value +} + +export function blocksDeploy(path) { + // A `@ref` suffix never appears on run paths today; stripping it keeps the check fail-closed. + const file = String(path ?? '') + .split('@')[0] + .split('/') + .at(-1) + return ( + file.startsWith(RELAY_WORKFLOW_FILE_PREFIX) && + file.endsWith('.yml') && + !NON_BLOCKING_WORKFLOWS.has(file) + ) +} + +export function membershipInput(cells) { + return cells.length === 0 ? 'none' : [...cells].sort().join(',') +} + +export function parseSelector(value, label) { + const selector = { + generation: requireGeneration(value?.generation, `${label} generation`), + membership: {} + } + for (const key of MEMBERSHIP_KEYS) { + const cells = value?.membership?.[key] + if (!Array.isArray(cells) || cells.some((cell) => !PRODUCTION_CELL.test(cell))) { + throw new Error(`${label} ${key} membership is invalid`) + } + selector.membership[key] = [...cells].sort() + } + const all = MEMBERSHIP_KEYS.flatMap((key) => selector.membership[key]) + if (new Set(all).size !== all.length) throw new Error(`${label} membership has duplicates`) + return selector +} + +function selectorInputs(selector) { + return { + 'expected-selector-generation': String(selector.generation), + 'expected-existing-only-cells': membershipInput(selector.membership.existingOnly), + 'expected-migration-only-cells': membershipInput(selector.membership.migrationOnly), + 'expected-general-cells': membershipInput(selector.membership.general) + } +} + +export function parseControl(value, label) { + const control = value ?? {} + const integers = [ + 'generation', + 'notBefore', + 'ratePerMinute', + 'preferenceMaxAgeMs', + 'drainGraceMs' + ] + if ( + typeof control.enabled !== 'boolean' || + integers.some((key) => !Number.isSafeInteger(control[key])) + ) { + throw new Error(`${label} is not a complete regional rehome control`) + } + if (control.hostCooldownMs !== undefined && !Number.isSafeInteger(control.hostCooldownMs)) { + throw new Error(`${label} host cooldown is invalid`) + } + return Object.fromEntries( + [...integers, 'enabled', 'hostCooldownMs'].map((key) => [key, control[key]]) + ) +} + +const INTEGER_INPUT = /^(0|[1-9][0-9]*)$/ + +/** + * The last check before `gh workflow run`. Builders also render dry-run placeholders such as + * ``; this guarantees none of them, or a malformed digest, is ever dispatched. + */ +export function validateDispatchInputs(inputs) { + for (const [key, value] of Object.entries(inputs)) { + if (typeof value !== 'string' || value.startsWith('<')) + throw new Error(`input ${key} is not resolved`) + if (key.endsWith('digest') && !DIGEST.test(value)) + throw new Error(`input ${key} is not a sha256 digest`) + if ( + (key.includes('generation') || + ['not-before', 'monitor-run-id', 'monitor-run-attempt'].includes(key)) && + !INTEGER_INPUT.test(value) + ) { + throw new Error(`input ${key} is not an integer`) + } + } + return inputs +} + +export function admissionInspectInputs(servingDigest) { + return { + environment: 'production', + mode: 'inspect', + 'cell-ids': ADMISSION_INSPECT_CELLS, + 'image-digest': servingDigest + } +} + +function rehomeInputs(mode, { director, selector, controlGeneration }) { + return { + mode, + 'director-image-digest': director.servingDigest, + 'rollback-image-digest': director.rollbackDigest, + ...selectorInputs(selector), + 'expected-control-generation': String(controlGeneration) + } +} + +export function rehomeInspectInputs(context) { + return rehomeInputs('inspect', context) +} + +// Pause keeps every durable field as inspected; only `enabled` and the generation change. +// Every `confirmation` is the phrase the operator typed, never filled in here. +export function rehomePauseInputs({ director, selector, control, confirmation }) { + return { + ...rehomeInputs('pause', { director, selector, controlGeneration: control.generation }), + 'not-before': String(control.notBefore), + 'rate-per-minute': String(control.ratePerMinute), + 'preference-max-age-ms': String(control.preferenceMaxAgeMs), + 'host-cooldown-ms': String(control.hostCooldownMs ?? 604_800_000), + 'drain-grace-ms': String(control.drainGraceMs), + confirmation + } +} + +export function rehomeEnableInputs({ + director, + selector, + control, + controlGeneration, + notBefore, + monitor, + confirmation +}) { + return { + ...rehomeInputs('enable', { director, selector, controlGeneration }), + 'not-before': String(notBefore), + // The enable job accepts exactly 10 per minute. + 'rate-per-minute': '10', + 'preference-max-age-ms': String(control.preferenceMaxAgeMs), + 'host-cooldown-ms': String(control.hostCooldownMs), + 'drain-grace-ms': String(control.drainGraceMs), + 'monitor-run-id': String(monitor.runId), + 'monitor-run-attempt': String(monitor.attempt), + confirmation + } +} + +export function publishInputs() { + return { mode: 'publish' } +} + +export function directorDeployInputs({ imageDigest, predecessorDigest, rehomeGeneration }) { + return { + 'image-digest': imageDigest, + 'regional-placement-mode': 'preserve', + 'region-correction-cohort-percent': 'preserve', + 'prune-incompatible-revisions': 'false', + 'expected-rehome-generation': String(rehomeGeneration), + 'bootstrap-runtime-identity': 'false', + // Required by the form even without the bootstrap; it is only format-checked then. + 'predecessor-image-digest': predecessorDigest + } +} + +export function configureInputs({ + cells, + cellImageDigest, + directorDigest, + selectorGeneration, + confirmation +}) { + return { + environment: 'production', + mode: 'configure', + 'cell-ids': cells.join(','), + 'selector-generation': String(selectorGeneration), + 'image-digest': cellImageDigest, + 'director-image-digest': directorDigest, + confirmation + } +} + +export function monitorDryRunInputs(selector) { + return { + mode: 'dry-run', + ...selectorInputs(selector), + 'migration-policy': 'strict', + 'recovery-source-cell-id': 'none', + 'capacity-cell-id': 'none' + } +} + +export function parseConfigureWave(value) { + const separator = String(value).lastIndexOf('=') + const cells = String(value) + .slice(0, separator) + .split(',') + .map((cell) => cell.trim()) + .filter(Boolean) + const cellImageDigest = String(value).slice(separator + 1) + if ( + separator < 1 || + cells.length === 0 || + new Set(cells).size !== cells.length || + cells.some((cell) => !PRODUCTION_CELL.test(cell)) + ) { + throw new Error('--configure must be [,...]=sha256:') + } + return { cells, cellImageDigest: requireDigest(cellImageDigest, '--configure cell image digest') } +} + +/** The single 100% revision and the selector-rollback revision of `gcloud run services describe`. */ +export function directorRevisions(service) { + const traffic = Array.isArray(service?.status?.traffic) ? service.status.traffic : [] + const serving = traffic.filter((entry) => (entry.percent ?? 0) > 0) + const rollback = traffic.filter((entry) => entry.tag === ROLLBACK_TAG) + if (serving.length !== 1 || serving[0].percent !== 100 || !serving[0].revisionName) { + throw new Error('director does not serve one revision at 100%') + } + if (rollback.length !== 1 || !rollback[0].revisionName) { + throw new Error(`director has no single ${ROLLBACK_TAG} revision`) + } + return { servingRevision: serving[0].revisionName, rollbackRevision: rollback[0].revisionName } +} + +export function revisionDigest(revision, label) { + const image = revision?.spec?.containers?.[0]?.image + const [repository, digest] = String(image ?? '').split('@') + if (repository !== IMAGE_REPOSITORY) + throw new Error(`${label} does not run the production relay image`) + return requireDigest(digest, `${label} image`) +} + +/** The relay push line, `sha-: digest: sha256:… size: …`, must name the registry digest. */ +export function logConfirmsPublishedDigest(log, commit, digest) { + return String(log) + .split('\n') + .some((line) => line.includes(`sha-${commit}: digest: ${digest} size:`)) +} + +/** The last `relay_regional_rehome_control` JSON line a rehome run printed, of any mode. */ +export function rehomeResultFromLog(log) { + let found + for (const line of String(log).split('\n')) { + const start = line.indexOf('{"event":"relay_regional_rehome_control"') + if (start < 0) continue + try { + found = JSON.parse(line.slice(start).trim()) + } catch { + // A truncated or echoed line is not the result. + } + } + if (!found) throw new Error('the rehome run printed no control result') + return { + mode: found.mode, + recovered: found.recovered, + control: parseControl(found.control, `rehome ${found.mode} control`) + } +} + +/** + * The generation a run paused rehome at, or undefined. Only two lines prove a run paused it: a + * `pause`, and a failed enable's `recover-enable` that itself disabled rehome (`recovered: true`). + * `recovered: false` means rehome was already disabled, possibly by a director safety pause. + */ +export function pausedGeneration(result) { + const paused = + !result.control.enabled && + (result.mode === 'pause' || (result.mode === 'recover-enable' && result.recovered === true)) + return paused ? result.control.generation : undefined +} + +export function admissionInspectResult(result) { + if (result?.mode !== 'inspect') throw new Error('admission result is not an inspect') + return parseSelector(result, 'admission selector') +} diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index be7c1b0a645..748db4af05d 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -693,6 +693,89 @@ Their typed confirmations are `PAUSE_REGIONAL_REHOMING` and `DISABLE_REGIONAL_RE default 3,600,000 ms drain grace so existing splices can finish. The job summary contains only fresh aggregate active, receipt, registration, completion, and abort counts. +### Director deploy driver + +`dev/scripts/drive-relay-director-deploy.mjs` runs a whole director deploy from an operator machine +with `gh` and `gcloud` logged in. It only dispatches the workflows above and reads their results; it +holds no credentials and changes no workflow. It never fills in a workflow's typed confirmation: the +operator types each one when the driver reaches it. + +```bash +cd cloud +node dev/scripts/drive-relay-director-deploy.mjs --commit --dry-run +node dev/scripts/drive-relay-director-deploy.mjs --commit \ + [--configure production-gce-c34=sha256:] +``` + +It keeps no state between runs. Every decision comes from live state read at the start of each run: + +- the serving director's digest and configured cells, from `gcloud`; +- the admission selector, from an `Operate Relay Asia Admission` `inspect`; +- the rehome control, from a rehome `inspect` at the generation the newest rehome run printed, or + at `--rehome-generation`. + +Steps already done are skipped: a serving digest that matches is not deployed again, and cells +already configured are not configured again. It always reads rehome, even when nothing is left to +do, so it never reports success over a pause it cannot explain. + +The sequence: + +1. **Preflight, read-only.** No `cloud-*` workflow is queued or running (all pages; the hourly + clock-skew monitor and `cloud-verify` excepted), and `main` is the reviewed commit. +2. **Publish**, after the operator types `DEPLOY `. It runs before rehome is touched, + so a moved `main` or a bad build needs no cleanup. The digest is the registry digest of + `relay:sha-`, and the run's own push line must name the same digest. +3. **Pause**, only if rehome is enabled, after the operator types `PAUSE_REGIONAL_REHOMING`. +4. **Deploy** with that digest, the paused generation, `preserve` for both regional inputs, no + prune, and the old serving digest as predecessor. +5. **Soak**, with `--configure` only, while no wave is configured yet. It watches 5 minutes of + director 5xx and stops if they exceed twice the 5 minutes before the new revision existed, plus + 25. The window starts at the traffic switch (a minute before the deploy run completed) when this + run deployed, otherwise at the time of the run, so a re-run judges fresh traffic. It is read a + minute late, to allow for log lag. Then the operator types `CONFIGURE_ASIA_DIRECTOR` and each + pending wave is configured. +6. **Digest check.** A rehome `inspect` bound to the serving and rollback digests that `gcloud` + reports now. A wrong digest fails here, read-only, before 15 minutes of monitor evidence is + spent on it. +7. **Monitor.** The operator types `ENABLE_REGIONAL_REHOMING`. The prompt says this arms an + automatic enable, sent about 17 minutes later, and only if the monitor is green and its evidence + is at most 150 s old. The monitor dry-run then starts. Its artifact passes the same + `relay-monitor-evidence.mjs verify-authority` check the enable job runs. +8. **Enable** with the verified digests, within 150 s of the monitor completing. + +Steps 6 to 8 run only for a pause this driver owns. + +**Ownership.** The driver owns a pause only if it can name the run that made it, and the live +control is still at that run's generation. Two kinds of line in a run's log prove it paused +rehome: + +- `pause`; +- `recover-enable` with `recovered: true`, meaning a failed enable that disabled rehome again itself. + +The run must be a rehome-control run by the same GitHub user. A `recover-enable` with +`recovered: false` found rehome already disabled, for example by a director safety pause, and is +never adopted. A failed enable run is never counted as an enable, whatever it printed last. A fresh run that finds rehome paused stops. It goes ahead only +with: + +- `--pause-run `, which an earlier run of this driver printed; or +- `--leave-rehome-paused`, which deploys and leaves rehome paused. It refuses an enabled switch. + +A pause made by anything else is never lifted. + +**Stops.** On any failure, Ctrl-C, SIGTERM or SIGHUP, the driver prints what changed: + +- `REHOME IS CHANGING` when a pause or enable run is in flight and will apply on its own; +- `PAUSE UNCONFIRMED` or `ENABLE UNCONFIRMED` when such a run printed no usable result; +- `REHOME IS PAUSED by this driver` with the owning run; +- the serving director, re-read; +- the published digest; +- the rollback point, with the `gh workflow run` command that redeploys it. + +It ends with the single command that finishes from where it stopped. That command carries +`--publish-run` and `--pause-run`, and the driver re-verifies both against the runs' logs and live +state. Each run writes a timestamped log under `~/.orca/relay-director-deploy/` +(`--log-directory` overrides it). + ## Mobile push gateway `Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path diff --git a/cloud/package.json b/cloud/package.json index e2c02ba2416..bb9a853d9a2 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -22,7 +22,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", - "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" }, "devDependencies": {