From 74ad08ec66605fa32cba55f70d668f9596dfed63 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:54:40 -0400 Subject: [PATCH] fix(relay-ops): accept monitor evidence from an ancestor commit with identical monitor code (#18754) --- ...d-deploy-relay-production-same-cap-job.yml | 4 + ...cloud-deploy-relay-production-same-cap.yml | 7 +- ...ud-operate-relay-production-rehome-job.yml | 4 + .../relay-evidence-code-provenance.mjs | 94 ++++++++++ cloud/dev/scripts/relay-monitor-evidence.mjs | 23 ++- .../scripts/relay-monitor-evidence.test.mjs | 171 +++++++++++++++++- .../relay-production-same-cap-wave.mjs | 13 +- .../relay-production-same-cap-wave.test.mjs | 67 +++++++ cloud/dev/scripts/relay-repository.mjs | 15 ++ 9 files changed, 385 insertions(+), 13 deletions(-) create mode 100644 cloud/dev/scripts/relay-evidence-code-provenance.mjs diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 6430c3f4793..afcaf3c066e 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -91,7 +91,11 @@ jobs: test -n "${CAPACITY_SERVICE_ACCOUNT}" test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" + # Full history: the monitor evidence this job verifies is sealed at an ancestor commit, + # and the provenance check fails closed on a commit a shallow clone left out. - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: pnpm/action-setup@v4 with: { package_json_file: cloud/package.json } diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml index 1994966d083..fba5df0dcb9 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap.yml @@ -87,13 +87,18 @@ jobs: gate: if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }} runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 10 + # Headroom for the full-history checkout the canary provenance check needs. + timeout-minutes: 15 environment: production outputs: cells: ${{ steps.wave.outputs.cells }} job-mode: ${{ steps.wave.outputs.job-mode }} steps: + # Full history: the canary authority a batch verifies is sealed at an ancestor commit, and + # the provenance check fails closed on a commit a shallow clone left out. - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-node@v4 with: { node-version: 24 } diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml index 682953af5e7..fdb1aca45e0 100644 --- a/.github/workflows/cloud-operate-relay-production-rehome-job.yml +++ b/.github/workflows/cloud-operate-relay-production-rehome-job.yml @@ -95,7 +95,11 @@ jobs: ;; esac + # Full history: the monitor evidence this job verifies is sealed at an ancestor commit, + # and the provenance check fails closed on a commit a shallow clone left out. - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-node@v4 with: diff --git a/cloud/dev/scripts/relay-evidence-code-provenance.mjs b/cloud/dev/scripts/relay-evidence-code-provenance.mjs new file mode 100644 index 00000000000..233a8139b85 --- /dev/null +++ b/cloud/dev/scripts/relay-evidence-code-provenance.mjs @@ -0,0 +1,94 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { + RELAY_REPOSITORY_ROOT, + relayTreePath, + relayWorkflowPath +} from './relay-repository.mjs' + +const SHA = /^[a-f0-9]{40}$/ + +// Every file that decides how relay evidence is produced, sealed, verified, and then spent against +// production; identical content across two commits is what makes the older commit's verdict binding. +export const TRUSTED_EVIDENCE_CODE_PATHS = [ + // Produces and seals the 15-minute dry-run evidence. + relayWorkflowPath('monitor-relay-production.yml'), + relayWorkflowPath('monitor-relay-production-job.yml'), + // Download it, verify its authority, and mutate production on it. + relayWorkflowPath('deploy-relay-production-same-cap.yml'), + relayWorkflowPath('deploy-relay-production-same-cap-job.yml'), + relayWorkflowPath('operate-relay-production-rehome.yml'), + relayWorkflowPath('operate-relay-production-rehome-job.yml'), + // Sealing, verification, the wave/canary authority, and the path constants below. + relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'), + relayTreePath('dev/scripts/relay-monitor-evidence.mjs'), + relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'), + relayTreePath('dev/scripts/relay-repository.mjs'), + // Every other script those jobs run against live production. + relayTreePath('dev/scripts/infra.mjs'), + relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'), + relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'), + relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'), + relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'), + relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'), + // The monitor itself and the live preflight recheck, plus anything that changes their behaviour. + relayTreePath('apps/relay-ops'), + relayTreePath('package.json'), + relayTreePath('pnpm-lock.yaml'), + relayTreePath('pnpm-workspace.yaml'), + // The Cloud SQL rollout lease every mutation job takes and releases. + '.github/actions/cloud-sql-rollout-lease' +] + +function git(root, args) { + const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' }) + if (result.error) throw new Error('relay evidence provenance cannot run git') + return result +} + +/** + * Accepts evidence sealed at a different commit only when the current commit descends from it and + * every trusted path is byte-identical, so the verdict provably came from this exact code. Anything + * git cannot answer (no checkout, unknown commit, shallow clone) fails closed. + */ +export function requireSameEvidenceCode({ + sealedSha, + currentSha, + label, + repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT) +}) { + if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) { + throw new Error(`${label} commit is invalid`) + } + if (sealedSha === currentSha) return + if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) { + throw new Error(`${label} commit cannot be compared without a git checkout`) + } + for (const sha of [sealedSha, currentSha]) { + if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) { + throw new Error( + `${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0` + ) + } + } + const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha]) + if (ancestry.status === 1) { + throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`) + } + if (ancestry.status !== 0) { + throw new Error(`${label} commit ancestry could not be determined`) + } + const diff = git(repositoryRoot, [ + 'diff', + '--name-only', + sealedSha, + currentSha, + '--', + ...TRUSTED_EVIDENCE_CODE_PATHS + ]) + if (diff.status !== 0) throw new Error(`${label} commit comparison failed`) + const changed = diff.stdout.split('\n').filter(Boolean) + if (changed.length > 0) { + throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`) + } +} diff --git a/cloud/dev/scripts/relay-monitor-evidence.mjs b/cloud/dev/scripts/relay-monitor-evidence.mjs index 7f387663f60..26eb37d0d4d 100644 --- a/cloud/dev/scripts/relay-monitor-evidence.mjs +++ b/cloud/dev/scripts/relay-monitor-evidence.mjs @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto' import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises' import { basename, join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' +import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/ const SHA = /^[a-f0-9]{40}$/ @@ -102,7 +103,7 @@ export async function createEvidenceManifest(argv) { return manifest } -async function readAndVerifyManifest(directory, expected) { +async function readAndVerifyManifest(directory, expected, sameCodeCommit) { const manifest = JSON.parse( await readFile(join(directory, 'evidence-manifest.json'), 'utf8') ) @@ -111,11 +112,23 @@ async function readAndVerifyManifest(directory, expected) { manifest.incidentId !== expected.incidentId || manifest.runId !== expected.runId || manifest.runAttempt !== expected.runAttempt || - manifest.commitSha !== expected.commitSha || - manifest.mode !== expected.mode + !SHA.test(manifest.commitSha ?? '') || + manifest.mode !== expected.mode || + (!sameCodeCommit && manifest.commitSha !== expected.commitSha) ) { throw new Error('relay monitor evidence provenance does not match') } + // Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than + // the monitor it must trust; identical monitor and mutation code is the property the SHA stood in + // for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them. + if (sameCodeCommit) { + requireSameEvidenceCode({ + sealedSha: manifest.commitSha, + currentSha: expected.commitSha, + label: 'relay monitor evidence', + ...sameCodeCommit + }) + } const names = Object.keys(manifest.files ?? {}) if (!names.includes(`${expected.incidentId}.state.json`)) { throw new Error('relay monitor evidence has no durable state') @@ -209,12 +222,12 @@ function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) { ) } -export async function verifyDryRunAuthority(argv, now = Date.now) { +export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) { const values = argumentsByName(argv) const directory = resolve(values.directory ?? '') const expected = provenance(values) if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence') - const manifest = await readAndVerifyManifest(directory, expected) + const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot }) const state = JSON.parse( await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8') ) diff --git a/cloud/dev/scripts/relay-monitor-evidence.test.mjs b/cloud/dev/scripts/relay-monitor-evidence.test.mjs index 45116761119..43d2ac02763 100644 --- a/cloud/dev/scripts/relay-monitor-evidence.test.mjs +++ b/cloud/dev/scripts/relay-monitor-evidence.test.mjs @@ -1,9 +1,15 @@ import assert from 'node:assert/strict' -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import test from 'node:test' -import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs' +import { TRUSTED_EVIDENCE_CODE_PATHS } from './relay-evidence-code-provenance.mjs' +import { + RELAY_REPOSITORY_ROOT, + relayWorkflowPath, + relayWorkflowUrl +} from './relay-repository.mjs' import { createEvidenceManifest, verifyDryRunAuthority, @@ -12,7 +18,7 @@ import { } from './relay-monitor-evidence.mjs' const now = Date.parse('2026-07-28T12:00:00.000Z') -const provenance = [ +const provenanceFor = (commitSha) => [ '--incident-id', 'relay-123', '--run-id', @@ -20,10 +26,11 @@ const provenance = [ '--run-attempt', '1', '--commit-sha', - 'a'.repeat(40), + commitSha, '--mode', 'dry-run' ] +const provenance = provenanceFor('a'.repeat(40)) const selector = { generation: 2, membership: { @@ -513,3 +520,157 @@ test('monitor uses a reusable job so exact job_workflow_ref is present', async ( assert.match(job, /workflow_call:/) assert.match(job, /environment: production/) }) + +function gitIn(root, ...args) { + return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() +} + +// A real repository shaped like main under unrelated merge traffic: one sealed commit, a +// descendant that only touched untrusted files, a descendant that touched the monitor, and a +// sibling that never descended from the seal. +async function trustedCodeRepository() { + const root = await mkdtemp(join(tmpdir(), 'relay-evidence-repository-')) + gitIn(root, 'init', '--quiet') + gitIn(root, 'config', 'user.email', 'relay@example.test') + gitIn(root, 'config', 'user.name', 'Relay Evidence Test') + gitIn(root, 'config', 'commit.gpgsign', 'false') + const commit = async (path, body, message) => { + await mkdir(dirname(join(root, path)), { recursive: true }) + await writeFile(join(root, path), body) + gitIn(root, 'add', '--all') + gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message) + return gitIn(root, 'rev-parse', 'HEAD') + } + const base = await commit( + 'cloud/apps/relay-ops/src/incident-monitor.ts', + 'export const v = 1\n', + 'monitor' + ) + const sealed = await commit('README.md', 'base\n', 'base') + const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated') + const changedCode = await commit( + 'cloud/apps/relay-ops/src/incident-monitor.ts', + 'export const v = 2\n', + 'monitor change' + ) + // Branches before the seal, so the seal is not in its history even though its code matches. + gitIn(root, 'checkout', '--quiet', '--detach', base) + const sibling = await commit('README.md', 'a divergent line\n', 'divergent') + return { root, sealed, sameCode, changedCode, sibling } +} + +const authorityAt = (directory, commitSha, repositoryRoot) => verifyDryRunAuthority( + [ + '--directory', + directory, + ...provenanceFor(commitSha), + '--required-migration-policy', + 'strict' + ], + () => now, + repositoryRoot +) + +test('accepts dry-run evidence sealed by identical code at an ancestor commit', async () => { + const repository = await trustedCodeRepository() + const directory = await evidenceDirectory() + try { + await createEvidenceManifest([ + '--directory', + directory, + ...provenanceFor(repository.sealed) + ]) + // An exact match never consults git: a root with no checkout at all still verifies. + await assert.doesNotReject(authorityAt(directory, repository.sealed, directory)) + await assert.doesNotReject(authorityAt(directory, repository.sameCode, repository.root)) + } finally { + await rm(repository.root, { recursive: true, force: true }) + await rm(directory, { recursive: true, force: true }) + } +}) + +test('rejects dry-run evidence whose monitor code or lineage differs', async () => { + const repository = await trustedCodeRepository() + const directory = await evidenceDirectory() + try { + await createEvidenceManifest([ + '--directory', + directory, + ...provenanceFor(repository.sealed) + ]) + await assert.rejects( + authorityAt(directory, repository.changedCode, repository.root), + /code changed after it was sealed: cloud\/apps\/relay-ops\/src\/incident-monitor\.ts/ + ) + await assert.rejects( + authorityAt(directory, repository.sibling, repository.root), + /is not an ancestor of/ + ) + // Fails closed: a shallow clone that never fetched the sealed commit proves nothing. + await assert.rejects( + authorityAt(directory, 'f'.repeat(40), repository.root), + /unknown to this checkout/ + ) + // Fails closed: no checkout to compare against. + await assert.rejects( + authorityAt(directory, repository.sameCode, directory), + /cannot be compared without a git checkout/ + ) + } finally { + await rm(repository.root, { recursive: true, force: true }) + await rm(directory, { recursive: true, force: true }) + } +}) + +test('keeps restore and mutation bound to the exact sealing commit', async () => { + const repository = await trustedCodeRepository() + const directory = await evidenceDirectory() + try { + await createEvidenceManifest([ + '--directory', + directory, + ...provenanceFor(repository.sealed) + ]) + await assert.rejects( + verifyRestoredEvidence([ + '--directory', + directory, + ...provenanceFor(repository.sameCode) + ]), + /provenance does not match/ + ) + await assert.rejects( + verifyMutationEvidence( + [ + '--directory', + directory, + ...provenanceFor(repository.sameCode), + '--mutation-mode', + 'execute', + '--source-cell-id', + 'c1', + '--director-origin', + 'https://relay.example' + ], + { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }, + async () => Response.json({ selector }), + () => now + ), + /provenance does not match/ + ) + } finally { + await rm(repository.root, { recursive: true, force: true }) + await rm(directory, { recursive: true, force: true }) + } +}) + +// A trusted path that no longer exists silently stops being compared, so the same-code rule would +// pass over code it was written to pin. +test('every trusted provenance path exists in this checkout', async () => { + for (const path of TRUSTED_EVIDENCE_CODE_PATHS) { + await assert.doesNotReject( + stat(new URL(path, RELAY_REPOSITORY_ROOT)), + `${path} is missing` + ) + } +}) diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs index e391c4c4381..6e84c1c9104 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.mjs @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs' import { pathToFileURL } from 'node:url' +import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' export const SAME_CAP_CELLS = [ 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', @@ -85,10 +86,10 @@ export function canaryAuthority(input) { } } -export function verifyCanaryAuthority(authority, expected) { +export function verifyCanaryAuthority(authority, expected, repositoryRoot) { if ( authority?.v !== 1 || - authority.commitSha !== expected.commitSha || + !/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') || authority.runId !== expected.runId || authority.targetDigest !== expected.targetDigest || authority.rollbackDigest !== expected.rollbackDigest || @@ -96,6 +97,14 @@ export function verifyCanaryAuthority(authority, expected) { authority.rehomeGeneration !== Number(expected.rehomeGeneration) || !SAME_CAP_CELLS.includes(authority.cellId) ) throw new Error('canary authority does not match this batch') + // The batch dispatch resolves main after the canary sealed, so bind to the same code, not the + // same SHA; every field above still pins this batch to that exact canary. + requireSameEvidenceCode({ + sealedSha: authority.commitSha, + currentSha: expected.commitSha, + label: 'relay same-cap canary authority', + repositoryRoot + }) return authority } diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs index 0b45ae85a99..d636c324b33 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs @@ -1,4 +1,8 @@ import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' import { test } from 'node:test' import { canaryAuthority, @@ -104,3 +108,66 @@ test('seals and verifies canary authority for later batches', () => { rehomeGeneration: '4' }), /does not match/) }) + +function gitIn(root, ...args) { + return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() +} + +async function canaryRepository() { + const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-')) + gitIn(root, 'init', '--quiet') + gitIn(root, 'config', 'user.email', 'relay@example.test') + gitIn(root, 'config', 'user.name', 'Relay Wave Test') + gitIn(root, 'config', 'commit.gpgsign', 'false') + const commit = async (path, body, message) => { + await mkdir(dirname(join(root, path)), { recursive: true }) + await writeFile(join(root, path), body) + gitIn(root, 'add', '--all') + gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message) + return gitIn(root, 'rev-parse', 'HEAD') + } + const sealed = await commit( + 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', + 'export const v = 1\n', + 'wave' + ) + const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated') + const changedCode = await commit( + 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', + 'export const v = 2\n', + 'wave change' + ) + return { root, sealed, sameCode, changedCode } +} + +test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => { + const repository = await canaryRepository() + try { + const authority = canaryAuthority({ + cellIds: 'production-gce-c7', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, + commitSha: repository.sealed, + runId: '42', + selectorGeneration: '11', + rehomeGeneration: '4' + }) + const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, { + commitSha, + runId: '42', + targetDigest, + rollbackDigest, + selectorGeneration: '13', + rehomeGeneration: '4' + }, repositoryRoot) + assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7') + assert.throws( + () => verifyAt(repository.changedCode, repository.root), + /code changed after it was sealed/ + ) + assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/) + } finally { + await rm(repository.root, { recursive: true, force: true }) + } +}) diff --git a/cloud/dev/scripts/relay-repository.mjs b/cloud/dev/scripts/relay-repository.mjs index 7e8b01e4799..040acef5fe5 100644 --- a/cloud/dev/scripts/relay-repository.mjs +++ b/cloud/dev/scripts/relay-repository.mjs @@ -1,4 +1,6 @@ import { readFileSync } from 'node:fs' +import { relative } from 'node:path' +import { fileURLToPath } from 'node:url' // Single place naming the repository the Relay workflows live in and where their files sit. The // public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the @@ -11,6 +13,19 @@ export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-' // this tree moves under cloud/, so the depth changes at the copy even though the layout does not. export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url) +// Repository root, derived from the one directory above that already tracks the copy's depth. +export const RELAY_REPOSITORY_ROOT = new URL('../../', RELAY_WORKFLOW_DIRECTORY) + +// Repository-relative path for a file in this tree. The prefix is 'cloud/' here and empty where +// the tree is the repository root, so callers naming git paths never restate the layout. +export function relayTreePath(suffix) { + const prefix = relative( + fileURLToPath(RELAY_REPOSITORY_ROOT), + fileURLToPath(new URL('../../', import.meta.url)) + ).split(/[\\/]/).filter(Boolean) + return [...prefix, suffix].join('/') +} + export function relayWorkflowFile(name) { return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` }