fix(relay-ops): accept monitor evidence from an ancestor commit with identical monitor code (#18754)

.github/workflows/cloud-deploy-relay-production-same-cap-job.yml

@@ -91,7 +91,11 @@ jobs:
           test -n "${CAPACITY_SERVICE_ACCOUNT}"
           test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
 
+      # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+      # and the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: pnpm/action-setup@v4
         with: { package_json_file: cloud/package.json }

.github/workflows/cloud-deploy-relay-production-same-cap.yml

@@ -87,13 +87,18 @@ jobs:
   gate:
     if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
     runs-on: blacksmith-2vcpu-ubuntu-2204
-    timeout-minutes: 10
+    # Headroom for the full-history checkout the canary provenance check needs.
+    timeout-minutes: 15
     environment: production
     outputs:
       cells: ${{ steps.wave.outputs.cells }}
       job-mode: ${{ steps.wave.outputs.job-mode }}
     steps:
+      # Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
+      # the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with: { node-version: 24 }

.github/workflows/cloud-operate-relay-production-rehome-job.yml

@@ -95,7 +95,11 @@ jobs:
               ;;
           esac
 
+      # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+      # and the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with:

cloud/dev/scripts/relay-evidence-code-provenance.mjs

@@ -0,0 +1,94 @@
+import { spawnSync } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import {
+  RELAY_REPOSITORY_ROOT,
+  relayTreePath,
+  relayWorkflowPath
+} from './relay-repository.mjs'
+
+const SHA = /^[a-f0-9]{40}$/
+
+// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
+// production; identical content across two commits is what makes the older commit's verdict binding.
+export const TRUSTED_EVIDENCE_CODE_PATHS = [
+  // Produces and seals the 15-minute dry-run evidence.
+  relayWorkflowPath('monitor-relay-production.yml'),
+  relayWorkflowPath('monitor-relay-production-job.yml'),
+  // Download it, verify its authority, and mutate production on it.
+  relayWorkflowPath('deploy-relay-production-same-cap.yml'),
+  relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
+  relayWorkflowPath('operate-relay-production-rehome.yml'),
+  relayWorkflowPath('operate-relay-production-rehome-job.yml'),
+  // Sealing, verification, the wave/canary authority, and the path constants below.
+  relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
+  relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
+  relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
+  relayTreePath('dev/scripts/relay-repository.mjs'),
+  // Every other script those jobs run against live production.
+  relayTreePath('dev/scripts/infra.mjs'),
+  relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
+  relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
+  relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
+  relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
+  relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
+  // The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
+  relayTreePath('apps/relay-ops'),
+  relayTreePath('package.json'),
+  relayTreePath('pnpm-lock.yaml'),
+  relayTreePath('pnpm-workspace.yaml'),
+  // The Cloud SQL rollout lease every mutation job takes and releases.
+  '.github/actions/cloud-sql-rollout-lease'
+]
+
+function git(root, args) {
+  const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
+  if (result.error) throw new Error('relay evidence provenance cannot run git')
+  return result
+}
+
+/**
+ * Accepts evidence sealed at a different commit only when the current commit descends from it and
+ * every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
+ * git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
+ */
+export function requireSameEvidenceCode({
+  sealedSha,
+  currentSha,
+  label,
+  repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
+}) {
+  if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
+    throw new Error(`${label} commit is invalid`)
+  }
+  if (sealedSha === currentSha) return
+  if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
+    throw new Error(`${label} commit cannot be compared without a git checkout`)
+  }
+  for (const sha of [sealedSha, currentSha]) {
+    if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
+      throw new Error(
+        `${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
+      )
+    }
+  }
+  const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
+  if (ancestry.status === 1) {
+    throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
+  }
+  if (ancestry.status !== 0) {
+    throw new Error(`${label} commit ancestry could not be determined`)
+  }
+  const diff = git(repositoryRoot, [
+    'diff',
+    '--name-only',
+    sealedSha,
+    currentSha,
+    '--',
+    ...TRUSTED_EVIDENCE_CODE_PATHS
+  ])
+  if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
+  const changed = diff.stdout.split('\n').filter(Boolean)
+  if (changed.length > 0) {
+    throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
+  }
+}

cloud/dev/scripts/relay-monitor-evidence.mjs

@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'
 import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
 import { basename, join, resolve } from 'node:path'
 import { pathToFileURL } from 'node:url'
+import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
 
 const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
 const SHA = /^[a-f0-9]{40}$/
@@ -102,7 +103,7 @@ export async function createEvidenceManifest(argv) {
   return manifest
 }
 
-async function readAndVerifyManifest(directory, expected) {
+async function readAndVerifyManifest(directory, expected, sameCodeCommit) {
   const manifest = JSON.parse(
     await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
   )
@@ -111,11 +112,23 @@ async function readAndVerifyManifest(directory, expected) {
     manifest.incidentId !== expected.incidentId ||
     manifest.runId !== expected.runId ||
     manifest.runAttempt !== expected.runAttempt ||
-    manifest.commitSha !== expected.commitSha ||
-    manifest.mode !== expected.mode
+    !SHA.test(manifest.commitSha ?? '') ||
+    manifest.mode !== expected.mode ||
+    (!sameCodeCommit && manifest.commitSha !== expected.commitSha)
   ) {
     throw new Error('relay monitor evidence provenance does not match')
   }
+  // Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than
+  // the monitor it must trust; identical monitor and mutation code is the property the SHA stood in
+  // for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them.
+  if (sameCodeCommit) {
+    requireSameEvidenceCode({
+      sealedSha: manifest.commitSha,
+      currentSha: expected.commitSha,
+      label: 'relay monitor evidence',
+      ...sameCodeCommit
+    })
+  }
   const names = Object.keys(manifest.files ?? {})
   if (!names.includes(`${expected.incidentId}.state.json`)) {
     throw new Error('relay monitor evidence has no durable state')
@@ -209,12 +222,12 @@ function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
   )
 }
 
-export async function verifyDryRunAuthority(argv, now = Date.now) {
+export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) {
   const values = argumentsByName(argv)
   const directory = resolve(values.directory ?? '')
   const expected = provenance(values)
   if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
-  const manifest = await readAndVerifyManifest(directory, expected)
+  const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot })
   const state = JSON.parse(
     await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
   )

cloud/dev/scripts/relay-monitor-evidence.test.mjs

@@ -1,9 +1,15 @@
 import assert from 'node:assert/strict'
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { execFileSync } from 'node:child_process'
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
-import { join } from 'node:path'
+import { dirname, join } from 'node:path'
 import test from 'node:test'
-import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs'
+import { TRUSTED_EVIDENCE_CODE_PATHS } from './relay-evidence-code-provenance.mjs'
+import {
+  RELAY_REPOSITORY_ROOT,
+  relayWorkflowPath,
+  relayWorkflowUrl
+} from './relay-repository.mjs'
 import {
   createEvidenceManifest,
   verifyDryRunAuthority,
@@ -12,7 +18,7 @@ import {
 } from './relay-monitor-evidence.mjs'
 
 const now = Date.parse('2026-07-28T12:00:00.000Z')
-const provenance = [
+const provenanceFor = (commitSha) => [
   '--incident-id',
   'relay-123',
   '--run-id',
@@ -20,10 +26,11 @@ const provenance = [
   '--run-attempt',
   '1',
   '--commit-sha',
-  'a'.repeat(40),
+  commitSha,
   '--mode',
   'dry-run'
 ]
+const provenance = provenanceFor('a'.repeat(40))
 const selector = {
   generation: 2,
   membership: {
@@ -513,3 +520,157 @@ test('monitor uses a reusable job so exact job_workflow_ref is present', async (
   assert.match(job, /workflow_call:/)
   assert.match(job, /environment: production/)
 })
+
+function gitIn(root, ...args) {
+  return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
+}
+
+// A real repository shaped like main under unrelated merge traffic: one sealed commit, a
+// descendant that only touched untrusted files, a descendant that touched the monitor, and a
+// sibling that never descended from the seal.
+async function trustedCodeRepository() {
+  const root = await mkdtemp(join(tmpdir(), 'relay-evidence-repository-'))
+  gitIn(root, 'init', '--quiet')
+  gitIn(root, 'config', 'user.email', 'relay@example.test')
+  gitIn(root, 'config', 'user.name', 'Relay Evidence Test')
+  gitIn(root, 'config', 'commit.gpgsign', 'false')
+  const commit = async (path, body, message) => {
+    await mkdir(dirname(join(root, path)), { recursive: true })
+    await writeFile(join(root, path), body)
+    gitIn(root, 'add', '--all')
+    gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
+    return gitIn(root, 'rev-parse', 'HEAD')
+  }
+  const base = await commit(
+    'cloud/apps/relay-ops/src/incident-monitor.ts',
+    'export const v = 1\n',
+    'monitor'
+  )
+  const sealed = await commit('README.md', 'base\n', 'base')
+  const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
+  const changedCode = await commit(
+    'cloud/apps/relay-ops/src/incident-monitor.ts',
+    'export const v = 2\n',
+    'monitor change'
+  )
+  // Branches before the seal, so the seal is not in its history even though its code matches.
+  gitIn(root, 'checkout', '--quiet', '--detach', base)
+  const sibling = await commit('README.md', 'a divergent line\n', 'divergent')
+  return { root, sealed, sameCode, changedCode, sibling }
+}
+
+const authorityAt = (directory, commitSha, repositoryRoot) => verifyDryRunAuthority(
+  [
+    '--directory',
+    directory,
+    ...provenanceFor(commitSha),
+    '--required-migration-policy',
+    'strict'
+  ],
+  () => now,
+  repositoryRoot
+)
+
+test('accepts dry-run evidence sealed by identical code at an ancestor commit', async () => {
+  const repository = await trustedCodeRepository()
+  const directory = await evidenceDirectory()
+  try {
+    await createEvidenceManifest([
+      '--directory',
+      directory,
+      ...provenanceFor(repository.sealed)
+    ])
+    // An exact match never consults git: a root with no checkout at all still verifies.
+    await assert.doesNotReject(authorityAt(directory, repository.sealed, directory))
+    await assert.doesNotReject(authorityAt(directory, repository.sameCode, repository.root))
+  } finally {
+    await rm(repository.root, { recursive: true, force: true })
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('rejects dry-run evidence whose monitor code or lineage differs', async () => {
+  const repository = await trustedCodeRepository()
+  const directory = await evidenceDirectory()
+  try {
+    await createEvidenceManifest([
+      '--directory',
+      directory,
+      ...provenanceFor(repository.sealed)
+    ])
+    await assert.rejects(
+      authorityAt(directory, repository.changedCode, repository.root),
+      /code changed after it was sealed: cloud\/apps\/relay-ops\/src\/incident-monitor\.ts/
+    )
+    await assert.rejects(
+      authorityAt(directory, repository.sibling, repository.root),
+      /is not an ancestor of/
+    )
+    // Fails closed: a shallow clone that never fetched the sealed commit proves nothing.
+    await assert.rejects(
+      authorityAt(directory, 'f'.repeat(40), repository.root),
+      /unknown to this checkout/
+    )
+    // Fails closed: no checkout to compare against.
+    await assert.rejects(
+      authorityAt(directory, repository.sameCode, directory),
+      /cannot be compared without a git checkout/
+    )
+  } finally {
+    await rm(repository.root, { recursive: true, force: true })
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('keeps restore and mutation bound to the exact sealing commit', async () => {
+  const repository = await trustedCodeRepository()
+  const directory = await evidenceDirectory()
+  try {
+    await createEvidenceManifest([
+      '--directory',
+      directory,
+      ...provenanceFor(repository.sealed)
+    ])
+    await assert.rejects(
+      verifyRestoredEvidence([
+        '--directory',
+        directory,
+        ...provenanceFor(repository.sameCode)
+      ]),
+      /provenance does not match/
+    )
+    await assert.rejects(
+      verifyMutationEvidence(
+        [
+          '--directory',
+          directory,
+          ...provenanceFor(repository.sameCode),
+          '--mutation-mode',
+          'execute',
+          '--source-cell-id',
+          'c1',
+          '--director-origin',
+          'https://relay.example'
+        ],
+        { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
+        async () => Response.json({ selector }),
+        () => now
+      ),
+      /provenance does not match/
+    )
+  } finally {
+    await rm(repository.root, { recursive: true, force: true })
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+// A trusted path that no longer exists silently stops being compared, so the same-code rule would
+// pass over code it was written to pin.
+test('every trusted provenance path exists in this checkout', async () => {
+  for (const path of TRUSTED_EVIDENCE_CODE_PATHS) {
+    await assert.doesNotReject(
+      stat(new URL(path, RELAY_REPOSITORY_ROOT)),
+      `${path} is missing`
+    )
+  }
+})

cloud/dev/scripts/relay-production-same-cap-wave.mjs

@@ -1,5 +1,6 @@
 import { readFileSync } from 'node:fs'
 import { pathToFileURL } from 'node:url'
+import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
 
 export const SAME_CAP_CELLS = [
   'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
@@ -85,10 +86,10 @@ export function canaryAuthority(input) {
   }
 }
 
-export function verifyCanaryAuthority(authority, expected) {
+export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
   if (
     authority?.v !== 1 ||
-    authority.commitSha !== expected.commitSha ||
+    !/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
     authority.runId !== expected.runId ||
     authority.targetDigest !== expected.targetDigest ||
     authority.rollbackDigest !== expected.rollbackDigest ||
@@ -96,6 +97,14 @@ export function verifyCanaryAuthority(authority, expected) {
     authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
     !SAME_CAP_CELLS.includes(authority.cellId)
   ) throw new Error('canary authority does not match this batch')
+  // The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
+  // same SHA; every field above still pins this batch to that exact canary.
+  requireSameEvidenceCode({
+    sealedSha: authority.commitSha,
+    currentSha: expected.commitSha,
+    label: 'relay same-cap canary authority',
+    repositoryRoot
+  })
   return authority
 }
 

cloud/dev/scripts/relay-production-same-cap-wave.test.mjs

@@ -1,4 +1,8 @@
 import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
 import { test } from 'node:test'
 import {
   canaryAuthority,
@@ -104,3 +108,66 @@ test('seals and verifies canary authority for later batches', () => {
     rehomeGeneration: '4'
   }), /does not match/)
 })
+
+function gitIn(root, ...args) {
+  return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
+}
+
+async function canaryRepository() {
+  const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-'))
+  gitIn(root, 'init', '--quiet')
+  gitIn(root, 'config', 'user.email', 'relay@example.test')
+  gitIn(root, 'config', 'user.name', 'Relay Wave Test')
+  gitIn(root, 'config', 'commit.gpgsign', 'false')
+  const commit = async (path, body, message) => {
+    await mkdir(dirname(join(root, path)), { recursive: true })
+    await writeFile(join(root, path), body)
+    gitIn(root, 'add', '--all')
+    gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
+    return gitIn(root, 'rev-parse', 'HEAD')
+  }
+  const sealed = await commit(
+    'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
+    'export const v = 1\n',
+    'wave'
+  )
+  const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
+  const changedCode = await commit(
+    'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
+    'export const v = 2\n',
+    'wave change'
+  )
+  return { root, sealed, sameCode, changedCode }
+}
+
+test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => {
+  const repository = await canaryRepository()
+  try {
+    const authority = canaryAuthority({
+      cellIds: 'production-gce-c7',
+      targetDigest,
+      rollbackDigest,
+      confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
+      commitSha: repository.sealed,
+      runId: '42',
+      selectorGeneration: '11',
+      rehomeGeneration: '4'
+    })
+    const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
+      commitSha,
+      runId: '42',
+      targetDigest,
+      rollbackDigest,
+      selectorGeneration: '13',
+      rehomeGeneration: '4'
+    }, repositoryRoot)
+    assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
+    assert.throws(
+      () => verifyAt(repository.changedCode, repository.root),
+      /code changed after it was sealed/
+    )
+    assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/)
+  } finally {
+    await rm(repository.root, { recursive: true, force: true })
+  }
+})

cloud/dev/scripts/relay-repository.mjs

@@ -1,4 +1,6 @@
 import { readFileSync } from 'node:fs'
+import { relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
 
 // Single place naming the repository the Relay workflows live in and where their files sit. The
 // public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the
@@ -11,6 +13,19 @@ export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-'
 // this tree moves under cloud/, so the depth changes at the copy even though the layout does not.
 export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url)
 
+// Repository root, derived from the one directory above that already tracks the copy's depth.
+export const RELAY_REPOSITORY_ROOT = new URL('../../', RELAY_WORKFLOW_DIRECTORY)
+
+// Repository-relative path for a file in this tree. The prefix is 'cloud/' here and empty where
+// the tree is the repository root, so callers naming git paths never restate the layout.
+export function relayTreePath(suffix) {
+  const prefix = relative(
+    fileURLToPath(RELAY_REPOSITORY_ROOT),
+    fileURLToPath(new URL('../../', import.meta.url))
+  ).split(/[\\/]/).filter(Boolean)
+  return [...prefix, suffix].join('/')
+}
+
 export function relayWorkflowFile(name) {
   return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
 }