Files
orca/cloud/dev/scripts/relay-evidence-code-provenance.mjs
T
Jinwoo Hong 7c119465b0 fix(relay): define restart-safe by the cell runtime, and refuse waves without headroom (#24259)
* fix(relay): let a same-cap drain finish when only unplaceable hosts remain

The c28 canary on 2026-10-01 drained the cell to zero live connections, but four
hosts with no free slot anywhere kept redialling and held director leases on it,
so the restart-safe wait timed out and left the cell isolated and empty.

The drain wait now also passes once the runtime has carried nothing for a
sustained quiet window while a small, capped number of leases remain, and logs
the escape. Apply modes also refuse a cell whose hosts exceed 80% of the free
slots on the other general cells, so a wave cannot strand hosts in the first place.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): define restart-safe by the cell runtime, not director leases

Replaces the opt-in stranded-host escape with a corrected definition. A
restart is safe when the cell runtime carries nothing live and no migration
is open, sustained for the drain pace window. Director activity leases lag
hosts that already left or cannot be placed, so they are reported in a
progress line and the verified result instead of blocking the restart.

The same-cap drain passes its existing pace window. The headroom script is
added to the trusted evidence code paths with the other production scripts.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): print stranded director counts on every restart-safe sample

Each restart-safe poll now prints its sample count and the director's
restart-blocking leases, request units, reserved remainder, and migrations
under `stranded`; the verified line carries the same object. Open migrations
still block because each is pinned to the cell incarnation a restart replaces.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): require the pace window for every live restart-safe wait

Pre-auth and total connections no longer reset the restart-safe window:
on drained c28 they flickered with unauthenticated redials in a third of
samples, which a restart does not lose. They stay in the progress output.

Every live restart-safe call must now pass --pace-window-ms. The capacity
job and staging proof drain unpaced, so they pass the production 300000 ms
window, and the calls that relied on the 180000 ms default get 480000 ms.

Headroom free slots now follow the director's placement rule: the admission
pause minus the larger of observed and enforced units, minus outstanding
control reservations.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
2026-09-30 22:25:36 -04:00

96 lines
4.0 KiB
JavaScript

import { spawnSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import {
RELAY_REPOSITORY_ROOT,
relayTreePath,
relayWorkflowPath
} from './relay-repository.mjs'
const SHA = /^[a-f0-9]{40}$/
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
// production; identical content across two commits is what makes the older commit's verdict binding.
export const TRUSTED_EVIDENCE_CODE_PATHS = [
// Produces and seals the 15-minute dry-run evidence.
relayWorkflowPath('monitor-relay-production.yml'),
relayWorkflowPath('monitor-relay-production-job.yml'),
// Download it, verify its authority, and mutate production on it.
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
relayWorkflowPath('operate-relay-production-rehome.yml'),
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
// Sealing, verification, the wave/canary authority, and the path constants below.
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
relayTreePath('dev/scripts/relay-repository.mjs'),
// Every other script those jobs run against live production.
relayTreePath('dev/scripts/check-relay-same-cap-headroom.mjs'),
relayTreePath('dev/scripts/infra.mjs'),
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
// The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
relayTreePath('apps/relay-ops'),
relayTreePath('package.json'),
relayTreePath('pnpm-lock.yaml'),
relayTreePath('pnpm-workspace.yaml'),
// The Cloud SQL rollout lease every mutation job takes and releases.
'.github/actions/cloud-sql-rollout-lease'
]
function git(root, args) {
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
if (result.error) throw new Error('relay evidence provenance cannot run git')
return result
}
/**
* Accepts evidence sealed at a different commit only when the current commit descends from it and
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
*/
export function requireSameEvidenceCode({
sealedSha,
currentSha,
label,
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
}) {
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
throw new Error(`${label} commit is invalid`)
}
if (sealedSha === currentSha) return
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
throw new Error(`${label} commit cannot be compared without a git checkout`)
}
for (const sha of [sealedSha, currentSha]) {
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
throw new Error(
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
)
}
}
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
if (ancestry.status === 1) {
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
}
if (ancestry.status !== 0) {
throw new Error(`${label} commit ancestry could not be determined`)
}
const diff = git(repositoryRoot, [
'diff',
'--name-only',
sealedSha,
currentSha,
'--',
...TRUSTED_EVIDENCE_CODE_PATHS
])
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
const changed = diff.stdout.split('\n').filter(Boolean)
if (changed.length > 0) {
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
}
}