mirror of
https://github.com/stablyai/orca.git
synced 2026-10-03 16:02:11 +00:00
* refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run A same-cap wave no longer consumes a 15-minute monitor dry-run and its sealed, single-use, five-minute-fresh evidence. Each apply wave now samples fleet health itself right before isolation, with the monitor's evaluator, thresholds, and tolerances, for a window sized to the cell's host count (3/5/8 min), plus three lookback rules: no cell container exit in 10 min, no minute over 500 director 503s in 10 min, and director concurrency p99 within the monitor bar over 4 min. Removes the monitor-run inputs, the gate's consume/authorize steps, the break-glass override, and the same-cap-only authorization shapes in relay-monitor-evidence.mjs. The monitor workflow and the rehome enable path are unchanged. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): bound the pre-drain sample overrun and keep the drain token fresh Review follow-ups: alternating tolerated readings could hold the sample open until its step timeout, so cap the overrun at three samples past the window; record why a read failed; mint a fresh admin ID token for the drain after the sample; raise the job timeout to 90 min so a long sample cannot cancel the job past the failsafe. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * feat(relay): exempt the rolled cell and existing-only cells from the pre-drain crash rule The exit rule counted every relay container exit fleet-wide, so a cell that crashes every few hours (c25, 12 a week) blocked the very roll that fixes it, and existing-only legacy cells (c5, 15 a week) blocked rolls they take no part in. Exits are now grouped by instance, each instance is named by its own newest runtime-metrics log line, and only exits on general or migration-only cells other than the target count. An exit no configured cell can be named for trips the rule; a failed lookup is a failed read. relay-observability.tf joins the evidence-code set because the rule depends on its filter. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * test(relay): cover re-asking for an unnamed exiting instance; note the boot-exit risk Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
100 lines
4.3 KiB
JavaScript
100 lines
4.3 KiB
JavaScript
import { spawnSync } from 'node:child_process'
|
|
import { fileURLToPath } from 'node:url'
|
|
import {
|
|
RELAY_REPOSITORY_ROOT,
|
|
relayTreePath,
|
|
relayWorkflowPath
|
|
} from './relay-repository.mjs'
|
|
|
|
const SHA = /^[a-f0-9]{40}$/
|
|
|
|
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
|
|
// production; identical content across two commits is what makes the older commit's verdict binding.
|
|
export const TRUSTED_EVIDENCE_CODE_PATHS = [
|
|
// Produces and seals the 15-minute dry-run evidence.
|
|
relayWorkflowPath('monitor-relay-production.yml'),
|
|
relayWorkflowPath('monitor-relay-production-job.yml'),
|
|
// Rehome enable downloads it, verifies its authority, and mutates production on it.
|
|
relayWorkflowPath('operate-relay-production-rehome.yml'),
|
|
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
|
|
// Same-cap seals and spends canary authority, which a later batch accepts only from this code.
|
|
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
|
|
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
|
|
// Sealing, verification, the wave/canary authority, and the path constants below.
|
|
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
|
|
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
|
|
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
|
|
relayTreePath('dev/scripts/relay-repository.mjs'),
|
|
// Every other script those jobs run against live production.
|
|
relayTreePath('dev/scripts/check-relay-same-cap-headroom.mjs'),
|
|
relayTreePath('dev/scripts/infra.mjs'),
|
|
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
|
|
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
|
|
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
|
|
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
|
|
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
|
|
// The exit metric filter the same-cap pre-drain sample's crash rule reads.
|
|
relayTreePath('infra/terraform/relay-observability.tf'),
|
|
// The monitor, the live preflight recheck, and the same-cap pre-drain sample, plus anything that
|
|
// changes their behaviour.
|
|
relayTreePath('apps/relay-ops'),
|
|
relayTreePath('package.json'),
|
|
relayTreePath('pnpm-lock.yaml'),
|
|
relayTreePath('pnpm-workspace.yaml'),
|
|
// The Cloud SQL rollout lease every mutation job takes and releases.
|
|
'.github/actions/cloud-sql-rollout-lease'
|
|
]
|
|
|
|
function git(root, args) {
|
|
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
|
|
if (result.error) throw new Error('relay evidence provenance cannot run git')
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Accepts evidence sealed at a different commit only when the current commit descends from it and
|
|
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
|
|
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
|
|
*/
|
|
export function requireSameEvidenceCode({
|
|
sealedSha,
|
|
currentSha,
|
|
label,
|
|
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
|
|
}) {
|
|
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
|
|
throw new Error(`${label} commit is invalid`)
|
|
}
|
|
if (sealedSha === currentSha) return
|
|
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
|
|
throw new Error(`${label} commit cannot be compared without a git checkout`)
|
|
}
|
|
for (const sha of [sealedSha, currentSha]) {
|
|
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
|
|
throw new Error(
|
|
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
|
|
)
|
|
}
|
|
}
|
|
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
|
|
if (ancestry.status === 1) {
|
|
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
|
|
}
|
|
if (ancestry.status !== 0) {
|
|
throw new Error(`${label} commit ancestry could not be determined`)
|
|
}
|
|
const diff = git(repositoryRoot, [
|
|
'diff',
|
|
'--name-only',
|
|
sealedSha,
|
|
currentSha,
|
|
'--',
|
|
...TRUSTED_EVIDENCE_CODE_PATHS
|
|
])
|
|
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
|
|
const changed = diff.stdout.split('\n').filter(Boolean)
|
|
if (changed.length > 0) {
|
|
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
|
|
}
|
|
}
|