mirror of
https://github.com/stablyai/orca.git
synced 2026-10-02 16:02:15 +00:00
* fix(relay): let a same-cap drain finish when only unplaceable hosts remain The c28 canary on 2026-10-01 drained the cell to zero live connections, but four hosts with no free slot anywhere kept redialling and held director leases on it, so the restart-safe wait timed out and left the cell isolated and empty. The drain wait now also passes once the runtime has carried nothing for a sustained quiet window while a small, capped number of leases remain, and logs the escape. Apply modes also refuse a cell whose hosts exceed 80% of the free slots on the other general cells, so a wave cannot strand hosts in the first place. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): define restart-safe by the cell runtime, not director leases Replaces the opt-in stranded-host escape with a corrected definition. A restart is safe when the cell runtime carries nothing live and no migration is open, sustained for the drain pace window. Director activity leases lag hosts that already left or cannot be placed, so they are reported in a progress line and the verified result instead of blocking the restart. The same-cap drain passes its existing pace window. The headroom script is added to the trusted evidence code paths with the other production scripts. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): print stranded director counts on every restart-safe sample Each restart-safe poll now prints its sample count and the director's restart-blocking leases, request units, reserved remainder, and migrations under `stranded`; the verified line carries the same object. Open migrations still block because each is pinned to the cell incarnation a restart replaces. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): require the pace window for every live restart-safe wait Pre-auth and total connections no longer reset the restart-safe window: on drained c28 they flickered with unauthenticated redials in a third of samples, which a restart does not lose. They stay in the progress output. Every live restart-safe call must now pass --pace-window-ms. The capacity job and staging proof drain unpaced, so they pass the production 300000 ms window, and the calls that relied on the 180000 ms default get 480000 ms. Headroom free slots now follow the director's placement rule: the admission pause minus the larger of observed and enforced units, minus outstanding control reservations. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
96 lines
4.0 KiB
JavaScript
96 lines
4.0 KiB
JavaScript
import { spawnSync } from 'node:child_process'
|
|
import { fileURLToPath } from 'node:url'
|
|
import {
|
|
RELAY_REPOSITORY_ROOT,
|
|
relayTreePath,
|
|
relayWorkflowPath
|
|
} from './relay-repository.mjs'
|
|
|
|
const SHA = /^[a-f0-9]{40}$/
|
|
|
|
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
|
|
// production; identical content across two commits is what makes the older commit's verdict binding.
|
|
export const TRUSTED_EVIDENCE_CODE_PATHS = [
|
|
// Produces and seals the 15-minute dry-run evidence.
|
|
relayWorkflowPath('monitor-relay-production.yml'),
|
|
relayWorkflowPath('monitor-relay-production-job.yml'),
|
|
// Download it, verify its authority, and mutate production on it.
|
|
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
|
|
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
|
|
relayWorkflowPath('operate-relay-production-rehome.yml'),
|
|
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
|
|
// Sealing, verification, the wave/canary authority, and the path constants below.
|
|
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
|
|
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
|
|
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
|
|
relayTreePath('dev/scripts/relay-repository.mjs'),
|
|
// Every other script those jobs run against live production.
|
|
relayTreePath('dev/scripts/check-relay-same-cap-headroom.mjs'),
|
|
relayTreePath('dev/scripts/infra.mjs'),
|
|
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
|
|
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
|
|
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
|
|
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
|
|
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
|
|
// The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
|
|
relayTreePath('apps/relay-ops'),
|
|
relayTreePath('package.json'),
|
|
relayTreePath('pnpm-lock.yaml'),
|
|
relayTreePath('pnpm-workspace.yaml'),
|
|
// The Cloud SQL rollout lease every mutation job takes and releases.
|
|
'.github/actions/cloud-sql-rollout-lease'
|
|
]
|
|
|
|
function git(root, args) {
|
|
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
|
|
if (result.error) throw new Error('relay evidence provenance cannot run git')
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Accepts evidence sealed at a different commit only when the current commit descends from it and
|
|
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
|
|
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
|
|
*/
|
|
export function requireSameEvidenceCode({
|
|
sealedSha,
|
|
currentSha,
|
|
label,
|
|
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
|
|
}) {
|
|
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
|
|
throw new Error(`${label} commit is invalid`)
|
|
}
|
|
if (sealedSha === currentSha) return
|
|
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
|
|
throw new Error(`${label} commit cannot be compared without a git checkout`)
|
|
}
|
|
for (const sha of [sealedSha, currentSha]) {
|
|
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
|
|
throw new Error(
|
|
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
|
|
)
|
|
}
|
|
}
|
|
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
|
|
if (ancestry.status === 1) {
|
|
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
|
|
}
|
|
if (ancestry.status !== 0) {
|
|
throw new Error(`${label} commit ancestry could not be determined`)
|
|
}
|
|
const diff = git(repositoryRoot, [
|
|
'diff',
|
|
'--name-only',
|
|
sealedSha,
|
|
currentSha,
|
|
'--',
|
|
...TRUSTED_EVIDENCE_CODE_PATHS
|
|
])
|
|
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
|
|
const changed = diff.stdout.split('\n').filter(Boolean)
|
|
if (changed.length > 0) {
|
|
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
|
|
}
|
|
}
|