mirror of
https://github.com/stablyai/orca.git
synced 2026-10-14 00:02:42 +00:00
* refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run A same-cap wave no longer consumes a 15-minute monitor dry-run and its sealed, single-use, five-minute-fresh evidence. Each apply wave now samples fleet health itself right before isolation, with the monitor's evaluator, thresholds, and tolerances, for a window sized to the cell's host count (3/5/8 min), plus three lookback rules: no cell container exit in 10 min, no minute over 500 director 503s in 10 min, and director concurrency p99 within the monitor bar over 4 min. Removes the monitor-run inputs, the gate's consume/authorize steps, the break-glass override, and the same-cap-only authorization shapes in relay-monitor-evidence.mjs. The monitor workflow and the rehome enable path are unchanged. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): bound the pre-drain sample overrun and keep the drain token fresh Review follow-ups: alternating tolerated readings could hold the sample open until its step timeout, so cap the overrun at three samples past the window; record why a read failed; mint a fresh admin ID token for the drain after the sample; raise the job timeout to 90 min so a long sample cannot cancel the job past the failsafe. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * feat(relay): exempt the rolled cell and existing-only cells from the pre-drain crash rule The exit rule counted every relay container exit fleet-wide, so a cell that crashes every few hours (c25, 12 a week) blocked the very roll that fixes it, and existing-only legacy cells (c5, 15 a week) blocked rolls they take no part in. Exits are now grouped by instance, each instance is named by its own newest runtime-metrics log line, and only exits on general or migration-only cells other than the target count. An exit no configured cell can be named for trips the rule; a failed lookup is a failed read. relay-observability.tf joins the evidence-code set because the rule depends on its filter. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * test(relay): cover re-asking for an unnamed exiting instance; note the boot-exit risk Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
159 lines
5.5 KiB
TypeScript
159 lines
5.5 KiB
TypeScript
import { pathToFileURL } from 'node:url'
|
|
import { relayOpsEnvironment } from './environment-config.js'
|
|
import { createGcloudClient } from './gcloud-client.js'
|
|
import {
|
|
livePreflightGcloud,
|
|
readDispatchSelector,
|
|
waveAdjustedSelector
|
|
} from './incident-live-preflight-cli.js'
|
|
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
|
|
import type { IncidentSample } from './incident-monitor.js'
|
|
import { createPreDrainHardRuleReader } from './pre-drain-hard-rule-sources.js'
|
|
import {
|
|
preDrainSampleWindowMinutes,
|
|
PreDrainSampleTripped,
|
|
runPreDrainSample,
|
|
type PreDrainHardRuleReadings
|
|
} from './pre-drain-sample.js'
|
|
|
|
const USAGE =
|
|
'usage: --target-cell-id <cell> --target-hosts <n> --expected-selector-generation <n>' +
|
|
' --selector-membership-file <json> --wave-index <0-9> --selector-wave-delta <0|2>'
|
|
|
|
const OPTIONS = [
|
|
'--target-cell-id',
|
|
'--target-hosts',
|
|
'--expected-selector-generation',
|
|
'--selector-membership-file',
|
|
'--wave-index',
|
|
'--selector-wave-delta'
|
|
] as const
|
|
|
|
type Option = (typeof OPTIONS)[number]
|
|
|
|
const COUNT = /^(0|[1-9][0-9]*)$/
|
|
|
|
function isOption(value: string): value is Option {
|
|
return (OPTIONS as readonly string[]).includes(value)
|
|
}
|
|
|
|
// Every option is required exactly once, so a typo can never silently shorten the window.
|
|
export function parsePreDrainSampleArgs(argv: string[]): Record<Option, string> {
|
|
const args = argv[0] === '--' ? argv.slice(1) : argv
|
|
const values = new Map<Option, string>()
|
|
for (let index = 0; index < args.length; index += 2) {
|
|
const name = args[index] ?? ''
|
|
const value = args[index + 1]
|
|
if (!isOption(name) || values.has(name) || !value) throw new Error(USAGE)
|
|
values.set(name, value)
|
|
}
|
|
const get = (name: Option): string => {
|
|
const value = values.get(name)
|
|
if (value === undefined) throw new Error(USAGE)
|
|
return value
|
|
}
|
|
const parsed = {
|
|
'--target-cell-id': get('--target-cell-id'),
|
|
'--target-hosts': get('--target-hosts'),
|
|
'--expected-selector-generation': get('--expected-selector-generation'),
|
|
'--selector-membership-file': get('--selector-membership-file'),
|
|
'--wave-index': get('--wave-index'),
|
|
'--selector-wave-delta': get('--selector-wave-delta')
|
|
}
|
|
if (
|
|
!COUNT.test(parsed['--target-hosts']) ||
|
|
!COUNT.test(parsed['--expected-selector-generation'])
|
|
) throw new Error(USAGE)
|
|
return parsed
|
|
}
|
|
|
|
export async function runPreDrainSampleCli(
|
|
argv: string[],
|
|
dependencies: {
|
|
now?: () => number
|
|
wait?: (ms: number) => Promise<void>
|
|
collect?: () => Promise<IncidentSample>
|
|
readHardRules?: () => Promise<PreDrainHardRuleReadings>
|
|
gcloud?: ReturnType<typeof createGcloudClient>
|
|
environment?: NodeJS.ProcessEnv
|
|
print?: (line: string) => void
|
|
} = {}
|
|
): Promise<void> {
|
|
const options = parsePreDrainSampleArgs(argv)
|
|
const environment = relayOpsEnvironment('production')
|
|
const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId))
|
|
const targetCellId = options['--target-cell-id']
|
|
// An unknown target would exempt nothing and make every one of its exits unattributed.
|
|
if (!configuredCellIds.has(targetCellId)) throw new Error('pre-drain sample target cell is unknown')
|
|
const windowMinutes = preDrainSampleWindowMinutes(Number(options['--target-hosts']))
|
|
const expectedSelector = waveAdjustedSelector(
|
|
await readDispatchSelector(
|
|
options['--expected-selector-generation'],
|
|
options['--selector-membership-file']
|
|
),
|
|
options['--wave-index'],
|
|
options['--selector-wave-delta']
|
|
)
|
|
const print = dependencies.print ?? ((line: string) => console.log(line))
|
|
const gcloud = livePreflightGcloud(
|
|
dependencies.gcloud ?? createGcloudClient(),
|
|
dependencies.environment
|
|
)
|
|
const collect = dependencies.collect ?? createIncidentSampleCollector(gcloud, {
|
|
environment: 'production',
|
|
expectedSelector,
|
|
...(dependencies.now ? { now: dependencies.now } : {})
|
|
})
|
|
const { membership } = expectedSelector
|
|
const readHardRules = dependencies.readHardRules ?? createPreDrainHardRuleReader(
|
|
environment,
|
|
() => gcloud.accessToken(),
|
|
{
|
|
targetCellId,
|
|
placementCellIds: new Set([...membership.general, ...membership.migrationOnly]),
|
|
configuredCellIds
|
|
},
|
|
dependencies.now ? { now: dependencies.now } : {}
|
|
)
|
|
print(JSON.stringify({
|
|
event: 'relay_pre_drain_sample_window',
|
|
targetCellId,
|
|
targetHosts: Number(options['--target-hosts']),
|
|
windowMinutes
|
|
}))
|
|
try {
|
|
const result = await runPreDrainSample({
|
|
windowMinutes,
|
|
collect,
|
|
readHardRules,
|
|
...(dependencies.now ? { now: dependencies.now } : {}),
|
|
...(dependencies.wait ? { wait: dependencies.wait } : {}),
|
|
log: (record) => print(JSON.stringify({ event: 'relay_pre_drain_sample', ...record }))
|
|
})
|
|
print(JSON.stringify({
|
|
event: 'relay_pre_drain_sample_passed',
|
|
windowMinutes: result.windowMinutes,
|
|
startedAt: result.startedAt,
|
|
completedAt: result.completedAt,
|
|
samples: result.samples.length
|
|
}))
|
|
} catch (error) {
|
|
if (error instanceof PreDrainSampleTripped) {
|
|
print(JSON.stringify({
|
|
event: 'relay_pre_drain_sample_tripped',
|
|
windowMinutes,
|
|
samples: error.samples.length,
|
|
failures: error.failures
|
|
}))
|
|
}
|
|
throw error
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
runPreDrainSampleCli(process.argv.slice(2)).catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.message : 'relay pre-drain sample failed')
|
|
process.exitCode = 1
|
|
})
|
|
}
|