Files
orca/cloud/apps/relay-ops/src/pre-drain-sample-cli.ts
T
Jinwoo Hong 07dad6739a refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run (#24443)
* refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run

A same-cap wave no longer consumes a 15-minute monitor dry-run and its sealed,
single-use, five-minute-fresh evidence. Each apply wave now samples fleet health
itself right before isolation, with the monitor's evaluator, thresholds, and
tolerances, for a window sized to the cell's host count (3/5/8 min), plus three
lookback rules: no cell container exit in 10 min, no minute over 500 director
503s in 10 min, and director concurrency p99 within the monitor bar over 4 min.

Removes the monitor-run inputs, the gate's consume/authorize steps, the
break-glass override, and the same-cap-only authorization shapes in
relay-monitor-evidence.mjs. The monitor workflow and the rehome enable path are
unchanged.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): bound the pre-drain sample overrun and keep the drain token fresh

Review follow-ups: alternating tolerated readings could hold the sample open
until its step timeout, so cap the overrun at three samples past the window;
record why a read failed; mint a fresh admin ID token for the drain after the
sample; raise the job timeout to 90 min so a long sample cannot cancel the
job past the failsafe.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* feat(relay): exempt the rolled cell and existing-only cells from the pre-drain crash rule

The exit rule counted every relay container exit fleet-wide, so a cell that
crashes every few hours (c25, 12 a week) blocked the very roll that fixes it,
and existing-only legacy cells (c5, 15 a week) blocked rolls they take no part
in. Exits are now grouped by instance, each instance is named by its own newest
runtime-metrics log line, and only exits on general or migration-only cells
other than the target count. An exit no configured cell can be named for trips
the rule; a failed lookup is a failed read. relay-observability.tf joins the
evidence-code set because the rule depends on its filter.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* test(relay): cover re-asking for an unnamed exiting instance; note the boot-exit risk

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
2026-10-01 15:36:17 -04:00

159 lines
5.5 KiB
TypeScript

import { pathToFileURL } from 'node:url'
import { relayOpsEnvironment } from './environment-config.js'
import { createGcloudClient } from './gcloud-client.js'
import {
livePreflightGcloud,
readDispatchSelector,
waveAdjustedSelector
} from './incident-live-preflight-cli.js'
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
import type { IncidentSample } from './incident-monitor.js'
import { createPreDrainHardRuleReader } from './pre-drain-hard-rule-sources.js'
import {
preDrainSampleWindowMinutes,
PreDrainSampleTripped,
runPreDrainSample,
type PreDrainHardRuleReadings
} from './pre-drain-sample.js'
const USAGE =
'usage: --target-cell-id <cell> --target-hosts <n> --expected-selector-generation <n>' +
' --selector-membership-file <json> --wave-index <0-9> --selector-wave-delta <0|2>'
const OPTIONS = [
'--target-cell-id',
'--target-hosts',
'--expected-selector-generation',
'--selector-membership-file',
'--wave-index',
'--selector-wave-delta'
] as const
type Option = (typeof OPTIONS)[number]
const COUNT = /^(0|[1-9][0-9]*)$/
function isOption(value: string): value is Option {
return (OPTIONS as readonly string[]).includes(value)
}
// Every option is required exactly once, so a typo can never silently shorten the window.
export function parsePreDrainSampleArgs(argv: string[]): Record<Option, string> {
const args = argv[0] === '--' ? argv.slice(1) : argv
const values = new Map<Option, string>()
for (let index = 0; index < args.length; index += 2) {
const name = args[index] ?? ''
const value = args[index + 1]
if (!isOption(name) || values.has(name) || !value) throw new Error(USAGE)
values.set(name, value)
}
const get = (name: Option): string => {
const value = values.get(name)
if (value === undefined) throw new Error(USAGE)
return value
}
const parsed = {
'--target-cell-id': get('--target-cell-id'),
'--target-hosts': get('--target-hosts'),
'--expected-selector-generation': get('--expected-selector-generation'),
'--selector-membership-file': get('--selector-membership-file'),
'--wave-index': get('--wave-index'),
'--selector-wave-delta': get('--selector-wave-delta')
}
if (
!COUNT.test(parsed['--target-hosts']) ||
!COUNT.test(parsed['--expected-selector-generation'])
) throw new Error(USAGE)
return parsed
}
export async function runPreDrainSampleCli(
argv: string[],
dependencies: {
now?: () => number
wait?: (ms: number) => Promise<void>
collect?: () => Promise<IncidentSample>
readHardRules?: () => Promise<PreDrainHardRuleReadings>
gcloud?: ReturnType<typeof createGcloudClient>
environment?: NodeJS.ProcessEnv
print?: (line: string) => void
} = {}
): Promise<void> {
const options = parsePreDrainSampleArgs(argv)
const environment = relayOpsEnvironment('production')
const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId))
const targetCellId = options['--target-cell-id']
// An unknown target would exempt nothing and make every one of its exits unattributed.
if (!configuredCellIds.has(targetCellId)) throw new Error('pre-drain sample target cell is unknown')
const windowMinutes = preDrainSampleWindowMinutes(Number(options['--target-hosts']))
const expectedSelector = waveAdjustedSelector(
await readDispatchSelector(
options['--expected-selector-generation'],
options['--selector-membership-file']
),
options['--wave-index'],
options['--selector-wave-delta']
)
const print = dependencies.print ?? ((line: string) => console.log(line))
const gcloud = livePreflightGcloud(
dependencies.gcloud ?? createGcloudClient(),
dependencies.environment
)
const collect = dependencies.collect ?? createIncidentSampleCollector(gcloud, {
environment: 'production',
expectedSelector,
...(dependencies.now ? { now: dependencies.now } : {})
})
const { membership } = expectedSelector
const readHardRules = dependencies.readHardRules ?? createPreDrainHardRuleReader(
environment,
() => gcloud.accessToken(),
{
targetCellId,
placementCellIds: new Set([...membership.general, ...membership.migrationOnly]),
configuredCellIds
},
dependencies.now ? { now: dependencies.now } : {}
)
print(JSON.stringify({
event: 'relay_pre_drain_sample_window',
targetCellId,
targetHosts: Number(options['--target-hosts']),
windowMinutes
}))
try {
const result = await runPreDrainSample({
windowMinutes,
collect,
readHardRules,
...(dependencies.now ? { now: dependencies.now } : {}),
...(dependencies.wait ? { wait: dependencies.wait } : {}),
log: (record) => print(JSON.stringify({ event: 'relay_pre_drain_sample', ...record }))
})
print(JSON.stringify({
event: 'relay_pre_drain_sample_passed',
windowMinutes: result.windowMinutes,
startedAt: result.startedAt,
completedAt: result.completedAt,
samples: result.samples.length
}))
} catch (error) {
if (error instanceof PreDrainSampleTripped) {
print(JSON.stringify({
event: 'relay_pre_drain_sample_tripped',
windowMinutes,
samples: error.samples.length,
failures: error.failures
}))
}
throw error
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
runPreDrainSampleCli(process.argv.slice(2)).catch((error: unknown) => {
console.error(error instanceof Error ? error.message : 'relay pre-drain sample failed')
process.exitCode = 1
})
}