mirror of
https://github.com/stablyai/orca.git
synced 2026-10-05 16:02:22 +00:00
Jinwoo Hong
07dad6739a
* refactor(relay): sample fleet health inside the same-cap roll instead of a separate monitor run A same-cap wave no longer consumes a 15-minute monitor dry-run and its sealed, single-use, five-minute-fresh evidence. Each apply wave now samples fleet health itself right before isolation, with the monitor's evaluator, thresholds, and tolerances, for a window sized to the cell's host count (3/5/8 min), plus three lookback rules: no cell container exit in 10 min, no minute over 500 director 503s in 10 min, and director concurrency p99 within the monitor bar over 4 min. Removes the monitor-run inputs, the gate's consume/authorize steps, the break-glass override, and the same-cap-only authorization shapes in relay-monitor-evidence.mjs. The monitor workflow and the rehome enable path are unchanged. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * fix(relay): bound the pre-drain sample overrun and keep the drain token fresh Review follow-ups: alternating tolerated readings could hold the sample open until its step timeout, so cap the overrun at three samples past the window; record why a read failed; mint a fresh admin ID token for the drain after the sample; raise the job timeout to 90 min so a long sample cannot cancel the job past the failsafe. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * feat(relay): exempt the rolled cell and existing-only cells from the pre-drain crash rule The exit rule counted every relay container exit fleet-wide, so a cell that crashes every few hours (c25, 12 a week) blocked the very roll that fixes it, and existing-only legacy cells (c5, 15 a week) blocked rolls they take no part in. Exits are now grouped by instance, each instance is named by its own newest runtime-metrics log line, and only exits on general or migration-only cells other than the target count. An exit no configured cell can be named for trips the rule; a failed lookup is a failed read. relay-observability.tf joins the evidence-code set because the rule depends on its filter. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 * test(relay): cover re-asking for an unnamed exiting instance; note the boot-exit risk Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
102 lines
4.0 KiB
TypeScript
102 lines
4.0 KiB
TypeScript
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
import { relayOpsEnvironment } from './environment-config.js'
|
|
import { parsePreDrainSampleArgs, runPreDrainSampleCli } from './pre-drain-sample-cli.js'
|
|
|
|
const directories: string[] = []
|
|
|
|
afterEach(() => {
|
|
for (const directory of directories.splice(0)) {
|
|
rmSync(directory, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
function membershipFile(): string {
|
|
const directory = mkdtempSync(join(tmpdir(), 'relay-pre-drain-selector-'))
|
|
directories.push(directory)
|
|
const path = join(directory, 'selector.json')
|
|
writeFileSync(path, JSON.stringify({
|
|
existingOnly: relayOpsEnvironment('production').cells.map((cell) => cell.cellId),
|
|
migrationOnly: [],
|
|
general: []
|
|
}))
|
|
return path
|
|
}
|
|
|
|
function args(overrides: Record<string, string> = {}): string[] {
|
|
const values: Record<string, string> = {
|
|
'--target-cell-id': 'production-gce-c25',
|
|
'--target-hosts': '1200',
|
|
'--expected-selector-generation': '40',
|
|
'--selector-membership-file': membershipFile(),
|
|
'--wave-index': '2',
|
|
'--selector-wave-delta': '2',
|
|
...overrides
|
|
}
|
|
return Object.entries(values).flatMap(([name, value]) => [name, value])
|
|
}
|
|
|
|
describe('pre-drain sample CLI', () => {
|
|
it('requires every option exactly once and nothing else', () => {
|
|
expect(() => parsePreDrainSampleArgs(['--', ...args()])).not.toThrow()
|
|
const full = args()
|
|
for (let index = 0; index < full.length; index += 2) {
|
|
const missing = [...full.slice(0, index), ...full.slice(index + 2)]
|
|
expect(() => parsePreDrainSampleArgs(missing)).toThrow('usage')
|
|
}
|
|
expect(() => parsePreDrainSampleArgs([...full, '--wave-index', '2'])).toThrow('usage')
|
|
expect(() => parsePreDrainSampleArgs([...full, '--skip-window', '1'])).toThrow('usage')
|
|
expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '-1' }))).toThrow('usage')
|
|
expect(() => parsePreDrainSampleArgs(args({ '--target-hosts': '' }))).toThrow('usage')
|
|
})
|
|
|
|
it('rejects a target cell the environment does not configure', async () => {
|
|
await expect(runPreDrainSampleCli(args({ '--target-cell-id': 'production-gce-c99' }), {
|
|
collect: async () => { throw new Error('must not sample') },
|
|
readHardRules: async () => { throw new Error('must not sample') }
|
|
})).rejects.toThrow('target cell is unknown')
|
|
})
|
|
|
|
it('rejects a wave index or selector delta no wave produces', async () => {
|
|
for (const overrides of [{ '--wave-index': '10' }, { '--selector-wave-delta': '1' }]) {
|
|
await expect(runPreDrainSampleCli(args(overrides), {
|
|
collect: async () => { throw new Error('must not sample') },
|
|
readHardRules: async () => { throw new Error('must not sample') }
|
|
})).rejects.toThrow('wave index or selector wave delta is invalid')
|
|
}
|
|
})
|
|
|
|
it('sizes the window from the target hosts and reports the verdict', async () => {
|
|
let clock = Date.parse('2026-10-01T12:00:00.000Z')
|
|
const lines: string[] = []
|
|
let collected = 0
|
|
// A sample that cannot be judged green trips the run, which is enough to see the window.
|
|
await expect(runPreDrainSampleCli(args(), {
|
|
now: () => clock,
|
|
wait: async (ms) => { clock += ms },
|
|
collect: async () => {
|
|
collected += 1
|
|
throw new Error('collector down')
|
|
},
|
|
readHardRules: async () => ({
|
|
cellProcessExits: 0,
|
|
unattributedExitInstances: [],
|
|
director503PeakPerMinute: 0,
|
|
directorConcurrencyP99: 1
|
|
}),
|
|
print: (line) => lines.push(line)
|
|
})).rejects.toThrow('relay pre-drain sample tripped')
|
|
const events = lines.map((line) => JSON.parse(line))
|
|
expect(events[0]).toEqual({
|
|
event: 'relay_pre_drain_sample_window',
|
|
targetCellId: 'production-gce-c25',
|
|
targetHosts: 1200,
|
|
windowMinutes: 5
|
|
})
|
|
expect(events.at(-1)).toMatchObject({ event: 'relay_pre_drain_sample_tripped' })
|
|
expect(collected).toBe(3)
|
|
})
|
|
})
|