mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(relay-ops): stop the reserve watch cleanly when the cell leaves reserve (#30972)
evaluateWatchSample's two early returns (waiting for reserve to apply, and a cell flipped back to db by hand) carried no pages, so watchSegment threw 'result.pages is not iterable' and the run ended as failed instead of stopped (c29, 10-20 02:25Z, run 39354895994).
This commit is contained in:
@@ -360,14 +360,14 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
|
||||
if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) {
|
||||
// Not judged, and the streak holds: the reserve write may not have applied yet.
|
||||
const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) }
|
||||
return { state: next, rules, line: sampleLine(next, sample, rules) }
|
||||
return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] }
|
||||
}
|
||||
// Someone else flipped it back (or it never became reserve): nothing left to watch.
|
||||
next.state = seen ? 'stopped' : 'refused'
|
||||
next.verdict = seen
|
||||
? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})`
|
||||
: `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})`
|
||||
return { state: next, rules: {}, line: sampleLine(next, sample, {}) }
|
||||
return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] }
|
||||
}
|
||||
if (cell.ok) {
|
||||
next.reserveSeenAt = state.reserveSeenAt ?? at
|
||||
|
||||
@@ -788,6 +788,33 @@ test('a quiet fleet with zero director 503s watches on and never trips', async (
|
||||
}
|
||||
})
|
||||
|
||||
// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable").
|
||||
test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
const config = parseAt(
|
||||
watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() })
|
||||
)
|
||||
const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' }
|
||||
const lines = []
|
||||
const state = await watchSegment(config, {
|
||||
...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }),
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
assert.equal(lines[0].rules.reserve.status, 'pending')
|
||||
assert.equal(state.state, 'stopped')
|
||||
assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/)
|
||||
assert.equal(lines.length, 4)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early.
|
||||
const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } })
|
||||
|
||||
|
||||
Reference in New Issue
Block a user