fix(relay-ops): stop the reserve watch cleanly when the cell leaves reserve (#30972)

evaluateWatchSample's two early returns (waiting for reserve to apply, and
a cell flipped back to db by hand) carried no pages, so watchSegment threw
'result.pages is not iterable' and the run ended as failed instead of
stopped (c29, 10-20 02:25Z, run 39354895994).
This commit is contained in:
Jinwoo Hong
2026-10-19 22:36:42 -04:00
committed by GitHub
parent c0202b460c
commit db89ced0f5
2 changed files with 29 additions and 2 deletions
@@ -360,14 +360,14 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) {
// Not judged, and the streak holds: the reserve write may not have applied yet.
const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) }
return { state: next, rules, line: sampleLine(next, sample, rules) }
return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] }
}
// Someone else flipped it back (or it never became reserve): nothing left to watch.
next.state = seen ? 'stopped' : 'refused'
next.verdict = seen
? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})`
: `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})`
return { state: next, rules: {}, line: sampleLine(next, sample, {}) }
return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] }
}
if (cell.ok) {
next.reserveSeenAt = state.reserveSeenAt ?? at
@@ -788,6 +788,33 @@ test('a quiet fleet with zero director 503s watches on and never trips', async (
}
})
// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable").
test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => {
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
try {
let clock = START
const config = parseAt(
watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() })
)
const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' }
const lines = []
const state = await watchSegment(config, {
...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }),
idToken: 'id',
retryDelayMs: 0,
now: () => clock,
sleep: async (ms) => { clock += ms },
log: (line) => lines.push(JSON.parse(line))
})
assert.equal(lines[0].rules.reserve.status, 'pending')
assert.equal(state.state, 'stopped')
assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/)
assert.equal(lines.length, 4)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early.
const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } })