From db89ced0f502c282ad70e7d6e63659bf8eac1b36 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:36:42 -0400 Subject: [PATCH] fix(relay-ops): stop the reserve watch cleanly when the cell leaves reserve (#30972) evaluateWatchSample's two early returns (waiting for reserve to apply, and a cell flipped back to db by hand) carried no pages, so watchSegment threw 'result.pages is not iterable' and the run ended as failed instead of stopped (c29, 10-20 02:25Z, run 39354895994). --- .../dev/scripts/relay-reserve-watch-rules.mjs | 4 +-- .../scripts/watch-relay-reserve-cell.test.mjs | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs index 5ade25766ad..9da9d1ce40b 100644 --- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs +++ b/cloud/dev/scripts/relay-reserve-watch-rules.mjs @@ -360,14 +360,14 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) { // Not judged, and the streak holds: the reserve write may not have applied yet. const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) } - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] } } // Someone else flipped it back (or it never became reserve): nothing left to watch. next.state = seen ? 'stopped' : 'refused' next.verdict = seen ? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})` : `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})` - return { state: next, rules: {}, line: sampleLine(next, sample, {}) } + return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] } } if (cell.ok) { next.reserveSeenAt = state.reserveSeenAt ?? at diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs index 98c9829f8fa..0f5b1467c0c 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs @@ -788,6 +788,33 @@ test('a quiet fleet with zero director 503s watches on and never trips', async ( } }) +// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable"). +test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const config = parseAt( + watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() }) + ) + const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' } + const lines = [] + const state = await watchSegment(config, { + ...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }), + idToken: 'id', + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(JSON.parse(line)) + }) + assert.equal(lines[0].rules.reserve.status, 'pending') + assert.equal(state.state, 'stopped') + assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/) + assert.equal(lines.length, 4) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + // 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early. const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } })