From 76f9978880752f2b3ed63bb8c2698b8a7236440d Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:40:33 -0400 Subject: [PATCH] fix(relay): judge the reserve watch's 503 rule on non-drain 503s, not a host's own re-dials (#30487) * fix(relay): judge the reserve watch's 503 rule on non-drain 503s, not a host's own re-dials The watch counted every director 503, and on 2026-10-18 ~98% were host-rate-limited re-dials that grow with traffic, so it flipped production-gce-c7 back twice with no fault. It now uses the roll ladder's non-drain measure (request_count 503s less drain-return deferrals and host-rate-limited / host-in-flight re-dials from the director's runtime metrics) for both the background and the live samples. A judged minute no runtime-metrics line covers is unverifiable, not zero. Co-Authored-By: Claude * fix(relay): state the reserve watch's metrics coverage honestly and share the metrics read limit Co-Authored-By: Claude --------- Co-authored-by: Claude --- ...operate-relay-production-reserve-watch.yml | 9 +- .../dev/scripts/relay-reserve-watch-rules.mjs | 28 ++-- .../scripts/relay-same-cap-shadow-gate.mjs | 4 +- .../dev/scripts/watch-relay-reserve-cell.mjs | 53 +++++-- .../scripts/watch-relay-reserve-cell.test.mjs | 130 +++++++++++++++--- cloud/docs/relay-workflows.md | 12 +- 6 files changed, 189 insertions(+), 47 deletions(-) diff --git a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml index e045d10928e..d747843b7ca 100644 --- a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml +++ b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml @@ -25,15 +25,16 @@ on: type: string flipped-at: description: >- - When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503 - background is the median of the 10 minutes before it + When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The non-drain + 503 background is the median of the 10 minutes before it required: false default: '' type: string background-503-per-minute: description: >- - Director 503s per minute before the flip; overrides the one measured before flipped-at. - One of the two is required + Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals + and a host's own early or duplicate re-dials, from the director's runtime metrics); + overrides the one measured before flipped-at. One of the two is required required: false default: '' type: string diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs index 7fd9aae89f2..60f40f3b69e 100644 --- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs +++ b/cloud/dev/scripts/relay-reserve-watch-rules.mjs @@ -6,9 +6,9 @@ import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-ga const MINUTE_MS = 60_000 export const RESERVE_WATCH_RULES = { - // The roll ladder's warn line: max(1.5x, background + 20), so a zero background is not tripped - // by one 503. Three complete minutes in a row, not two. The read spans more minutes than that so - // a late publishing front still leaves three published minutes to judge. + // The roll ladder's warn line on non-drain 503s: max(1.5x, background + 20), so a zero background + // is not tripped by one 503. Three complete minutes in a row, not two. The read spans more minutes + // than that so a late publishing front still leaves three published minutes to judge. director503: { multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple, marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute, @@ -54,12 +54,15 @@ export function initialWatchState({ cellId, startedAt, durationMinutes, backgrou } } -// The background from the per-minute 503 counts before the flip. `publishedThrough` is the newest -// minute any response code has a point for: a 503 minute at or before it with no point had none, -// and the background waits (null) until it covers the last minute. -export function measuredBackground(perMinute, minutes, publishedThrough) { +// The background from the per-minute non-drain 503s before the flip, the live samples' measure. +// `publishedThrough` is the newest minute any response code has a point for: a 503 minute at or +// before it with no point had none, and the background waits (null) until it covers the last +// minute. Unmetered minutes are left out; under half the minutes metered is no background. +export function measuredBackground({ perMinute, minutes, publishedThrough, unmetered }) { if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null - const background = backgroundOf({ minutes, series: minutes.map((minute) => perMinute[minute] ?? 0), unverified: false }) + const metered = minutes.filter((minute) => !unmetered.includes(minute)) + if (metered.length * 2 < minutes.length) return null + const background = backgroundOf({ minutes: metered, series: metered.map((minute) => perMinute[minute] ?? 0), unverified: false }) return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' } } @@ -96,8 +99,8 @@ function judgeOverrun(cell) { return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap }) } -// Fleet-wide director 503s: no existing signal attributes a 503 to the cell a host was bound for, -// so a problem local to one cell can stay under the fleet's line. This proxy can miss. +// Fleet-wide non-drain director 503s: no existing signal attributes a 503 to the cell a host was +// bound for, so a problem local to one cell can stay under the fleet's line. This proxy can miss. function judgeDirector503(read, state, at) { const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs } = RESERVE_WATCH_RULES.director503 const { background } = state @@ -116,6 +119,8 @@ function judgeDirector503(read, state, at) { if (recent.length < consecutiveMinutes) { return unverifiable(`director requests are published only through ${read.publishedThrough}`) } + const unmetered = recent.filter((minute) => read.unmetered.includes(minute)) + if (unmetered.length > 0) return unverifiable(`no director runtime metrics for ${unmetered.join(', ')}`) const counts = recent.map((minute) => read.perMinute[minute] ?? 0) const detail = { backgroundPerMinute: perMinute, above, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) } if (counts.every((count) => count > above)) { @@ -183,7 +188,8 @@ function judgeLedgerDrift(cell, ledger, aboveSince, at) { /** * One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of - * director 503s with the request series' publishing front (`publishedThrough`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this + * non-drain director 503s with the request series' publishing front (`publishedThrough`) and the + * minutes no runtime metrics cover (`unmetered`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this * cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read. */ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs index 2914b282095..3815a6b965f 100644 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs +++ b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs @@ -232,9 +232,9 @@ const DIRECTOR_DRAIN_FIELDS = [ ] // Five instances at one sample per 30 s is ~100 per 10-min sub-window; this many is truncation. -const DIRECTOR_METRICS_LIMIT = 1000 +export const DIRECTOR_METRICS_LIMIT = 1000 -async function readDirectorDrainReturn(reader, { config, window }) { +export async function readDirectorDrainReturn(reader, { config, window }) { const projection = `json(timestamp,${DIRECTOR_DRAIN_FIELDS .map((field) => `jsonPayload.${field}`) .join(',')})` diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.mjs index 6f5cc38e279..5809b4f1f1d 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.mjs @@ -21,8 +21,19 @@ import { readRuntime, WRITE_CONFIRMATION } from './operate-relay-cell-flags.mjs' -import { readDirectorRequestsByCodePerMinute, readLogEntries } from './relay-same-cap-shadow-gate.mjs' -import { minutesOf } from './relay-same-cap-shadow-gate-verdict.mjs' +import { + DIRECTOR_METRICS_LIMIT, + readDirectorDrainReturn, + readDirectorRequestsByCodePerMinute, + readLogEntries +} from './relay-same-cap-shadow-gate.mjs' +import { + DIRECTOR_METRICS_INTERVAL_MS, + drainReturnByMinute, + minuteKey, + minutesOf, + withoutDrainDeferrals +} from './relay-same-cap-shadow-gate-verdict.mjs' import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs' import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs' import { @@ -130,28 +141,46 @@ export async function readCellSample(fetchImpl, request, idToken, wait) { } const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS -const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null }) +const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null, unmetered: [] }) -// The `minutes` whole minutes that end at the minute boundary `endedAt`: the 503 counts, and the -// newest minute any response code has a point for. The director always serves something, so that -// front is how far the 503 series is published; no front at all is no evidence. A code publishing -// late only delays a trip: a later read sees its points. +// The `minutes` whole minutes that end at the minute boundary `endedAt`: the director's non-drain +// 503s, the roll ladder's measure its line comes from. Cloud Run's 503 count less what the +// director's runtime metrics attribute to drain-return deferrals and to a host re-dialling inside +// its own retry interval or while its last dial is in flight (10-18: ~98% of 503s), since those +// grow with traffic, not trouble. `publishedThrough` is the newest minute any response code has a +// point for (the director always serves something, so no front at all is no evidence); a minute +// with no metrics line from any instance is `unmetered`, since subtracting nothing there is a guess. +// Coverage is any instance, not each: one instance's missing lines under-subtract, biased to a trip. export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes }) { const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } + const config = { projectId, directorService } let read try { - read = await readDirectorRequestsByCodePerMinute(reader, { config: { projectId, directorService }, span }) + read = await readDirectorRequestsByCodePerMinute(reader, { config, span }) } catch (error) { // A point with an unparseable end time has no minute. return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`) } if (read.failed) return failed503Read('director 503 metric read failed') + // A line covers the 30 s before it, so the last minute's tail is in the next minute's lines. + const metrics = await readDirectorDrainReturn(reader, { + config, + window: { startedAt: span.startedAt, endedAt: new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS) } + }) + const lineTimes = metrics.flatMap((part) => part.samples.map((sample) => Date.parse(sample.timestamp))) + if (lineTimes.some((at) => !Number.isFinite(at))) return failed503Read('director runtime metrics line with an unparseable timestamp') + const scheduled = drainReturnByMinute(metrics, DIRECTOR_METRICS_LIMIT) + if (scheduled.truncated) return failed503Read('director runtime metrics read failed, truncated, or short') + const keys = minutesOf(span) + const metered = new Set(lineTimes.flatMap((at) => [minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS), minuteKey(at - 1)])) + const nonDrain = withoutDrainDeferrals({ perMinute: read.byCode['503'] ?? {} }, scheduled, keys) const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort() return { ok: true, - perMinute: read.byCode['503'] ?? {}, - minutes: minutesOf(span), - publishedThrough: published.at(-1) ?? null + perMinute: Object.fromEntries(keys.map((minute, index) => [minute, nonDrain.series[index]])), + minutes: keys, + publishedThrough: published.at(-1) ?? null, + unmetered: keys.filter((minute) => !metered.has(minute)) } } @@ -342,7 +371,7 @@ export async function watchSegment(config, dependencies) { endedAt: baselineEndsAt, minutes: BACKGROUND_MINUTES }) - state = { ...state, background: read.ok ? measuredBackground(read.perMinute, read.minutes, read.publishedThrough) : null } + state = { ...state, background: read.ok ? measuredBackground(read) : null } } const reader = createReader(deps, request.projectId) // Serialised: concurrent Logging reads are what earn a 429. diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs index 28d34ab4723..41a25eb4ead 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs @@ -47,7 +47,7 @@ const minutesBefore = (at, count) => ) const quiet503 = (at, perMinute = 10) => { const minutes = minutesBefore(at, 3) - return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1) } + return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1), unmetered: [] } } const sampleAt = (at, overrides = {}) => ({ at, @@ -93,7 +93,8 @@ test('503s trip only after three complete minutes over max(1.5x background, back ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])), - publishedThrough: minutes.at(-1) + publishedThrough: minutes.at(-1), + unmetered: [] } } const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT) @@ -115,12 +116,12 @@ test('503s trip only after three complete minutes over max(1.5x background, back test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => { const minutes = minutesBefore(START, 10) - const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1) } + const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1), unmetered: [] } const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } })) assert.equal(result.rules.director503.status, 'pass') assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0]) assert.equal(result.state.unverifiableStreak, 0) - assert.deepEqual(measuredBackground({}, minutes, minutes.at(-1)), { medianPerMinute: 0, minutes: 10, source: 'measured' }) + assert.deepEqual(measuredBackground({ perMinute: {}, minutes, publishedThrough: minutes.at(-1), unmetered: [] }), { medianPerMinute: 0, minutes: 10, source: 'measured' }) }) test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => { @@ -129,20 +130,20 @@ test('503 minutes past the publishing front are left out, and only no front at a const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT) // The front is two minutes behind: the three minutes before it are judged, the two after are not. const over = Object.fromEntries(minutes.map((minute) => [minute, 500])) - const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7] }) + const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7], unmetered: [] }) assert.equal(behind.rules.director503.status, 'trip') assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8)) // A sparse 503 series: minutes after its last point but within the front are zero, not late. - const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1) }) + const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1), unmetered: [] }) assert.equal(sparse.rules.director503.status, 'pass') assert.equal(sparse.state.unverifiableStreak, 0) // No request series at all. - const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null }) + const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null, unmetered: [] }) assert.equal(empty.rules.director503.status, 'unverifiable') assert.match(empty.rules.director503.reason, /request series is empty/) assert.equal(empty.state.unverifiableStreak, 1) // A front too far behind to leave three published minutes in the read. - const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1] }) + const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1], unmetered: [] }) assert.equal(stale.rules.director503.status, 'unverifiable') assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`)) }) @@ -215,16 +216,21 @@ test('the duplicate read carries a positive control: no fresh ledger summary in assert.equal(gap.state.unverifiableStreak, 1) }) -test('the measured background is the median of the minutes before the flip, published gaps as zero', () => { +test('the measured background is the median of the metered minutes before the flip, published gaps as zero', () => { const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49'] const front = minutes[2] - assert.equal(measuredBackground({ '2026-10-14T11:48': 30 }, minutes, front).medianPerMinute, 0) - assert.equal(measuredBackground({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, minutes, front).medianPerMinute, 20) - assert.equal(measuredBackground({}, minutes, '2026-10-14T11:52').medianPerMinute, 0) + const background = (perMinute, publishedThrough, readMinutes = minutes, unmetered = []) => + measuredBackground({ perMinute, minutes: readMinutes, publishedThrough, unmetered }) + assert.equal(background({ '2026-10-14T11:48': 30 }, front).medianPerMinute, 0) + assert.equal(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front).medianPerMinute, 20) + assert.equal(background({}, '2026-10-14T11:52').medianPerMinute, 0) // Not published through the last minute yet, or no request series: no background yet. - assert.equal(measuredBackground({ [minutes[0]]: 10 }, minutes, minutes[1]), null) - assert.equal(measuredBackground({}, minutes, null), null) - assert.equal(measuredBackground({}, [], front), null) + assert.equal(background({ [minutes[0]]: 10 }, minutes[1]), null) + assert.equal(background({}, null), null) + assert.equal(background({}, front, []), null) + // An unmetered minute is left out, not read as zero; under half metered is no background. + assert.deepEqual(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front, minutes, [minutes[0]]), { medianPerMinute: 25, minutes: 2, source: 'measured' }) + assert.equal(background({}, front, minutes, minutes.slice(1)), null) }) test('any overrun trips at once', () => { @@ -391,6 +397,17 @@ test('the 503 background needs flipped-at or an explicit value; flipped-at must }) // A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample. +const METRICS_EVENT = 'orca_relay_runtime_metrics' +// Each instance's line every 30 s inside the read's timestamp bounds, newest first. +function metricsLines(filter, metrics) { + const [, from, to] = filter.match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) + const lines = [] + for (let at = Math.ceil(Date.parse(from) / 30_000) * 30_000; at < Date.parse(to); at += 30_000) { + for (const deltas of metrics(at)) lines.push({ timestamp: new Date(at).toISOString(), jsonPayload: deltas }) + } + return lines.reverse() +} + function fakeSources({ cell = () => ({}), // Logged at each sample's minute, so it stays fresh for the positive control. @@ -399,7 +416,9 @@ function fakeSources({ // Requests of every other code per minute; zero is a minute with no traffic at all. perOther = () => 1_000, seatControl = true, - rolloutLease = null + rolloutLease = null, + // Each director instance's runtime-metrics deltas for the 30 s ending at `endedAt`. + metrics = () => [{}] } = {}) { let samples = 0 const monitoringSpans = [] @@ -447,6 +466,7 @@ function fakeSources({ const summaries = ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } })) if (filter.includes(SEAT_EVENT)) return { stdout: JSON.stringify(seatControl ? summaries : []) } if (filter.includes(SUMMARY_EVENT)) return { stdout: JSON.stringify(ledger(samples)) } + if (filter.includes(METRICS_EVENT)) return { stdout: JSON.stringify(metricsLines(filter, metrics)) } return { stdout: '[]' } } return { fetchImpl, runGcloud, monitoringSpans, readRolloutLease: async () => rolloutLease } @@ -755,6 +775,84 @@ test('a quiet fleet with zero director 503s watches on and never trips', async ( } }) +// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early. +const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } }) + +async function watchFor(minutes, sources, flippedAt = START - 2 * MINUTE) { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const config = parseAt( + watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes), 'flipped-at': new Date(flippedAt).toISOString() }) + ) + const lines = [] + const state = await watchSegment(config, { + ...sources, + idToken: 'id', + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(JSON.parse(line)) + }) + return { state, lines } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +test('a host re-dialling inside its own interval never trips the 503 rule, however much it grows', async () => { + // Background 20/min, all re-dials; then 50/min, 48 of them re-dials. All 503s would trip at 40. + const flippedAt = START - 2 * MINUTE + const { state, lines } = await watchFor(20, fakeSources({ + per503: (minute) => (minute < flippedAt ? 20 : 50), + metrics: (at) => [rateLimited(at <= flippedAt ? 10 : 24)] + }), flippedAt) + assert.equal(state.state, 'watching') + assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' }) + assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass')) + assert.ok(lines.slice(1).every((line) => Object.values(line.rules.director503.recent).every((count) => count <= 2))) +}) + +test('503s the director could not serve trip after three minutes over the line, beside the re-dials', async () => { + const flippedAt = START - 2 * MINUTE + const { state } = await watchFor(10, fakeSources({ + per503: (minute) => (minute < flippedAt ? 20 : 60), + // 10 re-dials a minute throughout; the 40 more after the flip are queue waits that timed out. + metrics: (at) => [{ + ...rateLimited(5), + stickyRejectionsByReasonDelta: at <= flippedAt ? {} : { 'wait-timeout': 20 } + }] + }), flippedAt) + assert.equal(state.state, 'tripped') + assert.deepEqual(state.background, { medianPerMinute: 10, minutes: 10, source: 'measured' }) + assert.match(state.verdict, /director503: director 503s above 30\/min for 3 min \(50, 50, 50\)/) +}) + +test('a minute no director instance logged runtime metrics for is unverifiable, not zero', async () => { + // Two instances, both silent for three minutes in the middle of the read. + const endedAt = START + 2 * MINUTE + const silent = (at) => at > endedAt - 5 * MINUTE && at <= endedAt - 2 * MINUTE + const reader = fakeSources({ per503: () => 50, metrics: (at) => (silent(at) ? [] : [rateLimited(12), rateLimited(12)]) }) + const read = await readDirector503Minutes( + { ...fakeReader(), fetch: reader.fetchImpl, runGcloud: reader.runGcloud }, + { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } + ) + assert.equal(read.ok, true) + assert.deepEqual(read.unmetered, read.minutes.slice(5, 8)) + assert.equal(read.perMinute[read.minutes[0]], 2) + const judged = evaluateWatchSample(freshState({ background: { medianPerMinute: 0, source: 'measured' } }), sampleAt(START, { director503: read }), LIMIT) + assert.equal(judged.rules.director503.status, 'unverifiable') + assert.match(judged.rules.director503.reason, /no director runtime metrics for/) + // No line at all in the read is short, not a quiet fleet. + const unlogged = fakeSources({ per503: () => 50, metrics: () => [] }) + const none = await readDirector503Minutes( + { ...fakeReader(), fetch: unlogged.fetchImpl, runGcloud: unlogged.runGcloud }, + { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } + ) + assert.equal(none.ok, false) + assert.match(none.error, /runtime metrics read failed, truncated, or short/) +}) + test('no request series at all is unverifiable and trips at the limit', async () => { const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) try { diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index edab96a4277..72b749b79de 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -826,7 +826,7 @@ this order: |---|---|---| | Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact | | Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units | -| 503s | Director `request_count` 503s above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line | +| 503s | Director non-drain 503s (the roll ladder's measure: `request_count` 503s less the drain-return deferrals and the `host-rate-limited` and `host-in-flight` re-dials the director's `orca_relay_runtime_metrics` count) above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line | | Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat | | Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact | | Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell | @@ -837,7 +837,15 @@ newest minute any code has a point for is how far the series is published: a 503 before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable timestamp is unverifiable. This assumes the codes publish together; one code lagging the others -delays a trip by its lag but cannot hide one, because a later read sees its points. Until the +delays a trip by its lag but cannot hide one, because a later read sees its points. A host's own +early or duplicate re-dial says nothing about a cell and grows with traffic: on 2026-10-18 it was +~98% of 503s and tripped c7 back twice with no fault. The subtraction is only trusted where the +director logged: a judged minute no runtime-metrics line covers is unverifiable, a 10-minute read +with fewer lines than a single instance logs (20) or at its entry limit is unverifiable, and the +measured background leaves such minutes out (and is not measured with fewer than half of its 10 +metered). Coverage is any instance, not every one: an instance whose lines are missing while others +log under-subtracts its re-dials, which can only push toward a trip, never hide one. The same measure is the +background and `background-503-per-minute`. Until the minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which every placing director logs each minute, in the same query and window: with no summary from the