diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts index 9cfae74910b..1a57a50f213 100644 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts @@ -154,9 +154,9 @@ describe('relay incident live preflight', () => { )).resolves.toBeUndefined() }) - it('rejects monitor evidence beyond the 25-minute lineage bound', async () => { + it('rejects monitor evidence beyond the 35-minute lineage bound', async () => { const path = stateFile('strict', { - startedAt: new Date(now - 26 * 60_000 - 1).toISOString() + startedAt: new Date(now - 36 * 60_000 - 1).toISOString() }) await expect(runIncidentLivePreflight( ['--state-file', path], diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.ts index b4991a3b8b3..0f24459c9fd 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-cli.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-cli.ts @@ -477,6 +477,7 @@ export async function runIncidentMonitorCli( await wait(ms) }, collect: segmentedCollect, + warn: (message) => console.warn(message), persist: async (nextState) => await persistState(options.stateFile, nextState), checkpoint: async (checkpoint) => { await appendCheckpoint(options.summaryFile, checkpoint) diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts index f8feb17b78e..7ba81cb4f28 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -813,6 +813,150 @@ describe('incident monitor lifecycle', () => { expect(preDrainDryRunPassed(result)).toBe(true) }) + // Why: dry-run 35258662628 read a healthy fleet clean for 13 minutes, then one + // unreadable Cloud Monitoring sample restarted the window and the restart blew + // the lineage cap, so a green fleet produced no verdict. + it('carries a 15-minute window through a single collector failure', async () => { + let now = startedAt + const warnings: string[] = [] + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + if (now === startedAt + 10 * 60_000) { + throw new Error('cloud monitoring read failed') + } + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async () => {}, + warn: (message) => { + warnings.push(message) + } + }) + expect(result.windowSequence).toBe(0) + expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString()) + expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString()) + expect(result.sampleCount).toBe(16) + expect(result.frozenAt).toBeNull() + expect(result.failures).toEqual([]) + expect(result.continuityEvents).toEqual([{ + recordedAt: new Date(startedAt + 10 * 60_000).toISOString(), + windowSequence: 0, + tolerated: true, + failures: [{ code: 'collector_failed', source: 'cloud-monitoring' }] + }]) + expect(warnings).toEqual([ + 'incident monitor collector failed: cloud monitoring read failed' + ]) + expect(preDrainDryRunPassed(result)).toBe(true) + }) + + it('restarts the window after three consecutive collector failures', async () => { + let now = startedAt + let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + if (failures > 0 && now >= startedAt + 10 * 60_000) { + failures-- + throw new Error('cloud monitoring read failed') + } + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async () => {}, + warn: () => {} + }) + const restartMinute = 10 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 + expect(result.windowSequence).toBe(1) + expect(result.windowStartedAt).toBe( + new Date(startedAt + restartMinute * 60_000).toISOString() + ) + expect(result.completedAt).toBe( + new Date(startedAt + (restartMinute + 15) * 60_000).toISOString() + ) + expect(result.sampleCount).toBe(16) + expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([ + ...Array(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true), + false + ]) + expect(result.continuityEvents.at(-1)!.failures).toEqual([ + { code: 'collector_failed', source: 'cloud-monitoring' } + ]) + expect(result.frozenAt).toBeNull() + expect(preDrainDryRunPassed(result)).toBe(true) + }) + + it('still reaches a dry-run verdict after a restart on the last sample', async () => { + let now = startedAt + let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + if (failures > 0 && now >= startedAt + 13 * 60_000) { + failures-- + throw new Error('cloud monitoring read failed') + } + return healthySample(now) + }, + persist: async () => {}, + checkpoint: async () => {}, + warn: () => {} + }) + expect(result.windowSequence).toBe(1) + expect(result.windowStartedAt).toBe(new Date(startedAt + 16 * 60_000).toISOString()) + expect(result.completedAt).toBe(new Date(startedAt + 31 * 60_000).toISOString()) + expect(result.sampleCount).toBe(16) + expect(result.frozenAt).toBeNull() + expect(result.failures).toEqual([]) + expect(preDrainDryRunPassed(result)).toBe(true) + }) + it('gives a signal a fresh budget only after it reads fresh again', async () => { let now = startedAt const staleMinutes = new Set([3, 5, 6, 9, 10]) @@ -977,7 +1121,7 @@ describe('incident monitor lifecycle', () => { expect(result.sampleCount).toBe(16) }) - it('fails a dry run after 25 total minutes of continuity resets', async () => { + it('fails a dry run after 35 total minutes of continuity resets', async () => { let now = startedAt const state = initialIncidentMonitorState({ incidentId: 'incident-1', @@ -991,15 +1135,16 @@ describe('incident monitor lifecycle', () => { durationMinutes: 15, intervalMs: 60_000 }) - let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 + // Two restarts: the first on the window's last sample, the second far enough + // into the replacement window that no third window can finish in the lineage. + const staleMinutes = new Set([13, 14, 15, 24, 25, 26]) const result = await runIncidentMonitor(state, { now: () => now, wait: async (ms) => { now += ms }, collect: async () => { - if (staleSamples > 0 && now >= startedAt + 10 * 60_000) { - staleSamples-- + if (staleMinutes.has((now - startedAt) / 60_000)) { return healthySample( now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 ) @@ -1014,8 +1159,8 @@ describe('incident monitor lifecycle', () => { new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString() ) expect(result.frozenAt).not.toBeNull() - expect(result.windowSequence).toBe(1) - expect(result.sampleCount).toBe(13) + expect(result.windowSequence).toBe(2) + expect(result.sampleCount).toBe(9) expect(result.failures).toContainEqual({ code: 'continuity_deadline_exceeded', source: 'active-probe', diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts index e21a1126e51..875fa7e81b4 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.ts @@ -143,7 +143,11 @@ export const INCIDENT_MONITOR_THRESHOLDS = { } as const export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const -export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000 +// Why: 35 minutes, raised 2026-09-17 from 25. A 15-minute window plus one +// restart must fit: a continuity reset on the window's last sample restarts at +// minute 16 and finishes at 31. Under 25 a reset past minute 9 cost the whole +// verdict, which is what run 35258662628 hit on a healthy fleet. +export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 35 * 60_000 export type IncidentSourceName = | 'active-probe' @@ -678,6 +682,7 @@ export type IncidentMonitorDependencies = { collect(): Promise persist(state: IncidentMonitorState): Promise checkpoint(summary: IncidentCheckpoint): Promise + warn?(message: string): void } function checkpointMinutes(durationMinutes: number): number[] { @@ -699,10 +704,19 @@ const CONTINUITY_FAILURE_CODES = new Set([ ...FRESHNESS_FAILURE_CODES ]) +// A whole sample we could not read gets the same consecutive-sample budget as an +// unread signal, for the same reason: one failed collector round trip is evidence +// about that round trip, not about the fleet. `monitor_gap` is excluded because it +// means the run itself stopped sampling, so the window genuinely has a hole. +const TOLERABLE_CONTINUITY_FAILURE_CODES = new Set([ + 'collector_failed', + ...FRESHNESS_FAILURE_CODES +]) + // Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is -// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart -// past minute 10 costs the entire verdict, so a healthy fleet produced none on -// 2026-09-05. A signal may miss this many consecutive samples before the window +// not evidence of an unhealthy fleet. Under the 25-minute lineage cap in force +// then, a restart past minute 10 cost the entire verdict, so a healthy fleet +// produced none on 2026-09-05. A signal may miss this many consecutive samples before the window // restarts; the sample is still evaluated against every threshold it can read, // and a threshold breach still freezes the run outright. export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2 @@ -838,7 +852,12 @@ export async function runIncidentMonitor( state.recoverySourceCellId, state.capacityCellId ) - } catch { + } catch (error) { + dependencies.warn?.( + `incident monitor collector failed: ${ + error instanceof Error ? error.message : 'unknown error' + }` + ) evaluation = { status: 'freeze', evaluatedAt: new Date(dependencies.now()).toISOString(), @@ -859,7 +878,8 @@ export async function runIncidentMonitor( const toleratedKeys = new Set( state.windowStartedAt !== null && continuityFailures.length > 0 && - continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code)) + continuityFailures.every((failure) => + TOLERABLE_CONTINUITY_FAILURE_CODES.has(failure.code)) ? continuityFailures.map(freshnessKey) : [] ) diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md index 92347ad3c1d..ee499c3e95b 100644 --- a/cloud/docs/relay-incident-monitor.md +++ b/cloud/docs/relay-incident-monitor.md @@ -39,9 +39,11 @@ days. No tokens, request bodies, logs, user IDs, host IDs, or relay device IDs a Reruns keep one stable incident ID, restore the immediately preceding private artifact, verify its commit/run/attempt provenance and content hashes, and pass `--restart`. A missing or mismatched artifact fails closed. A missing, stale, -or collector-failed sample is durably recorded and resets the active continuous -window. The next fresh sample starts a new 15- or 90-minute window under the -same incident lineage. +or collector-failed sample is durably recorded. Up to two consecutive such +samples per source are tolerated and the window keeps running; a third resets +the active continuous window, and the next fresh sample starts a new 15- or +90-minute window under the same incident lineage. A pre-drain dry run must reach +a verdict within 35 minutes of its lineage start. Exit code `2` means the gate froze or a dry run failed. Missing, stale, malformed, unauthorized, or unavailable telemetry fails closed. @@ -206,6 +208,18 @@ without its segment is a compile error in relay-contract, not a silent gap. ## Implementation log +- Gave `collector_failed` the same two-consecutive-sample tolerance as an unread + signal and raised the pre-drain lineage cap from 25 to 35 minutes + (2026-09-17). Basis: dry-run 35258662628 sampled a healthy fleet clean for + 13 minutes, then a single unreadable Cloud Monitoring sample restarted the + window, and the restarted window ran past the 25-minute cap at 1 500 002 ms, + so a healthy fleet produced no verdict. One failed collector round trip is + evidence about that round trip, not about the fleet, and it cannot freeze the + gate on its own because it carries no threshold breach. `monitor_gap` keeps + zero tolerance: it means the run stopped sampling, so the window has a real + hole. 35 minutes fits a 15-minute window plus one restart: a reset on the + window's last sample restarts at minute 16 and finishes at 31. The job + timeout is already 100 minutes. - Recalibrated the Cloud SQL backends freeze from 250 to 320, the unexpected director 5xx freeze from 3 to 15, and gave per-cell endpoint probes a two-consecutive-sample tolerance (2026-09-17). Basis: the pre-roll dry-run had