From ba4bbacd6b209e0d67730ac1497bbd504457397f Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sat, 5 Sep 2026 01:18:21 -0400 Subject: [PATCH] fix(relay-ops): align the cloud-data freshness bar with Cloud Monitoring publish lag (#18798) --- .../src/incident-live-preflight-cli.test.ts | 15 +- .../src/incident-live-preflight-cli.ts | 7 +- .../relay-ops/src/incident-monitor-cli.ts | 2 + .../src/incident-monitor-sources.test.ts | 4 +- .../relay-ops/src/incident-monitor-sources.ts | 2 +- .../relay-ops/src/incident-monitor.test.ts | 258 +++++++++++++++++- cloud/apps/relay-ops/src/incident-monitor.ts | 105 ++++++- cloud/docs/relay-incident-monitor.md | 29 +- 8 files changed, 388 insertions(+), 34 deletions(-) diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts index 3252ec7645e..412c2905408 100644 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts @@ -6,7 +6,10 @@ import { livePreflightGcloud, runIncidentLivePreflight } from './incident-live-preflight-cli.js' -import type { IncidentSample } from './incident-monitor.js' +import { + INCIDENT_MONITOR_THRESHOLDS, + type IncidentSample +} from './incident-monitor.js' import type { AdmissionSelector } from './incident-selector.js' const directories: string[] = [] @@ -313,7 +316,7 @@ describe('relay incident live preflight', () => { it('retries freshness-only failures when explicitly requested', async () => { const stale = sample() stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - 180_001).toISOString() + new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() const missing = sample() delete missing.sources['relay-logs'] const collect = vi.fn() @@ -334,7 +337,7 @@ describe('relay incident live preflight', () => { it('retries a first-wave stale sample and passes on the fresh one', async () => { const stale = sample() stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - 180_001).toISOString() + new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample()) const wait = vi.fn(async () => undefined) await expect(runIncidentLivePreflight( @@ -348,7 +351,7 @@ describe('relay incident live preflight', () => { it('stops retrying when the next wait would exceed the evidence-age bound', async () => { const completedAt = now - 290_000 const stale = sample() - stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString() + stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() const collect = vi.fn(async () => stale) const wait = vi.fn(async () => undefined) await expect(runIncidentLivePreflight( @@ -368,7 +371,7 @@ describe('relay incident live preflight', () => { const unhealthy = sample() unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9 unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt = - new Date(now - 180_001).toISOString() + new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() const collect = vi.fn(async () => unhealthy) const wait = vi.fn(async () => undefined) await expect(runIncidentLivePreflight( @@ -381,7 +384,7 @@ describe('relay incident live preflight', () => { it('fails closed after the bounded freshness retry window', async () => { const stale = sample() - stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString() + stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString() const collect = vi.fn(async () => stale) const wait = vi.fn(async () => undefined) await expect(runIncidentLivePreflight( diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts index fc2c3a99751..2fcce3ed85d 100644 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts @@ -7,6 +7,7 @@ import { suppliedIdentityToken } from './incident-monitor-cli.js' import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js' import { evaluateIncidentSample, + FRESHNESS_FAILURE_CODES, preDrainDryRunPassed, type IncidentSample } from './incident-monitor.js' @@ -18,12 +19,6 @@ const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave. const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 const WAVE_INDEX_PATTERN = /^[0-3]$/ -const FRESHNESS_FAILURE_CODES = new Set([ - 'signal_missing', - 'signal_stale', - 'source_missing', - 'source_stale' -]) export function livePreflightGcloud( gcloud: ReturnType, diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.ts index e090be7ea58..adfe6cad480 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-cli.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-cli.ts @@ -50,6 +50,8 @@ const StateSchema = z.object({ continuityEvents: z.array(z.object({ recordedAt: z.string(), windowSequence: z.number().int().nonnegative(), + // Pre-2026-09-05 state files predate tolerated freshness gaps. + tolerated: z.boolean().default(false), failures: z.array(z.object({ code: z.string(), source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']), diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts index 09b7b16fa45..74054b6c0ba 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts @@ -93,7 +93,7 @@ describe('incident monitor sources', () => { }) it('zero-fills an expired sparse lock-wait point', async () => { - let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs const fetchImpl: typeof fetch = async () => Response.json({ timeSeries: [{ points: [{ @@ -141,7 +141,7 @@ describe('incident monitor sources', () => { it('freshens a sparse zero without masking a recent nonzero lock wait', async () => { let value = 0 - const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs const readAt = now + 11_879 const fetchImpl: typeof fetch = async () => Response.json({ timeSeries: [{ diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts index 0b78c2c4f6b..a97bfe3df43 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts @@ -95,7 +95,7 @@ export const GOOGLE_METRICS: GoogleMetricDefinition[] = [ 'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"', aggregation: 'latest-max', emptyIsZero: true, - zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs }, { signal: 'cloud_sql.deadlocks', diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts index 4e1da9fab26..076cff3de3b 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { evaluateIncidentSample, INCIDENT_CHECKPOINT_MINUTES, + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES, INCIDENT_MONITOR_THRESHOLDS, INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS, initialIncidentMonitorState, @@ -182,12 +183,49 @@ describe('incident monitor evaluator', () => { code: 'source_missing', source: 'relay-logs' }) - const stale = healthySample(startedAt - 180_001) + const stale = healthySample( + startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 + ) const failures = evaluateIncidentSample(stale, startedAt).failures expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true) expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true) }) + // Why: production run 33944873727 at 2026-09-05T04:46:09Z read + // cloud_sql.lock_waits 189 286 ms old and restarted a 15-minute window on + // Google's publish lag. Cloud SQL documents 60 s sampling plus up to 165 s of + // invisibility, so that age is Google's clock, not our fleet. + it('reads a 189-second cloud signal as fresh and holds the other sources at 180 s', () => { + const lagged = healthySample() + lagged.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = + signal(0, startedAt - 189_286) + expect(evaluateIncidentSample(lagged, startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + const laggedDirector = healthySample() + laggedDirector.sources['director-admin']!.observedAt = + new Date(startedAt - 189_286).toISOString() + expect(evaluateIncidentSample(laggedDirector, startedAt).failures).toContainEqual( + expect.objectContaining({ code: 'source_stale', source: 'director-admin' }) + ) + }) + + it('still fails a cloud signal past the documented publish lag', () => { + const dark = healthySample() + dark.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal( + 0, + startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 + ) + expect(evaluateIncidentSample(dark, startedAt).failures).toContainEqual( + expect.objectContaining({ + code: 'signal_stale', + source: 'cloud-monitoring', + signal: 'cloud_sql.lock_waits' + }) + ) + }) + it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => { const sample = healthySample() sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81) @@ -592,7 +630,7 @@ describe('incident monitor lifecycle', () => { 'restarts a %i-minute continuous window after stale telemetry', async (durationMinutes) => { let now = startedAt - let staleInjected = false + let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 const checkpoints: Array<[number, number]> = [] const state = initialIncidentMonitorState({ incidentId: 'incident-1', @@ -612,9 +650,11 @@ describe('incident monitor lifecycle', () => { now += ms }, collect: async () => { - if (!staleInjected && now === startedAt + 5 * 60_000) { - staleInjected = true - return healthySample(now - 180_001) + if (staleSamples > 0 && now >= startedAt + 5 * 60_000) { + staleSamples-- + return healthySample( + now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 + ) } return healthySample(now) }, @@ -623,16 +663,20 @@ describe('incident monitor lifecycle', () => { checkpoints.push([summary.windowSequence, summary.checkpointMinute]) } }) + const restartMinute = 5 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 expect(result.windowSequence).toBe(1) expect(result.windowStartedAt).toBe( - new Date(startedAt + 6 * 60_000).toISOString() + new Date(startedAt + restartMinute * 60_000).toISOString() ) expect(result.completedAt).toBe( - new Date(startedAt + (durationMinutes + 6) * 60_000).toISOString() + new Date(startedAt + (durationMinutes + restartMinute) * 60_000).toISOString() ) expect(result.sampleCount).toBe(durationMinutes + 1) - expect(result.continuityEvents).toHaveLength(1) - expect(result.continuityEvents[0]!.failures).toEqual( + expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([ + ...Array(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true), + false + ]) + expect(result.continuityEvents.at(-1)!.failures).toEqual( expect.arrayContaining([ expect.objectContaining({ code: 'source_stale' }) ]) @@ -642,6 +686,188 @@ describe('incident monitor lifecycle', () => { } ) + // Why: run 33944873727 on 2026-09-05 restarted at 04:46:09Z on a single + // 189-second cloud reading and then blew the 25-minute lineage cap, so a + // green fleet produced no verdict at all. One unread sample now continues the + // window; the sample is still checked against every threshold it can read. + it('carries a 15-minute window through a single stale cloud sample', async () => { + let now = startedAt + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + const sample = healthySample(now) + if (now === startedAt + 10 * 60_000) { + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = + signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) + } + return sample + }, + persist: async () => {}, + checkpoint: async () => {} + }) + expect(result.windowSequence).toBe(0) + expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString()) + expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString()) + expect(result.sampleCount).toBe(16) + expect(result.frozenAt).toBeNull() + expect(result.continuityEvents).toEqual([{ + recordedAt: new Date(startedAt + 10 * 60_000).toISOString(), + windowSequence: 0, + tolerated: true, + failures: [expect.objectContaining({ + code: 'signal_stale', + source: 'cloud-monitoring', + signal: 'cloud_sql.lock_waits' + })] + }]) + expect(preDrainDryRunPassed(result)).toBe(true) + }) + + it('gives a signal a fresh budget only after it reads fresh again', async () => { + let now = startedAt + const staleMinutes = new Set([3, 5, 6, 9, 10]) + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + const sample = healthySample(now) + if (staleMinutes.has((now - startedAt) / 60_000)) { + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = + signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) + } + return sample + }, + persist: async () => {}, + checkpoint: async () => {} + }) + expect(result.windowSequence).toBe(0) + expect(result.continuityEvents).toHaveLength(staleMinutes.size) + expect(result.continuityEvents.every((event) => event.tolerated)).toBe(true) + expect(preDrainDryRunPassed(result)).toBe(true) + }) + + it('does not hand a resumed monitor a fresh tolerance budget', async () => { + let now = startedAt + 3 * 60_000 + const resumed = { + ...initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }), + windowStartedAt: new Date(startedAt).toISOString(), + lastSampleAt: new Date(startedAt + 2 * 60_000).toISOString(), + sampleCount: 3, + totalSampleCount: 3, + continuityEvents: Array.from( + { length: INCIDENT_FRESHNESS_TOLERANCE_SAMPLES }, + (_, index) => ({ + recordedAt: new Date(startedAt + (index + 1) * 60_000).toISOString(), + windowSequence: 0, + tolerated: true, + failures: [{ + code: 'signal_stale', + source: 'cloud-monitoring' as const, + signal: 'cloud_sql.lock_waits' + }] + }) + ) + } + const stop = new Error('stop after the resumed sample') + await expect(runIncidentMonitor(resumed, { + now: () => now, + wait: async () => { + throw stop + }, + collect: async () => { + const sample = healthySample(now) + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = + signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) + return sample + }, + persist: async (state) => { + expect(state.windowSequence).toBe(1) + expect(state.windowStartedAt).toBeNull() + expect(state.continuityEvents.at(-1)!.tolerated).toBe(false) + }, + checkpoint: async () => {} + })).rejects.toThrow(stop) + }) + + it('freezes on a threshold breach that arrives with a tolerated stale signal', async () => { + let now = startedAt + const state = initialIncidentMonitorState({ + incidentId: 'incident-1', + environment: 'production', + expectedSelector: selector, + preDrainDryRun: true, + migrationPolicy: 'strict', + recoverySourceCellId: null, + capacityCellId: null, + startedAt: new Date(startedAt).toISOString(), + durationMinutes: 15, + intervalMs: 60_000 + }) + const result = await runIncidentMonitor(state, { + now: () => now, + wait: async (ms) => { + now += ms + }, + collect: async () => { + const sample = healthySample(now) + if (now === startedAt + 2 * 60_000) { + sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = + signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1) + sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81, now) + } + return sample + }, + persist: async () => {}, + checkpoint: async () => {} + }) + expect(result.frozenAt).toBe(new Date(startedAt + 2 * 60_000).toISOString()) + expect(result.failures).toContainEqual(expect.objectContaining({ + code: 'threshold_max', + signal: 'cloud_sql.cpu' + })) + expect(preDrainDryRunPassed(result)).toBe(false) + }) + it('resets at the next fresh sample after a runner gap', async () => { let now = startedAt + 10 * 60_000 const state = { @@ -690,13 +916,21 @@ describe('incident monitor lifecycle', () => { durationMinutes: 15, intervalMs: 60_000 }) + let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1 const result = await runIncidentMonitor(state, { now: () => now, wait: async (ms) => { now += ms }, - collect: async () => - healthySample(now === startedAt + 10 * 60_000 ? now - 180_001 : now), + collect: async () => { + if (staleSamples > 0 && now >= startedAt + 10 * 60_000) { + staleSamples-- + return healthySample( + now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1 + ) + } + return healthySample(now) + }, persist: async () => {}, checkpoint: async () => {} }) @@ -706,7 +940,7 @@ describe('incident monitor lifecycle', () => { ) expect(result.frozenAt).not.toBeNull() expect(result.windowSequence).toBe(1) - expect(result.sampleCount).toBe(15) + expect(result.sampleCount).toBe(13) expect(result.failures).toContainEqual({ code: 'continuity_deadline_exceeded', source: 'active-probe', diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts index a121568d918..2785eb573af 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.ts @@ -6,7 +6,27 @@ import { export const INCIDENT_MONITOR_THRESHOLDS = { activeProbeMaxAgeMs: 60_000, - cloudDataMaxAgeMs: 180_000, + // Why: Cloud Monitoring publishes on Google's clock, not ours. Per the metric + // list read 2026-09-05, Cloud Run instance_count / cpu / memory / + // max_request_concurrencies / request_count are "Sampled every 60 seconds. + // After sampling, data is not visible for up to 120 seconds" (60+120=180 s), + // and Cloud SQL cpu / memory / num_backends / backends_in_wait / + // deadlock_count say "up to 165 seconds" (60+165=225 s). Window-sum signals + // age differently: observedAt is the newest point in the 5-minute query + // window, so a label series that stops emitting reads as 300 s old while its + // summed value is still complete. 330 s clears the worst of the three (the + // 300 s query window) plus ~30 s of collect-to-evaluate latency. The old + // 180 s bar restarted healthy 15-minute windows at 181 s, 189 s and 255 s on + // 2026-09-04/05, once burning the whole 25-minute lineage with no verdict. + cloudDataMaxAgeMs: 330_000, + // Why: the director admin API answers live on our own request, so hold its + // freshness bar where it sat while it shared cloudDataMaxAgeMs. + directorAdminMaxAgeMs: 180_000, + // Why: how long a nonzero backends-in-wait point is carried before it reads as + // zero. Held at the pre-2026-09-05 cloud bar: carrying it for the full + // cloudDataMaxAgeMs would hand the evaluator a point older than its own + // freshness bar as soon as collection latency is added. + cloudLockWaitCarryMs: 180_000, relayLogMaxAgeMs: 180_000, heartbeatMaxAgeMs: 45_000, endpointLatencyMs: 2_000, @@ -175,6 +195,7 @@ export type IncidentMonitorState = { continuityEvents: { recordedAt: string windowSequence: number + tolerated: boolean failures: IncidentFailure[] }[] frozenAt: string | null @@ -307,7 +328,7 @@ const SOURCE_MAX_AGE: Record = { 'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs, 'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs, 'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs, - 'director-admin': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 'director-admin': INCIDENT_MONITOR_THRESHOLDS.directorAdminMaxAgeMs } function ageMs(timestamp: string, nowMs: number): number { @@ -608,14 +629,59 @@ function checkpointMinutes(durationMinutes: number): number[] { return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes) } -const CONTINUITY_FAILURE_CODES = new Set([ - 'collector_failed', - 'monitor_gap', +// Freshness-only failures: we could not read a signal this sample. Distinct from +// collector_failed / monitor_gap, where the whole sample is absent. +export const FRESHNESS_FAILURE_CODES = new Set([ + 'signal_missing', 'signal_stale', 'source_missing', 'source_stale' ]) +const CONTINUITY_FAILURE_CODES = new Set([ + 'collector_failed', + 'monitor_gap', + ...FRESHNESS_FAILURE_CODES +]) + +// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is +// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart +// past minute 10 costs the entire verdict, so a healthy fleet produced none on +// 2026-09-05. A signal may miss this many consecutive samples before the window +// restarts; the sample is still evaluated against every threshold it can read, +// and a threshold breach still freezes the run outright. +export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2 + +function freshnessKey(failure: IncidentFailure): string { + return `${failure.source}/${failure.signal ?? '*'}` +} + +// Rebuild the per-signal tolerated streak from the trailing continuity events so a +// resumed monitor cannot hand a signal a fresh budget. +function resumeFreshnessStreaks( + state: IncidentMonitorState +): Map { + const events = state.continuityEvents + const streaks = new Map() + const last = events[events.length - 1] + if (!last?.tolerated) return streaks + for (const key of new Set(last.failures.map(freshnessKey))) { + let streak = 0 + let laterAt: number | null = null + for (let index = events.length - 1; index >= 0; index--) { + const event = events[index]! + const recordedAt = Date.parse(event.recordedAt) + if (!event.tolerated) break + if (laterAt !== null && laterAt - recordedAt > state.intervalMs * 1.5) break + if (!event.failures.some((failure) => freshnessKey(failure) === key)) break + streak++ + laterAt = recordedAt + } + streaks.set(key, streak) + } + return streaks +} + function resetContinuousWindow( state: IncidentMonitorState, recordedAt: string, @@ -631,6 +697,7 @@ function resetContinuousWindow( state.continuityEvents.push({ recordedAt, windowSequence: state.windowSequence, + tolerated: false, failures }) } @@ -681,6 +748,7 @@ export async function runIncidentMonitor( await dependencies.persist(state) return state } + const freshnessStreaks = resumeFreshnessStreaks(state) while (state.completedAt === null) { if (dependencies.now() > lineageDeadlineMs) { completeContinuityDeadline(state, dependencies.now(), lineageStartMs) @@ -715,9 +783,34 @@ export async function runIncidentMonitor( const thresholdFailures = evaluation.failures.filter((failure) => !CONTINUITY_FAILURE_CODES.has(failure.code) ) - if (continuityFailures.length > 0) { + const toleratedKeys = new Set( + state.windowStartedAt !== null && + continuityFailures.length > 0 && + continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code)) + ? continuityFailures.map(freshnessKey) + : [] + ) + for (const key of [...freshnessStreaks.keys()]) { + if (!toleratedKeys.has(key)) freshnessStreaks.delete(key) + } + let tolerated = toleratedKeys.size > 0 + for (const key of toleratedKeys) { + const streak = (freshnessStreaks.get(key) ?? 0) + 1 + freshnessStreaks.set(key, streak) + if (streak > INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) tolerated = false + } + if (continuityFailures.length > 0 && !tolerated) { + freshnessStreaks.clear() resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures) } else { + if (tolerated) { + state.continuityEvents.push({ + recordedAt: evaluation.evaluatedAt, + windowSequence: state.windowSequence, + tolerated: true, + failures: continuityFailures + }) + } if (state.windowStartedAt === null) { state.windowStartedAt = evaluation.evaluatedAt } diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md index 870c95dd413..8a8dfda1495 100644 --- a/cloud/docs/relay-incident-monitor.md +++ b/cloud/docs/relay-incident-monitor.md @@ -73,6 +73,13 @@ for a committed forward-recovery gate. Durable files default to gap resets the active window at the next fresh sample and preserves the prior window evidence. A threshold freeze never clears automatically. +A signal that reads missing or stale may miss up to two consecutive samples +without restarting the window. The sample still counts and is still checked +against every threshold it can read, and each tolerated gap is recorded in +`continuityEvents` with `tolerated: true`. A third consecutive miss of the same +signal, a failed collector, a runner gap, or any threshold breach restarts or +freezes as before. + A production candidate or multi-target mutation must download the exact dry-run artifact by workflow run ID and attempt. It verifies the artifact hashes and provenance, requires a green completed 15-minute state no older @@ -89,7 +96,8 @@ durably marked consumed before mutation and cannot authorize another run. | Signal | Freeze condition | | --- | ---: | | Active probe age | over 60 seconds | -| Cloud/log data age | over 180 seconds | +| Cloud Monitoring data age | over 330 seconds | +| Relay log and director admin data age | over 180 seconds | | Cell heartbeat age | over 45 seconds | | Endpoint latency | over 2,000 ms | | Cloud SQL CPU | over 80% | @@ -155,6 +163,25 @@ heartbeats, and matching live admission. separate it from today's baseline; the exhausted-retry bar (incident peak 467 vs bar 300), director concurrency, and the pool bars carry that role. Re-tighten after the fleet is on the 500 ms lock wait. +- Raised the Cloud Monitoring freshness bar from 180 s to 330 s and let a + freshness-only failure miss up to two consecutive samples without restarting + the window (2026-09-05). Basis: Google's metric list documents Cloud Run + `request_count`, `container/instance_count`, `container/cpu/utilizations`, + `container/memory/utilizations` and `container/max_request_concurrencies` as + "Sampled every 60 seconds. After sampling, data is not visible for up to 120 + seconds", and Cloud SQL `database/cpu/utilization`, + `database/memory/utilization`, `database/postgresql/num_backends`, + `database/postgresql/backends_in_wait` and `database/postgresql/deadlock_count` + as "up to 165 seconds", so the newest visible point is up to 180 s and 225 s + old respectively. Window-sum signals age further: `observedAt` is the newest + point in the 5-minute query window, so a label series that stops emitting + reads as 300 s old while its summed value is complete. The old bar sat under + all three. Production on 2026-09-04/05 restarted healthy 15-minute windows at + 181 s and 255 s (`auth.errors`, run 33928912676) and at 189 s + (`cloud_sql.lock_waits`, run 33944873727), and the last of those then blew the + 25-minute lineage cap at 1 500 004 ms, so a green fleet produced no verdict. + The director admin bar stays at 180 s and the nonzero lock-wait carry window + stays at 180 s; both publish on our own cadence. - Recalibrated the exhausted-PostgreSQL-retry freeze from 0 to 300 per five minutes (2026-09-04). Basis: #18521 cut the request-path cell-inventory lock wait from the 1 s pool `lock_timeout` to 500 ms, so contended waiters