fix(relay-ops): align the cloud-data freshness bar with Cloud Monitoring publish lag (#18798)

cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts

@@ -6,7 +6,10 @@ import {
   livePreflightGcloud,
   runIncidentLivePreflight
 } from './incident-live-preflight-cli.js'
-import type { IncidentSample } from './incident-monitor.js'
+import {
+  INCIDENT_MONITOR_THRESHOLDS,
+  type IncidentSample
+} from './incident-monitor.js'
 import type { AdmissionSelector } from './incident-selector.js'
 
 const directories: string[] = []
@@ -313,7 +316,7 @@ describe('relay incident live preflight', () => {
   it('retries freshness-only failures when explicitly requested', async () => {
     const stale = sample()
     stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
-      new Date(now - 180_001).toISOString()
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const missing = sample()
     delete missing.sources['relay-logs']
     const collect = vi.fn()
@@ -334,7 +337,7 @@ describe('relay incident live preflight', () => {
   it('retries a first-wave stale sample and passes on the fresh one', async () => {
     const stale = sample()
     stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
-      new Date(now - 180_001).toISOString()
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample())
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(
@@ -348,7 +351,7 @@ describe('relay incident live preflight', () => {
   it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
     const completedAt = now - 290_000
     const stale = sample()
-    stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString()
+    stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn(async () => stale)
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(
@@ -368,7 +371,7 @@ describe('relay incident live preflight', () => {
     const unhealthy = sample()
     unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
     unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
-      new Date(now - 180_001).toISOString()
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn(async () => unhealthy)
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(
@@ -381,7 +384,7 @@ describe('relay incident live preflight', () => {
 
   it('fails closed after the bounded freshness retry window', async () => {
     const stale = sample()
-    stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString()
+    stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn(async () => stale)
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(

cloud/apps/relay-ops/src/incident-live-preflight-cli.ts

@@ -7,6 +7,7 @@ import { suppliedIdentityToken } from './incident-monitor-cli.js'
 import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
 import {
   evaluateIncidentSample,
+  FRESHNESS_FAILURE_CODES,
   preDrainDryRunPassed,
   type IncidentSample
 } from './incident-monitor.js'
@@ -18,12 +19,6 @@ const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
 const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
 const WAVE_INDEX_PATTERN = /^[0-3]$/
-const FRESHNESS_FAILURE_CODES = new Set([
-  'signal_missing',
-  'signal_stale',
-  'source_missing',
-  'source_stale'
-])
 
 export function livePreflightGcloud(
   gcloud: ReturnType<typeof createGcloudClient>,

cloud/apps/relay-ops/src/incident-monitor-cli.ts

@@ -50,6 +50,8 @@ const StateSchema = z.object({
   continuityEvents: z.array(z.object({
     recordedAt: z.string(),
     windowSequence: z.number().int().nonnegative(),
+    // Pre-2026-09-05 state files predate tolerated freshness gaps.
+    tolerated: z.boolean().default(false),
     failures: z.array(z.object({
       code: z.string(),
       source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),

cloud/apps/relay-ops/src/incident-monitor-sources.test.ts

@@ -93,7 +93,7 @@ describe('incident monitor sources', () => {
   })
 
   it('zero-fills an expired sparse lock-wait point', async () => {
-    let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
     const fetchImpl: typeof fetch = async () => Response.json({
       timeSeries: [{
         points: [{
@@ -141,7 +141,7 @@ describe('incident monitor sources', () => {
 
   it('freshens a sparse zero without masking a recent nonzero lock wait', async () => {
     let value = 0
-    const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
     const readAt = now + 11_879
     const fetchImpl: typeof fetch = async () => Response.json({
       timeSeries: [{

cloud/apps/relay-ops/src/incident-monitor-sources.ts

@@ -95,7 +95,7 @@ export const GOOGLE_METRICS: GoogleMetricDefinition[] = [
       'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"',
     aggregation: 'latest-max',
     emptyIsZero: true,
-    zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
   },
   {
     signal: 'cloud_sql.deadlocks',

cloud/apps/relay-ops/src/incident-monitor.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'
 import {
   evaluateIncidentSample,
   INCIDENT_CHECKPOINT_MINUTES,
+  INCIDENT_FRESHNESS_TOLERANCE_SAMPLES,
   INCIDENT_MONITOR_THRESHOLDS,
   INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS,
   initialIncidentMonitorState,
@@ -182,12 +183,49 @@ describe('incident monitor evaluator', () => {
       code: 'source_missing',
       source: 'relay-logs'
     })
-    const stale = healthySample(startedAt - 180_001)
+    const stale = healthySample(
+      startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+    )
     const failures = evaluateIncidentSample(stale, startedAt).failures
     expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true)
     expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true)
   })
 
+  // Why: production run 33944873727 at 2026-09-05T04:46:09Z read
+  // cloud_sql.lock_waits 189 286 ms old and restarted a 15-minute window on
+  // Google's publish lag. Cloud SQL documents 60 s sampling plus up to 165 s of
+  // invisibility, so that age is Google's clock, not our fleet.
+  it('reads a 189-second cloud signal as fresh and holds the other sources at 180 s', () => {
+    const lagged = healthySample()
+    lagged.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+      signal(0, startedAt - 189_286)
+    expect(evaluateIncidentSample(lagged, startedAt)).toMatchObject({
+      status: 'green',
+      failures: []
+    })
+    const laggedDirector = healthySample()
+    laggedDirector.sources['director-admin']!.observedAt =
+      new Date(startedAt - 189_286).toISOString()
+    expect(evaluateIncidentSample(laggedDirector, startedAt).failures).toContainEqual(
+      expect.objectContaining({ code: 'source_stale', source: 'director-admin' })
+    )
+  })
+
+  it('still fails a cloud signal past the documented publish lag', () => {
+    const dark = healthySample()
+    dark.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(
+      0,
+      startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+    )
+    expect(evaluateIncidentSample(dark, startedAt).failures).toContainEqual(
+      expect.objectContaining({
+        code: 'signal_stale',
+        source: 'cloud-monitoring',
+        signal: 'cloud_sql.lock_waits'
+      })
+    )
+  })
+
   it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
     const sample = healthySample()
     sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
@@ -592,7 +630,7 @@ describe('incident monitor lifecycle', () => {
     'restarts a %i-minute continuous window after stale telemetry',
     async (durationMinutes) => {
       let now = startedAt
-      let staleInjected = false
+      let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
       const checkpoints: Array<[number, number]> = []
       const state = initialIncidentMonitorState({
         incidentId: 'incident-1',
@@ -612,9 +650,11 @@ describe('incident monitor lifecycle', () => {
           now += ms
         },
         collect: async () => {
-          if (!staleInjected && now === startedAt + 5 * 60_000) {
-            staleInjected = true
-            return healthySample(now - 180_001)
+          if (staleSamples > 0 && now >= startedAt + 5 * 60_000) {
+            staleSamples--
+            return healthySample(
+              now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+            )
           }
           return healthySample(now)
         },
@@ -623,16 +663,20 @@ describe('incident monitor lifecycle', () => {
           checkpoints.push([summary.windowSequence, summary.checkpointMinute])
         }
       })
+      const restartMinute = 5 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
       expect(result.windowSequence).toBe(1)
       expect(result.windowStartedAt).toBe(
-        new Date(startedAt + 6 * 60_000).toISOString()
+        new Date(startedAt + restartMinute * 60_000).toISOString()
       )
       expect(result.completedAt).toBe(
-        new Date(startedAt + (durationMinutes + 6) * 60_000).toISOString()
+        new Date(startedAt + (durationMinutes + restartMinute) * 60_000).toISOString()
       )
       expect(result.sampleCount).toBe(durationMinutes + 1)
-      expect(result.continuityEvents).toHaveLength(1)
-      expect(result.continuityEvents[0]!.failures).toEqual(
+      expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
+        ...Array<boolean>(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
+        false
+      ])
+      expect(result.continuityEvents.at(-1)!.failures).toEqual(
         expect.arrayContaining([
           expect.objectContaining({ code: 'source_stale' })
         ])
@@ -642,6 +686,188 @@ describe('incident monitor lifecycle', () => {
     }
   )
 
+  // Why: run 33944873727 on 2026-09-05 restarted at 04:46:09Z on a single
+  // 189-second cloud reading and then blew the 25-minute lineage cap, so a
+  // green fleet produced no verdict at all. One unread sample now continues the
+  // window; the sample is still checked against every threshold it can read.
+  it('carries a 15-minute window through a single stale cloud sample', async () => {
+    let now = startedAt
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (now === startedAt + 10 * 60_000) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.windowSequence).toBe(0)
+    expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
+    expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
+    expect(result.sampleCount).toBe(16)
+    expect(result.frozenAt).toBeNull()
+    expect(result.continuityEvents).toEqual([{
+      recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
+      windowSequence: 0,
+      tolerated: true,
+      failures: [expect.objectContaining({
+        code: 'signal_stale',
+        source: 'cloud-monitoring',
+        signal: 'cloud_sql.lock_waits'
+      })]
+    }])
+    expect(preDrainDryRunPassed(result)).toBe(true)
+  })
+
+  it('gives a signal a fresh budget only after it reads fresh again', async () => {
+    let now = startedAt
+    const staleMinutes = new Set([3, 5, 6, 9, 10])
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (staleMinutes.has((now - startedAt) / 60_000)) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.windowSequence).toBe(0)
+    expect(result.continuityEvents).toHaveLength(staleMinutes.size)
+    expect(result.continuityEvents.every((event) => event.tolerated)).toBe(true)
+    expect(preDrainDryRunPassed(result)).toBe(true)
+  })
+
+  it('does not hand a resumed monitor a fresh tolerance budget', async () => {
+    let now = startedAt + 3 * 60_000
+    const resumed = {
+      ...initialIncidentMonitorState({
+        incidentId: 'incident-1',
+        environment: 'production',
+        expectedSelector: selector,
+        preDrainDryRun: true,
+        migrationPolicy: 'strict',
+        recoverySourceCellId: null,
+        capacityCellId: null,
+        startedAt: new Date(startedAt).toISOString(),
+        durationMinutes: 15,
+        intervalMs: 60_000
+      }),
+      windowStartedAt: new Date(startedAt).toISOString(),
+      lastSampleAt: new Date(startedAt + 2 * 60_000).toISOString(),
+      sampleCount: 3,
+      totalSampleCount: 3,
+      continuityEvents: Array.from(
+        { length: INCIDENT_FRESHNESS_TOLERANCE_SAMPLES },
+        (_, index) => ({
+          recordedAt: new Date(startedAt + (index + 1) * 60_000).toISOString(),
+          windowSequence: 0,
+          tolerated: true,
+          failures: [{
+            code: 'signal_stale',
+            source: 'cloud-monitoring' as const,
+            signal: 'cloud_sql.lock_waits'
+          }]
+        })
+      )
+    }
+    const stop = new Error('stop after the resumed sample')
+    await expect(runIncidentMonitor(resumed, {
+      now: () => now,
+      wait: async () => {
+        throw stop
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+          signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        return sample
+      },
+      persist: async (state) => {
+        expect(state.windowSequence).toBe(1)
+        expect(state.windowStartedAt).toBeNull()
+        expect(state.continuityEvents.at(-1)!.tolerated).toBe(false)
+      },
+      checkpoint: async () => {}
+    })).rejects.toThrow(stop)
+  })
+
+  it('freezes on a threshold breach that arrives with a tolerated stale signal', async () => {
+    let now = startedAt
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (now === startedAt + 2 * 60_000) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81, now)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.frozenAt).toBe(new Date(startedAt + 2 * 60_000).toISOString())
+    expect(result.failures).toContainEqual(expect.objectContaining({
+      code: 'threshold_max',
+      signal: 'cloud_sql.cpu'
+    }))
+    expect(preDrainDryRunPassed(result)).toBe(false)
+  })
+
   it('resets at the next fresh sample after a runner gap', async () => {
     let now = startedAt + 10 * 60_000
     const state = {
@@ -690,13 +916,21 @@ describe('incident monitor lifecycle', () => {
       durationMinutes: 15,
       intervalMs: 60_000
     })
+    let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
     const result = await runIncidentMonitor(state, {
       now: () => now,
       wait: async (ms) => {
         now += ms
       },
-      collect: async () =>
-        healthySample(now === startedAt + 10 * 60_000 ? now - 180_001 : now),
+      collect: async () => {
+        if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
+          staleSamples--
+          return healthySample(
+            now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+          )
+        }
+        return healthySample(now)
+      },
       persist: async () => {},
       checkpoint: async () => {}
     })
@@ -706,7 +940,7 @@ describe('incident monitor lifecycle', () => {
     )
     expect(result.frozenAt).not.toBeNull()
     expect(result.windowSequence).toBe(1)
-    expect(result.sampleCount).toBe(15)
+    expect(result.sampleCount).toBe(13)
     expect(result.failures).toContainEqual({
       code: 'continuity_deadline_exceeded',
       source: 'active-probe',

cloud/apps/relay-ops/src/incident-monitor.ts

@@ -6,7 +6,27 @@ import {
 
 export const INCIDENT_MONITOR_THRESHOLDS = {
   activeProbeMaxAgeMs: 60_000,
-  cloudDataMaxAgeMs: 180_000,
+  // Why: Cloud Monitoring publishes on Google's clock, not ours. Per the metric
+  // list read 2026-09-05, Cloud Run instance_count / cpu / memory /
+  // max_request_concurrencies / request_count are "Sampled every 60 seconds.
+  // After sampling, data is not visible for up to 120 seconds" (60+120=180 s),
+  // and Cloud SQL cpu / memory / num_backends / backends_in_wait /
+  // deadlock_count say "up to 165 seconds" (60+165=225 s). Window-sum signals
+  // age differently: observedAt is the newest point in the 5-minute query
+  // window, so a label series that stops emitting reads as 300 s old while its
+  // summed value is still complete. 330 s clears the worst of the three (the
+  // 300 s query window) plus ~30 s of collect-to-evaluate latency. The old
+  // 180 s bar restarted healthy 15-minute windows at 181 s, 189 s and 255 s on
+  // 2026-09-04/05, once burning the whole 25-minute lineage with no verdict.
+  cloudDataMaxAgeMs: 330_000,
+  // Why: the director admin API answers live on our own request, so hold its
+  // freshness bar where it sat while it shared cloudDataMaxAgeMs.
+  directorAdminMaxAgeMs: 180_000,
+  // Why: how long a nonzero backends-in-wait point is carried before it reads as
+  // zero. Held at the pre-2026-09-05 cloud bar: carrying it for the full
+  // cloudDataMaxAgeMs would hand the evaluator a point older than its own
+  // freshness bar as soon as collection latency is added.
+  cloudLockWaitCarryMs: 180_000,
   relayLogMaxAgeMs: 180_000,
   heartbeatMaxAgeMs: 45_000,
   endpointLatencyMs: 2_000,
@@ -175,6 +195,7 @@ export type IncidentMonitorState = {
   continuityEvents: {
     recordedAt: string
     windowSequence: number
+    tolerated: boolean
     failures: IncidentFailure[]
   }[]
   frozenAt: string | null
@@ -307,7 +328,7 @@ const SOURCE_MAX_AGE: Record<IncidentSourceName, number> = {
   'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs,
   'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs,
   'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs,
-  'director-admin': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+  'director-admin': INCIDENT_MONITOR_THRESHOLDS.directorAdminMaxAgeMs
 }
 
 function ageMs(timestamp: string, nowMs: number): number {
@@ -608,14 +629,59 @@ function checkpointMinutes(durationMinutes: number): number[] {
   return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes)
 }
 
-const CONTINUITY_FAILURE_CODES = new Set([
-  'collector_failed',
-  'monitor_gap',
+// Freshness-only failures: we could not read a signal this sample. Distinct from
+// collector_failed / monitor_gap, where the whole sample is absent.
+export const FRESHNESS_FAILURE_CODES = new Set([
+  'signal_missing',
   'signal_stale',
   'source_missing',
   'source_stale'
 ])
 
+const CONTINUITY_FAILURE_CODES = new Set([
+  'collector_failed',
+  'monitor_gap',
+  ...FRESHNESS_FAILURE_CODES
+])
+
+// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
+// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
+// past minute 10 costs the entire verdict, so a healthy fleet produced none on
+// 2026-09-05. A signal may miss this many consecutive samples before the window
+// restarts; the sample is still evaluated against every threshold it can read,
+// and a threshold breach still freezes the run outright.
+export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
+
+function freshnessKey(failure: IncidentFailure): string {
+  return `${failure.source}/${failure.signal ?? '*'}`
+}
+
+// Rebuild the per-signal tolerated streak from the trailing continuity events so a
+// resumed monitor cannot hand a signal a fresh budget.
+function resumeFreshnessStreaks(
+  state: IncidentMonitorState
+): Map<string, number> {
+  const events = state.continuityEvents
+  const streaks = new Map<string, number>()
+  const last = events[events.length - 1]
+  if (!last?.tolerated) return streaks
+  for (const key of new Set(last.failures.map(freshnessKey))) {
+    let streak = 0
+    let laterAt: number | null = null
+    for (let index = events.length - 1; index >= 0; index--) {
+      const event = events[index]!
+      const recordedAt = Date.parse(event.recordedAt)
+      if (!event.tolerated) break
+      if (laterAt !== null && laterAt - recordedAt > state.intervalMs * 1.5) break
+      if (!event.failures.some((failure) => freshnessKey(failure) === key)) break
+      streak++
+      laterAt = recordedAt
+    }
+    streaks.set(key, streak)
+  }
+  return streaks
+}
+
 function resetContinuousWindow(
   state: IncidentMonitorState,
   recordedAt: string,
@@ -631,6 +697,7 @@ function resetContinuousWindow(
   state.continuityEvents.push({
     recordedAt,
     windowSequence: state.windowSequence,
+    tolerated: false,
     failures
   })
 }
@@ -681,6 +748,7 @@ export async function runIncidentMonitor(
     await dependencies.persist(state)
     return state
   }
+  const freshnessStreaks = resumeFreshnessStreaks(state)
   while (state.completedAt === null) {
     if (dependencies.now() > lineageDeadlineMs) {
       completeContinuityDeadline(state, dependencies.now(), lineageStartMs)
@@ -715,9 +783,34 @@ export async function runIncidentMonitor(
     const thresholdFailures = evaluation.failures.filter((failure) =>
       !CONTINUITY_FAILURE_CODES.has(failure.code)
     )
-    if (continuityFailures.length > 0) {
+    const toleratedKeys = new Set(
+      state.windowStartedAt !== null &&
+        continuityFailures.length > 0 &&
+        continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
+        ? continuityFailures.map(freshnessKey)
+        : []
+    )
+    for (const key of [...freshnessStreaks.keys()]) {
+      if (!toleratedKeys.has(key)) freshnessStreaks.delete(key)
+    }
+    let tolerated = toleratedKeys.size > 0
+    for (const key of toleratedKeys) {
+      const streak = (freshnessStreaks.get(key) ?? 0) + 1
+      freshnessStreaks.set(key, streak)
+      if (streak > INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) tolerated = false
+    }
+    if (continuityFailures.length > 0 && !tolerated) {
+      freshnessStreaks.clear()
       resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures)
     } else {
+      if (tolerated) {
+        state.continuityEvents.push({
+          recordedAt: evaluation.evaluatedAt,
+          windowSequence: state.windowSequence,
+          tolerated: true,
+          failures: continuityFailures
+        })
+      }
       if (state.windowStartedAt === null) {
         state.windowStartedAt = evaluation.evaluatedAt
       }

cloud/docs/relay-incident-monitor.md

@@ -73,6 +73,13 @@ for a committed forward-recovery gate. Durable files default to
 gap resets the active window at the next fresh sample and preserves the prior
 window evidence. A threshold freeze never clears automatically.
 
+A signal that reads missing or stale may miss up to two consecutive samples
+without restarting the window. The sample still counts and is still checked
+against every threshold it can read, and each tolerated gap is recorded in
+`continuityEvents` with `tolerated: true`. A third consecutive miss of the same
+signal, a failed collector, a runner gap, or any threshold breach restarts or
+freezes as before.
+
 A production candidate or multi-target mutation must download the exact
 dry-run artifact by workflow run ID and attempt. It verifies the artifact
 hashes and provenance, requires a green completed 15-minute state no older
@@ -89,7 +96,8 @@ durably marked consumed before mutation and cannot authorize another run.
 | Signal | Freeze condition |
 | --- | ---: |
 | Active probe age | over 60 seconds |
-| Cloud/log data age | over 180 seconds |
+| Cloud Monitoring data age | over 330 seconds |
+| Relay log and director admin data age | over 180 seconds |
 | Cell heartbeat age | over 45 seconds |
 | Endpoint latency | over 2,000 ms |
 | Cloud SQL CPU | over 80% |
@@ -155,6 +163,25 @@ heartbeats, and matching live admission.
   separate it from today's baseline; the exhausted-retry bar (incident peak
   467 vs bar 300), director concurrency, and the pool bars carry that role.
   Re-tighten after the fleet is on the 500 ms lock wait.
+- Raised the Cloud Monitoring freshness bar from 180 s to 330 s and let a
+  freshness-only failure miss up to two consecutive samples without restarting
+  the window (2026-09-05). Basis: Google's metric list documents Cloud Run
+  `request_count`, `container/instance_count`, `container/cpu/utilizations`,
+  `container/memory/utilizations` and `container/max_request_concurrencies` as
+  "Sampled every 60 seconds. After sampling, data is not visible for up to 120
+  seconds", and Cloud SQL `database/cpu/utilization`,
+  `database/memory/utilization`, `database/postgresql/num_backends`,
+  `database/postgresql/backends_in_wait` and `database/postgresql/deadlock_count`
+  as "up to 165 seconds", so the newest visible point is up to 180 s and 225 s
+  old respectively. Window-sum signals age further: `observedAt` is the newest
+  point in the 5-minute query window, so a label series that stops emitting
+  reads as 300 s old while its summed value is complete. The old bar sat under
+  all three. Production on 2026-09-04/05 restarted healthy 15-minute windows at
+  181 s and 255 s (`auth.errors`, run 33928912676) and at 189 s
+  (`cloud_sql.lock_waits`, run 33944873727), and the last of those then blew the
+  25-minute lineage cap at 1 500 004 ms, so a green fleet produced no verdict.
+  The director admin bar stays at 180 s and the nonzero lock-wait carry window
+  stays at 180 s; both publish on our own cadence.
 - Recalibrated the exhausted-PostgreSQL-retry freeze from 0 to 300 per five
   minutes (2026-09-04). Basis: #18521 cut the request-path cell-inventory
   lock wait from the 1 s pool `lock_timeout` to 500 ms, so contended waiters