fix(cloud): tolerate sparse director errors in rollout monitor (#20238)

cloud/apps/relay-ops/src/incident-monitor.test.ts

@@ -273,6 +273,26 @@ describe('incident monitor evaluator', () => {
     )
   })
 
+  it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
+    for (const errors of [1, 2, 3]) {
+      const sample = healthySample()
+      sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
+      expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
+    }
+    const excess = healthySample()
+    excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
+    expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
+      expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
+    )
+    const auth = healthySample()
+    auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
+    expect(evaluateIncidentSample(auth, startedAt).status).toBe('freeze')
+    const pressure = healthySample()
+    pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1)
+    pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
+    expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze')
+  })
+
   it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
     const sample = healthySample()
     sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)

cloud/apps/relay-ops/src/incident-monitor.ts

@@ -101,7 +101,8 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
   directorCpuUtilization: 0.8,
   directorMemoryUtilization: 0.8,
   directorConcurrency: 64,
-  directorErrors: 0,
+  // Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
+  directorErrors: 3,
   authErrors: 0,
   // Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
   // the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).

cloud/docs/relay-incident-monitor.md

@@ -112,7 +112,8 @@ durably marked consumed before mutation and cannot authorize another run.
 | Director instances | outside 5–6 |
 | Director CPU or memory | over 80% |
 | Director concurrency | over 64 |
-| Unexpected director 5xx or auth 5xx in five minutes | over 0 |
+| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
+| Auth 5xx in five minutes | over 0 |
 | Connections per cell process | over 500 |
 | Queued bytes per cell process | over 48 MiB |
 | Blocked or expired/unregistered migration | over 0 |
@@ -276,3 +277,7 @@ without its segment is a compile error in relay-contract, not a silent gap.
   load the director's three-connection database pool.
 - Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence.
 - Added the manual production workflow. It has not been dispatched.
+
+### Director error allowance (2026-09-12)
+
+The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.