From 1a9a5f9bc720bebd06a5dd190a4ea0000e59ccb4 Mon Sep 17 00:00:00 2001
From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com>
Date: Sat, 12 Sep 2026 01:15:15 -0400
Subject: [PATCH] fix(cloud): tolerate sparse director errors in rollout
 monitor (#20238)

---
 .../relay-ops/src/incident-monitor.test.ts    | 20 +++++++++++++++++++
 cloud/apps/relay-ops/src/incident-monitor.ts  |  3 ++-
 cloud/docs/relay-incident-monitor.md          |  7 ++++++-
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts
index c1a073cde4a..be35b55f65b 100644
--- a/cloud/apps/relay-ops/src/incident-monitor.test.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts
@@ -273,6 +273,26 @@ describe('incident monitor evaluator', () => {
     )
   })
 
+  it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
+    for (const errors of [1, 2, 3]) {
+      const sample = healthySample()
+      sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
+      expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
+    }
+    const excess = healthySample()
+    excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
+    expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
+      expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
+    )
+    const auth = healthySample()
+    auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
+    expect(evaluateIncidentSample(auth, startedAt).status).toBe('freeze')
+    const pressure = healthySample()
+    pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1)
+    pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
+    expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze')
+  })
+
   it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
     const sample = healthySample()
     sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts
index 0887bb2d1ee..a48e100a6d8 100644
--- a/cloud/apps/relay-ops/src/incident-monitor.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor.ts
@@ -101,7 +101,8 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
   directorCpuUtilization: 0.8,
   directorMemoryUtilization: 0.8,
   directorConcurrency: 64,
-  directorErrors: 0,
+  // Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
+  directorErrors: 3,
   authErrors: 0,
   // Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
   // the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
diff --git a/cloud/docs/relay-incident-monitor.md b/cloud/docs/relay-incident-monitor.md
index 696efb85296..a97e92949e8 100644
--- a/cloud/docs/relay-incident-monitor.md
+++ b/cloud/docs/relay-incident-monitor.md
@@ -112,7 +112,8 @@ durably marked consumed before mutation and cannot authorize another run.
 | Director instances | outside 5–6 |
 | Director CPU or memory | over 80% |
 | Director concurrency | over 64 |
-| Unexpected director 5xx or auth 5xx in five minutes | over 0 |
+| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
+| Auth 5xx in five minutes | over 0 |
 | Connections per cell process | over 500 |
 | Queued bytes per cell process | over 48 MiB |
 | Blocked or expired/unregistered migration | over 0 |
@@ -276,3 +277,7 @@ without its segment is a compile error in relay-contract, not a silent gap.
   load the director's three-connection database pool.
 - Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence.
 - Added the manual production workflow. It has not been dispatched.
+
+### Director error allowance (2026-09-12)
+
+The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.