fix(relay): keep pool pressure a per-cell rehome exclusion, not a fleet stop (#21126)

The fleet safety gate returned database_pool_pressure whenever the
Math.max of database_pool_waiters_max or database_pool_wait_ms_max
across every general cell crossed 16 waiters or 250ms. Measured
2026-09-16, the asia-east2 cells breach continuously at 94-156 waiters
and ~2000ms while their server-side execution is 0.2ms, which is a
client pool too narrow for a 176ms round trip rather than database
distress, and the us-central1 cells breach in bursts on about a third of
polls. Worse, the bar flaps: the pre-check passes, the commit re-check
reads fresh rows seconds later and trips, and that path durably disables
the control instead of merely deferring.

Drop the pool check from the fleet gate. Pool pressure stays a per-cell
exclusion in regionalRehomeCellSafetyIsClean, which already drops a
breaching cell as both source and target on selection and again on the
commit path. The fleet bars that remain (stale monitoring, sql failure
storms, control-recovery failures, reconnect storms) all signal
database-wide distress. Nothing cells publish, no stored row and no
exported constant changes.

cloud/apps/relay/src/regional-rehome-safety.test.ts

@@ -1,7 +1,5 @@
 import { describe, expect, it } from 'vitest'
 import {
-  REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT,
-  REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT,
   REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
   REGIONAL_REHOME_SQL_FAILURES_LIMIT,
   regionalRehomeSafetyFailure
@@ -41,14 +39,38 @@ describe('regionalRehomeSafetyFailure', () => {
     ).toBeNull()
   })
 
-  it('still fails closed on each pool pressure bound', () => {
-    for (const overrides of [
-      { databasePoolWaitersMax: REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT + 1 },
-      { databasePoolWaitMsMax: REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT + 1 }
-    ]) {
-      expect(regionalRehomeSafetyFailure(safety(overrides), NOW, 19)).toBe(
-        'database_pool_pressure'
+  it('passes asia-scale pool pressure, which excludes a cell rather than the fleet', () => {
+    // Measured 2026-09-16 on the asia-east2 cells: a client pool too narrow for
+    // a 176ms round trip, with 0.2ms server-side execution. The fleet snapshot
+    // is a Math.max, so gating on it here stops every region.
+    expect(
+      regionalRehomeSafetyFailure(
+        safety({ databasePoolWaitersMax: 150, databasePoolWaitMsMax: 2_005 }),
+        NOW,
+        19
       )
+    ).toBeNull()
+    // Every other bar still fails closed at that same pool pressure.
+    for (const [overrides, reason] of [
+      [{ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1 }, 'sql_failures'],
+      [{ controlActivityRecoveryFailures: 1 }, 'control_recovery_failures'],
+      [
+        { reconnects: 19 * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1 },
+        'elevated_reconnects'
+      ],
+      [{ observedAt: 0 }, 'monitoring_stale']
+    ] as const) {
+      expect(
+        regionalRehomeSafetyFailure(
+          safety({
+            databasePoolWaitersMax: 150,
+            databasePoolWaitMsMax: 2_005,
+            ...overrides
+          }),
+          NOW,
+          19
+        )
+      ).toBe(reason)
     }
   })
 

cloud/apps/relay/src/regional-rehome-safety.ts

@@ -2,12 +2,20 @@ import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
 
 // Limits sit well above the healthy-fleet baseline measured in production on
 // 2026-08-28 (peak 2 waiters / 1ms pool waits on every cell; up to ~80
-// reconnects per published two-window row on the busiest cell). Sustained
-// pool saturation still trips: waiters-max 16 is 8x baseline yet far under a
-// backed-up pool, and 250ms peak wait is 1/10 of the incident-monitor alert.
-// Instantaneous databasePoolWaiting is not checked separately: it is bounded
-// by databasePoolWaitersMax within every published window.
+// reconnects per published two-window row on the busiest cell).
 export const REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT = 250
+
+// Pool pressure gates one cell, never the fleet. The combined snapshot is a
+// Math.max, so a single narrow client pool stops rehoming everywhere: measured
+// 2026-09-16, the asia-east2 cells sit at 94-156 waiters and ~2000ms waits
+// against 0.2ms server-side execution -- a pool too narrow for a 176ms round
+// trip, not database distress -- while us-central1 cells breach in bursts on
+// ~33% of polls, and a bar that flaps between the pre-check and the commit
+// re-check latches the worker off. regionalRehomeCellSafetyIsClean excludes a
+// breaching cell as both source and target; the bars below stay fleet-wide
+// because sql failures, control-recovery failures and reconnect storms mean
+// database-wide distress. Instantaneous databasePoolWaiting is not checked
+// separately: databasePoolWaitersMax bounds it within every published window.
 export const REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT = 16
 export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250
 
@@ -19,7 +27,7 @@ export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250
 // over four days; genuine database distress produced 395-457. The combined
 // snapshot spans up to two 30s windows per process (pathological ambient
 // alignment ~164), so 250 stays clear of noise while storms still trip.
-// Terminal outages also trip the pool bars and the worker's own
+// Terminal outages also trip the per-cell pool bars and the worker's own
 // dispatch-failure budget; the sql bar only needs to catch storms.
 export const REGIONAL_REHOME_SQL_FAILURES_LIMIT = 250
 // Per-cell candidate cleanliness is a soft skip, not a durable latch; the
@@ -47,9 +55,6 @@ export function regionalRehomeSafetyFailure(
     return 'monitoring_stale'
   }
   if (safety.sqlFailures > REGIONAL_REHOME_SQL_FAILURES_LIMIT) return 'sql_failures'
-  if (regionalRehomePoolPressure(safety)) {
-    return 'database_pool_pressure'
-  }
   if (safety.controlActivityRecoveryFailures > 0) {
     return 'control_recovery_failures'
   }

cloud/apps/relay/src/regional-rehome-store.test.ts

@@ -34,8 +34,12 @@ const target = {
   connectionHardCap: 1_000 as const,
   connectionUnobservedBound: 60
 }
+// A third general cell in the source region: never a source or target here,
+// but it is in the fleet whose safety the gate reads.
+const bystander = { ...source, id: 'us-c2', url: 'https://us-c2.relay.example.test' }
 const sourceIncarnation = '11111111-1111-4111-8111-111111111111'
 const targetIncarnation = '22222222-2222-4222-8222-222222222222'
+const bystanderIncarnation = '33333333-3333-4333-8333-333333333333'
 
 describe('regional rehome assignment state', () => {
   it('advances past a full candidate page whose destination lacks capacity', async () => {
@@ -424,7 +428,7 @@ describe('regional rehome assignment state', () => {
     await context.database.close()
   })
 
-  it('latches off on sustained pool pressure and logs the disable exactly once', async () => {
+  it('defers on target pool pressure without disabling, and claims once it clears', async () => {
     const context = await setup()
     await activatePreferredSource(context, {
       userId: 'user-1',
@@ -451,17 +455,59 @@ describe('regional rehome assignment state', () => {
       expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({
         outcome: 'deferred'
       })
-      // Already disabled: the next tick returns before the gate and stays silent.
-      expect(await context.store.tryIdleRehome()).toBeNull()
     } finally {
       warnings.restore()
     }
+    // A pool bar crossed between the scan and the commit skips the cell; it must
+    // not turn the durable switch off, or the worker never comes back.
     expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
-      generation: 2,
-      enabled: false
+      generation: 1,
+      enabled: true
     })
-    expect(warnings.entries).toMatchObject([
-      { reason: 'database_pool_pressure', databasePoolWaitersMax: 17 }
+    expect(warnings.entries).toEqual([])
+
+    await context.database.query(
+      `UPDATE relay_cell_rehome_safety SET database_pool_waiters_max = 0 WHERE cell_id = ?`,
+      [target.id]
+    )
+    expect(await context.store.tryIdleRehome()).toMatchObject({
+      userId: 'user-1',
+      sourceCellId: source.id,
+      targetCellId: target.id
+    })
+    await context.database.close()
+  })
+
+  it('keeps selecting candidates while an unrelated cell is over the pool bar', async () => {
+    // Production case: the fleet snapshot is a Math.max, so one cell with a
+    // narrow client pool used to empty every page.
+    const context = await setup()
+    await activatePreferredSource(context, {
+      userId: 'user-1',
+      relayHostId: 'abcdefghijklmnop'
+    })
+    await context.store.reconcileCells([source, target, bystander])
+    await heartbeat(context.store, bystander, bystanderIncarnation, 3, 2, {
+      observedAt: context.now(),
+      sqlFailures: 0,
+      reconnects: 0,
+      controlActivityRecoveryFailures: 0,
+      databasePoolWaiting: 150,
+      databasePoolWaitersMax: 150,
+      databasePoolWaitMsMax: 2_005
+    })
+
+    const safety: RegionalRehomeSafetySnapshot = {
+      observedAt: context.now(),
+      sqlFailures: 0,
+      reconnects: 0,
+      controlActivityRecoveryFailures: 0,
+      databasePoolWaiting: 0,
+      databasePoolWaitersMax: 0,
+      databasePoolWaitMsMax: 0
+    }
+    expect(await context.store.selectIdleRegionalRehomeCandidates(safety)).toMatchObject([
+      { sourceCellId: source.id, targetCellId: target.id }
     ])
     await context.database.close()
   })