diff --git a/cloud/apps/relay/src/regional-rehome-safety.test.ts b/cloud/apps/relay/src/regional-rehome-safety.test.ts index 5c7d10d37a0..4da072943d2 100644 --- a/cloud/apps/relay/src/regional-rehome-safety.test.ts +++ b/cloud/apps/relay/src/regional-rehome-safety.test.ts @@ -1,7 +1,5 @@ import { describe, expect, it } from 'vitest' import { - REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT, - REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT, REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT, REGIONAL_REHOME_SQL_FAILURES_LIMIT, regionalRehomeSafetyFailure @@ -41,14 +39,38 @@ describe('regionalRehomeSafetyFailure', () => { ).toBeNull() }) - it('still fails closed on each pool pressure bound', () => { - for (const overrides of [ - { databasePoolWaitersMax: REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT + 1 }, - { databasePoolWaitMsMax: REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT + 1 } - ]) { - expect(regionalRehomeSafetyFailure(safety(overrides), NOW, 19)).toBe( - 'database_pool_pressure' + it('passes asia-scale pool pressure, which excludes a cell rather than the fleet', () => { + // Measured 2026-09-16 on the asia-east2 cells: a client pool too narrow for + // a 176ms round trip, with 0.2ms server-side execution. The fleet snapshot + // is a Math.max, so gating on it here stops every region. + expect( + regionalRehomeSafetyFailure( + safety({ databasePoolWaitersMax: 150, databasePoolWaitMsMax: 2_005 }), + NOW, + 19 ) + ).toBeNull() + // Every other bar still fails closed at that same pool pressure. + for (const [overrides, reason] of [ + [{ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1 }, 'sql_failures'], + [{ controlActivityRecoveryFailures: 1 }, 'control_recovery_failures'], + [ + { reconnects: 19 * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1 }, + 'elevated_reconnects' + ], + [{ observedAt: 0 }, 'monitoring_stale'] + ] as const) { + expect( + regionalRehomeSafetyFailure( + safety({ + databasePoolWaitersMax: 150, + databasePoolWaitMsMax: 2_005, + ...overrides + }), + NOW, + 19 + ) + ).toBe(reason) } }) diff --git a/cloud/apps/relay/src/regional-rehome-safety.ts b/cloud/apps/relay/src/regional-rehome-safety.ts index 4da51f71b0d..072e82f7621 100644 --- a/cloud/apps/relay/src/regional-rehome-safety.ts +++ b/cloud/apps/relay/src/regional-rehome-safety.ts @@ -2,12 +2,20 @@ import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' // Limits sit well above the healthy-fleet baseline measured in production on // 2026-08-28 (peak 2 waiters / 1ms pool waits on every cell; up to ~80 -// reconnects per published two-window row on the busiest cell). Sustained -// pool saturation still trips: waiters-max 16 is 8x baseline yet far under a -// backed-up pool, and 250ms peak wait is 1/10 of the incident-monitor alert. -// Instantaneous databasePoolWaiting is not checked separately: it is bounded -// by databasePoolWaitersMax within every published window. +// reconnects per published two-window row on the busiest cell). export const REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT = 250 + +// Pool pressure gates one cell, never the fleet. The combined snapshot is a +// Math.max, so a single narrow client pool stops rehoming everywhere: measured +// 2026-09-16, the asia-east2 cells sit at 94-156 waiters and ~2000ms waits +// against 0.2ms server-side execution -- a pool too narrow for a 176ms round +// trip, not database distress -- while us-central1 cells breach in bursts on +// ~33% of polls, and a bar that flaps between the pre-check and the commit +// re-check latches the worker off. regionalRehomeCellSafetyIsClean excludes a +// breaching cell as both source and target; the bars below stay fleet-wide +// because sql failures, control-recovery failures and reconnect storms mean +// database-wide distress. Instantaneous databasePoolWaiting is not checked +// separately: databasePoolWaitersMax bounds it within every published window. export const REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT = 16 export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250 @@ -19,7 +27,7 @@ export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250 // over four days; genuine database distress produced 395-457. The combined // snapshot spans up to two 30s windows per process (pathological ambient // alignment ~164), so 250 stays clear of noise while storms still trip. -// Terminal outages also trip the pool bars and the worker's own +// Terminal outages also trip the per-cell pool bars and the worker's own // dispatch-failure budget; the sql bar only needs to catch storms. export const REGIONAL_REHOME_SQL_FAILURES_LIMIT = 250 // Per-cell candidate cleanliness is a soft skip, not a durable latch; the @@ -47,9 +55,6 @@ export function regionalRehomeSafetyFailure( return 'monitoring_stale' } if (safety.sqlFailures > REGIONAL_REHOME_SQL_FAILURES_LIMIT) return 'sql_failures' - if (regionalRehomePoolPressure(safety)) { - return 'database_pool_pressure' - } if (safety.controlActivityRecoveryFailures > 0) { return 'control_recovery_failures' } diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts index e4a355699da..9fae78c8afd 100644 --- a/cloud/apps/relay/src/regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -34,8 +34,12 @@ const target = { connectionHardCap: 1_000 as const, connectionUnobservedBound: 60 } +// A third general cell in the source region: never a source or target here, +// but it is in the fleet whose safety the gate reads. +const bystander = { ...source, id: 'us-c2', url: 'https://us-c2.relay.example.test' } const sourceIncarnation = '11111111-1111-4111-8111-111111111111' const targetIncarnation = '22222222-2222-4222-8222-222222222222' +const bystanderIncarnation = '33333333-3333-4333-8333-333333333333' describe('regional rehome assignment state', () => { it('advances past a full candidate page whose destination lacks capacity', async () => { @@ -424,7 +428,7 @@ describe('regional rehome assignment state', () => { await context.database.close() }) - it('latches off on sustained pool pressure and logs the disable exactly once', async () => { + it('defers on target pool pressure without disabling, and claims once it clears', async () => { const context = await setup() await activatePreferredSource(context, { userId: 'user-1', @@ -451,17 +455,59 @@ describe('regional rehome assignment state', () => { expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({ outcome: 'deferred' }) - // Already disabled: the next tick returns before the gate and stays silent. - expect(await context.store.tryIdleRehome()).toBeNull() } finally { warnings.restore() } + // A pool bar crossed between the scan and the commit skips the cell; it must + // not turn the durable switch off, or the worker never comes back. expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ - generation: 2, - enabled: false + generation: 1, + enabled: true }) - expect(warnings.entries).toMatchObject([ - { reason: 'database_pool_pressure', databasePoolWaitersMax: 17 } + expect(warnings.entries).toEqual([]) + + await context.database.query( + `UPDATE relay_cell_rehome_safety SET database_pool_waiters_max = 0 WHERE cell_id = ?`, + [target.id] + ) + expect(await context.store.tryIdleRehome()).toMatchObject({ + userId: 'user-1', + sourceCellId: source.id, + targetCellId: target.id + }) + await context.database.close() + }) + + it('keeps selecting candidates while an unrelated cell is over the pool bar', async () => { + // Production case: the fleet snapshot is a Math.max, so one cell with a + // narrow client pool used to empty every page. + const context = await setup() + await activatePreferredSource(context, { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop' + }) + await context.store.reconcileCells([source, target, bystander]) + await heartbeat(context.store, bystander, bystanderIncarnation, 3, 2, { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 150, + databasePoolWaitersMax: 150, + databasePoolWaitMsMax: 2_005 + }) + + const safety: RegionalRehomeSafetySnapshot = { + observedAt: context.now(), + sqlFailures: 0, + reconnects: 0, + controlActivityRecoveryFailures: 0, + databasePoolWaiting: 0, + databasePoolWaitersMax: 0, + databasePoolWaitMsMax: 0 + } + expect(await context.store.selectIdleRegionalRehomeCandidates(safety)).toMatchObject([ + { sourceCellId: source.id, targetCellId: target.id } ]) await context.database.close() })