mirror of
https://github.com/stablyai/orca.git
synced 2026-11-03 10:37:35 +00:00
Jinwoo Hong
3062b9d8d0
* fix(relay): refuse host hellos fast when the database pool is saturated; renewals jump the queue A reconnect herd (2026-10-06 18:34Z, 1,046 asia-east2 hosts dropped by the load balancer at once) queued hundreds of host hellos behind each cell's 16-connection pool. They timed out after 2 s, the desktops redialled into the same queue, and control-lease renewals starved behind them. - Host control upgrades get an immediate 503 (Retry-After: 2) once 32 callers wait for a pooled connection. Shipped desktops already treat that as a connect error and back off with jittered exponential retry. - General work now queues in PostgresPoolPressure instead of pg-pool, so renewal statements (priority lane) take the next released connection. - A renewal batch that never got a connection no longer fans out into one statement per host. - hostHellosShedDelta counts refusals in the runtime metrics. * fix(relay): shed host hellos on the pool's oldest wait, not its queue length asia-east2 queues of 50-196 waiters are routine and keep moving, so a 32-waiter limit would refuse hellos that were going to succeed. Shed only while the oldest pool waiter has waited 1.5 s of the 2 s acquire timeout, when a new hello would time out anyway; a refused desktop gets the same connect error and backoff it gets today, 2 s sooner. Rebinds over a live control (lease rotation) are never refused. * fix(relay): shed hellos only when a full pool queue has aged 1 s Per review: require both an oldest waiter of at least 1 s and a pool's worth of waiters, so one slow waiter cannot trip it, and state the real baseline (p99 of 2 waiters on 10-07).
49 lines
2.1 KiB
TypeScript
49 lines
2.1 KiB
TypeScript
import { afterAll, describe, expect, it } from 'vitest'
|
|
import type { PostgresDatabase } from './database.js'
|
|
import { isPostgresPoolConnectTimeout } from './postgres-pool-pressure.js'
|
|
import { openDelayedPostgresDatabase } from './test-fixtures/delayed-postgres-database.js'
|
|
|
|
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
|
const describePostgres = databaseUrl ? describe : describe.skip
|
|
|
|
// The 2026-10-06 asia-east2 shape at test scale: a reconnect herd far wider
|
|
// than the pool, every statement paying a cross-region round trip, and a
|
|
// control renewal arriving behind it.
|
|
describePostgres('PostgreSQL priority lane under a reconnect herd', () => {
|
|
const databases: PostgresDatabase[] = []
|
|
const delay = { enabled: true, delayMs: 400 }
|
|
|
|
afterAll(async () => {
|
|
delay.enabled = false
|
|
for (const database of databases) await database.close()
|
|
})
|
|
|
|
it('renews ahead of the herd while queued general work times out', async () => {
|
|
const database = openDelayedPostgresDatabase(databaseUrl!, delay, 4)
|
|
databases.push(database)
|
|
const herd = Array.from({ length: 40 }, () =>
|
|
database.query('SELECT 1').then(
|
|
() => 'ok',
|
|
(error: unknown) => (isPostgresPoolConnectTimeout(error) ? 'timeout' : 'other')
|
|
)
|
|
)
|
|
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
const startedAt = performance.now()
|
|
const general = database
|
|
.query('SELECT 1 AS renewed')
|
|
.catch(() => undefined)
|
|
.then(() => performance.now() - startedAt)
|
|
const renewed = await database.queryPriority('SELECT 1 AS renewed')
|
|
const renewedMs = performance.now() - startedAt
|
|
|
|
expect(renewed).toEqual([{ renewed: 1 }])
|
|
// The next released connection plus one statement, never a turn in the herd's queue.
|
|
expect(renewedMs).toBeLessThan(1_500)
|
|
// The same statement on the general lane waits out the herd's acquire timeout.
|
|
expect(await general).toBeGreaterThan(1_900)
|
|
const outcomes = await Promise.all(herd)
|
|
expect(outcomes.filter((outcome) => outcome === 'timeout').length).toBeGreaterThan(20)
|
|
expect(outcomes).not.toContain('other')
|
|
}, 15_000)
|
|
})
|