mirror of
https://github.com/stablyai/orca.git
synced 2026-11-03 10:37:35 +00:00
* fix(relay): refuse host hellos fast when the database pool is saturated; renewals jump the queue A reconnect herd (2026-10-06 18:34Z, 1,046 asia-east2 hosts dropped by the load balancer at once) queued hundreds of host hellos behind each cell's 16-connection pool. They timed out after 2 s, the desktops redialled into the same queue, and control-lease renewals starved behind them. - Host control upgrades get an immediate 503 (Retry-After: 2) once 32 callers wait for a pooled connection. Shipped desktops already treat that as a connect error and back off with jittered exponential retry. - General work now queues in PostgresPoolPressure instead of pg-pool, so renewal statements (priority lane) take the next released connection. - A renewal batch that never got a connection no longer fans out into one statement per host. - hostHellosShedDelta counts refusals in the runtime metrics. * fix(relay): shed host hellos on the pool's oldest wait, not its queue length asia-east2 queues of 50-196 waiters are routine and keep moving, so a 32-waiter limit would refuse hellos that were going to succeed. Shed only while the oldest pool waiter has waited 1.5 s of the 2 s acquire timeout, when a new hello would time out anyway; a refused desktop gets the same connect error and backoff it gets today, 2 s sooner. Rebinds over a live control (lease rotation) are never refused. * fix(relay): shed hellos only when a full pool queue has aged 1 s Per review: require both an oldest waiter of at least 1 s and a pool's worth of waiters, so one slow waiter cannot trip it, and state the real baseline (p99 of 2 waiters on 10-07).
150 lines
4.8 KiB
TypeScript
150 lines
4.8 KiB
TypeScript
import { EventEmitter } from 'node:events'
|
|
import { describe, expect, it, vi } from 'vitest'
|
|
import { isPostgresPoolConnectTimeout, PostgresPoolPressure } from './postgres-pool-pressure.js'
|
|
|
|
describe('PostgreSQL pool pressure', () => {
|
|
it('reports current waiters and interval high-water marks', async () => {
|
|
let now = 1_000
|
|
let resolveConnection!: (client: unknown) => void
|
|
const connection = new Promise((resolve) => {
|
|
resolveConnection = resolve
|
|
})
|
|
const pool = {
|
|
totalCount: 3,
|
|
idleCount: 0,
|
|
waitingCount: 0,
|
|
connect: vi.fn(() => {
|
|
pool.waitingCount++
|
|
return connection
|
|
})
|
|
}
|
|
const pressure = new PostgresPoolPressure(pool as never, () => now)
|
|
const pending = pressure.connect()
|
|
now = 1_750
|
|
|
|
expect(pressure.consumeCounts()).toMatchObject({
|
|
databasePoolTotal: 3,
|
|
databasePoolIdle: 0,
|
|
databasePoolWaiting: 1,
|
|
databasePoolWaitersMax: 1,
|
|
databasePoolOldestWaitMs: 750,
|
|
databasePoolWaitMsMax: 750
|
|
})
|
|
|
|
now = 2_250
|
|
pool.waitingCount--
|
|
// An EventEmitter because the acquire path now attaches an `error` listener.
|
|
resolveConnection(Object.assign(new EventEmitter(), { query: vi.fn(), release: vi.fn() }))
|
|
await pending
|
|
expect(pressure.consumeCounts()).toMatchObject({
|
|
databasePoolWaiting: 0,
|
|
databasePoolWaitersMax: 1,
|
|
databasePoolOldestWaitMs: 0,
|
|
databasePoolWaitMsMax: 1_250
|
|
})
|
|
expect(pressure.consumeCounts()).toMatchObject({
|
|
databasePoolWaitersMax: 0,
|
|
databasePoolWaitMsMax: 0
|
|
})
|
|
})
|
|
|
|
it('reads the oldest wait as it is now, not the interval maximum', async () => {
|
|
let now = 1_000
|
|
const pool = {
|
|
totalCount: 1,
|
|
idleCount: 0,
|
|
waitingCount: 0,
|
|
connect: vi.fn(() => {
|
|
pool.waitingCount++
|
|
return new Promise(() => {})
|
|
})
|
|
}
|
|
const pressure = new PostgresPoolPressure(pool as never, () => now)
|
|
expect(pressure.oldestWaitMs()).toBe(0)
|
|
void pressure.connect()
|
|
now = 2_600
|
|
expect(pressure.oldestWaitMs()).toBe(1_600)
|
|
})
|
|
})
|
|
|
|
describe('PostgreSQL pool priority lane', () => {
|
|
// pg-pool's acquire contract with a hand-driven supply of free connections.
|
|
function controlledPool(max: number, connectionTimeoutMillis = 0) {
|
|
const pending: Array<(client: unknown) => void> = []
|
|
const pool = {
|
|
options: { max, connectionTimeoutMillis },
|
|
totalCount: 0,
|
|
idleCount: 0,
|
|
waitingCount: 0,
|
|
connect: vi.fn(() => {
|
|
if (pool.totalCount < max) {
|
|
pool.totalCount++
|
|
return Promise.resolve(client())
|
|
}
|
|
pool.waitingCount++
|
|
return new Promise((resolve) => pending.push(resolve))
|
|
})
|
|
}
|
|
const client = () =>
|
|
Object.assign(new EventEmitter(), {
|
|
query: vi.fn(),
|
|
release: vi.fn(() => {
|
|
const next = pending.shift()
|
|
if (next) {
|
|
pool.waitingCount--
|
|
next(client())
|
|
} else pool.totalCount--
|
|
})
|
|
})
|
|
return pool
|
|
}
|
|
|
|
it('hands the next released connection to priority work ahead of queued general work', async () => {
|
|
const pool = controlledPool(2)
|
|
const pressure = new PostgresPoolPressure(pool as never)
|
|
const general = [await pressure.connect(), await pressure.connect()]
|
|
const order: string[] = []
|
|
const queuedGeneral = pressure.connect().then((client) => {
|
|
order.push('general')
|
|
return client
|
|
})
|
|
const priority = pressure.connect('priority').then((client) => {
|
|
order.push('priority')
|
|
return client
|
|
})
|
|
await Promise.resolve()
|
|
expect(pressure.peekCounts().databasePoolWaiting).toBe(2)
|
|
|
|
general[0]!.release()
|
|
const priorityClient = await priority
|
|
expect(order).toEqual(['priority'])
|
|
priorityClient.release()
|
|
const generalClient = await queuedGeneral
|
|
expect(order).toEqual(['priority', 'general'])
|
|
for (const client of [general[1]!, generalClient]) client.release()
|
|
expect(pool.totalCount).toBe(0)
|
|
expect(pressure.peekCounts().databasePoolWaiting).toBe(0)
|
|
})
|
|
|
|
it('fails a queued general caller with the pool timeout every classifier knows', async () => {
|
|
vi.useFakeTimers()
|
|
try {
|
|
const pressure = new PostgresPoolPressure(controlledPool(1, 2_000) as never)
|
|
const held = await pressure.connect()
|
|
const queued = pressure.connect().catch((error: unknown) => error)
|
|
await vi.advanceTimersByTimeAsync(2_000)
|
|
|
|
const error = await queued
|
|
expect(isPostgresPoolConnectTimeout(error)).toBe(true)
|
|
expect(pressure.peekCounts().databasePoolWaiting).toBe(0)
|
|
// The timed-out caller left the queue, so the slot goes to the next one in line.
|
|
held.release()
|
|
const next = await pressure.connect()
|
|
next.release()
|
|
} finally {
|
|
vi.useRealTimers()
|
|
}
|
|
})
|
|
|
|
})
|