mirror of
https://github.com/stablyai/orca.git
synced 2026-09-23 16:02:24 +00:00
Jinwoo Hong
a3046cd27b
* fix(relay): treat pool connect failures as transient, not director faults pg-pool raises connection-acquire failures as a plain Error with no SQLSTATE, so the transient classifier matched only one of the three messages it can produce. The other two reached the routes unclassified and became HTTP 500s, which is what the rollout safety gate counts. The acquire boundary now marks the errors it produces, so "Connection terminated unexpectedly" counts as transient when the socket died during the handshake and stays a hard failure mid-statement, where a retry could repeat a commit whose outcome is unknown. /v1/regions and /v1/admin/evacuation-status gain the transient handling /v1/assign and /v1/resolve already had. * fix(relay): mirror the pool-connect verdict in failure diagnostics The query-failure event's connectionTimeout boolean matched one of the two messages connectionTimeoutMillis can produce, so the 210 dialling timeouts in the last day logged as false and were invisible to the field meant to find them. The pool-connect vocabulary now lives beside the acquire boundary that owns it, and both the router's classifier and the diagnostics read it from there, so the two cannot drift. The event also carries the routing verdict the caller already computed, making "how much of this burst reached users as a 500" one field. * fix(relay): null-safe transient classification and honest transient docs The classifier now runs inside the query catch, where a thrown null or undefined would have turned a database failure into a TypeError that buried it. The diagnostics doc claimed transient maps to a 503 or a 500. Sweeps, startup reconciliation, and admin routes that answer 409 all emit the same event, so counting the false ones over-states user-facing hard failures.
69 lines
2.4 KiB
TypeScript
69 lines
2.4 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest'
|
|
import { isRelayDatabaseTransientError } from './database.js'
|
|
import { PostgresPoolPressure } from './postgres-pool-pressure.js'
|
|
|
|
// The only way to mark an error as an acquire failure is to fail a real
|
|
// acquire, so the gated cases go through the pressure wrapper the pool uses.
|
|
async function failedAcquire(message: string): Promise<unknown> {
|
|
const pool = {
|
|
totalCount: 0,
|
|
idleCount: 0,
|
|
waitingCount: 0,
|
|
connect: vi.fn(async () => {
|
|
throw new Error(message)
|
|
})
|
|
}
|
|
return await new PostgresPoolPressure(pool as never).connect().catch((error: unknown) => error)
|
|
}
|
|
|
|
describe('relay database transient errors', () => {
|
|
it.each(['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'])(
|
|
'classifies PostgreSQL code %s as retryable overload',
|
|
(code) => {
|
|
expect(isRelayDatabaseTransientError({ code })).toBe(true)
|
|
}
|
|
)
|
|
|
|
it('classifies pool acquisition timeout without hiding programming failures', () => {
|
|
expect(
|
|
isRelayDatabaseTransientError(new Error('timeout exceeded when trying to connect'))
|
|
).toBe(true)
|
|
expect(isRelayDatabaseTransientError(new TypeError('broken invariant'))).toBe(false)
|
|
})
|
|
|
|
it('classifies a pool connect timeout that node-postgres reports with no code', () => {
|
|
// pg-pool raises this only from its own connect path, so no statement ran.
|
|
expect(
|
|
isRelayDatabaseTransientError(
|
|
new Error('Connection terminated due to connection timeout')
|
|
)
|
|
).toBe(true)
|
|
})
|
|
|
|
it('classifies an early-ended socket only when it ended during the acquire', async () => {
|
|
expect(
|
|
isRelayDatabaseTransientError(await failedAcquire('Connection terminated unexpectedly'))
|
|
).toBe(true)
|
|
// The same message mid-statement leaves the commit outcome unknown, so it
|
|
// must stay a hard failure rather than invite a retry.
|
|
expect(
|
|
isRelayDatabaseTransientError(new Error('Connection terminated unexpectedly'))
|
|
).toBe(false)
|
|
})
|
|
|
|
it.each([null, undefined, 'a thrown string'])(
|
|
'survives %s reaching it instead of an error object',
|
|
(thrown) => {
|
|
expect(isRelayDatabaseTransientError(thrown)).toBe(false)
|
|
}
|
|
)
|
|
|
|
it('keeps a failed acquire that is not transient out of the retry path', async () => {
|
|
expect(
|
|
isRelayDatabaseTransientError(
|
|
await failedAcquire('password authentication failed for user "relay"')
|
|
)
|
|
).toBe(false)
|
|
})
|
|
})
|