mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
Jinwoo Hong
a3046cd27b
* fix(relay): treat pool connect failures as transient, not director faults pg-pool raises connection-acquire failures as a plain Error with no SQLSTATE, so the transient classifier matched only one of the three messages it can produce. The other two reached the routes unclassified and became HTTP 500s, which is what the rollout safety gate counts. The acquire boundary now marks the errors it produces, so "Connection terminated unexpectedly" counts as transient when the socket died during the handshake and stays a hard failure mid-statement, where a retry could repeat a commit whose outcome is unknown. /v1/regions and /v1/admin/evacuation-status gain the transient handling /v1/assign and /v1/resolve already had. * fix(relay): mirror the pool-connect verdict in failure diagnostics The query-failure event's connectionTimeout boolean matched one of the two messages connectionTimeoutMillis can produce, so the 210 dialling timeouts in the last day logged as false and were invisible to the field meant to find them. The pool-connect vocabulary now lives beside the acquire boundary that owns it, and both the router's classifier and the diagnostics read it from there, so the two cannot drift. The event also carries the routing verdict the caller already computed, making "how much of this burst reached users as a 500" one field. * fix(relay): null-safe transient classification and honest transient docs The classifier now runs inside the query catch, where a thrown null or undefined would have turned a database failure into a TypeError that buried it. The diagnostics doc claimed transient maps to a 503 or a 500. Sweeps, startup reconciliation, and admin routes that answer 409 all emit the same event, so counting the false ones over-states user-facing hard failures.
73 lines
2.2 KiB
TypeScript
73 lines
2.2 KiB
TypeScript
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
|
|
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
|
|
|
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
|
const describePostgres = databaseUrl ? describe : describe.skip
|
|
|
|
describePostgres('real PostgreSQL query failure phases', () => {
|
|
let database: RelayDatabase
|
|
|
|
beforeAll(async () => {
|
|
database = await openRelayDatabase({
|
|
databaseUrl,
|
|
dataDir: '',
|
|
poolMax: 1,
|
|
statementTimeoutMs: 50
|
|
})
|
|
})
|
|
afterAll(async () => {
|
|
await database.close()
|
|
})
|
|
afterEach(() => {
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
it('distinguishes a server statement timeout and leaves the pool usable', async () => {
|
|
const log = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
|
await expect(database.query('SELECT pg_sleep(0.2)')).rejects.toMatchObject({ code: '57014' })
|
|
expect(JSON.parse(log.mock.calls[0]![0] as string)).toMatchObject({
|
|
event: 'orca_relay_postgres_query_failed',
|
|
phase: 'execute',
|
|
code: '57014',
|
|
connectionTimeout: false,
|
|
transient: true
|
|
})
|
|
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
|
|
})
|
|
|
|
it('distinguishes queue acquisition timeout without running the statement', async () => {
|
|
const log = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
|
let acquired!: () => void
|
|
const ready = new Promise<void>((resolve) => {
|
|
acquired = resolve
|
|
})
|
|
let release!: () => void
|
|
const wait = new Promise<void>((resolve) => {
|
|
release = resolve
|
|
})
|
|
const holder = database.transaction(async () => {
|
|
acquired()
|
|
await wait
|
|
})
|
|
await ready
|
|
try {
|
|
await expect(database.query('SELECT pg_sleep(0.2)')).rejects.toThrow(
|
|
'timeout exceeded when trying to connect'
|
|
)
|
|
expect(JSON.parse(log.mock.calls[0]![0] as string)).toMatchObject({
|
|
event: 'orca_relay_postgres_query_failed',
|
|
phase: 'acquire',
|
|
code: 'unknown',
|
|
connectionTimeout: true,
|
|
transient: true,
|
|
poolTotal: 1,
|
|
poolIdle: 0
|
|
})
|
|
} finally {
|
|
release()
|
|
await holder
|
|
}
|
|
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
|
|
})
|
|
})
|