mirror of
https://github.com/stablyai/orca.git
synced 2026-10-26 00:03:55 +00:00
Jinwoo Hong
ce5d8c02d4
* fix(relay): wait out a cold proxy at boot instead of exiting the cell A cell container starts its relay process beside a cloud-sql-proxy that is itself still dialling. The first pool acquire therefore competes with a proxy cold start, and the 2s connect timeout that protects the request path fires before the proxy is listening. `openRelayDatabase` rejects out of the region backfill, the top-level await rejects, and the process exits; COS restarts the container and the next boot succeeds 1-3s later. The 2026-09-18 fleet roll saw 0-7 of these per cell, including on cells with zero hosts, so it is a property of the boot sequence rather than of database load. The boot open now retries on transient errors only, inside a 45s wall-clock window with exponential backoff from 250ms to 4s. The classifier is the one the request path already uses, so a rejected credential or a bad URL still exits on the first attempt. Each wait logs `orca_relay_boot_database_retry` and a give-up logs `orca_relay_boot_database_failed`, both with the bounded error category, so a rollout can tell a slow boot from a stuck one without reading container exit codes. The bounded startup retry is lifted out of `reconcileCellAdmissionAtStartup`, which had the same loop; its attempt budget, flat delay, and both log events are unchanged (a flat delay is a cap equal to the base). * fix(relay): retry the boot open only when Postgres is unreachable The boot open re-runs the schema apply, and applyPostgresSchema refuses to repeat a DDL lock timeout on purpose: relation locks are granted in queue order, so a repeat parks every writer behind the same statement again. Gating the boot retry on the full request-path classifier would have re-queued it up to 16 times in 45s on sustained 55P03 - the mechanism behind the 2026-09-16 outage. The boot call site now has its own predicate: pool connect failures (both connect-timeout messages and an acquire-marked early-ended socket) plus 08001 and 08006. Lock and overload SQLSTATEs - 55P03, 57014, 53300 - exit on the first attempt. The retry predicate moves onto the policy because what a step re-runs, not the request path, decides what it may repeat; the startup reconcile keeps the full classifier, which is what lets it wait out 55P03.
53 lines
2.1 KiB
TypeScript
53 lines
2.1 KiB
TypeScript
import { isRelayDatabaseTransientError } from './database.js'
|
|
|
|
export type DatabaseStartupRetryPolicy = {
|
|
attempts: number
|
|
windowMs: number
|
|
baseDelayMs: number
|
|
maxDelayMs: number
|
|
jitterMs: number
|
|
// Which failures this particular startup step may repeat. Not every caller can
|
|
// repeat everything the request path calls transient: what the retry re-runs
|
|
// decides that, so the call site owns it.
|
|
isRetryable?: (error: unknown) => boolean
|
|
}
|
|
|
|
export type DatabaseStartupRetryObserver = {
|
|
onRetry?: (event: { attempt: number; delayMs: number; error: unknown }) => void
|
|
onRecovered?: (event: { attempts: number }) => void
|
|
onGaveUp?: (event: { attempts: number; error: unknown; retryable: boolean }) => void
|
|
}
|
|
|
|
function retryDelayMs(policy: DatabaseStartupRetryPolicy, attempt: number): number {
|
|
const backoffMs = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs)
|
|
return backoffMs + Math.floor(Math.random() * (policy.jitterMs + 1))
|
|
}
|
|
|
|
// Startup work that a cold dependency - a proxy sidecar that just started, a
|
|
// database still accepting the fleet back - can fail once and serve a moment
|
|
// later. The wall-clock window, not the attempt count, is the real bound.
|
|
export async function retryTransientDatabaseStartup<T>(
|
|
operation: () => Promise<T>,
|
|
policy: DatabaseStartupRetryPolicy,
|
|
observer: DatabaseStartupRetryObserver = {}
|
|
): Promise<T> {
|
|
const retryDeadline = Date.now() + policy.windowMs
|
|
for (let attempt = 1; ; attempt += 1) {
|
|
try {
|
|
const result = await operation()
|
|
if (attempt > 1) observer.onRecovered?.({ attempts: attempt })
|
|
return result
|
|
} catch (error) {
|
|
const remainingMs = retryDeadline - Date.now()
|
|
const retryable = (policy.isRetryable ?? isRelayDatabaseTransientError)(error)
|
|
if (attempt === policy.attempts || remainingMs <= 0 || !retryable) {
|
|
observer.onGaveUp?.({ attempts: attempt, error, retryable })
|
|
throw error
|
|
}
|
|
const delayMs = Math.min(retryDelayMs(policy, attempt), remainingMs)
|
|
observer.onRetry?.({ attempt, delayMs, error })
|
|
await new Promise((resolve) => setTimeout(resolve, delayMs))
|
|
}
|
|
}
|
|
}
|