mirror of
https://github.com/stablyai/orca.git
synced 2026-11-01 09:37:33 +00:00
Jinwoo Hong
2a15a50691
Director only. When one pooled connection is far slower than its peers (median over 25 ms and 3x theirs, or third-slowest statement in 60 s over 200 ms and 5x every peer's slowest), it is destroyed on release and the pool dials a fresh one. At most one per pool per 10 s; never mid-transaction; no change in connection count. Fixes the 10-26 director stalls where one connection stalled the sticky lane.
1579 lines
60 KiB
TypeScript
1579 lines
60 KiB
TypeScript
import { mkdirSync } from 'node:fs'
|
||
import { performance } from 'node:perf_hooks'
|
||
import { join } from 'node:path'
|
||
import { DatabaseSync } from 'node:sqlite'
|
||
import pg from 'pg'
|
||
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||
import {
|
||
emptyPostgresPoolPressureCounts,
|
||
isPostgresPoolConnectFailure,
|
||
PostgresPoolPressure,
|
||
type PostgresPoolLane,
|
||
type PostgresPoolPressureCounts
|
||
} from './postgres-pool-pressure.js'
|
||
import { applyPostgresSchema } from './postgres-schema-startup.js'
|
||
import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
|
||
import { reportPostgresQueryFailure } from './postgres-query-failure.js'
|
||
import { noteStickySlotPoolWait, noteStickySlotStatement } from './sticky-slot-statements.js'
|
||
import { noteConnectionStatement, SlowConnectionRetirement } from './postgres-connection-latency.js'
|
||
import {
|
||
isPostgresReadTimeout,
|
||
isRelayDatabaseLayerError,
|
||
markRelayDatabaseError
|
||
} from './relay-database-rejection-fence.js'
|
||
import {
|
||
CellInventoryHoldSamples,
|
||
emptyCellInventoryHoldCounts,
|
||
type CellInventoryHoldCounts,
|
||
type CellLockHoldSite
|
||
} from './cell-inventory-hold-samples.js'
|
||
|
||
export const POSTGRES_LOCK_TIMEOUT_MS = 1_000
|
||
|
||
function setLocalLockTimeout(milliseconds: number): string {
|
||
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
|
||
throw new Error('invalid_lock_timeout')
|
||
}
|
||
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
|
||
}
|
||
|
||
// Region CHECK lists come from the contract so a new region cannot leave a
|
||
// column rejecting values the rest of the relay already accepts.
|
||
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
|
||
|
||
// A host that was just moved is not a candidate again for this long, so a
|
||
// desktop whose region probe flips cannot walk itself back and forth.
|
||
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
|
||
|
||
export type SqlRow = Record<string, unknown>
|
||
export type RelayLockOptions = {
|
||
failIfUnavailable?: boolean
|
||
// Only honoured inside a transaction: SET LOCAL is a no-op in autocommit.
|
||
lockTimeoutMs?: number
|
||
// Report how long this lock is held to COMMIT. The hold, not the wait, is what
|
||
// forms the queue, and nothing measured it before.
|
||
measureHoldMs?: boolean
|
||
// Labels the measured hold; unset reads as the full-inventory lock.
|
||
holdSite?: CellLockHoldSite
|
||
// The statement carries its own row-lock clause (a CTE locks inside the
|
||
// statement), so none is appended. failIfUnavailable must match that clause.
|
||
lockClauseInStatement?: boolean
|
||
}
|
||
|
||
type MeasuredHold = { holdMs: number; site: CellLockHoldSite }
|
||
|
||
// A transaction that can report how long it held a measured lock before COMMIT.
|
||
type HoldMeasuringTransaction = { consumeHold(): MeasuredHold | undefined }
|
||
|
||
function recordMeasuredHold(holds: CellInventoryHoldSamples, transaction: unknown): void {
|
||
const hold = (transaction as HoldMeasuringTransaction).consumeHold?.()
|
||
if (hold) holds.record(hold.holdMs, hold.site)
|
||
}
|
||
|
||
function lockedStatement(sql: string, options: RelayLockOptions): string {
|
||
if (options.lockClauseInStatement) return sql
|
||
return `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`
|
||
}
|
||
export type RelayTransactionOptions = { reportRetries?: boolean }
|
||
|
||
export interface RelayDatabase {
|
||
readonly dialect?: 'sqlite' | 'postgres'
|
||
query(sql: string, params?: unknown[]): Promise<SqlRow[]>
|
||
// Ahead of every queued query() for the next free connection; see
|
||
// PostgresPoolPressure. Absent means plain query().
|
||
queryPriority?(sql: string, params?: unknown[]): Promise<SqlRow[]>
|
||
queryLocked(
|
||
sql: string,
|
||
params?: unknown[],
|
||
options?: RelayLockOptions
|
||
): Promise<SqlRow[]>
|
||
transaction<T>(
|
||
operation: (transaction: RelayDatabase) => Promise<T>,
|
||
options?: RelayTransactionOptions
|
||
): Promise<T>
|
||
// Only on a PostgreSQL transaction handle; see commitWithFinalWrite.
|
||
commitWithFinal?(sql: string, params?: unknown[]): Promise<boolean>
|
||
close(): Promise<void>
|
||
}
|
||
|
||
// Final RETURNING write and COMMIT share one PostgreSQL message, avoiding a lock-held round trip; false means already rolled back.
|
||
export async function commitWithFinalWrite(
|
||
database: RelayDatabase,
|
||
sql: string,
|
||
params: unknown[] = []
|
||
): Promise<boolean> {
|
||
if (database.commitWithFinal) return await database.commitWithFinal(sql, params)
|
||
return (await database.query(sql, params)).length > 0
|
||
}
|
||
|
||
// RULE - no new index and no new column on `relay_control_connection_reservations`,
|
||
// `relay_confirm_results`, `relay_audit_events`, `relay_connection_bases`, or any other large
|
||
// table may be added to SCHEMA or to POSTGRES_SCHEMA_MIGRATIONS. The catalog pre-check skips a
|
||
// lock-taking statement only once the object exists, so a brand-new one reports missing on every
|
||
// director at once and each runs a non-concurrent build over the whole table. POSTGRES_LOCK_TIMEOUT_MS
|
||
// bounds how long that build waits for its lock, not how long it holds it. Build the index out of
|
||
// band with CREATE INDEX CONCURRENTLY first, then add it here, where the pre-check skips it forever
|
||
// after. relay-schema-lock-targets.test.ts pins the current list, so an addition fails CI.
|
||
// Constraint swaps are matched by NAME in pg_constraint, never by body, because the CHECK list is
|
||
// generated from REGION_LIST. Changing a constraint's definition under the same name therefore does
|
||
// nothing on boot: an operator drops it, and the next boot adds the current definition back.
|
||
// relay_confirmable_splices, relay_cell_drain_attempts and relay_migration_leases are no longer
|
||
// created; nothing ever wrote them. Databases that have them keep them empty until a drop is safe:
|
||
// an older image still creates them at boot, and a drop racing that CREATE can fail its schema step.
|
||
// Account erasure in orca-cloud must be deployed with retired-table support (orca-cloud#493) first.
|
||
const SCHEMA = `
|
||
CREATE TABLE IF NOT EXISTS relay_invites (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
relay_device_id TEXT NOT NULL,
|
||
token_hash TEXT NOT NULL UNIQUE,
|
||
state TEXT NOT NULL,
|
||
attempt_count BIGINT NOT NULL,
|
||
max_attempts BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL,
|
||
reservation_id TEXT,
|
||
reservation_expires_at BIGINT,
|
||
cooldown_until BIGINT,
|
||
created_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, relay_device_id, token_hash)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_invites_device
|
||
ON relay_invites(user_id, relay_host_id, relay_device_id);
|
||
|
||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||
-- Why: the credential sweep matches (state, expires_at) every cycle while invites in a terminal
|
||
-- state accumulate for the life of the database. Unindexed it seq-scans the whole table inside the
|
||
-- maintenance transaction. Partial, so the index holds only the states the sweep can act on.
|
||
CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry
|
||
ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown');
|
||
|
||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||
-- Why: the second sweep pass matches (state, reservation_expires_at) over the same table.
|
||
CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation
|
||
ON relay_invites(reservation_expires_at) WHERE state = 'reserved';
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_devices (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
relay_device_id TEXT NOT NULL,
|
||
current_hash TEXT NOT NULL,
|
||
current_version BIGINT NOT NULL,
|
||
current_expires_at BIGINT NOT NULL,
|
||
grace_hash TEXT,
|
||
grace_version BIGINT,
|
||
grace_expires_at BIGINT,
|
||
revoked_at BIGINT,
|
||
updated_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, relay_device_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_devices_current_hash ON relay_devices(relay_host_id, current_hash);
|
||
CREATE INDEX IF NOT EXISTS relay_devices_grace_hash ON relay_devices(relay_host_id, grace_hash);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_install_results (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
relay_device_id TEXT NOT NULL,
|
||
req_id TEXT NOT NULL,
|
||
authorization_mode TEXT NOT NULL,
|
||
result_json TEXT NOT NULL,
|
||
committed_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, relay_device_id, req_id)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_connection_bases (
|
||
basis_conn_id TEXT PRIMARY KEY,
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
relay_device_id TEXT NOT NULL,
|
||
owning_control_generation BIGINT NOT NULL,
|
||
credential_kind TEXT NOT NULL,
|
||
invite_token_hash TEXT,
|
||
accepted_credential_version BIGINT,
|
||
accepted_as TEXT,
|
||
deadline BIGINT NOT NULL,
|
||
active BIGINT NOT NULL,
|
||
created_at BIGINT NOT NULL
|
||
);
|
||
|
||
-- Why: the maintenance sweep matches (active, deadline) while inactive bases
|
||
-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle
|
||
-- and holds the maintenance transaction open long enough to time out
|
||
-- assignment lock waits.
|
||
-- Why not a partial index on active = 1: a basis is inserted active and flipped to 0, so each
|
||
-- deactivation leaves a dead entry in that index too. Measured on production-shaped history it
|
||
-- carries the same dead entries as this one, the planner picks this one in every state, and it
|
||
-- costs ~65 bytes of WAL per insert. Bloat here is cured by reaping and vacuum, not by a narrower
|
||
-- index.
|
||
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
|
||
ON relay_connection_bases(active, deadline);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_direct_authorizations (
|
||
direct_auth_id TEXT PRIMARY KEY,
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
relay_device_id TEXT NOT NULL,
|
||
owning_control_generation BIGINT NOT NULL,
|
||
deadline BIGINT NOT NULL,
|
||
consumed_at BIGINT
|
||
);
|
||
|
||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||
-- Why: the sweep expires pending authorizations by (consumed_at IS NULL, deadline), and consumed
|
||
-- rows are never deleted. Partial, so the index stays the size of the pending set.
|
||
CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline
|
||
ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL;
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_confirm_results (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
req_id TEXT NOT NULL,
|
||
basis_conn_id TEXT NOT NULL,
|
||
tuple_json TEXT NOT NULL,
|
||
result_json TEXT NOT NULL,
|
||
committed_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, req_id)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_assignments (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
cell_id TEXT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
lease_expires_at BIGINT NOT NULL,
|
||
last_activity_at BIGINT NOT NULL,
|
||
reserved_controls BIGINT NOT NULL,
|
||
reserved_splices BIGINT NOT NULL,
|
||
reserved_invites BIGINT NOT NULL,
|
||
pending_installs BIGINT NOT NULL,
|
||
pending_confirmations BIGINT NOT NULL,
|
||
migration_leases BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
preferred_region TEXT NOT NULL
|
||
CHECK (preferred_region IN (${REGION_LIST})),
|
||
observed_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
|
||
ON relay_assignment_region_preferences(observed_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_region_decisions (
|
||
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
|
||
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
|
||
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
|
||
cohort_bucket BIGINT NOT NULL DEFAULT 0,
|
||
last_considered_at BIGINT NOT NULL DEFAULT 0,
|
||
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
|
||
PRIMARY KEY (user_id, relay_host_id)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
|
||
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
|
||
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
|
||
finish_existing BIGINT NOT NULL,
|
||
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
|
||
PRIMARY KEY (user_id, relay_host_id, activity_id)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
|
||
worker_id TEXT PRIMARY KEY,
|
||
next_dispatch_at BIGINT NOT NULL,
|
||
paused_until BIGINT NOT NULL,
|
||
consecutive_failures BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
|
||
control_id TEXT PRIMARY KEY,
|
||
generation BIGINT NOT NULL,
|
||
enabled BIGINT NOT NULL,
|
||
observation_started_at BIGINT NOT NULL,
|
||
not_before BIGINT NOT NULL,
|
||
rate_per_minute BIGINT NOT NULL,
|
||
preference_max_age_ms BIGINT NOT NULL,
|
||
host_cooldown_ms BIGINT NOT NULL
|
||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
|
||
drain_grace_ms BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
preferred_region TEXT NOT NULL
|
||
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||
CHECK (preferred_region IN (${REGION_LIST})),
|
||
source_cell_id TEXT NOT NULL,
|
||
source_cell_incarnation TEXT NOT NULL,
|
||
source_generation BIGINT NOT NULL DEFAULT 0,
|
||
target_cell_id TEXT NOT NULL,
|
||
target_cell_incarnation TEXT NOT NULL,
|
||
previous_epoch BIGINT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
drain_grace_ms BIGINT NOT NULL,
|
||
send_attempts BIGINT NOT NULL,
|
||
last_send_attempt_at BIGINT,
|
||
drain_receipt_at BIGINT,
|
||
drain_outcome TEXT CHECK (
|
||
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
|
||
),
|
||
completed_at BIGINT,
|
||
aborted_at BIGINT,
|
||
abort_reason TEXT,
|
||
created_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL,
|
||
UNIQUE (user_id, relay_host_id, assignment_epoch)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
|
||
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
|
||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
|
||
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cells (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_url TEXT NOT NULL UNIQUE,
|
||
enabled BIGINT NOT NULL,
|
||
capacity_requests BIGINT NOT NULL,
|
||
reserved_requests BIGINT NOT NULL,
|
||
observed_requests BIGINT NOT NULL,
|
||
last_heartbeat_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_regions (
|
||
cell_id TEXT PRIMARY KEY,
|
||
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_admission (
|
||
cell_id TEXT PRIMARY KEY,
|
||
admission_state TEXT NOT NULL
|
||
CHECK (admission_state IN ('existing-only', 'migration-only', 'general')),
|
||
updated_at BIGINT NOT NULL,
|
||
roll_isolated_at BIGINT
|
||
);
|
||
|
||
-- Step 5: written by the flag workflow. A cell with no row is in database mode.
|
||
CREATE TABLE IF NOT EXISTS relay_cell_admit_modes (
|
||
cell_id TEXT PRIMARY KEY,
|
||
admit_mode TEXT NOT NULL CHECK (admit_mode IN ('db', 'reserve')),
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
-- Step 5: written by each cell, its raw mode (switch file and dead-man latch). The database
|
||
-- path places on a reserve cell only while this row, from its current process, says db.
|
||
CREATE TABLE IF NOT EXISTS relay_cell_admit_effective (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
mode TEXT NOT NULL CHECK (mode IN ('db', 'reserve')),
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_admission_selectors (
|
||
selector_id TEXT PRIMARY KEY,
|
||
generation BIGINT NOT NULL,
|
||
attempt_id TEXT,
|
||
membership_json TEXT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_admission_selector_intents (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
expected_generation BIGINT NOT NULL,
|
||
intended_generation BIGINT NOT NULL,
|
||
previous_membership_json TEXT NOT NULL,
|
||
membership_json TEXT NOT NULL,
|
||
created_at BIGINT NOT NULL,
|
||
committed_at BIGINT
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_admission_selector_cell_additions (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
cells_json TEXT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_runtime (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_url TEXT NOT NULL,
|
||
cell_incarnation TEXT NOT NULL,
|
||
started_at BIGINT NOT NULL,
|
||
ready BIGINT NOT NULL,
|
||
observed_requests BIGINT NOT NULL,
|
||
last_heartbeat_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_runtime_heartbeat
|
||
ON relay_cell_runtime(ready, last_heartbeat_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_capabilities (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
regional_rehome_protocol BIGINT NOT NULL,
|
||
last_heartbeat_at BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_rehome_safety (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
observed_at BIGINT NOT NULL,
|
||
sql_failures BIGINT NOT NULL,
|
||
reconnects BIGINT NOT NULL,
|
||
control_activity_recovery_failures BIGINT NOT NULL,
|
||
database_pool_waiting BIGINT NOT NULL,
|
||
database_pool_waiters_max BIGINT NOT NULL,
|
||
database_pool_wait_ms_max BIGINT NOT NULL
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_connection_limits (
|
||
cell_id TEXT PRIMARY KEY,
|
||
hard_cap BIGINT NOT NULL,
|
||
unobserved_bound BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL,
|
||
reported_hard_cap BIGINT
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_connection_runtime (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
total_connections BIGINT NOT NULL,
|
||
in_flight_connections BIGINT NOT NULL,
|
||
reserved_connection_units BIGINT NOT NULL,
|
||
enforced_connection_units BIGINT NOT NULL,
|
||
last_heartbeat_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_connection_runtime_heartbeat
|
||
ON relay_cell_connection_runtime(last_heartbeat_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_connection_snapshots (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
inclusion_watermark BIGINT NOT NULL,
|
||
total_connections BIGINT NOT NULL,
|
||
in_flight_connections BIGINT NOT NULL,
|
||
reserved_connection_units BIGINT NOT NULL,
|
||
enforced_connection_units BIGINT NOT NULL,
|
||
snapshot_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_connection_snapshot_freshness
|
||
ON relay_cell_connection_snapshots(snapshot_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_fences (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
attested_at BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_fences_expiry
|
||
ON relay_cell_fences(expires_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_committed_fences (
|
||
cell_id TEXT PRIMARY KEY,
|
||
attempt_id TEXT NOT NULL UNIQUE,
|
||
cell_incarnation TEXT NOT NULL,
|
||
attested_at BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_committed_fences_expiry
|
||
ON relay_cell_committed_fences(expires_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_legacy_fence_adoptions (
|
||
cell_id TEXT PRIMARY KEY,
|
||
cell_incarnation TEXT NOT NULL,
|
||
attested_at BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_legacy_fence_adoptions_expiry
|
||
ON relay_cell_legacy_fence_adoptions(expires_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_fence_attempts (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
environment TEXT NOT NULL,
|
||
cell_id TEXT NOT NULL,
|
||
cell_incarnation TEXT NOT NULL,
|
||
mig_name TEXT NOT NULL,
|
||
instance_group TEXT NOT NULL,
|
||
generation_identity TEXT NOT NULL,
|
||
fence_commit TEXT NOT NULL,
|
||
plan_sha256 TEXT NOT NULL,
|
||
gce_operation TEXT,
|
||
created_at BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL,
|
||
apply_started_at BIGINT,
|
||
completed_at BIGINT,
|
||
aborted_at BIGINT
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_fence_attempts_expiry
|
||
ON relay_cell_fence_attempts(expires_at);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_fence_attempts_cell
|
||
ON relay_cell_fence_attempts(cell_id, created_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_fence_plan_bindings (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
plan_object_name TEXT NOT NULL,
|
||
plan_object_generation TEXT,
|
||
var_file_sha256 TEXT NOT NULL,
|
||
terraform_state_lineage TEXT NOT NULL,
|
||
terraform_state_serial BIGINT NOT NULL,
|
||
terraform_state_object_generation TEXT NOT NULL,
|
||
terraform_state_object_sha256 TEXT NOT NULL,
|
||
request_reason TEXT NOT NULL,
|
||
FOREIGN KEY (attempt_id) REFERENCES relay_cell_fence_attempts(attempt_id)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_fence_apply_invocations (
|
||
invocation_id TEXT PRIMARY KEY,
|
||
attempt_id TEXT NOT NULL,
|
||
request_reason TEXT NOT NULL UNIQUE,
|
||
started_at BIGINT NOT NULL,
|
||
gce_operation TEXT,
|
||
FOREIGN KEY (attempt_id) REFERENCES relay_cell_fence_attempts(attempt_id)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_fence_apply_invocations_attempt
|
||
ON relay_cell_fence_apply_invocations(attempt_id, started_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_drain_attempt_states (
|
||
attempt_id TEXT PRIMARY KEY,
|
||
cell_id TEXT NOT NULL,
|
||
cell_incarnation TEXT NOT NULL,
|
||
trace_value TEXT NOT NULL UNIQUE,
|
||
planned_grace_ms BIGINT NOT NULL,
|
||
state TEXT NOT NULL CHECK (
|
||
state IN (
|
||
'prepared',
|
||
'send-may-have-started',
|
||
'application-receipt',
|
||
'proven-not-delivered'
|
||
)
|
||
),
|
||
prepared_at BIGINT NOT NULL,
|
||
send_may_have_started_at BIGINT,
|
||
send_permit_expires_at BIGINT,
|
||
application_receipt_at BIGINT,
|
||
backend_success_status BIGINT,
|
||
backend_instance TEXT,
|
||
receipt_cell_incarnation TEXT,
|
||
retry_after BIGINT,
|
||
recover_forward_attempted_at BIGINT,
|
||
proven_not_delivered_at BIGINT
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_cell_drain_attempt_states_cell
|
||
ON relay_cell_drain_attempt_states(cell_id, prepared_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_cell_drain_recovery_attempts (
|
||
drain_attempt_id TEXT NOT NULL,
|
||
cell_incarnation TEXT NOT NULL,
|
||
attempted_at BIGINT NOT NULL,
|
||
PRIMARY KEY (drain_attempt_id, cell_incarnation),
|
||
FOREIGN KEY (drain_attempt_id) REFERENCES relay_cell_drain_attempt_states(attempt_id)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
activity_id TEXT NOT NULL,
|
||
activity_kind TEXT NOT NULL,
|
||
cell_id TEXT NOT NULL,
|
||
request_units BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, activity_id)
|
||
);
|
||
-- expires_at is deliberately unindexed: every control renewal writes it (~471/s), so an index on
|
||
-- it makes each renewal a non-HOT update that rewrites index entries. Its only reader is the 30s
|
||
-- expiry sweep, which seq-scans 14.8k rows / 7MB in a few milliseconds.
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_control_connection_reservations (
|
||
reservation_id TEXT PRIMARY KEY,
|
||
idempotency_key TEXT NOT NULL UNIQUE,
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
cell_id TEXT NOT NULL,
|
||
state TEXT NOT NULL
|
||
CHECK (state IN ('reserved', 'late-arrival-debt', 'claimed', 'released')),
|
||
inclusion_watermark BIGINT,
|
||
claim_activity_id TEXT,
|
||
created_at BIGINT NOT NULL,
|
||
timeout_at BIGINT NOT NULL,
|
||
claimed_at BIGINT,
|
||
released_at BIGINT,
|
||
updated_at BIGINT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_control_connection_reservation_headroom
|
||
ON relay_control_connection_reservations(cell_id, state);
|
||
CREATE INDEX IF NOT EXISTS relay_control_connection_reservation_assignment
|
||
ON relay_control_connection_reservations(
|
||
user_id, relay_host_id, assignment_epoch, cell_id, created_at
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_rate_windows (
|
||
scope_key TEXT NOT NULL,
|
||
window_kind TEXT NOT NULL,
|
||
window_started_at BIGINT NOT NULL,
|
||
count BIGINT NOT NULL,
|
||
PRIMARY KEY (scope_key, window_kind, window_started_at)
|
||
);
|
||
|
||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||
-- Why: window_started_at is the PRIMARY KEY's last column, so the sweep's 24h retention delete
|
||
-- cannot use it and seq-scans instead.
|
||
CREATE INDEX IF NOT EXISTS relay_rate_windows_started
|
||
ON relay_rate_windows(window_started_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_assignment_migrations (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
source_cell_id TEXT NOT NULL,
|
||
target_cell_id TEXT NOT NULL,
|
||
previous_epoch BIGINT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
source_request_units BIGINT NOT NULL,
|
||
target_reserved_units BIGINT NOT NULL,
|
||
expires_at BIGINT NOT NULL,
|
||
target_registered_at BIGINT,
|
||
completed_at BIGINT,
|
||
aborted_at BIGINT,
|
||
created_at BIGINT NOT NULL,
|
||
updated_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, assignment_epoch)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_assignment_migrations_active
|
||
ON relay_assignment_migrations(expires_at, completed_at, aborted_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_assignment_migration_incarnations (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
source_cell_incarnation TEXT NOT NULL,
|
||
target_cell_incarnation TEXT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, assignment_epoch)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_post_drain_migration_pins (
|
||
user_id TEXT NOT NULL,
|
||
relay_host_id TEXT NOT NULL,
|
||
assignment_epoch BIGINT NOT NULL,
|
||
drain_attempt_id TEXT NOT NULL,
|
||
source_cell_id TEXT NOT NULL,
|
||
source_cell_incarnation TEXT NOT NULL,
|
||
target_cell_id TEXT NOT NULL,
|
||
target_cell_incarnation TEXT NOT NULL,
|
||
source_request_units BIGINT NOT NULL,
|
||
target_reserved_units BIGINT NOT NULL,
|
||
pinned_at BIGINT NOT NULL,
|
||
PRIMARY KEY (user_id, relay_host_id, assignment_epoch)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_post_drain_migration_pins_attempt
|
||
ON relay_post_drain_migration_pins(drain_attempt_id);
|
||
|
||
CREATE TABLE IF NOT EXISTS relay_audit_events (
|
||
id TEXT PRIMARY KEY,
|
||
at BIGINT NOT NULL,
|
||
type TEXT NOT NULL,
|
||
user_id TEXT,
|
||
relay_host_id TEXT,
|
||
relay_device_id TEXT,
|
||
detail_json TEXT NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
|
||
`
|
||
|
||
// Replace auto-named legacy single-region checks with named bidirectional checks; current-schema databases no-op without either statement dropping the other.
|
||
export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||
POSTGRES_STATEMENT_STATS_MIGRATION,
|
||
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
|
||
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
|
||
`ALTER TABLE relay_region_rehome_attempts
|
||
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
|
||
`ALTER TABLE relay_region_rehome_attempts
|
||
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||
CHECK (preferred_region IN (${REGION_LIST}))`,
|
||
`ALTER TABLE relay_region_rehome_control
|
||
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
|
||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
|
||
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
|
||
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
|
||
// Nullable with no default, so the rewrite is catalog-only; every row
|
||
// aborted before this column existed reads as an unattributed abort.
|
||
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS abort_reason TEXT`,
|
||
// Only same-cap roll stamps migration-only; other parked hosts stay pinned. Nullable/no-default is catalog-only, and pre-column isolates retain unmarked legacy behavior.
|
||
`ALTER TABLE relay_cell_admission ADD COLUMN IF NOT EXISTS roll_isolated_at BIGINT`,
|
||
// The cap the cell last reported enforcing; see cell-connection-cap.ts. Nullable with no default,
|
||
// so the rewrite is catalog-only, and a director that predates it neither reads nor clears it.
|
||
`ALTER TABLE relay_cell_connection_limits ADD COLUMN IF NOT EXISTS reported_hard_cap BIGINT`,
|
||
// Deferrable DROP: one boot must win ACCESS EXCLUSIVE against continuous writes; restarting 28 directors would repeat the lock queue. Once absent, catalog precheck skips it.
|
||
`-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s
|
||
DROP INDEX IF EXISTS relay_assignment_activity_expiry`,
|
||
// Dropping expiry index permits HOT; fillfactor 70 leaves room for versions (100% HOT vs 0.5% at 100). Catalog-only SHARE UPDATE EXCLUSIVE blocks vacuum/DDL, not reads/writes; defer with the drop and let pages converge through renewals.
|
||
`-- schema-deferrable: buys nothing until the drop above lands
|
||
ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`,
|
||
// Slow ~5.5GB-index autovacuums from ~3 to ~15min at 20ms/200 cost units to reduce shared-DB cache eviction. Deferrable: vacuum holds the same lock, and boot needs no change.
|
||
`-- schema-deferrable: a running vacuum holds the lock this needs
|
||
ALTER TABLE relay_control_connection_reservations SET (autovacuum_vacuum_cost_delay = 20)`,
|
||
`-- schema-deferrable: a running vacuum holds the lock this needs
|
||
ALTER TABLE relay_control_connection_reservations SET (autovacuum_vacuum_cost_limit = 200)`
|
||
]
|
||
|
||
// The exact statement list a Postgres boot applies, in order, so the lock-target census can read
|
||
// what production runs rather than a copy of it. The SQLite path keeps SCHEMA on its own.
|
||
export function relayPostgresSchemaStatements(): string[] {
|
||
return [...SCHEMA.split(';'), ...POSTGRES_SCHEMA_MIGRATIONS]
|
||
.map((statement) => statement.trim())
|
||
.filter((statement) => statement.length > 0)
|
||
}
|
||
|
||
function postgresSql(sql: string): string {
|
||
let index = 0
|
||
return sql.replace(/\?/g, () => `$${++index}`)
|
||
}
|
||
|
||
function returnsRows(sql: string): boolean {
|
||
return /^\s*(select|with)/i.test(sql) || /returning/i.test(sql)
|
||
}
|
||
|
||
const POSTGRES_TRANSACTION_PHASES = [
|
||
['relay_region_rehome_', 'regional-rehome'],
|
||
['relay_assignment_activity_leases', 'activity-lease'],
|
||
['relay_assignment_migration', 'migration'],
|
||
['relay_post_drain_migration_pins', 'migration'],
|
||
['relay_cell_connection_runtime', 'cell-runtime'],
|
||
['relay_cell_connection_snapshots', 'cell-runtime'],
|
||
['relay_cell_runtime', 'cell-runtime'],
|
||
['relay_cell_drain_', 'cell-operation'],
|
||
['relay_cell_fence', 'cell-operation'],
|
||
['relay_cell_committed_fences', 'cell-operation'],
|
||
['relay_cell_legacy_fence_adoptions', 'cell-operation'],
|
||
['relay_admission_selector', 'admission'],
|
||
['relay_cell_admission', 'admission'],
|
||
['relay_control_connection_reservations', 'connection'],
|
||
['relay_connection_bases', 'connection'],
|
||
['relay_direct_authorizations', 'connection'],
|
||
['relay_confirm_results', 'connection'],
|
||
['relay_assignment_region_preferences', 'assignment'],
|
||
['relay_assignments', 'assignment'],
|
||
['relay_cell_', 'cell-inventory'],
|
||
['relay_cells', 'cell-inventory'],
|
||
['relay_invites', 'credential'],
|
||
['relay_devices', 'credential'],
|
||
['relay_install_results', 'credential'],
|
||
['relay_rate_windows', 'rate-limit'],
|
||
['relay_audit_events', 'audit']
|
||
] as const
|
||
|
||
const postgresTransactionPhaseByError = new WeakMap<object, string>()
|
||
|
||
function postgresTransactionPhase(sql: string): string {
|
||
const normalized = sql.toLowerCase()
|
||
return POSTGRES_TRANSACTION_PHASES.find(([table]) => normalized.includes(table))?.[1] ?? 'other'
|
||
}
|
||
|
||
function rememberPostgresTransactionPhase(error: unknown, sql: string): void {
|
||
if (typeof error === 'object' && error !== null) {
|
||
postgresTransactionPhaseByError.set(error, postgresTransactionPhase(sql))
|
||
}
|
||
}
|
||
|
||
function postgresTransactionErrorPhase(error: unknown): string {
|
||
return typeof error === 'object' && error !== null
|
||
? (postgresTransactionPhaseByError.get(error) ?? 'transaction')
|
||
: 'transaction'
|
||
}
|
||
|
||
class SqliteTransaction implements RelayDatabase {
|
||
readonly dialect = 'sqlite' as const
|
||
private held: { fromMs: number; site: CellLockHoldSite } | undefined
|
||
|
||
constructor(protected readonly database: DatabaseSync) {}
|
||
|
||
consumeHold(): MeasuredHold | undefined {
|
||
if (this.held === undefined) return undefined
|
||
const hold = { holdMs: performance.now() - this.held.fromMs, site: this.held.site }
|
||
this.held = undefined
|
||
return hold
|
||
}
|
||
|
||
protected noteHeld(options: RelayLockOptions): void {
|
||
if (options.measureHoldMs && this.held === undefined) {
|
||
this.held = { fromMs: performance.now(), site: options.holdSite ?? 'inventory' }
|
||
}
|
||
}
|
||
|
||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||
const statement = this.database.prepare(sql)
|
||
const bound = params.map((value) => (value === undefined ? null : value)) as never[]
|
||
if (returnsRows(sql)) return statement.all(...bound) as SqlRow[]
|
||
const result = statement.run(...bound)
|
||
return [{ changes: Number(result.changes) }]
|
||
}
|
||
|
||
async queryLocked(
|
||
sql: string,
|
||
params: unknown[] = [],
|
||
options: RelayLockOptions = {}
|
||
): Promise<SqlRow[]> {
|
||
const rows = await this.query(sql, params)
|
||
this.noteHeld(options)
|
||
return rows
|
||
}
|
||
|
||
async transaction<T>(
|
||
operation: (transaction: RelayDatabase) => Promise<T>,
|
||
_options: RelayTransactionOptions = {}
|
||
): Promise<T> {
|
||
return await operation(this)
|
||
}
|
||
|
||
async close(): Promise<void> {}
|
||
}
|
||
|
||
class SqliteDatabase extends SqliteTransaction {
|
||
private tail: Promise<void> = Promise.resolve()
|
||
private readonly holds = new CellInventoryHoldSamples()
|
||
|
||
consumeHoldCounts(): CellInventoryHoldCounts {
|
||
return this.holds.consumeCounts()
|
||
}
|
||
|
||
override async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||
await this.tail
|
||
return await super.query(sql, params)
|
||
}
|
||
|
||
override async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
|
||
const previous = this.tail
|
||
let release!: () => void
|
||
this.tail = new Promise((resolve) => (release = resolve))
|
||
await previous
|
||
try {
|
||
this.database.exec('BEGIN IMMEDIATE')
|
||
const transaction = new SqliteTransaction(this.database)
|
||
try {
|
||
const result = await operation(transaction)
|
||
this.database.exec('COMMIT')
|
||
recordMeasuredHold(this.holds, transaction)
|
||
return result
|
||
} catch (error) {
|
||
this.database.exec('ROLLBACK')
|
||
throw error
|
||
}
|
||
} finally {
|
||
release()
|
||
}
|
||
}
|
||
|
||
override async close(): Promise<void> {
|
||
await this.tail
|
||
this.database.close()
|
||
}
|
||
}
|
||
|
||
// Literals for a simple-query message, which carries no bind parameters. Only safe
|
||
// integers and strings: anything else is a caller bug, not something to stringify.
|
||
function inlinePostgresParameters(sql: string, params: unknown[], client: pg.PoolClient): string {
|
||
let index = 0
|
||
const inlined = sql.replace(/\?/g, () => {
|
||
const value = params[index++]
|
||
if (typeof value === 'number' && Number.isSafeInteger(value)) return String(value)
|
||
if (typeof value === 'string') return client.escapeLiteral(value)
|
||
throw new Error('unsupported_inline_parameter')
|
||
})
|
||
if (index !== params.length) throw new Error('inline_parameter_count_mismatch')
|
||
return inlined
|
||
}
|
||
|
||
// A per-query read timeout, when the cell's switch overrides the pool's (pg reads
|
||
// `query_timeout` from a query config object first). Undefined keeps the pool's.
|
||
type PostgresQueryTimeout = () => number | undefined
|
||
|
||
async function timedQuery(
|
||
client: pg.PoolClient,
|
||
connectionId: number,
|
||
text: string,
|
||
values: unknown[] | undefined,
|
||
queryTimeout: PostgresQueryTimeout | undefined
|
||
): Promise<pg.QueryResult> {
|
||
const startedAt = performance.now()
|
||
try {
|
||
const timeout = queryTimeout?.()
|
||
if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values)
|
||
// pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig.
|
||
const config: pg.QueryConfig & { query_timeout: number } = {
|
||
text,
|
||
...(values === undefined ? {} : { values }),
|
||
query_timeout: timeout
|
||
}
|
||
return await client.query(config)
|
||
} finally {
|
||
const durationMs = performance.now() - startedAt
|
||
noteStickySlotStatement(durationMs, connectionId)
|
||
noteConnectionStatement(client, durationMs)
|
||
}
|
||
}
|
||
|
||
class PostgresTransaction implements RelayDatabase {
|
||
readonly dialect = 'postgres' as const
|
||
private held: { fromMs: number; site: CellLockHoldSite } | undefined
|
||
private lockUnavailable = 0
|
||
private lockTimeouts = 0
|
||
private state: 'open' | 'committed' | 'rolled-back' = 'open'
|
||
// Lost replies leave pg’s active query running; queued COMMIT could commit caller-failed work, so poison and destroy this connection.
|
||
private lostReply: Error | undefined
|
||
|
||
constructor(
|
||
protected readonly client: pg.PoolClient,
|
||
private readonly connectionId: number,
|
||
private readonly queryTimeout?: PostgresQueryTimeout
|
||
) {}
|
||
|
||
get poisonedBy(): Error | undefined {
|
||
return this.lostReply
|
||
}
|
||
|
||
private failIfPoisoned(): void {
|
||
if (this.lostReply) throw this.lostReply
|
||
}
|
||
|
||
private noteFailure(error: unknown): void {
|
||
markRelayDatabaseError(error)
|
||
if (isPostgresReadTimeout(error)) this.lostReply ??= error
|
||
}
|
||
|
||
get open(): boolean {
|
||
return this.state === 'open'
|
||
}
|
||
|
||
async commitWithFinal(sql: string, params: unknown[] = []): Promise<boolean> {
|
||
this.assertNotCommitted()
|
||
this.failIfPoisoned()
|
||
// Zero rows divides by zero, so the message stops before COMMIT exactly when the write missed.
|
||
const message =
|
||
`WITH final_write AS (${inlinePostgresParameters(sql, params, this.client)}) ` +
|
||
'SELECT 1 / (SELECT count(*)::int FROM final_write); COMMIT'
|
||
try {
|
||
await timedQuery(this.client, this.connectionId, message, undefined, this.queryTimeout)
|
||
} catch (error) {
|
||
// Simple query stops on first error, before COMMIT; retry server errors normally, but never unknown outcomes after connection loss.
|
||
this.noteFailure(error)
|
||
if (String((error as { code?: unknown }).code) !== '22012') {
|
||
rememberPostgresTransactionPhase(error, sql)
|
||
throw error
|
||
}
|
||
await this.client.query('ROLLBACK').catch((rollbackError: unknown) => {
|
||
this.noteFailure(rollbackError)
|
||
throw rollbackError
|
||
})
|
||
this.state = 'rolled-back'
|
||
return false
|
||
}
|
||
this.state = 'committed'
|
||
return true
|
||
}
|
||
|
||
private assertNotCommitted(): void {
|
||
// A later statement would run in autocommit, outside the work it belongs to.
|
||
if (this.state === 'committed') throw new Error('postgres_transaction_already_committed')
|
||
}
|
||
|
||
consumeHold(): MeasuredHold | undefined {
|
||
if (this.held === undefined) return undefined
|
||
const hold = { holdMs: performance.now() - this.held.fromMs, site: this.held.site }
|
||
this.held = undefined
|
||
return hold
|
||
}
|
||
|
||
// Drain metrics on rollback too or contended 55P03 attempts disappear from counts.
|
||
consumeLockUnavailable(): number {
|
||
const count = this.lockUnavailable
|
||
this.lockUnavailable = 0
|
||
return count
|
||
}
|
||
|
||
consumeLockTimeouts(): number {
|
||
const count = this.lockTimeouts
|
||
this.lockTimeouts = 0
|
||
return count
|
||
}
|
||
|
||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||
this.assertNotCommitted()
|
||
this.failIfPoisoned()
|
||
try {
|
||
const result = await timedQuery(
|
||
this.client,
|
||
this.connectionId,
|
||
postgresSql(sql),
|
||
params,
|
||
this.queryTimeout
|
||
)
|
||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||
} catch (error) {
|
||
this.noteFailure(error)
|
||
rememberPostgresTransactionPhase(error, sql)
|
||
throw error
|
||
}
|
||
}
|
||
|
||
async queryLocked(
|
||
sql: string,
|
||
params: unknown[] = [],
|
||
options: RelayLockOptions = {}
|
||
): Promise<SqlRow[]> {
|
||
// Restore SET LOCAL after lock or later statements inherit the bound and misattribute 55P03s.
|
||
const bounded = options.lockTimeoutMs !== undefined && !options.failIfUnavailable
|
||
try {
|
||
// A blocked waiter holds its pooled client for the whole lock_timeout, so
|
||
// hot tiny-table locks bound their own wait well under the pool default.
|
||
if (bounded) await this.query(setLocalLockTimeout(options.lockTimeoutMs!))
|
||
const rows = await this.query(lockedStatement(sql, options), params)
|
||
if (options.measureHoldMs && this.held === undefined) {
|
||
this.held = { fromMs: performance.now(), site: options.holdSite ?? 'inventory' }
|
||
}
|
||
return rows
|
||
} catch (error) {
|
||
if (
|
||
options.failIfUnavailable &&
|
||
String((error as { code?: unknown }).code) === '55P03'
|
||
) {
|
||
if (options.measureHoldMs) this.lockUnavailable += 1
|
||
throw new Error('database_lock_unavailable')
|
||
}
|
||
// A bounded wait that expires raises the same 55P03 without NOWAIT. This is
|
||
// the request path, so it is counted apart from by-design sweep deferrals.
|
||
if (bounded && options.measureHoldMs && String((error as { code?: unknown }).code) === '55P03') {
|
||
this.lockTimeouts += 1
|
||
}
|
||
throw error
|
||
} finally {
|
||
// Restore on the error path too: the transaction may still be retried or
|
||
// continue with unrelated locks after a caught lock failure.
|
||
if (bounded) await this.query(setLocalLockTimeout(POSTGRES_LOCK_TIMEOUT_MS)).catch(() => undefined)
|
||
}
|
||
}
|
||
|
||
async transaction<T>(
|
||
operation: (transaction: RelayDatabase) => Promise<T>,
|
||
_options: RelayTransactionOptions = {}
|
||
): Promise<T> {
|
||
return await operation(this)
|
||
}
|
||
|
||
async close(): Promise<void> {}
|
||
}
|
||
|
||
const POSTGRES_TRANSACTION_ATTEMPTS = 3
|
||
const POSTGRES_RETRY_MAX_DELAY_MS = 25
|
||
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
|
||
// Three attempts × 5s timeout = 15s, half the 30s renewal tick, leaving connect-timeout room.
|
||
export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
|
||
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
|
||
// Allow routine 5–7s VM stalls to deliver already-timed replies; shorter read deadlines would destroy otherwise healthy clients (connect-burst RCA).
|
||
export const POSTGRES_READ_TIMEOUT_MARGIN_MS = 10_000
|
||
const POSTGRES_IDLE_SESSION_TIMEOUT_MS = 30_000
|
||
|
||
export function relayPostgresStatementTimeoutMs(
|
||
env: NodeJS.ProcessEnv = process.env
|
||
): number {
|
||
const configured = env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
|
||
if (configured === undefined || configured === '') return POSTGRES_STATEMENT_TIMEOUT_MS
|
||
const milliseconds = Number(configured)
|
||
// 0 is PostgreSQL's "no timeout"; refusing it keeps the deadline this exists
|
||
// to enforce from being disabled by a typo in an environment variable.
|
||
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
|
||
throw new Error('invalid_statement_timeout')
|
||
}
|
||
return milliseconds
|
||
}
|
||
|
||
export function relayPostgresReadTimeoutMarginMs(
|
||
env: NodeJS.ProcessEnv = process.env
|
||
): number {
|
||
const configured = env.ORCA_RELAY_POSTGRES_READ_TIMEOUT_MARGIN_MS
|
||
if (configured === undefined || configured === '') return POSTGRES_READ_TIMEOUT_MARGIN_MS
|
||
const milliseconds = Number(configured)
|
||
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
|
||
throw new Error('invalid_read_timeout_margin')
|
||
}
|
||
return milliseconds
|
||
}
|
||
|
||
function retryablePostgresTransactionError(error: unknown): boolean {
|
||
const code = String((error as { code?: unknown }).code)
|
||
// 57014 aborts like lock timeout; use bounded transaction retry instead of terminal failure.
|
||
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
|
||
}
|
||
|
||
export function isRelayDatabaseTransientError(error: unknown): boolean {
|
||
// Runs inside the query catch, where a thrown null or undefined would turn a
|
||
// database failure into a TypeError that buries it.
|
||
const code = String((error as { code?: unknown } | null)?.code)
|
||
if (
|
||
[
|
||
'40P01',
|
||
'40001',
|
||
'55P03',
|
||
'57014',
|
||
'53300',
|
||
'57P03',
|
||
'08000',
|
||
'08001',
|
||
'08003',
|
||
'08006',
|
||
// The server ended the session: pg_terminate_backend or a fast/immediate shutdown.
|
||
'57P01',
|
||
'57P02',
|
||
// The server ended a session idle in a transaction past its limit (a database stall).
|
||
'25P03'
|
||
].includes(code)
|
||
) {
|
||
return true
|
||
}
|
||
// The connection dropped mid-statement: pg's own message, or the socket's reset. Only from
|
||
// the database layer, since any other socket can raise the same errno.
|
||
if (
|
||
isRelayDatabaseLayerError(error) &&
|
||
(code === 'ECONNRESET' ||
|
||
code === 'EPIPE' ||
|
||
String((error as { message?: unknown }).message).startsWith('Connection terminated'))
|
||
) {
|
||
return true
|
||
}
|
||
// Lost reply answers 503 but may have committed: no transaction retry. Startup reconcile alone retries convergent upserts (cell-admission-startup.ts).
|
||
if (isPostgresReadTimeout(error)) return true
|
||
// A pool that cannot hand out a client reports no SQLSTATE at all, so the
|
||
// acquire boundary owns that vocabulary.
|
||
return isPostgresPoolConnectFailure(error)
|
||
}
|
||
|
||
// pg may still call ended sessions queryable until socket end; destroy on release or the next caller inherits a dead connection.
|
||
const SESSION_ENDING_CODES = new Set(['57P01', '57P02', '57P03', '57P04', '57P05', '25P03'])
|
||
function endsPostgresSession(error: unknown): boolean {
|
||
if (typeof error !== 'object' || error === null) return false
|
||
const code = 'code' in error ? String(error.code) : ''
|
||
if (SESSION_ENDING_CODES.has(code) || code.startsWith('08')) return true
|
||
return error instanceof Error && error.message.startsWith('Connection terminated')
|
||
}
|
||
|
||
async function waitForPostgresRetry(random: () => number = Math.random): Promise<void> {
|
||
const delayMs = Math.floor(random() * (POSTGRES_RETRY_MAX_DELAY_MS + 1))
|
||
await new Promise((resolve) => setTimeout(resolve, delayMs))
|
||
}
|
||
|
||
export class PostgresDatabase implements RelayDatabase {
|
||
readonly dialect = 'postgres' as const
|
||
private readonly pressure: PostgresPoolPressure
|
||
private readonly holds = new CellInventoryHoldSamples()
|
||
// pg-pool hands the same client object back for the same connection.
|
||
private readonly connectionIds = new WeakMap<pg.PoolClient, number>()
|
||
private nextConnectionId = 1
|
||
private readonly slowConnections: SlowConnectionRetirement | undefined
|
||
|
||
consumeHoldCounts(): CellInventoryHoldCounts {
|
||
return this.holds.consumeCounts()
|
||
}
|
||
|
||
constructor(
|
||
private readonly pool: pg.Pool,
|
||
private readonly queryTimeout?: PostgresQueryTimeout,
|
||
options: { retiresSlowConnections?: boolean } = {}
|
||
) {
|
||
this.pressure = new PostgresPoolPressure(pool)
|
||
this.slowConnections = options.retiresSlowConnections ? new SlowConnectionRetirement() : undefined
|
||
}
|
||
|
||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||
return await this.queryOnLane('general', sql, params)
|
||
}
|
||
|
||
async queryPriority(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||
return await this.queryOnLane('priority', sql, params)
|
||
}
|
||
|
||
private async queryOnLane(
|
||
lane: PostgresPoolLane,
|
||
sql: string,
|
||
params: unknown[]
|
||
): Promise<SqlRow[]> {
|
||
const startedAt = performance.now()
|
||
let phase: 'acquire' | 'execute' = 'acquire'
|
||
let client: pg.PoolClient | undefined
|
||
let lostReply: Error | undefined
|
||
try {
|
||
client = await this.connect(lane)
|
||
phase = 'execute'
|
||
const result = await timedQuery(
|
||
client,
|
||
this.idOfConnection(client),
|
||
postgresSql(sql),
|
||
params,
|
||
this.queryTimeout
|
||
)
|
||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||
} catch (error) {
|
||
markRelayDatabaseError(error)
|
||
if (isPostgresReadTimeout(error)) lostReply = error
|
||
else if (error instanceof Error && endsPostgresSession(error)) lostReply = error
|
||
reportPostgresQueryFailure({
|
||
error,
|
||
phase,
|
||
sql,
|
||
// Passed in rather than re-derived: the log has to say what the routes
|
||
// actually did, and one classifier cannot drift from itself.
|
||
transient: isRelayDatabaseTransientError(error),
|
||
elapsedMs: performance.now() - startedAt,
|
||
pool: this.pool
|
||
})
|
||
throw error
|
||
} finally {
|
||
// An error destroys the client rather than handing the next caller a dead connection.
|
||
if (client) {
|
||
this.release(client, lostReply)
|
||
}
|
||
}
|
||
}
|
||
|
||
async queryLocked(
|
||
sql: string,
|
||
params: unknown[] = [],
|
||
options: RelayLockOptions = {}
|
||
): Promise<SqlRow[]> {
|
||
try {
|
||
// No transaction here, so options.lockTimeoutMs cannot apply: SET LOCAL
|
||
// would be discarded at the autocommit boundary before the lock is taken.
|
||
return await this.query(lockedStatement(sql, options), params)
|
||
} catch (error) {
|
||
if (
|
||
options.failIfUnavailable &&
|
||
String((error as { code?: unknown }).code) === '55P03'
|
||
) {
|
||
if (options.measureHoldMs) this.holds.recordUnavailable()
|
||
throw new Error('database_lock_unavailable')
|
||
}
|
||
throw error
|
||
}
|
||
}
|
||
|
||
async transaction<T>(
|
||
operation: (transaction: RelayDatabase) => Promise<T>,
|
||
options: RelayTransactionOptions = {}
|
||
): Promise<T> {
|
||
for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
|
||
const client = await this.connect()
|
||
const connectionId = this.idOfConnection(client)
|
||
const transaction = new PostgresTransaction(client, connectionId, this.queryTimeout)
|
||
let lostReply: Error | undefined
|
||
let sessionEnded: Error | undefined
|
||
try {
|
||
await timedQuery(client, connectionId, 'BEGIN', undefined, this.queryTimeout)
|
||
const result = await operation(transaction)
|
||
// Even when the operation caught it: COMMIT must not run after a lost reply.
|
||
if (transaction.poisonedBy) throw transaction.poisonedBy
|
||
if (transaction.open) {
|
||
await timedQuery(client, connectionId, 'COMMIT', undefined, this.queryTimeout)
|
||
}
|
||
recordMeasuredHold(this.holds, transaction)
|
||
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
|
||
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
|
||
return result
|
||
} catch (error) {
|
||
markRelayDatabaseError(error)
|
||
// No ROLLBACK behind a lost reply: it would queue behind the dead statement. Destroying
|
||
// the client ends the session, and the server drops its locks.
|
||
lostReply = transaction.poisonedBy ?? (isPostgresReadTimeout(error) ? error : undefined)
|
||
if (error instanceof Error && endsPostgresSession(error)) sessionEnded = error
|
||
const open = transaction.open
|
||
// An ended session cannot roll back; the untimed ROLLBACK would only wait on a dead socket.
|
||
if (open && !lostReply && !sessionEnded) {
|
||
await client.query('ROLLBACK').catch(() => undefined)
|
||
}
|
||
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
|
||
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
|
||
// Once the fused commit has ended the transaction, a retry would apply the work twice.
|
||
if (
|
||
!open ||
|
||
lostReply ||
|
||
!retryablePostgresTransactionError(error) ||
|
||
attempt === POSTGRES_TRANSACTION_ATTEMPTS
|
||
) {
|
||
if (retryablePostgresTransactionError(error) && options.reportRetries !== false) {
|
||
console.warn(
|
||
JSON.stringify({
|
||
event: 'orca_relay_postgres_transaction_exhausted',
|
||
code: String((error as { code?: unknown }).code),
|
||
attempts: attempt,
|
||
phase: postgresTransactionErrorPhase(error)
|
||
})
|
||
)
|
||
}
|
||
throw error
|
||
}
|
||
if (options.reportRetries !== false) {
|
||
console.warn(
|
||
JSON.stringify({
|
||
event: 'orca_relay_postgres_transaction_retry',
|
||
code: String((error as { code?: unknown }).code),
|
||
attempt,
|
||
phase: postgresTransactionErrorPhase(error)
|
||
})
|
||
)
|
||
}
|
||
} finally {
|
||
this.release(client, lostReply ?? sessionEnded)
|
||
}
|
||
// A PostgreSQL transaction is unusable after an abort, so retry all work
|
||
// on a fresh pooled client with a small full-jitter delay.
|
||
await waitForPostgresRetry()
|
||
}
|
||
throw new Error('postgres_transaction_retry_exhausted')
|
||
}
|
||
|
||
async close(): Promise<void> {
|
||
await this.pool.end()
|
||
}
|
||
|
||
private async connect(lane?: PostgresPoolLane): Promise<pg.PoolClient> {
|
||
const startedAt = performance.now()
|
||
try {
|
||
const client = await this.pressure.connect(lane)
|
||
this.slowConnections?.track(client)
|
||
return client
|
||
} finally {
|
||
noteStickySlotPoolWait(performance.now() - startedAt)
|
||
}
|
||
}
|
||
|
||
// Destroys the client on an error, or when it has turned slow, so the pool dials a fresh one.
|
||
private release(client: pg.PoolClient, error: Error | undefined): void {
|
||
if (error) {
|
||
this.slowConnections?.forget(client)
|
||
client.release(error)
|
||
return
|
||
}
|
||
const slow = this.slowConnections?.shouldRetire(client, this.idOfConnection(client))
|
||
client.release(slow || undefined)
|
||
}
|
||
|
||
private idOfConnection(client: pg.PoolClient): number {
|
||
let id = this.connectionIds.get(client)
|
||
if (id === undefined) {
|
||
id = this.nextConnectionId++
|
||
this.connectionIds.set(client, id)
|
||
}
|
||
return id
|
||
}
|
||
|
||
consumePoolPressure(): PostgresPoolPressureCounts {
|
||
return this.pressure.consumeCounts()
|
||
}
|
||
|
||
peekPoolPressure(): PostgresPoolPressureCounts {
|
||
return this.pressure.peekCounts()
|
||
}
|
||
|
||
poolOldestWaitMs(): number {
|
||
return this.pressure.oldestWaitMs()
|
||
}
|
||
}
|
||
|
||
export function consumeRelayDatabasePoolPressure(
|
||
database: RelayDatabase
|
||
): PostgresPoolPressureCounts {
|
||
return database instanceof PostgresDatabase
|
||
? database.consumePoolPressure()
|
||
: emptyPostgresPoolPressureCounts()
|
||
}
|
||
|
||
export function consumeRelayCellInventoryHold(
|
||
database: RelayDatabase
|
||
): CellInventoryHoldCounts {
|
||
const holder = database as { consumeHoldCounts?: () => CellInventoryHoldCounts }
|
||
return holder.consumeHoldCounts?.() ?? emptyCellInventoryHoldCounts()
|
||
}
|
||
|
||
export function readRelayDatabasePoolPressure(
|
||
database: RelayDatabase
|
||
): PostgresPoolPressureCounts {
|
||
return database instanceof PostgresDatabase
|
||
? database.peekPoolPressure()
|
||
: emptyPostgresPoolPressureCounts()
|
||
}
|
||
|
||
// How long the longest-waiting pooled query has queued so far; 0 when none waits.
|
||
export function readRelayDatabasePoolOldestWaitMs(database: RelayDatabase): number {
|
||
return database instanceof PostgresDatabase ? database.poolOldestWaitMs() : 0
|
||
}
|
||
|
||
// Keep a listener across pg-pool checkout/release listener swaps so no error gap can crash the process; checked-out/idle listeners own logging.
|
||
export function keepPostgresClientErrorsHandled(pool: Pick<pg.Pool, 'on'>): void {
|
||
pool.on('connect', (client) => {
|
||
client.on('error', () => undefined)
|
||
})
|
||
}
|
||
|
||
// URL options can override pool defaults; report effective idle timeout without reading/logging the URL.
|
||
function reportPostgresIdleSessionTimeoutOnce(pool: Pick<pg.Pool, 'on'>): void {
|
||
let reported = false
|
||
pool.on('connect', (client) => {
|
||
if (reported) return
|
||
reported = true
|
||
client
|
||
.query("SELECT current_setting('idle_session_timeout') AS value")
|
||
.then((result: pg.QueryResult) => {
|
||
console.log(
|
||
JSON.stringify({
|
||
event: 'orca_relay_postgres_session_settings',
|
||
idleSessionTimeout: String(result.rows[0]?.value)
|
||
})
|
||
)
|
||
})
|
||
.catch(() => undefined)
|
||
})
|
||
}
|
||
|
||
export function absorbPostgresIdleClientErrors(pool: Pick<pg.Pool, 'on'>): void {
|
||
pool.on('error', () => {
|
||
// Why: node-postgres removes failed idle clients itself; leaving `error`
|
||
// unhandled would crash the cell and turn a SQL outage into autoheal churn.
|
||
console.warn('[orca-relay] idle PostgreSQL client failed')
|
||
})
|
||
}
|
||
|
||
async function applySchema(database: RelayDatabase): Promise<void> {
|
||
for (const statement of SCHEMA.split(';')) {
|
||
if (statement.trim()) await database.query(statement)
|
||
}
|
||
for (const [table, column, definition] of [
|
||
['relay_control_capabilities', 'idle_regional_rehome', 'BIGINT NOT NULL DEFAULT 0'],
|
||
['relay_region_rehome_attempts', 'source_generation', 'BIGINT NOT NULL DEFAULT 0'],
|
||
['relay_cell_connection_limits', 'reported_hard_cap', 'BIGINT']
|
||
]) {
|
||
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
|
||
if (!columns.some((existing) => existing.name === column)) {
|
||
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`)
|
||
}
|
||
}
|
||
}
|
||
|
||
// DDL can exceed request deadlines; use a separate untimed schema connection and close it before serving or every startup repeats the same failure.
|
||
async function applySchemaOnUntimedPool(
|
||
databaseUrl: string,
|
||
applicationName: string | undefined
|
||
): Promise<void> {
|
||
const pool = new pg.Pool({
|
||
connectionString: databaseUrl,
|
||
max: 1,
|
||
application_name: applicationName ? `${applicationName}/schema` : undefined,
|
||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||
statement_timeout: 0,
|
||
// Kept: a DDL blocked behind another director's ACCESS EXCLUSIVE lock must
|
||
// yield to the bounded schema retry instead of holding the connection.
|
||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
|
||
})
|
||
absorbPostgresIdleClientErrors(pool)
|
||
keepPostgresClientErrorsHandled(pool)
|
||
const database = new PostgresDatabase(pool)
|
||
try {
|
||
await applyPostgresSchema(
|
||
relayPostgresSchemaStatements(),
|
||
async (statement) => await database.query(statement),
|
||
// IF NOT EXISTS takes table locks before checking existence; catalog reads skip migrated objects without queueing writers behind boot DDL.
|
||
{ catalogQuery: async (sql, params) => await database.query(sql, params) }
|
||
)
|
||
} finally {
|
||
await database.close().catch(() => undefined)
|
||
}
|
||
}
|
||
|
||
async function backfillRelayCellRegions(database: RelayDatabase): Promise<void> {
|
||
await database.query(
|
||
`INSERT INTO relay_cell_regions (cell_id, region)
|
||
SELECT cell_id, 'us-central1' FROM relay_cells WHERE true
|
||
ON CONFLICT (cell_id) DO NOTHING`
|
||
)
|
||
}
|
||
|
||
export type RelayDatabaseOpenInput = {
|
||
databaseUrl?: string
|
||
dataDir: string
|
||
poolMax?: number
|
||
applicationName?: string
|
||
statementTimeoutMs?: number
|
||
readTimeoutMarginMs?: number
|
||
// Read per query, so a cell's switch moves the margin without a restart.
|
||
readTimeoutMarginOverrideMs?: () => number | undefined
|
||
// Directors own schema; cells skip boot DB access, listen through outages and stay unready until the first successful query.
|
||
appliesPostgresSchema?: boolean
|
||
// Directors only: on cells, lock waits and batch flushes could trip the tail rule, and cells have not shown this stall.
|
||
retiresSlowConnections?: boolean
|
||
}
|
||
|
||
export async function openRelayDatabase(input: RelayDatabaseOpenInput): Promise<RelayDatabase> {
|
||
let database: RelayDatabase
|
||
const appliesPostgresSchema = input.appliesPostgresSchema !== false
|
||
if (input.databaseUrl) {
|
||
if (appliesPostgresSchema) {
|
||
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
|
||
}
|
||
const statementTimeoutMs = input.statementTimeoutMs ?? relayPostgresStatementTimeoutMs()
|
||
const pool = new pg.Pool({
|
||
connectionString: input.databaseUrl,
|
||
max: input.poolMax ?? 10,
|
||
application_name: input.applicationName,
|
||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||
statement_timeout: statementTimeoutMs,
|
||
// The server answers every statement within statement_timeout, as 57014 at worst, so
|
||
// this fires only on a lost reply, or a database or event-loop stall past the margin.
|
||
query_timeout:
|
||
statementTimeoutMs + (input.readTimeoutMarginMs ?? relayPostgresReadTimeoutMarginMs()),
|
||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS,
|
||
// Bounds a backend whose client was destroyed outside a transaction (after a lost
|
||
// reply on an autocommit write or COMMIT). pg-pool closes healthy idle clients at 10 s.
|
||
options: `-c idle_session_timeout=${POSTGRES_IDLE_SESSION_TIMEOUT_MS}`
|
||
})
|
||
absorbPostgresIdleClientErrors(pool)
|
||
keepPostgresClientErrorsHandled(pool)
|
||
reportPostgresIdleSessionTimeoutOnce(pool)
|
||
const override = input.readTimeoutMarginOverrideMs
|
||
database = new PostgresDatabase(
|
||
pool,
|
||
override
|
||
? () => {
|
||
const marginMs = override()
|
||
return marginMs === undefined ? undefined : statementTimeoutMs + marginMs
|
||
}
|
||
: undefined,
|
||
{ retiresSlowConnections: input.retiresSlowConnections === true }
|
||
)
|
||
} else {
|
||
mkdirSync(input.dataDir, { recursive: true })
|
||
const sqlite = new DatabaseSync(join(input.dataDir, 'orca-relay.sqlite'))
|
||
sqlite.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;')
|
||
database = new SqliteDatabase(sqlite)
|
||
}
|
||
try {
|
||
if (!input.databaseUrl) await applySchema(database)
|
||
if (!input.databaseUrl || appliesPostgresSchema) await backfillRelayCellRegions(database)
|
||
return database
|
||
} catch (error) {
|
||
await database.close().catch(() => undefined)
|
||
throw error
|
||
}
|
||
}
|
||
|
||
export async function openInMemoryRelayDatabase(): Promise<RelayDatabase> {
|
||
const sqlite = new DatabaseSync(':memory:')
|
||
sqlite.exec('PRAGMA foreign_keys = ON;')
|
||
const database = new SqliteDatabase(sqlite)
|
||
await applySchema(database)
|
||
await backfillRelayCellRegions(database)
|
||
return database
|
||
}
|