Files
orca/cloud/apps/relay/src/postgres-schema-startup.ts
T
Jinwoo Hong dd9eaa9585 fix(cloud): retry the committed-winner collision codes in relay schema startup (#18553)
* fix(cloud): retry the committed-winner collision codes in relay schema startup

`CREATE TABLE IF NOT EXISTS` only checks the name before the catalog inserts, so
the loser of a concurrent CREATE fails in one of two ways depending on timing:
on the catalog unique index (23505, which the startup retry already handled) or,
when the winner has committed by the time the loser reaches TypeCreate /
heap_create_with_catalog, on the name check those routines repeat (42710
duplicate type, 42P07 duplicate relation). The predicate treated the latter as
fatal, so a director could fail startup on a table it was about to find present.

This is what turned `postgres-schema-concurrency-postgres.test.ts` red on main
and on every relay PR (CI's shared runner loses the race more often than a dev
box): a throwaway diagnostic run in CI reported 42710 from TypeCreate and 42P07
from heap_create_with_catalog as the only rejection reasons.

Treat 42710/42P07 as retryable for `CREATE TABLE IF NOT EXISTS` and 42P07 for
`CREATE [UNIQUE] INDEX IF NOT EXISTS`; every other statement shape still fails
fast. The concurrency test now runs ten rounds and reports the loser's SQLSTATE
instead of a bare boolean.

* chore(cloud): allowlist the RFC 6455 example Sec-WebSocket-Key for upgrade tests

Cloud Verify's Secret scan runs gitleaks over --all refs, so the raw-socket
upgrade test on fix/relay-upgrade-malformed-uri (#18547) trips every cloud PR's
scan until its allowlist reaches main. Land the allowlist here first.
2026-09-03 21:20:27 -04:00

106 lines
3.3 KiB
TypeScript

const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014'])
const DEFAULT_RETRY_DEADLINE_MS = 30_000
const RETRY_BASE_DELAY_MS = 250
const RETRY_MAX_DELAY_MS = 2_000
type SchemaStartupOptions = {
now?: () => number
random?: () => number
retryDeadlineMs?: number
wait?: (delayMs: number) => Promise<void>
}
function retryDelayMs(attempt: number, random: () => number): number {
const ceiling = Math.min(
RETRY_BASE_DELAY_MS * 2 ** (attempt - 1),
RETRY_MAX_DELAY_MS
)
return Math.ceil(ceiling * (0.5 + random() * 0.5))
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs))
}
const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
function concurrentCreateCollision(
value: { code?: unknown; constraint?: unknown },
statement: string
): boolean {
if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) {
return (
(value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
value.code === '42710' ||
value.code === '42P07'
)
}
if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) {
return (
(value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
value.code === '42P07'
)
}
return false
}
function retryableSchemaError(error: unknown, statement: string): boolean {
const value = error as { code?: unknown; constraint?: unknown }
return (
RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement)
)
}
export async function applyPostgresSchema(
statements: string[],
query: (statement: string) => Promise<unknown>,
options: SchemaStartupOptions = {}
): Promise<void> {
const now = options.now ?? Date.now
const random = options.random ?? Math.random
const pause = options.wait ?? wait
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
for (const statement of statements) {
let attempt = 1
while (true) {
try {
await query(statement)
break
} catch (error) {
const code = String((error as { code?: unknown }).code)
const remainingMs = deadlineAt - now()
const retryable = retryableSchemaError(error, statement)
if (!retryable || remainingMs <= 0) {
if (retryable) {
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_schema_retry_exhausted',
code,
attempts: attempt
})
)
}
throw error
}
const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random))
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_schema_retry',
code,
attempt,
delayMs
})
)
await pause(delayMs)
attempt += 1
}
}
}
}