From 9c41d9e8d80e02e3b8ff048f69dad8f844ef34e5 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:54:00 -0400 Subject: [PATCH] feat(relay): per-cell switches for the rejection fence and the lost-reply margin (#27057) * feat(relay): per-cell switches for the rejection fence and the lost-reply margin, and report ignored keys rejectionFence (default on) turns the unhandled-rejection fence off without a roll. readTimeoutMarginMs (1-60 s) replaces the boot margin per query through pg's query_timeout. The applied log line and flagsApplied name keys this image does not know. * feat(relay): runtime-status names the switches and values this image accepts The flag tool checks a write against it, so it can refuse ticketCheck=enforce (or any key) on an image that would void or drop it. --- cloud/apps/relay/src/app.ts | 6 +- cloud/apps/relay/src/cell-flags.ts | 59 ++++++++++++++++++- cloud/apps/relay/src/cell-seat-feed.test.ts | 20 ++++++- cloud/apps/relay/src/database.ts | 56 +++++++++++++++--- cloud/apps/relay/src/index.ts | 11 +++- .../src/postgres-lost-reply-postgres.test.ts | 35 ++++++++++- .../src/relay-control-flag-channel.test.ts | 28 +++++++-- .../relay/src/relay-control-flag-channel.ts | 14 +++-- .../relay-database-rejection-fence.test.ts | 9 +++ .../src/relay-database-rejection-fence.ts | 5 +- 10 files changed, 209 insertions(+), 34 deletions(-) diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts index ecb2d5a7f45..8dd8deeb4e4 100644 --- a/cloud/apps/relay/src/app.ts +++ b/cloud/apps/relay/src/app.ts @@ -40,7 +40,7 @@ import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window. import { CELL_ADMISSION_STATES } from './cell-admission-selector.js' import { registerCellSeatFeedRoute } from './cell-seat-feed-route.js' import type { CellSeatFeedPage } from './cell-seat-log.js' -import type { CellFlags } from './cell-flags.js' +import { type CellFlags, supportedCellFlags } from './cell-flags.js' import type { AppliedControlFlags } from './relay-control-flag-channel.js' import { RELAY_MAX_CELL_CAPACITY_REQUESTS, type RelayConfig } from './config.js' import type { RelayCredentialStore } from './credential-store.js' @@ -783,7 +783,9 @@ export function createRelayApp( draining: operations.isDraining?.() ?? false, regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0, // The flag workflow's read-back: applied switches, never the desired object. - ...(operations.cellFlags ? { flagsApplied: operations.cellFlags() } : {}), + ...(operations.cellFlags + ? { flagsApplied: operations.cellFlags(), supportedFlags: supportedCellFlags() } + : {}), connectionCapacity: config.connectionHardCap === undefined ? null diff --git a/cloud/apps/relay/src/cell-flags.ts b/cloud/apps/relay/src/cell-flags.ts index 4a127be8b29..f9e01b967f7 100644 --- a/cloud/apps/relay/src/cell-flags.ts +++ b/cloud/apps/relay/src/cell-flags.ts @@ -6,16 +6,63 @@ import type { ControlFlagParse } from './relay-control-flag-channel.js' export type CellFlags = { readinessLocal: boolean ticketCheck: 'off' | 'shadow' + // Off restores Node's default: any unhandled rejection ends the process (#26817 reverted). + rejectionFence: boolean + // Past statement_timeout before a lost reply ends its connection; absent keeps the boot value. + readTimeoutMarginMs?: number } -export const CELL_FLAG_DEFAULTS: CellFlags = { readinessLocal: false, ticketCheck: 'off' } +export const CELL_FLAG_DEFAULTS: CellFlags = { + readinessLocal: false, + ticketCheck: 'off', + rejectionFence: true +} + +export const READ_TIMEOUT_MARGIN_MIN_MS = 1_000 +export const READ_TIMEOUT_MARGIN_MAX_MS = 60_000 export const CELL_FLAGS_APPLIED_EVENT = 'orca_relay_cell_flags_applied' const CellFlagsSchema = z.object({ readinessLocal: z.boolean().default(CELL_FLAG_DEFAULTS.readinessLocal), - ticketCheck: z.enum(['off', 'shadow']).default(CELL_FLAG_DEFAULTS.ticketCheck) + ticketCheck: z.enum(['off', 'shadow']).default(CELL_FLAG_DEFAULTS.ticketCheck), + rejectionFence: z.boolean().default(CELL_FLAG_DEFAULTS.rejectionFence), + readTimeoutMarginMs: z + .number() + .int() + .min(READ_TIMEOUT_MARGIN_MIN_MS) + .max(READ_TIMEOUT_MARGIN_MAX_MS) + .optional() }) +const KNOWN_FLAG_KEYS = new Set(Object.keys(CellFlagsSchema.shape)) + +export type SupportedCellFlag = + | { type: 'boolean' } + | { type: 'enum'; values: string[] } + | { type: 'number'; min?: number; max?: number; integer?: boolean } + +// What this image accepts, read from the schema itself: the flag tool refuses a key or value +// the cell would drop or void before it writes. +export function supportedCellFlags(): Record { + const supported: Record = {} + for (const [key, field] of Object.entries(CellFlagsSchema.shape)) { + let inner: z.ZodTypeAny = field + while (inner instanceof z.ZodDefault || inner instanceof z.ZodOptional) { + inner = inner instanceof z.ZodDefault ? inner.removeDefault() : inner.unwrap() + } + if (inner instanceof z.ZodBoolean) supported[key] = { type: 'boolean' } + else if (inner instanceof z.ZodEnum) supported[key] = { type: 'enum', values: [...inner.options] } + else if (inner instanceof z.ZodNumber) { + supported[key] = { + type: 'number', + ...(inner.minValue === null ? {} : { min: inner.minValue }), + ...(inner.maxValue === null ? {} : { max: inner.maxValue }), + ...(inner.isInt ? { integer: true } : {}) + } + } + } + return supported +} export function cellFlagObjectName(cellId: string): string { return `cells/${cellId}.json` @@ -29,6 +76,12 @@ export function cellFlagParser(cellId: string): ControlFlagParse { }) return (body) => { const parsed = ObjectSchema.safeParse(body) - return parsed.success ? parsed.data.flags : null + if (!parsed.success) return null + const named = body !== null && typeof body === 'object' && 'flags' in body ? body.flags : {} + const ignoredKeys = + named !== null && typeof named === 'object' + ? Object.keys(named).filter((key) => !KNOWN_FLAG_KEYS.has(key)).sort() + : [] + return { flags: parsed.data.flags, ignoredKeys } } } diff --git a/cloud/apps/relay/src/cell-seat-feed.test.ts b/cloud/apps/relay/src/cell-seat-feed.test.ts index c384e9ea62f..9bde68090a3 100644 --- a/cloud/apps/relay/src/cell-seat-feed.test.ts +++ b/cloud/apps/relay/src/cell-seat-feed.test.ts @@ -67,7 +67,12 @@ const SeatFeedReplySchema = z.object({ flagsApplied: z .object({ generation: z.number(), - flags: z.object({ readinessLocal: z.boolean(), ticketCheck: z.enum(['off', 'shadow']) }) + flags: z.object({ + readinessLocal: z.boolean(), + ticketCheck: z.enum(['off', 'shadow']), + rejectionFence: z.boolean() + }), + ignoredKeys: z.array(z.string()).optional() }) .optional(), seq: z.number(), @@ -333,14 +338,23 @@ describe('cell seat feed', () => { let applied: AppliedControlFlags = { generation: 0, flags: CELL_FLAG_DEFAULTS } const { read, app } = createCell(config(), () => applied) expect((await read()).flagsApplied).toEqual({ generation: 0, flags: CELL_FLAG_DEFAULTS }) - applied = { generation: 12, flags: { readinessLocal: true, ticketCheck: 'shadow' } } + applied = { generation: 12, flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true, ticketCheck: 'shadow' } } expect((await read()).flagsApplied).toEqual(applied) const runtime = await app.request('/v1/admin/runtime-status', { method: 'POST', headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' }, body: JSON.stringify({ v: 1 }) }) - expect(await runtime.json()).toMatchObject({ flagsApplied: applied }) + expect(await runtime.json()).toMatchObject({ + flagsApplied: applied, + // What the flag tool checks a write against before it writes. + supportedFlags: { + readinessLocal: { type: 'boolean' }, + ticketCheck: { type: 'enum', values: ['off', 'shadow'] }, + rejectionFence: { type: 'boolean' }, + readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true } + } + }) }) it('accepts only the directors rehome identity, verified once per poll', async () => { diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts index 0f219b964e3..fa1ac57acad 100644 --- a/cloud/apps/relay/src/database.ts +++ b/cloud/apps/relay/src/database.ts @@ -888,6 +888,27 @@ function inlinePostgresParameters(sql: string, params: unknown[], client: pg.Poo return inlined } +// A per-query read timeout, when the cell's switch overrides the pool's (pg reads +// `query_timeout` from a query config object first). Undefined keeps the pool's. +type PostgresQueryTimeout = () => number | undefined + +async function timedQuery( + client: pg.PoolClient, + text: string, + values: unknown[] | undefined, + queryTimeout: PostgresQueryTimeout | undefined +): Promise { + const timeout = queryTimeout?.() + if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values) + // pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig. + const config: pg.QueryConfig & { query_timeout: number } = { + text, + ...(values === undefined ? {} : { values }), + query_timeout: timeout + } + return await client.query(config) +} + class PostgresTransaction implements RelayDatabase { readonly dialect = 'postgres' as const private held: { fromMs: number; site: CellLockHoldSite } | undefined @@ -899,7 +920,10 @@ class PostgresTransaction implements RelayDatabase { // later COMMIT would commit a statement its caller saw fail. The connection is finished. private lostReply: Error | undefined - constructor(protected readonly client: pg.PoolClient) {} + constructor( + protected readonly client: pg.PoolClient, + private readonly queryTimeout?: PostgresQueryTimeout + ) {} get poisonedBy(): Error | undefined { return this.lostReply @@ -926,7 +950,7 @@ class PostgresTransaction implements RelayDatabase { `WITH final_write AS (${inlinePostgresParameters(sql, params, this.client)}) ` + 'SELECT 1 / (SELECT count(*)::int FROM final_write); COMMIT' try { - await this.client.query(message) + await timedQuery(this.client, message, undefined, this.queryTimeout) } catch (error) { // Simple query stops at the first error, so any server error means COMMIT never ran: // retryable codes take the caller's normal rollback-and-retry path. A lost connection @@ -977,7 +1001,7 @@ class PostgresTransaction implements RelayDatabase { this.assertNotCommitted() this.failIfPoisoned() try { - const result = await this.client.query(postgresSql(sql), params) + const result = await timedQuery(this.client, postgresSql(sql), params, this.queryTimeout) return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] } catch (error) { this.noteFailure(error) @@ -1142,7 +1166,10 @@ export class PostgresDatabase implements RelayDatabase { return this.holds.consumeCounts() } - constructor(private readonly pool: pg.Pool) { + constructor( + private readonly pool: pg.Pool, + private readonly queryTimeout?: PostgresQueryTimeout + ) { this.pressure = new PostgresPoolPressure(pool) } @@ -1166,7 +1193,7 @@ export class PostgresDatabase implements RelayDatabase { try { client = await this.pressure.connect(lane) phase = 'execute' - const result = await client.query(postgresSql(sql), params) + const result = await timedQuery(client, postgresSql(sql), params, this.queryTimeout) return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }] } catch (error) { markRelayDatabaseError(error) @@ -1215,14 +1242,14 @@ export class PostgresDatabase implements RelayDatabase { ): Promise { for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) { const client = await this.pressure.connect() - const transaction = new PostgresTransaction(client) + const transaction = new PostgresTransaction(client, this.queryTimeout) let lostReply: Error | undefined try { - await client.query('BEGIN') + await timedQuery(client, 'BEGIN', undefined, this.queryTimeout) const result = await operation(transaction) // Even when the operation caught it: COMMIT must not run after a lost reply. if (transaction.poisonedBy) throw transaction.poisonedBy - if (transaction.open) await client.query('COMMIT') + if (transaction.open) await timedQuery(client, 'COMMIT', undefined, this.queryTimeout) recordMeasuredHold(this.holds, transaction) this.holds.recordUnavailable(transaction.consumeLockUnavailable()) this.holds.recordLockTimeout(transaction.consumeLockTimeouts()) @@ -1427,6 +1454,8 @@ export type RelayDatabaseOpenInput = { applicationName?: string statementTimeoutMs?: number readTimeoutMarginMs?: number + // Read per query, so a cell's switch moves the margin without a restart. + readTimeoutMarginOverrideMs?: () => number | undefined // Directors own the PostgreSQL schema. A cell skips it and never touches the database // at boot, so it starts listening while the database is down and stays unready until // its first successful query. @@ -1460,7 +1489,16 @@ export async function openRelayDatabase(input: RelayDatabaseOpenInput): Promise< absorbPostgresIdleClientErrors(pool) keepPostgresClientErrorsHandled(pool) reportPostgresIdleSessionTimeoutOnce(pool) - database = new PostgresDatabase(pool) + const override = input.readTimeoutMarginOverrideMs + database = new PostgresDatabase( + pool, + override + ? () => { + const marginMs = override() + return marginMs === undefined ? undefined : statementTimeoutMs + marginMs + } + : undefined + ) } else { mkdirSync(input.dataDir, { recursive: true }) const sqlite = new DatabaseSync(join(input.dataDir, 'orca-relay.sqlite')) diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index 8d914576f82..67547e4e781 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -9,6 +9,7 @@ import { startCellHeartbeat } from './cell-heartbeat-client.js' import { CELL_FLAG_DEFAULTS, CELL_FLAGS_APPLIED_EVENT, + type CellFlags, cellFlagObjectName, cellFlagParser } from './cell-flags.js' @@ -36,15 +37,20 @@ import { readRegisteredMigrationInventory } from './registered-migration-inventory.js' +// Both read the cell's switch file once it exists; until then, the boot values. +let cellFlagsApplied: (() => CellFlags) | null = null // Before anything can start a database promise. -process.on('unhandledRejection', handleRelayUnhandledRejection) +process.on('unhandledRejection', (reason) => + handleRelayUnhandledRejection(reason, cellFlagsApplied?.().rejectionFence ?? true) +) const config = loadRelayConfig() const database = await openRelayDatabaseAtBoot({ databaseUrl: config.databaseUrl, dataDir: config.dataDir, poolMax: config.databasePoolMax, applicationName: `orca-relay/${config.role}/${config.cellId}`, - appliesPostgresSchema: config.role !== 'cell' + appliesPostgresSchema: config.role !== 'cell', + readTimeoutMarginOverrideMs: () => cellFlagsApplied?.().readTimeoutMarginMs }) await reconcileCellAdmissionAtStartup(config, new RelayAssignmentStore(database)) // Never awaited: the cell listens on defaults (all off) until the first read lands. @@ -57,6 +63,7 @@ const cellFlagChannel = appliedEvent: CELL_FLAGS_APPLIED_EVENT }) : null +cellFlagsApplied = cellFlagChannel ? () => cellFlagChannel.applied().flags : null const { server, sessions, diff --git a/cloud/apps/relay/src/postgres-lost-reply-postgres.test.ts b/cloud/apps/relay/src/postgres-lost-reply-postgres.test.ts index e7c5fd1b6fa..d189e10f060 100644 --- a/cloud/apps/relay/src/postgres-lost-reply-postgres.test.ts +++ b/cloud/apps/relay/src/postgres-lost-reply-postgres.test.ts @@ -68,7 +68,11 @@ describePostgres('a lost PostgreSQL reply against a real server', () => { async function open( url: string, - options: { readTimeoutMarginMs?: number; poolMax?: number } = {} + options: { + readTimeoutMarginMs?: number + poolMax?: number + readTimeoutMarginOverrideMs?: () => number | undefined + } = {} ): Promise { const database = await openRelayDatabase({ databaseUrl: url, @@ -76,7 +80,8 @@ describePostgres('a lost PostgreSQL reply against a real server', () => { appliesPostgresSchema: false, statementTimeoutMs: STATEMENT_TIMEOUT_MS, readTimeoutMarginMs: options.readTimeoutMarginMs ?? READ_TIMEOUT_MARGIN_MS, - poolMax: options.poolMax + poolMax: options.poolMax, + readTimeoutMarginOverrideMs: options.readTimeoutMarginOverrideMs }) cleanups.push(async () => await database.close()) return database @@ -125,6 +130,32 @@ describePostgres('a lost PostgreSQL reply against a real server', () => { ).resolves.toEqual([{ id: 'c1' }]) }, 20_000) + // The per-cell switch moves the margin per query, with no restart and no new pool. + it('applies the switch file margin per query, over the pool margin', async () => { + const direct = await open(databaseUrl!) + await direct.query(`CREATE TABLE IF NOT EXISTS ${PROBE_TABLE} (id TEXT PRIMARY KEY)`) + cleanups.push(async () => { + await direct.query(`DROP TABLE IF EXISTS ${PROBE_TABLE}`) + }) + const proxy = await blackholeProxy(new URL(databaseUrl!), `FROM ${PROBE_TABLE}`) + cleanups.push(proxy.close) + let override: number | undefined = 1_000 + // The pool's own margin would hold the lost reply for 30 s. + const database = await open(proxy.url, { + readTimeoutMarginMs: 30_000, + readTimeoutMarginOverrideMs: () => override + }) + await expect(database.query('SELECT 1 AS one')).resolves.toEqual([{ one: 1 }]) + const startedAt = performance.now() + const failure = await database + .query(`SELECT id FROM ${PROBE_TABLE}`) + .catch((error: unknown) => error) + expect(isRelayDatabaseTransientError(failure)).toBe(true) + expect(performance.now() - startedAt).toBeLessThan(STATEMENT_TIMEOUT_MS + 1_000 + 1_000) + override = undefined + await expect(database.query('SELECT 1 AS one')).resolves.toEqual([{ one: 1 }]) + }, 20_000) + // A reply the server sent after statement_timeout, held up by a whole-VM stall, is still a // reply: the default margin must outlast the routine 5-7 s stalls. it('keeps a reply delayed 6.5 s by a stall, on the same connection', async () => { diff --git a/cloud/apps/relay/src/relay-control-flag-channel.test.ts b/cloud/apps/relay/src/relay-control-flag-channel.test.ts index 412472307cf..c99f230124c 100644 --- a/cloud/apps/relay/src/relay-control-flag-channel.test.ts +++ b/cloud/apps/relay/src/relay-control-flag-channel.test.ts @@ -88,7 +88,7 @@ describe('control flag channel', () => { expect(google.state.authorizations[0]).toBe(`Bearer ${ACCESS_TOKEN}`) expect(channel.applied()).toEqual({ generation: 5, - flags: { readinessLocal: true, ticketCheck: 'off' } + flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true } }) }) @@ -106,7 +106,7 @@ describe('control flag channel', () => { { event: CELL_FLAGS_APPLIED_EVENT, generation: 5, - flags: { readinessLocal: true, ticketCheck: 'shadow' } + flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true, ticketCheck: 'shadow' } }, { event: CELL_FLAGS_APPLIED_EVENT, generation: 7, flags: CELL_FLAG_DEFAULTS } ]) @@ -134,6 +134,16 @@ describe('control flag channel', () => { expect(output).not.toContain(ACCESS_TOKEN) }) + it('applies the fence and read-timeout switches within their bounds', async () => { + const google = fakeGoogle(cellObject(5, { rejectionFence: false, readTimeoutMarginMs: 4_000 })) + const channel = cellChannel(google.fetchImpl) + await channel.poll() + expect(channel.applied()).toEqual({ + generation: 5, + flags: { ...CELL_FLAG_DEFAULTS, rejectionFence: false, readTimeoutMarginMs: 4_000 } + }) + }) + it('ignores a generation below the applied one', async () => { const google = fakeGoogle(cellObject(9, { readinessLocal: true })) const channel = cellChannel(google.fetchImpl) @@ -142,7 +152,7 @@ describe('control flag channel', () => { await channel.poll() expect(channel.applied()).toEqual({ generation: 9, - flags: { readinessLocal: true, ticketCheck: 'off' } + flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true } }) }) @@ -150,11 +160,19 @@ describe('control flag channel', () => { const google = fakeGoogle(cellObject(5, { readinessLocal: true, admitMode: 'memory' })) const channel = cellChannel(google.fetchImpl) await channel.poll() - expect(channel.applied().flags).toEqual({ readinessLocal: true, ticketCheck: 'off' }) + expect(channel.applied()).toEqual({ + generation: 5, + flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true }, + ignoredKeys: ['admitMode'] + }) + expect(appliedLines().at(-1)).toMatchObject({ ignoredKeys: ['admitMode'] }) for (const [generation, object] of [ [6, cellObject(6, { readinessLocal: false, ticketCheck: 'enforce' })], [7, cellObject(7, { readinessLocal: 'no' })], - [8, cellObject(8, { readinessLocal: false }, 'production-gce-c8')] + [8, cellObject(8, { readTimeoutMarginMs: 500 })], + [9, cellObject(9, { readTimeoutMarginMs: 61_000 })], + [10, cellObject(10, { rejectionFence: 'off' })], + [11, cellObject(11, { readinessLocal: false }, 'production-gce-c8')] ] as const) { google.state.object = object await channel.poll() diff --git a/cloud/apps/relay/src/relay-control-flag-channel.ts b/cloud/apps/relay/src/relay-control-flag-channel.ts index 1ca435a47c7..827d5b6ee3f 100644 --- a/cloud/apps/relay/src/relay-control-flag-channel.ts +++ b/cloud/apps/relay/src/relay-control-flag-channel.ts @@ -14,9 +14,10 @@ export const CONTROL_FLAG_POLL_MS = 5_000 // Refresh a cached access token this long before Google says it expires. const ACCESS_TOKEN_EARLY_REFRESH_MS = 60_000 -export type ControlFlagParse = (body: unknown) => Flags | null +// `ignoredKeys` are names this image does not know: a newer writer's, or a typo. +export type ControlFlagParse = (body: unknown) => { flags: Flags; ignoredKeys: string[] } | null -export type AppliedControlFlags = { generation: number; flags: Flags } +export type AppliedControlFlags = { generation: number; flags: Flags; ignoredKeys?: string[] } export type ControlFlagChannel = { applied: () => AppliedControlFlags @@ -136,10 +137,11 @@ export function startControlFlagChannel(input: { // Only a body read whole is "seen": a read cut off mid-body must be asked for again, or // every later poll would answer 304 and strand that write. seenGeneration = generation - const flags = input.parse(body) - if (flags === null) throw new ControlFlagPollError('void') - applied = { generation, flags } - console.log(JSON.stringify({ event: input.appliedEvent, generation, flags })) + const parsed = input.parse(body) + if (parsed === null) throw new ControlFlagPollError('void') + const { flags, ignoredKeys } = parsed + applied = ignoredKeys.length > 0 ? { generation, flags, ignoredKeys } : { generation, flags } + console.log(JSON.stringify({ event: input.appliedEvent, ...applied })) } const poll = (): Promise => { diff --git a/cloud/apps/relay/src/relay-database-rejection-fence.test.ts b/cloud/apps/relay/src/relay-database-rejection-fence.test.ts index 53b0c04de37..28ee785d15e 100644 --- a/cloud/apps/relay/src/relay-database-rejection-fence.test.ts +++ b/cloud/apps/relay/src/relay-database-rejection-fence.test.ts @@ -87,6 +87,15 @@ describe('database rejection fence', () => { expect(error).not.toHaveBeenCalled() }) + it('keeps every rejection fatal while the cell switch has the fence off', async () => { + const duplicate = await rejectionFrom(() => + databaseFailingWith(serverError('23505', 'duplicate key')).query('SELECT 1') + ) + expect(() => handleRelayUnhandledRejection(duplicate, false)).toThrow() + expect(error).toHaveBeenCalledTimes(1) + expect(() => handleRelayUnhandledRejection(duplicate, true)).not.toThrow() + }) + // A whole-VM stall of 6-7 s outlasts the 5 s idle-in-transaction limit for any backend caught // idle mid-transaction, and the server ends that session with FATAL 25P03 on resume. it('fences an idle-in-transaction timeout but keeps the rest of class 25 fatal', async () => { diff --git a/cloud/apps/relay/src/relay-database-rejection-fence.ts b/cloud/apps/relay/src/relay-database-rejection-fence.ts index f8eaf84e184..2a745d713c0 100644 --- a/cloud/apps/relay/src/relay-database-rejection-fence.ts +++ b/cloud/apps/relay/src/relay-database-rejection-fence.ts @@ -67,8 +67,9 @@ function rejectionFields(reason: unknown): { code: string; message: string } { // Installed once by the process entry point. Fatal rejections are rethrown, which ends the // process exactly as Node's default would, after one line that names them. -export function handleRelayUnhandledRejection(reason: unknown): void { - if (isFencedRelayDatabaseRejection(reason)) { +// `fenced` false is the per-cell switch turned off: every rejection is fatal, as before #26817. +export function handleRelayUnhandledRejection(reason: unknown, fenced = true): void { + if (fenced && isFencedRelayDatabaseRejection(reason)) { console.warn( JSON.stringify({ event: 'orca_relay_database_rejection_fenced', ...rejectionFields(reason) }) )