From e2741fa4cdc1445bd0f8f2be35b43f43a411d1ed Mon Sep 17 00:00:00 2001 From: Neil Date: Sat, 7 Nov 2026 22:52:50 -0800 Subject: [PATCH] relay: failed edge fences recorded (relay_edge_fence_pending) and retried from the evacuation sweep to the host's current row --- cloud/apps/relay/src/database.ts | 12 ++ cloud/apps/relay/src/edge-relay-evacuation.ts | 6 +- cloud/apps/relay/src/edge-relay-fence.test.ts | 168 ++++++++++++++++-- cloud/apps/relay/src/edge-relay-fence.ts | 167 +++++++++++++++++ .../relay/src/edge-relay-postgres.test.ts | 38 ++++ cloud/apps/relay/src/edge-relay.ts | 21 +-- 6 files changed, 382 insertions(+), 30 deletions(-) create mode 100644 cloud/apps/relay/src/edge-relay-fence.ts diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts index 6436177617a..5349992092f 100644 --- a/cloud/apps/relay/src/database.ts +++ b/cloud/apps/relay/src/database.ts @@ -262,6 +262,18 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences ( CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed ON relay_assignment_region_preferences(observed_at); +-- Relay edge fences that did not land (edge-relay-fence.ts). Small and short-lived: a row is +-- deleted when its fence lands, the host has no row or is back on the edge, or after its last retry. +CREATE TABLE IF NOT EXISTS relay_edge_fence_pending ( + user_id TEXT NOT NULL, + relay_host_id TEXT NOT NULL, + cell_id TEXT NOT NULL, + assignment_epoch BIGINT NOT NULL, + attempts BIGINT NOT NULL, + next_attempt_at BIGINT NOT NULL, + PRIMARY KEY (user_id, relay_host_id) +); + CREATE TABLE IF NOT EXISTS relay_region_decisions ( user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, generation BIGINT NOT NULL, expires_at BIGINT NOT NULL, diff --git a/cloud/apps/relay/src/edge-relay-evacuation.ts b/cloud/apps/relay/src/edge-relay-evacuation.ts index c396d30fe4c..8642e9b5f8e 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.ts @@ -1,11 +1,13 @@ // Evacuation sweep: a desktop connected to the relay edge only asks the director again when its // lease ends, so a host that left the edge cohort would stay there for hours. The sweep finds edge // rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then -// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). +// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also +// retries fences that did not land after a host left (edge-relay-fence.ts). import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions' import type { RelayDatabase, SqlRow } from './database.js' import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js' import { inEdgeCohort, type EdgeRelayConfig } from './edge-relay.js' +import { retryPendingFences } from './edge-relay-fence.js' type Identity = { userId: string; relayHostId: string } @@ -81,6 +83,8 @@ export class EdgeEvacuation { console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message })) } } + // Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured. + if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true } return answered } diff --git a/cloud/apps/relay/src/edge-relay-fence.test.ts b/cloud/apps/relay/src/edge-relay-fence.test.ts index 5a7af19cf3a..c85c9d1f4ba 100644 --- a/cloud/apps/relay/src/edge-relay-fence.test.ts +++ b/cloud/apps/relay/src/edge-relay-fence.test.ts @@ -11,6 +11,8 @@ import { type EdgeAssignBody, type EdgeAssignResult } from './edge-admin-client.js' +import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js' +import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js' import { EDGE_CELL_ID, EDGE_LOCKED_CALLS_MAX, @@ -23,6 +25,7 @@ const EDGE_URL = 'https://edge.relay.example.test' const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!') const USER = 'user-1' const HOST = 'fencehost0000001' +const T0 = 1_700_000_000_000 const databases: RelayDatabase[] = [] afterEach(async () => { @@ -75,7 +78,7 @@ async function harness() { const database = await openInMemoryRelayDatabase() databases.push(database) const edge = new ScriptedEdge() - const clock = { now: 1_700_000_000_000 } + const clock = { now: T0 } const assignments = new EdgeAssignments(database, edgeConfig(), () => clock.now, edge) await assignments.ensureEdgeCell() const placeRow = async (cellId: string, epoch: number, relayHostId = HOST) => @@ -89,7 +92,16 @@ async function harness() { assignment_epoch = excluded.assignment_epoch`, [USER, relayHostId, cellId, epoch] ) - return { database, edge, clock, assignments, placeRow } + const pending = async () => + (await database.query(`SELECT * FROM relay_edge_fence_pending ORDER BY relay_host_id`)).map((row) => ({ + relayHostId: String(row.relay_host_id), + cellId: String(row.cell_id), + assignmentEpoch: Number(row.assignment_epoch), + attempts: Number(row.attempts), + nextAttemptAt: Number(row.next_attempt_at) + })) + const retry = async () => await retryPendingFences(database, edge, EDGE_CELL_ID, () => clock.now) + return { database, edge, clock, assignments, placeRow, pending, retry } } function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record[] { @@ -99,7 +111,7 @@ function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record { - it('treats a stale answer as a failed fence, logged with its outcome and not retried inline', async () => { + it('treats a stale answer as a failed fence: logged with its outcome, recorded, not retried inline', async () => { const error = vi.spyOn(console, 'error').mockImplementation(() => {}) const h = await harness() h.edge.answers = ['stale'] @@ -108,26 +120,158 @@ describe('relay edge fence after a host leaves', () => { expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([ { event: 'orca_relay_edge_fence_failed', outcome: 'stale', cellId: 'us-c1', assignmentEpoch: 9 } ]) + expect(await h.pending()).toEqual([ + { relayHostId: HOST, cellId: 'us-c1', assignmentEpoch: 9, attempts: 0, nextAttemptAt: T0 + EDGE_FENCE_RETRY_BASE_MS } + ]) }) - it('retries conflict and unavailable inline, then logs the last outcome', async () => { + it('retries conflict and unavailable inline, then records the last outcome', async () => { const error = vi.spyOn(console, 'error').mockImplementation(() => {}) const h = await harness() - h.edge.answers = ['conflict', 'conflict', 'down'] + h.edge.answers = ['conflict', 'down', 'conflict'] expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false) expect(h.edge.calls).toHaveLength(3) - expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({ - outcome: 'relay_edge_admin_unavailable:network' - }) + expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({ outcome: 'conflict' }) + expect(await h.pending()).toHaveLength(1) }) - it('lands on applied or unchanged', async () => { - const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + it('records nothing when the fence lands, and a newer failure replaces an older record', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) const h = await harness() h.edge.answers = ['down', 'unchanged'] expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true) - expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true) - expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([]) + expect(await h.pending()).toEqual([]) + h.edge.answers = ['down', 'down', 'down'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + await h.database.query(`UPDATE relay_edge_fence_pending SET attempts = 3`) + h.clock.now += 5_000 + h.edge.answers = ['stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c2', 12) + expect(await h.pending()).toEqual([ + { relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 12, attempts: 0, nextAttemptAt: h.clock.now + EDGE_FENCE_RETRY_BASE_MS } + ]) + }) + + it('still answers when the failure cannot be recorded', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + await h.database.query(`DROP TABLE relay_edge_fence_pending`) + h.edge.answers = ['stale'] + expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false) + expect(logged(error, 'orca_relay_edge_fence_record_failed')).toHaveLength(1) + }) +}) + +describe('relay edge pending fence retries', () => { + it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + h.edge.answers = ['stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + // The host moved again before the retry. + await h.placeRow('us-c2', 11) + h.edge.calls = [] + expect(await h.retry()).toBe(true) + expect(h.edge.calls).toEqual([]) + h.clock.now += EDGE_FENCE_RETRY_BASE_MS + expect(await h.retry()).toBe(false) + expect(h.edge.calls).toEqual([{ body: { userId: USER, relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 11 } }]) + expect(await h.pending()).toEqual([]) + expect(logged(warn, 'orca_relay_edge_fence_retried')).toEqual([ + { event: 'orca_relay_edge_fence_retried', outcome: 'applied', cellId: 'us-c2', assignmentEpoch: 11, attempts: 1 } + ]) + }) + + it('drops the record without a call when the host is back on the edge or has no row', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + h.edge.answers = ['stale', 'stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + await h.assignments.fenceAway({ userId: USER, relayHostId: 'fencehost0000002' }, 'us-c1', 4) + await h.placeRow(EDGE_CELL_ID, 12) + h.edge.calls = [] + h.clock.now += EDGE_FENCE_RETRY_BASE_MS + expect(await h.retry()).toBe(false) + expect(h.edge.calls).toEqual([]) + expect(await h.pending()).toEqual([]) + }) + + it('backs off on each failure, logging it, and gives up after the last attempt', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + await h.placeRow('us-c1', 9) + h.edge.answers = ['stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + h.clock.now += EDGE_FENCE_RETRY_BASE_MS + for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) { + h.edge.answers = [attempt % 2 === 0 ? 'down' : 'conflict'] + expect(await h.retry()).toBe(true) + expect(await h.pending()).toMatchObject([{ attempts: attempt, nextAttemptAt: h.clock.now + edgeFenceRetryDelayMs(attempt) }]) + // Not due yet: nothing is sent. + h.clock.now += edgeFenceRetryDelayMs(attempt) - 1 + const calls = h.edge.calls.length + await h.retry() + expect(h.edge.calls).toHaveLength(calls) + h.clock.now += 1 + } + h.edge.answers = ['stale'] + expect(await h.retry()).toBe(false) + expect(await h.pending()).toEqual([]) + const failures = logged(error, 'orca_relay_edge_fence_failed').filter((entry) => entry.retry === true) + expect(failures).toHaveLength(EDGE_FENCE_MAX_ATTEMPTS) + expect(failures[1]).toEqual({ + event: 'orca_relay_edge_fence_failed', + outcome: 'relay_edge_admin_unavailable:network', + cellId: 'us-c1', + assignmentEpoch: 9, + attempts: 2, + retry: true + }) + expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([ + { event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS } + ]) + expect(edgeFenceRetryDelayMs(1)).toBe(60_000) + expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000) + }) + + it('lets only one director claim an attempt', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + await h.placeRow('us-c1', 9) + h.edge.answers = ['stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + h.clock.now += EDGE_FENCE_RETRY_BASE_MS + h.edge.calls = [] + await Promise.all([h.retry(), h.retry()]) + expect(h.edge.calls).toHaveLength(1) + }) + + it('runs from the evacuation sweep, which keeps its busy cadence while fences are pending', async () => { + vi.useFakeTimers() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + const h = await harness() + await h.placeRow('us-c1', 9) + const config = { ...edgeConfig(), rolloutPercent: 0 } + const evacuation = new EdgeEvacuation(h.database, config, h.edge, EDGE_CELL_ID, () => h.clock.now, () => 0.5) + h.edge.answers = ['stale'] + await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9) + h.edge.calls = [] + h.edge.answers = ['down'] + const sweep = vi.spyOn(evacuation, 'sweep') + evacuation.start() + h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(h.edge.calls).toHaveLength(1) + // Still pending: the next pass comes after the busy interval, and lands. + h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(sweep).toHaveBeenCalledTimes(2) + expect(h.edge.calls).toHaveLength(2) + expect(await h.pending()).toEqual([]) + evacuation.stop() }) }) diff --git a/cloud/apps/relay/src/edge-relay-fence.ts b/cloud/apps/relay/src/edge-relay-fence.ts new file mode 100644 index 00000000000..63dd051a191 --- /dev/null +++ b/cloud/apps/relay/src/edge-relay-fence.ts @@ -0,0 +1,167 @@ +// Fences that did not land. After a host leaves the relay edge the director tells the edge where it +// went (EdgeAssignments.fenceAway); until that lands the edge object still claims the host and its +// phones get HOST_OFFLINE instead of re-resolving. A failed fence is recorded here, so any director's +// evacuation sweep retries it, always to the host's current row, never to the recorded one. +import type { RelayDatabase, SqlRow } from './database.js' +import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js' + +type Identity = { userId: string; relayHostId: string } + +function integer(row: SqlRow, field: string): number { + const value = Number(row[field]) + if (!Number.isSafeInteger(value)) { throw new Error(`invalid_${field}`) } + return value +} + +function text(row: SqlRow, field: string): string { + const value = row[field] + if (typeof value !== 'string') { throw new Error(`invalid_${field}`) } + return value +} + +// Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`, +// cannot land until the host moves again, and that move fences anew). +export const EDGE_FENCE_MAX_ATTEMPTS = 8 +// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all. +export const EDGE_FENCE_RETRY_BASE_MS = 60_000 +const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000 +// Pending fences retried per sweep (per director). +const EDGE_FENCE_RETRY_BATCH = 20 + +export function edgeFenceRetryDelayMs(attempts: number): number { + return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS) +} + +// A newer failure for the same host replaces the older one and starts its attempts again. +export async function recordPendingFence( + database: RelayDatabase, + identity: Identity, + cellId: string, + assignmentEpoch: number, + now: number +): Promise { + await database.query( + `INSERT INTO relay_edge_fence_pending + (user_id, relay_host_id, cell_id, assignment_epoch, attempts, next_attempt_at) + VALUES (?, ?, ?, ?, 0, ?) + ON CONFLICT (user_id, relay_host_id) DO UPDATE SET + cell_id = excluded.cell_id, assignment_epoch = excluded.assignment_epoch, + attempts = 0, next_attempt_at = excluded.next_attempt_at`, + [identity.userId, identity.relayHostId, cellId, assignmentEpoch, now + EDGE_FENCE_RETRY_BASE_MS] + ) +} + +// The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged +// fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the +// host); the failure is logged and recorded for retry. Returns whether the fence landed. +export async function fenceEdge( + database: RelayDatabase, + admin: EdgeAdmin, + fence: Identity & { cellId: string; assignmentEpoch: number }, + attempts: number, + now: number +): Promise { + const { cellId, assignmentEpoch } = fence + let outcome = 'unavailable' + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + outcome = (await admin.assign(fence)).outcome + } catch (error) { + if (!(error instanceof EdgeAdminUnavailableError)) { throw error } + outcome = error.message + } + if (outcome === 'applied' || outcome === 'unchanged') { return true } + // The same epoch stays stale; only a later move (a newer epoch) can land. + if (outcome === 'stale') { break } + } + console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch })) + try { + await recordPendingFence(database, fence, cellId, assignmentEpoch, now) + } catch (error) { + // Best effort, like the fence itself: the host is already placed and its answer still goes out. + console.error( + JSON.stringify({ + event: 'orca_relay_edge_fence_record_failed', + reason: error instanceof Error ? error.message : 'unknown' + }) + ) + } + return false +} + +type Retry = 'landed' | 'dropped' | 'failed' + +// One pass over the due pending fences. Returns whether any pending fence is left (the sweep then +// keeps its busy cadence). +export async function retryPendingFences( + database: RelayDatabase, + admin: EdgeAdmin, + edgeCellId: string, + now: () => number +): Promise { + const due = await database.query( + `SELECT user_id, relay_host_id, attempts, next_attempt_at FROM relay_edge_fence_pending + WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`, + [now(), EDGE_FENCE_RETRY_BATCH] + ) + for (const row of due) { + const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') } + const attempts = integer(row, 'attempts') + 1 + // Claim the attempt: a director that read the same row loses here and skips it. + const claimed = await database.query( + `UPDATE relay_edge_fence_pending SET attempts = ?, next_attempt_at = ? + WHERE user_id = ? AND relay_host_id = ? AND attempts = ? AND next_attempt_at = ? + RETURNING attempts`, + [attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId, + attempts - 1, integer(row, 'next_attempt_at')] + ) + if (claimed.length !== 1) { continue } + const result = await retryOne(database, admin, edgeCellId, identity, attempts) + if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) { + // Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0. + await database.query( + `DELETE FROM relay_edge_fence_pending WHERE user_id = ? AND relay_host_id = ? AND attempts = ?`, + [identity.userId, identity.relayHostId, attempts] + ) + } + if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) { + console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts })) + } + } + return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0 +} + +// Fences the edge to the host's CURRENT row; a host that is back on the edge (or has no row) needs +// no fence, since its next move away fences again. +async function retryOne( + database: RelayDatabase, + admin: EdgeAdmin, + edgeCellId: string, + identity: Identity, + attempts: number +): Promise { + const row = ( + await database.query( + `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + if (!row || text(row, 'cell_id') === edgeCellId) { return 'dropped' } + const cellId = text(row, 'cell_id') + const assignmentEpoch = integer(row, 'assignment_epoch') + let outcome: string + try { + outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome + } catch (error) { + if (!(error instanceof EdgeAdminUnavailableError)) { throw error } + outcome = error.message + } + if (outcome === 'applied' || outcome === 'unchanged') { + console.warn(JSON.stringify({ event: 'orca_relay_edge_fence_retried', outcome, cellId, assignmentEpoch, attempts })) + return 'landed' + } + console.error( + JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true }) + ) + return 'failed' +} diff --git a/cloud/apps/relay/src/edge-relay-postgres.test.ts b/cloud/apps/relay/src/edge-relay-postgres.test.ts index 4e39efd8edd..a26d843d4ef 100644 --- a/cloud/apps/relay/src/edge-relay-postgres.test.ts +++ b/cloud/apps/relay/src/edge-relay-postgres.test.ts @@ -10,6 +10,7 @@ import { EdgeMoveDeferredError } from './edge-relay.js' import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js' +import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js' const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip @@ -35,6 +36,7 @@ describePostgres('relay edge on PostgreSQL', () => { await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, ['edge-pg-user']) await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, ['edge-pg-user']) await database.query(`DELETE FROM relay_devices WHERE user_id = ?`, ['edge-pg-user']) + await database.query(`DELETE FROM relay_edge_fence_pending WHERE user_id = ?`, ['edge-pg-user']) // Leave the shared selector as other files expect it: without this file's cells. await database.transaction(async (transaction) => { const selector = ( @@ -198,4 +200,40 @@ describePostgres('relay edge on PostgreSQL', () => { await Promise.all(held) expect((await edge.moveToEdge(last, 0)).cellId).toBe(EDGE_CELL_ID) }) + + it('records a failed fence and lets exactly one director retry it to the current row', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const host = { userId: 'edge-pg-user', relayHostId: 'edgepgfence00001' } + const sent: EdgeAssignBody[] = [] + const admin = { + assign: async (body: EdgeAssignBody) => { + sent.push(body) + return { outcome: sent.length === 1 ? ('stale' as const) : ('applied' as const), assignmentEpoch: 30 } + }, + exportHost: async () => ({ outcome: 'busy' as const }), + drain: async () => ({ outcome: 'idle' as const }) + } + const clock = { now: 1_700_000_000_000 } + const edge = new EdgeAssignments(database, edgeConfig, () => clock.now, admin) + await database.query( + `INSERT INTO relay_assignments + (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, last_activity_at, + reserved_controls, reserved_splices, reserved_invites, pending_installs, + pending_confirmations, migration_leases) + VALUES (?, ?, ?, 31, 0, 0, 0, 0, 0, 0, 0, 0)`, + [host.userId, host.relayHostId, cell.id] + ) + expect(await edge.fenceAway(host, cell.id, 29)).toBe(false) + clock.now += EDGE_FENCE_RETRY_BASE_MS + const results = await Promise.all([ + retryPendingFences(database, admin, EDGE_CELL_ID, () => clock.now), + retryPendingFences(second, admin, EDGE_CELL_ID, () => clock.now) + ]) + // The winner saw nothing left; the loser may still have seen the claimed record. + expect(results).toContain(false) + expect(sent.slice(1)).toEqual([{ ...host, cellId: cell.id, assignmentEpoch: 31 }]) + expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([]) + vi.restoreAllMocks() + }) }) diff --git a/cloud/apps/relay/src/edge-relay.ts b/cloud/apps/relay/src/edge-relay.ts index cdaa015548c..b3996d29dbd 100644 --- a/cloud/apps/relay/src/edge-relay.ts +++ b/cloud/apps/relay/src/edge-relay.ts @@ -4,6 +4,7 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra import type { RelayAssignment } from './assignment/assignment-store.js' import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmission } from './cell-admission-selector.js' import type { RelayDatabase, SqlRow } from './database.js' +import { fenceEdge } from './edge-relay-fence.js' import { z } from 'zod' import { EDGE_ADMIN_LOCKED_TIMEOUT_MS, @@ -411,27 +412,13 @@ export class EdgeAssignments { } // After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve) - // rather than HOST_OFFLINE. Best effort; until it lands the edge's freeze does the same, and after - // that its phones see HOST_OFFLINE until the next move. Anything but applied/unchanged fails: - // unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the host). + // rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land + // is recorded for the evacuation sweep to retry (edge-relay-fence.ts). async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise { if (!this.admin) return true // An edge whose exports are failing gets one try, so a skipped export is not slowed further. const attempts = this.exportFailingSince === undefined ? 3 : 1 - let outcome = 'unavailable' - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - outcome = (await this.admin.assign({ ...identity, cellId, assignmentEpoch })).outcome - } catch (error) { - if (!(error instanceof EdgeAdminUnavailableError)) throw error - outcome = error.message - } - if (outcome === 'applied' || outcome === 'unchanged') return true - // The same epoch stays stale; only a later move (a newer epoch) can land. - if (outcome === 'stale') break - } - console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch })) - return false + return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now()) } private answer(identity: Identity, assignmentEpoch: number, leaseExpiresAt: number): RelayAssignment {