mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
relay: failed edge fences recorded (relay_edge_fence_pending) and retried from the evacuation sweep to the host's current row
This commit is contained in:
@@ -262,6 +262,18 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
|
||||
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
|
||||
ON relay_assignment_region_preferences(observed_at);
|
||||
|
||||
-- Relay edge fences that did not land (edge-relay-fence.ts). Small and short-lived: a row is
|
||||
-- deleted when its fence lands, the host has no row or is back on the edge, or after its last retry.
|
||||
CREATE TABLE IF NOT EXISTS relay_edge_fence_pending (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
cell_id TEXT NOT NULL,
|
||||
assignment_epoch BIGINT NOT NULL,
|
||||
attempts BIGINT NOT NULL,
|
||||
next_attempt_at BIGINT NOT NULL,
|
||||
PRIMARY KEY (user_id, relay_host_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_region_decisions (
|
||||
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
|
||||
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
// Evacuation sweep: a desktop connected to the relay edge only asks the director again when its
|
||||
// lease ends, so a host that left the edge cohort would stay there for hours. The sweep finds edge
|
||||
// rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then
|
||||
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts).
|
||||
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also
|
||||
// retries fences that did not land after a host left (edge-relay-fence.ts).
|
||||
import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions'
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
|
||||
import { inEdgeCohort, type EdgeRelayConfig } from './edge-relay.js'
|
||||
import { retryPendingFences } from './edge-relay-fence.js'
|
||||
|
||||
type Identity = { userId: string; relayHostId: string }
|
||||
|
||||
@@ -81,6 +83,8 @@ export class EdgeEvacuation {
|
||||
console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message }))
|
||||
}
|
||||
}
|
||||
// Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured.
|
||||
if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true }
|
||||
return answered
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type EdgeAssignBody,
|
||||
type EdgeAssignResult
|
||||
} from './edge-admin-client.js'
|
||||
import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js'
|
||||
import {
|
||||
EDGE_CELL_ID,
|
||||
EDGE_LOCKED_CALLS_MAX,
|
||||
@@ -23,6 +25,7 @@ const EDGE_URL = 'https://edge.relay.example.test'
|
||||
const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!')
|
||||
const USER = 'user-1'
|
||||
const HOST = 'fencehost0000001'
|
||||
const T0 = 1_700_000_000_000
|
||||
|
||||
const databases: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
@@ -75,7 +78,7 @@ async function harness() {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const edge = new ScriptedEdge()
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const clock = { now: T0 }
|
||||
const assignments = new EdgeAssignments(database, edgeConfig(), () => clock.now, edge)
|
||||
await assignments.ensureEdgeCell()
|
||||
const placeRow = async (cellId: string, epoch: number, relayHostId = HOST) =>
|
||||
@@ -89,7 +92,16 @@ async function harness() {
|
||||
assignment_epoch = excluded.assignment_epoch`,
|
||||
[USER, relayHostId, cellId, epoch]
|
||||
)
|
||||
return { database, edge, clock, assignments, placeRow }
|
||||
const pending = async () =>
|
||||
(await database.query(`SELECT * FROM relay_edge_fence_pending ORDER BY relay_host_id`)).map((row) => ({
|
||||
relayHostId: String(row.relay_host_id),
|
||||
cellId: String(row.cell_id),
|
||||
assignmentEpoch: Number(row.assignment_epoch),
|
||||
attempts: Number(row.attempts),
|
||||
nextAttemptAt: Number(row.next_attempt_at)
|
||||
}))
|
||||
const retry = async () => await retryPendingFences(database, edge, EDGE_CELL_ID, () => clock.now)
|
||||
return { database, edge, clock, assignments, placeRow, pending, retry }
|
||||
}
|
||||
|
||||
function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record<string, unknown>[] {
|
||||
@@ -99,7 +111,7 @@ function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record<st
|
||||
}
|
||||
|
||||
describe('relay edge fence after a host leaves', () => {
|
||||
it('treats a stale answer as a failed fence, logged with its outcome and not retried inline', async () => {
|
||||
it('treats a stale answer as a failed fence: logged with its outcome, recorded, not retried inline', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['stale']
|
||||
@@ -108,26 +120,158 @@ describe('relay edge fence after a host leaves', () => {
|
||||
expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([
|
||||
{ event: 'orca_relay_edge_fence_failed', outcome: 'stale', cellId: 'us-c1', assignmentEpoch: 9 }
|
||||
])
|
||||
expect(await h.pending()).toEqual([
|
||||
{ relayHostId: HOST, cellId: 'us-c1', assignmentEpoch: 9, attempts: 0, nextAttemptAt: T0 + EDGE_FENCE_RETRY_BASE_MS }
|
||||
])
|
||||
})
|
||||
|
||||
it('retries conflict and unavailable inline, then logs the last outcome', async () => {
|
||||
it('retries conflict and unavailable inline, then records the last outcome', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['conflict', 'conflict', 'down']
|
||||
h.edge.answers = ['conflict', 'down', 'conflict']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false)
|
||||
expect(h.edge.calls).toHaveLength(3)
|
||||
expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({
|
||||
outcome: 'relay_edge_admin_unavailable:network'
|
||||
})
|
||||
expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({ outcome: 'conflict' })
|
||||
expect(await h.pending()).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('lands on applied or unchanged', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
it('records nothing when the fence lands, and a newer failure replaces an older record', async () => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['down', 'unchanged']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
|
||||
expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([])
|
||||
expect(await h.pending()).toEqual([])
|
||||
h.edge.answers = ['down', 'down', 'down']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
await h.database.query(`UPDATE relay_edge_fence_pending SET attempts = 3`)
|
||||
h.clock.now += 5_000
|
||||
h.edge.answers = ['stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c2', 12)
|
||||
expect(await h.pending()).toEqual([
|
||||
{ relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 12, attempts: 0, nextAttemptAt: h.clock.now + EDGE_FENCE_RETRY_BASE_MS }
|
||||
])
|
||||
})
|
||||
|
||||
it('still answers when the failure cannot be recorded', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.database.query(`DROP TABLE relay_edge_fence_pending`)
|
||||
h.edge.answers = ['stale']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false)
|
||||
expect(logged(error, 'orca_relay_edge_fence_record_failed')).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('relay edge pending fence retries', () => {
|
||||
it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
// The host moved again before the retry.
|
||||
await h.placeRow('us-c2', 11)
|
||||
h.edge.calls = []
|
||||
expect(await h.retry()).toBe(true)
|
||||
expect(h.edge.calls).toEqual([])
|
||||
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
expect(await h.retry()).toBe(false)
|
||||
expect(h.edge.calls).toEqual([{ body: { userId: USER, relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 11 } }])
|
||||
expect(await h.pending()).toEqual([])
|
||||
expect(logged(warn, 'orca_relay_edge_fence_retried')).toEqual([
|
||||
{ event: 'orca_relay_edge_fence_retried', outcome: 'applied', cellId: 'us-c2', assignmentEpoch: 11, attempts: 1 }
|
||||
])
|
||||
})
|
||||
|
||||
it('drops the record without a call when the host is back on the edge or has no row', async () => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['stale', 'stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: 'fencehost0000002' }, 'us-c1', 4)
|
||||
await h.placeRow(EDGE_CELL_ID, 12)
|
||||
h.edge.calls = []
|
||||
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
expect(await h.retry()).toBe(false)
|
||||
expect(h.edge.calls).toEqual([])
|
||||
expect(await h.pending()).toEqual([])
|
||||
})
|
||||
|
||||
it('backs off on each failure, logging it, and gives up after the last attempt', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.placeRow('us-c1', 9)
|
||||
h.edge.answers = ['stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) {
|
||||
h.edge.answers = [attempt % 2 === 0 ? 'down' : 'conflict']
|
||||
expect(await h.retry()).toBe(true)
|
||||
expect(await h.pending()).toMatchObject([{ attempts: attempt, nextAttemptAt: h.clock.now + edgeFenceRetryDelayMs(attempt) }])
|
||||
// Not due yet: nothing is sent.
|
||||
h.clock.now += edgeFenceRetryDelayMs(attempt) - 1
|
||||
const calls = h.edge.calls.length
|
||||
await h.retry()
|
||||
expect(h.edge.calls).toHaveLength(calls)
|
||||
h.clock.now += 1
|
||||
}
|
||||
h.edge.answers = ['stale']
|
||||
expect(await h.retry()).toBe(false)
|
||||
expect(await h.pending()).toEqual([])
|
||||
const failures = logged(error, 'orca_relay_edge_fence_failed').filter((entry) => entry.retry === true)
|
||||
expect(failures).toHaveLength(EDGE_FENCE_MAX_ATTEMPTS)
|
||||
expect(failures[1]).toEqual({
|
||||
event: 'orca_relay_edge_fence_failed',
|
||||
outcome: 'relay_edge_admin_unavailable:network',
|
||||
cellId: 'us-c1',
|
||||
assignmentEpoch: 9,
|
||||
attempts: 2,
|
||||
retry: true
|
||||
})
|
||||
expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([
|
||||
{ event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS }
|
||||
])
|
||||
expect(edgeFenceRetryDelayMs(1)).toBe(60_000)
|
||||
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000)
|
||||
})
|
||||
|
||||
it('lets only one director claim an attempt', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.placeRow('us-c1', 9)
|
||||
h.edge.answers = ['stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
h.edge.calls = []
|
||||
await Promise.all([h.retry(), h.retry()])
|
||||
expect(h.edge.calls).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('runs from the evacuation sweep, which keeps its busy cadence while fences are pending', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.placeRow('us-c1', 9)
|
||||
const config = { ...edgeConfig(), rolloutPercent: 0 }
|
||||
const evacuation = new EdgeEvacuation(h.database, config, h.edge, EDGE_CELL_ID, () => h.clock.now, () => 0.5)
|
||||
h.edge.answers = ['stale']
|
||||
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
|
||||
h.edge.calls = []
|
||||
h.edge.answers = ['down']
|
||||
const sweep = vi.spyOn(evacuation, 'sweep')
|
||||
evacuation.start()
|
||||
h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(h.edge.calls).toHaveLength(1)
|
||||
// Still pending: the next pass comes after the busy interval, and lands.
|
||||
h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(2)
|
||||
expect(h.edge.calls).toHaveLength(2)
|
||||
expect(await h.pending()).toEqual([])
|
||||
evacuation.stop()
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
// Fences that did not land. After a host leaves the relay edge the director tells the edge where it
|
||||
// went (EdgeAssignments.fenceAway); until that lands the edge object still claims the host and its
|
||||
// phones get HOST_OFFLINE instead of re-resolving. A failed fence is recorded here, so any director's
|
||||
// evacuation sweep retries it, always to the host's current row, never to the recorded one.
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
|
||||
|
||||
type Identity = { userId: string; relayHostId: string }
|
||||
|
||||
function integer(row: SqlRow, field: string): number {
|
||||
const value = Number(row[field])
|
||||
if (!Number.isSafeInteger(value)) { throw new Error(`invalid_${field}`) }
|
||||
return value
|
||||
}
|
||||
|
||||
function text(row: SqlRow, field: string): string {
|
||||
const value = row[field]
|
||||
if (typeof value !== 'string') { throw new Error(`invalid_${field}`) }
|
||||
return value
|
||||
}
|
||||
|
||||
// Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`,
|
||||
// cannot land until the host moves again, and that move fences anew).
|
||||
export const EDGE_FENCE_MAX_ATTEMPTS = 8
|
||||
// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all.
|
||||
export const EDGE_FENCE_RETRY_BASE_MS = 60_000
|
||||
const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
|
||||
// Pending fences retried per sweep (per director).
|
||||
const EDGE_FENCE_RETRY_BATCH = 20
|
||||
|
||||
export function edgeFenceRetryDelayMs(attempts: number): number {
|
||||
return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS)
|
||||
}
|
||||
|
||||
// A newer failure for the same host replaces the older one and starts its attempts again.
|
||||
export async function recordPendingFence(
|
||||
database: RelayDatabase,
|
||||
identity: Identity,
|
||||
cellId: string,
|
||||
assignmentEpoch: number,
|
||||
now: number
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_edge_fence_pending
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, attempts, next_attempt_at)
|
||||
VALUES (?, ?, ?, ?, 0, ?)
|
||||
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
|
||||
cell_id = excluded.cell_id, assignment_epoch = excluded.assignment_epoch,
|
||||
attempts = 0, next_attempt_at = excluded.next_attempt_at`,
|
||||
[identity.userId, identity.relayHostId, cellId, assignmentEpoch, now + EDGE_FENCE_RETRY_BASE_MS]
|
||||
)
|
||||
}
|
||||
|
||||
// The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged
|
||||
// fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the
|
||||
// host); the failure is logged and recorded for retry. Returns whether the fence landed.
|
||||
export async function fenceEdge(
|
||||
database: RelayDatabase,
|
||||
admin: EdgeAdmin,
|
||||
fence: Identity & { cellId: string; assignmentEpoch: number },
|
||||
attempts: number,
|
||||
now: number
|
||||
): Promise<boolean> {
|
||||
const { cellId, assignmentEpoch } = fence
|
||||
let outcome = 'unavailable'
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try {
|
||||
outcome = (await admin.assign(fence)).outcome
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
outcome = error.message
|
||||
}
|
||||
if (outcome === 'applied' || outcome === 'unchanged') { return true }
|
||||
// The same epoch stays stale; only a later move (a newer epoch) can land.
|
||||
if (outcome === 'stale') { break }
|
||||
}
|
||||
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch }))
|
||||
try {
|
||||
await recordPendingFence(database, fence, cellId, assignmentEpoch, now)
|
||||
} catch (error) {
|
||||
// Best effort, like the fence itself: the host is already placed and its answer still goes out.
|
||||
console.error(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_edge_fence_record_failed',
|
||||
reason: error instanceof Error ? error.message : 'unknown'
|
||||
})
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Retry = 'landed' | 'dropped' | 'failed'
|
||||
|
||||
// One pass over the due pending fences. Returns whether any pending fence is left (the sweep then
|
||||
// keeps its busy cadence).
|
||||
export async function retryPendingFences(
|
||||
database: RelayDatabase,
|
||||
admin: EdgeAdmin,
|
||||
edgeCellId: string,
|
||||
now: () => number
|
||||
): Promise<boolean> {
|
||||
const due = await database.query(
|
||||
`SELECT user_id, relay_host_id, attempts, next_attempt_at FROM relay_edge_fence_pending
|
||||
WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`,
|
||||
[now(), EDGE_FENCE_RETRY_BATCH]
|
||||
)
|
||||
for (const row of due) {
|
||||
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
|
||||
const attempts = integer(row, 'attempts') + 1
|
||||
// Claim the attempt: a director that read the same row loses here and skips it.
|
||||
const claimed = await database.query(
|
||||
`UPDATE relay_edge_fence_pending SET attempts = ?, next_attempt_at = ?
|
||||
WHERE user_id = ? AND relay_host_id = ? AND attempts = ? AND next_attempt_at = ?
|
||||
RETURNING attempts`,
|
||||
[attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId,
|
||||
attempts - 1, integer(row, 'next_attempt_at')]
|
||||
)
|
||||
if (claimed.length !== 1) { continue }
|
||||
const result = await retryOne(database, admin, edgeCellId, identity, attempts)
|
||||
if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
|
||||
// Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0.
|
||||
await database.query(
|
||||
`DELETE FROM relay_edge_fence_pending WHERE user_id = ? AND relay_host_id = ? AND attempts = ?`,
|
||||
[identity.userId, identity.relayHostId, attempts]
|
||||
)
|
||||
}
|
||||
if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
|
||||
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts }))
|
||||
}
|
||||
}
|
||||
return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0
|
||||
}
|
||||
|
||||
// Fences the edge to the host's CURRENT row; a host that is back on the edge (or has no row) needs
|
||||
// no fence, since its next move away fences again.
|
||||
async function retryOne(
|
||||
database: RelayDatabase,
|
||||
admin: EdgeAdmin,
|
||||
edgeCellId: string,
|
||||
identity: Identity,
|
||||
attempts: number
|
||||
): Promise<Retry> {
|
||||
const row = (
|
||||
await database.query(
|
||||
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
if (!row || text(row, 'cell_id') === edgeCellId) { return 'dropped' }
|
||||
const cellId = text(row, 'cell_id')
|
||||
const assignmentEpoch = integer(row, 'assignment_epoch')
|
||||
let outcome: string
|
||||
try {
|
||||
outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
outcome = error.message
|
||||
}
|
||||
if (outcome === 'applied' || outcome === 'unchanged') {
|
||||
console.warn(JSON.stringify({ event: 'orca_relay_edge_fence_retried', outcome, cellId, assignmentEpoch, attempts }))
|
||||
return 'landed'
|
||||
}
|
||||
console.error(
|
||||
JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true })
|
||||
)
|
||||
return 'failed'
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
EdgeMoveDeferredError
|
||||
} from './edge-relay.js'
|
||||
import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js'
|
||||
import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -35,6 +36,7 @@ describePostgres('relay edge on PostgreSQL', () => {
|
||||
await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, ['edge-pg-user'])
|
||||
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, ['edge-pg-user'])
|
||||
await database.query(`DELETE FROM relay_devices WHERE user_id = ?`, ['edge-pg-user'])
|
||||
await database.query(`DELETE FROM relay_edge_fence_pending WHERE user_id = ?`, ['edge-pg-user'])
|
||||
// Leave the shared selector as other files expect it: without this file's cells.
|
||||
await database.transaction(async (transaction) => {
|
||||
const selector = (
|
||||
@@ -198,4 +200,40 @@ describePostgres('relay edge on PostgreSQL', () => {
|
||||
await Promise.all(held)
|
||||
expect((await edge.moveToEdge(last, 0)).cellId).toBe(EDGE_CELL_ID)
|
||||
})
|
||||
|
||||
it('records a failed fence and lets exactly one director retry it to the current row', async () => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const host = { userId: 'edge-pg-user', relayHostId: 'edgepgfence00001' }
|
||||
const sent: EdgeAssignBody[] = []
|
||||
const admin = {
|
||||
assign: async (body: EdgeAssignBody) => {
|
||||
sent.push(body)
|
||||
return { outcome: sent.length === 1 ? ('stale' as const) : ('applied' as const), assignmentEpoch: 30 }
|
||||
},
|
||||
exportHost: async () => ({ outcome: 'busy' as const }),
|
||||
drain: async () => ({ outcome: 'idle' as const })
|
||||
}
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const edge = new EdgeAssignments(database, edgeConfig, () => clock.now, admin)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, last_activity_at,
|
||||
reserved_controls, reserved_splices, reserved_invites, pending_installs,
|
||||
pending_confirmations, migration_leases)
|
||||
VALUES (?, ?, ?, 31, 0, 0, 0, 0, 0, 0, 0, 0)`,
|
||||
[host.userId, host.relayHostId, cell.id]
|
||||
)
|
||||
expect(await edge.fenceAway(host, cell.id, 29)).toBe(false)
|
||||
clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
const results = await Promise.all([
|
||||
retryPendingFences(database, admin, EDGE_CELL_ID, () => clock.now),
|
||||
retryPendingFences(second, admin, EDGE_CELL_ID, () => clock.now)
|
||||
])
|
||||
// The winner saw nothing left; the loser may still have seen the claimed record.
|
||||
expect(results).toContain(false)
|
||||
expect(sent.slice(1)).toEqual([{ ...host, cellId: cell.id, assignmentEpoch: 31 }])
|
||||
expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([])
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra
|
||||
import type { RelayAssignment } from './assignment/assignment-store.js'
|
||||
import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmission } from './cell-admission-selector.js'
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
import { fenceEdge } from './edge-relay-fence.js'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
EDGE_ADMIN_LOCKED_TIMEOUT_MS,
|
||||
@@ -411,27 +412,13 @@ export class EdgeAssignments {
|
||||
}
|
||||
|
||||
// After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve)
|
||||
// rather than HOST_OFFLINE. Best effort; until it lands the edge's freeze does the same, and after
|
||||
// that its phones see HOST_OFFLINE until the next move. Anything but applied/unchanged fails:
|
||||
// unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the host).
|
||||
// rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land
|
||||
// is recorded for the evacuation sweep to retry (edge-relay-fence.ts).
|
||||
async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise<boolean> {
|
||||
if (!this.admin) return true
|
||||
// An edge whose exports are failing gets one try, so a skipped export is not slowed further.
|
||||
const attempts = this.exportFailingSince === undefined ? 3 : 1
|
||||
let outcome = 'unavailable'
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try {
|
||||
outcome = (await this.admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) throw error
|
||||
outcome = error.message
|
||||
}
|
||||
if (outcome === 'applied' || outcome === 'unchanged') return true
|
||||
// The same epoch stays stale; only a later move (a newer epoch) can land.
|
||||
if (outcome === 'stale') break
|
||||
}
|
||||
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch }))
|
||||
return false
|
||||
return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now())
|
||||
}
|
||||
|
||||
private answer(identity: Identity, assignmentEpoch: number, leaseExpiresAt: number): RelayAssignment {
|
||||
|
||||
Reference in New Issue
Block a user