relay: failed edge fences recorded (relay_edge_fence_pending) and retried from the evacuation sweep to the host's current row

This commit is contained in:
Neil
2026-11-07 22:52:50 -08:00
parent 5527dc574a
commit e2741fa4cd
6 changed files with 382 additions and 30 deletions
+12
View File
@@ -262,6 +262,18 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
ON relay_assignment_region_preferences(observed_at);
-- Relay edge fences that did not land (edge-relay-fence.ts). Small and short-lived: a row is
-- deleted when its fence lands, the host has no row or is back on the edge, or after its last retry.
CREATE TABLE IF NOT EXISTS relay_edge_fence_pending (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
cell_id TEXT NOT NULL,
assignment_epoch BIGINT NOT NULL,
attempts BIGINT NOT NULL,
next_attempt_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id)
);
CREATE TABLE IF NOT EXISTS relay_region_decisions (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
@@ -1,11 +1,13 @@
// Evacuation sweep: a desktop connected to the relay edge only asks the director again when its
// lease ends, so a host that left the edge cohort would stay there for hours. The sweep finds edge
// rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts).
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also
// retries fences that did not land after a host left (edge-relay-fence.ts).
import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions'
import type { RelayDatabase, SqlRow } from './database.js'
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
import { inEdgeCohort, type EdgeRelayConfig } from './edge-relay.js'
import { retryPendingFences } from './edge-relay-fence.js'
type Identity = { userId: string; relayHostId: string }
@@ -81,6 +83,8 @@ export class EdgeEvacuation {
console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message }))
}
}
// Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured.
if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true }
return answered
}
+156 -12
View File
@@ -11,6 +11,8 @@ import {
type EdgeAssignBody,
type EdgeAssignResult
} from './edge-admin-client.js'
import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js'
import {
EDGE_CELL_ID,
EDGE_LOCKED_CALLS_MAX,
@@ -23,6 +25,7 @@ const EDGE_URL = 'https://edge.relay.example.test'
const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!')
const USER = 'user-1'
const HOST = 'fencehost0000001'
const T0 = 1_700_000_000_000
const databases: RelayDatabase[] = []
afterEach(async () => {
@@ -75,7 +78,7 @@ async function harness() {
const database = await openInMemoryRelayDatabase()
databases.push(database)
const edge = new ScriptedEdge()
const clock = { now: 1_700_000_000_000 }
const clock = { now: T0 }
const assignments = new EdgeAssignments(database, edgeConfig(), () => clock.now, edge)
await assignments.ensureEdgeCell()
const placeRow = async (cellId: string, epoch: number, relayHostId = HOST) =>
@@ -89,7 +92,16 @@ async function harness() {
assignment_epoch = excluded.assignment_epoch`,
[USER, relayHostId, cellId, epoch]
)
return { database, edge, clock, assignments, placeRow }
const pending = async () =>
(await database.query(`SELECT * FROM relay_edge_fence_pending ORDER BY relay_host_id`)).map((row) => ({
relayHostId: String(row.relay_host_id),
cellId: String(row.cell_id),
assignmentEpoch: Number(row.assignment_epoch),
attempts: Number(row.attempts),
nextAttemptAt: Number(row.next_attempt_at)
}))
const retry = async () => await retryPendingFences(database, edge, EDGE_CELL_ID, () => clock.now)
return { database, edge, clock, assignments, placeRow, pending, retry }
}
function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record<string, unknown>[] {
@@ -99,7 +111,7 @@ function logged(spy: { mock: { calls: unknown[][] } }, event: string): Record<st
}
describe('relay edge fence after a host leaves', () => {
it('treats a stale answer as a failed fence, logged with its outcome and not retried inline', async () => {
it('treats a stale answer as a failed fence: logged with its outcome, recorded, not retried inline', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['stale']
@@ -108,26 +120,158 @@ describe('relay edge fence after a host leaves', () => {
expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([
{ event: 'orca_relay_edge_fence_failed', outcome: 'stale', cellId: 'us-c1', assignmentEpoch: 9 }
])
expect(await h.pending()).toEqual([
{ relayHostId: HOST, cellId: 'us-c1', assignmentEpoch: 9, attempts: 0, nextAttemptAt: T0 + EDGE_FENCE_RETRY_BASE_MS }
])
})
it('retries conflict and unavailable inline, then logs the last outcome', async () => {
it('retries conflict and unavailable inline, then records the last outcome', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['conflict', 'conflict', 'down']
h.edge.answers = ['conflict', 'down', 'conflict']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false)
expect(h.edge.calls).toHaveLength(3)
expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({
outcome: 'relay_edge_admin_unavailable:network'
})
expect(logged(error, 'orca_relay_edge_fence_failed')[0]).toMatchObject({ outcome: 'conflict' })
expect(await h.pending()).toHaveLength(1)
})
it('lands on applied or unchanged', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
it('records nothing when the fence lands, and a newer failure replaces an older record', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['down', 'unchanged']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
expect(logged(error, 'orca_relay_edge_fence_failed')).toEqual([])
expect(await h.pending()).toEqual([])
h.edge.answers = ['down', 'down', 'down']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
await h.database.query(`UPDATE relay_edge_fence_pending SET attempts = 3`)
h.clock.now += 5_000
h.edge.answers = ['stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c2', 12)
expect(await h.pending()).toEqual([
{ relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 12, attempts: 0, nextAttemptAt: h.clock.now + EDGE_FENCE_RETRY_BASE_MS }
])
})
it('still answers when the failure cannot be recorded', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
await h.database.query(`DROP TABLE relay_edge_fence_pending`)
h.edge.answers = ['stale']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(false)
expect(logged(error, 'orca_relay_edge_fence_record_failed')).toHaveLength(1)
})
})
describe('relay edge pending fence retries', () => {
it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
// The host moved again before the retry.
await h.placeRow('us-c2', 11)
h.edge.calls = []
expect(await h.retry()).toBe(true)
expect(h.edge.calls).toEqual([])
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
expect(await h.retry()).toBe(false)
expect(h.edge.calls).toEqual([{ body: { userId: USER, relayHostId: HOST, cellId: 'us-c2', assignmentEpoch: 11 } }])
expect(await h.pending()).toEqual([])
expect(logged(warn, 'orca_relay_edge_fence_retried')).toEqual([
{ event: 'orca_relay_edge_fence_retried', outcome: 'applied', cellId: 'us-c2', assignmentEpoch: 11, attempts: 1 }
])
})
it('drops the record without a call when the host is back on the edge or has no row', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['stale', 'stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
await h.assignments.fenceAway({ userId: USER, relayHostId: 'fencehost0000002' }, 'us-c1', 4)
await h.placeRow(EDGE_CELL_ID, 12)
h.edge.calls = []
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
expect(await h.retry()).toBe(false)
expect(h.edge.calls).toEqual([])
expect(await h.pending()).toEqual([])
})
it('backs off on each failure, logging it, and gives up after the last attempt', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
await h.placeRow('us-c1', 9)
h.edge.answers = ['stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) {
h.edge.answers = [attempt % 2 === 0 ? 'down' : 'conflict']
expect(await h.retry()).toBe(true)
expect(await h.pending()).toMatchObject([{ attempts: attempt, nextAttemptAt: h.clock.now + edgeFenceRetryDelayMs(attempt) }])
// Not due yet: nothing is sent.
h.clock.now += edgeFenceRetryDelayMs(attempt) - 1
const calls = h.edge.calls.length
await h.retry()
expect(h.edge.calls).toHaveLength(calls)
h.clock.now += 1
}
h.edge.answers = ['stale']
expect(await h.retry()).toBe(false)
expect(await h.pending()).toEqual([])
const failures = logged(error, 'orca_relay_edge_fence_failed').filter((entry) => entry.retry === true)
expect(failures).toHaveLength(EDGE_FENCE_MAX_ATTEMPTS)
expect(failures[1]).toEqual({
event: 'orca_relay_edge_fence_failed',
outcome: 'relay_edge_admin_unavailable:network',
cellId: 'us-c1',
assignmentEpoch: 9,
attempts: 2,
retry: true
})
expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([
{ event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS }
])
expect(edgeFenceRetryDelayMs(1)).toBe(60_000)
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000)
})
it('lets only one director claim an attempt', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
await h.placeRow('us-c1', 9)
h.edge.answers = ['stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
h.edge.calls = []
await Promise.all([h.retry(), h.retry()])
expect(h.edge.calls).toHaveLength(1)
})
it('runs from the evacuation sweep, which keeps its busy cadence while fences are pending', async () => {
vi.useFakeTimers()
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
const h = await harness()
await h.placeRow('us-c1', 9)
const config = { ...edgeConfig(), rolloutPercent: 0 }
const evacuation = new EdgeEvacuation(h.database, config, h.edge, EDGE_CELL_ID, () => h.clock.now, () => 0.5)
h.edge.answers = ['stale']
await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)
h.edge.calls = []
h.edge.answers = ['down']
const sweep = vi.spyOn(evacuation, 'sweep')
evacuation.start()
h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(h.edge.calls).toHaveLength(1)
// Still pending: the next pass comes after the busy interval, and lands.
h.clock.now += EDGE_EVACUATION_BUSY_INTERVAL_MS
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(sweep).toHaveBeenCalledTimes(2)
expect(h.edge.calls).toHaveLength(2)
expect(await h.pending()).toEqual([])
evacuation.stop()
})
})
+167
View File
@@ -0,0 +1,167 @@
// Fences that did not land. After a host leaves the relay edge the director tells the edge where it
// went (EdgeAssignments.fenceAway); until that lands the edge object still claims the host and its
// phones get HOST_OFFLINE instead of re-resolving. A failed fence is recorded here, so any director's
// evacuation sweep retries it, always to the host's current row, never to the recorded one.
import type { RelayDatabase, SqlRow } from './database.js'
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
type Identity = { userId: string; relayHostId: string }
function integer(row: SqlRow, field: string): number {
const value = Number(row[field])
if (!Number.isSafeInteger(value)) { throw new Error(`invalid_${field}`) }
return value
}
function text(row: SqlRow, field: string): string {
const value = row[field]
if (typeof value !== 'string') { throw new Error(`invalid_${field}`) }
return value
}
// Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`,
// cannot land until the host moves again, and that move fences anew).
export const EDGE_FENCE_MAX_ATTEMPTS = 8
// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all.
export const EDGE_FENCE_RETRY_BASE_MS = 60_000
const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
// Pending fences retried per sweep (per director).
const EDGE_FENCE_RETRY_BATCH = 20
export function edgeFenceRetryDelayMs(attempts: number): number {
return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS)
}
// A newer failure for the same host replaces the older one and starts its attempts again.
export async function recordPendingFence(
database: RelayDatabase,
identity: Identity,
cellId: string,
assignmentEpoch: number,
now: number
): Promise<void> {
await database.query(
`INSERT INTO relay_edge_fence_pending
(user_id, relay_host_id, cell_id, assignment_epoch, attempts, next_attempt_at)
VALUES (?, ?, ?, ?, 0, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
cell_id = excluded.cell_id, assignment_epoch = excluded.assignment_epoch,
attempts = 0, next_attempt_at = excluded.next_attempt_at`,
[identity.userId, identity.relayHostId, cellId, assignmentEpoch, now + EDGE_FENCE_RETRY_BASE_MS]
)
}
// The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged
// fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the
// host); the failure is logged and recorded for retry. Returns whether the fence landed.
export async function fenceEdge(
database: RelayDatabase,
admin: EdgeAdmin,
fence: Identity & { cellId: string; assignmentEpoch: number },
attempts: number,
now: number
): Promise<boolean> {
const { cellId, assignmentEpoch } = fence
let outcome = 'unavailable'
for (let attempt = 0; attempt < attempts; attempt += 1) {
try {
outcome = (await admin.assign(fence)).outcome
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
outcome = error.message
}
if (outcome === 'applied' || outcome === 'unchanged') { return true }
// The same epoch stays stale; only a later move (a newer epoch) can land.
if (outcome === 'stale') { break }
}
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch }))
try {
await recordPendingFence(database, fence, cellId, assignmentEpoch, now)
} catch (error) {
// Best effort, like the fence itself: the host is already placed and its answer still goes out.
console.error(
JSON.stringify({
event: 'orca_relay_edge_fence_record_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
return false
}
type Retry = 'landed' | 'dropped' | 'failed'
// One pass over the due pending fences. Returns whether any pending fence is left (the sweep then
// keeps its busy cadence).
export async function retryPendingFences(
database: RelayDatabase,
admin: EdgeAdmin,
edgeCellId: string,
now: () => number
): Promise<boolean> {
const due = await database.query(
`SELECT user_id, relay_host_id, attempts, next_attempt_at FROM relay_edge_fence_pending
WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`,
[now(), EDGE_FENCE_RETRY_BATCH]
)
for (const row of due) {
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
const attempts = integer(row, 'attempts') + 1
// Claim the attempt: a director that read the same row loses here and skips it.
const claimed = await database.query(
`UPDATE relay_edge_fence_pending SET attempts = ?, next_attempt_at = ?
WHERE user_id = ? AND relay_host_id = ? AND attempts = ? AND next_attempt_at = ?
RETURNING attempts`,
[attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId,
attempts - 1, integer(row, 'next_attempt_at')]
)
if (claimed.length !== 1) { continue }
const result = await retryOne(database, admin, edgeCellId, identity, attempts)
if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
// Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0.
await database.query(
`DELETE FROM relay_edge_fence_pending WHERE user_id = ? AND relay_host_id = ? AND attempts = ?`,
[identity.userId, identity.relayHostId, attempts]
)
}
if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts }))
}
}
return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0
}
// Fences the edge to the host's CURRENT row; a host that is back on the edge (or has no row) needs
// no fence, since its next move away fences again.
async function retryOne(
database: RelayDatabase,
admin: EdgeAdmin,
edgeCellId: string,
identity: Identity,
attempts: number
): Promise<Retry> {
const row = (
await database.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (!row || text(row, 'cell_id') === edgeCellId) { return 'dropped' }
const cellId = text(row, 'cell_id')
const assignmentEpoch = integer(row, 'assignment_epoch')
let outcome: string
try {
outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
outcome = error.message
}
if (outcome === 'applied' || outcome === 'unchanged') {
console.warn(JSON.stringify({ event: 'orca_relay_edge_fence_retried', outcome, cellId, assignmentEpoch, attempts }))
return 'landed'
}
console.error(
JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true })
)
return 'failed'
}
@@ -10,6 +10,7 @@ import {
EdgeMoveDeferredError
} from './edge-relay.js'
import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js'
import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
@@ -35,6 +36,7 @@ describePostgres('relay edge on PostgreSQL', () => {
await database.query(`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`, ['edge-pg-user'])
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, ['edge-pg-user'])
await database.query(`DELETE FROM relay_devices WHERE user_id = ?`, ['edge-pg-user'])
await database.query(`DELETE FROM relay_edge_fence_pending WHERE user_id = ?`, ['edge-pg-user'])
// Leave the shared selector as other files expect it: without this file's cells.
await database.transaction(async (transaction) => {
const selector = (
@@ -198,4 +200,40 @@ describePostgres('relay edge on PostgreSQL', () => {
await Promise.all(held)
expect((await edge.moveToEdge(last, 0)).cellId).toBe(EDGE_CELL_ID)
})
it('records a failed fence and lets exactly one director retry it to the current row', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
const host = { userId: 'edge-pg-user', relayHostId: 'edgepgfence00001' }
const sent: EdgeAssignBody[] = []
const admin = {
assign: async (body: EdgeAssignBody) => {
sent.push(body)
return { outcome: sent.length === 1 ? ('stale' as const) : ('applied' as const), assignmentEpoch: 30 }
},
exportHost: async () => ({ outcome: 'busy' as const }),
drain: async () => ({ outcome: 'idle' as const })
}
const clock = { now: 1_700_000_000_000 }
const edge = new EdgeAssignments(database, edgeConfig, () => clock.now, admin)
await database.query(
`INSERT INTO relay_assignments
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at, last_activity_at,
reserved_controls, reserved_splices, reserved_invites, pending_installs,
pending_confirmations, migration_leases)
VALUES (?, ?, ?, 31, 0, 0, 0, 0, 0, 0, 0, 0)`,
[host.userId, host.relayHostId, cell.id]
)
expect(await edge.fenceAway(host, cell.id, 29)).toBe(false)
clock.now += EDGE_FENCE_RETRY_BASE_MS
const results = await Promise.all([
retryPendingFences(database, admin, EDGE_CELL_ID, () => clock.now),
retryPendingFences(second, admin, EDGE_CELL_ID, () => clock.now)
])
// The winner saw nothing left; the loser may still have seen the claimed record.
expect(results).toContain(false)
expect(sent.slice(1)).toEqual([{ ...host, cellId: cell.id, assignmentEpoch: 31 }])
expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([])
vi.restoreAllMocks()
})
})
+4 -17
View File
@@ -4,6 +4,7 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra
import type { RelayAssignment } from './assignment/assignment-store.js'
import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmission } from './cell-admission-selector.js'
import type { RelayDatabase, SqlRow } from './database.js'
import { fenceEdge } from './edge-relay-fence.js'
import { z } from 'zod'
import {
EDGE_ADMIN_LOCKED_TIMEOUT_MS,
@@ -411,27 +412,13 @@ export class EdgeAssignments {
}
// After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve)
// rather than HOST_OFFLINE. Best effort; until it lands the edge's freeze does the same, and after
// that its phones see HOST_OFFLINE until the next move. Anything but applied/unchanged fails:
// unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the host).
// rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land
// is recorded for the evacuation sweep to retry (edge-relay-fence.ts).
async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise<boolean> {
if (!this.admin) return true
// An edge whose exports are failing gets one try, so a skipped export is not slowed further.
const attempts = this.exportFailingSince === undefined ? 3 : 1
let outcome = 'unavailable'
for (let attempt = 0; attempt < attempts; attempt += 1) {
try {
outcome = (await this.admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) throw error
outcome = error.message
}
if (outcome === 'applied' || outcome === 'unchanged') return true
// The same epoch stays stale; only a later move (a newer epoch) can land.
if (outcome === 'stale') break
}
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch }))
return false
return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now())
}
private answer(identity: Identity, assignmentEpoch: number, leaseExpiresAt: number): RelayAssignment {