relay: export skip needs continuous failures; evacuation sweep runs soon after start, then every 10 min

Failures more than 30 s apart restart the unavailable clock, so one old failure cannot skip an
export. The sweep reads the edge rows (cell_id has no index): first pass 45-75 s after start (a
cohort change is a new revision), every 45-75 s while hosts are queued, else every 7.5-12.5 min.
This commit is contained in:
Neil
2026-11-07 19:49:39 -08:00
parent b185c5b6ef
commit b369be6e96
5 changed files with 62 additions and 12 deletions
@@ -26,6 +26,8 @@ export type EdgeRelayConfig = {
// An edge outage must not strand every leaving host on 503s: after this long of nothing but
// failed exports, a leaving host keeps its Postgres devices and is placed anyway.
export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000
// Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge).
export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000
export function edgeCohortBucket(relayHostId: string): number {
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100
@@ -9,7 +9,8 @@ import {
type EdgeAdmin,
type EdgeDrainResult
} from './edge-admin-client.js'
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS,
EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js'
const EDGE_URL = 'https://edge.relay.example.test'
@@ -140,14 +141,20 @@ describe('edge evacuation sweep', () => {
await h.insert(hostId(1))
const sweep = vi.spyOn(h.evacuation, 'sweep')
h.evacuation.start()
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - 1)
// First pass soon after start, then the slow interval once nothing more is queued.
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS - 1)
expect(sweep).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1)
expect(sweep).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(h.admin.drained).toEqual([hostId(1)]))
await vi.advanceTimersByTimeAsync(0)
expect(h.admin.drained).toEqual([hostId(1)])
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(sweep).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(sweep).toHaveBeenCalledTimes(2)
h.evacuation.stop()
await vi.advanceTimersByTimeAsync(10 * EDGE_EVACUATION_INTERVAL_MS)
expect(sweep).toHaveBeenCalledTimes(1)
expect(sweep).toHaveBeenCalledTimes(2)
})
})
+10 -3
View File
@@ -7,7 +7,11 @@ import type { RelayDatabase } from './database/database.js'
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
export const EDGE_EVACUATION_INTERVAL_MS = 60_000
// The cohort only changes when a new revision starts, so a pass soon after start and then a slow
// background pass are enough; each pass reads the edge rows (cell_id has no index).
export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000
// First pass after start, and the next pass while more leaving hosts are queued.
export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000
// Drains sent per tick (per director), so a cohort cut moves desktops back gradually.
export const EDGE_EVACUATION_BATCH = 20
// Edge rows read per tick; the cursor walks the table across ticks.
@@ -21,6 +25,7 @@ export class EdgeEvacuation {
private readonly drainedAt = new Map<string, number>()
private timer: ReturnType<typeof setTimeout> | undefined
private stopped = true
private busy = true
constructor(
private readonly database: RelayDatabase,
@@ -49,6 +54,7 @@ export class EdgeEvacuation {
if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) }
}
const leaving = await this.leavingHosts()
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined
let answered = 0
for (const identity of leaving) {
try {
@@ -105,8 +111,9 @@ export class EdgeEvacuation {
private schedule(): void {
if (this.stopped) { return }
// Jittered (45-75 s), so directors do not sweep together.
const delay = EDGE_EVACUATION_INTERVAL_MS * (0.75 + this.random() / 2)
// Jittered (±25%), so directors do not sweep together.
const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS
const delay = interval * (0.75 + this.random() / 2)
this.timer = setTimeout(() => {
void this.sweep()
.catch((error: unknown) =>
+29 -3
View File
@@ -179,6 +179,16 @@ async function harness(cohort: 'host' | 'account' = 'host', extra: Partial<EdgeR
return { database, fake, clock, assignments, assign, insertDevice, postgresDevices, row, leaveCohort }
}
// Asks every 20 s (desktops retry every 2 s; failures further apart than 30 s restart the clock)
// until `ms` after the current time, expecting the 503 each time.
async function keepFailing(h: { clock: { now: number }; assign: () => Promise<Response> }, ms: number) {
const end = h.clock.now + ms
while (h.clock.now < end) {
h.clock.now = Math.min(end, h.clock.now + 20_000)
if (h.clock.now < end) { expect((await h.assign()).status).toBe(503) }
}
}
describe('relay edge phase 2: the director moves host data with the host', () => {
it('pushes the host and its Postgres devices before answering with the edge, with a location hint', async () => {
const h = await harness()
@@ -256,7 +266,7 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
h.fake.down = true
h.leaveCohort()
expect((await h.assign()).status).toBe(503)
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
const early = await h.assign()
expect(early.status).toBe(503)
expect(early.headers.get('retry-after')).toBe('2')
@@ -289,9 +299,25 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
h.fake.connected = true
expect((await h.assign()).status).toBe(503)
h.fake.down = true
h.clock.now += 100_000
h.clock.now += 20_000
expect((await h.assign()).status).toBe(503)
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
expect((await h.assign()).status).toBe(503)
h.clock.now += 1
expect((await h.assign()).status).toBe(200)
})
it('does not count failures further apart than 30 s as one outage', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
await h.assign()
h.leaveCohort()
h.fake.down = true
expect((await h.assign()).status).toBe(503)
// One old failure, then quiet for longer than the skip threshold: the next failure starts over.
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS + 1
expect((await h.assign()).status).toBe(503)
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
expect((await h.assign()).status).toBe(503)
h.clock.now += 1
expect((await h.assign()).status).toBe(200)
+10 -2
View File
@@ -5,6 +5,7 @@ import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmissi
import type { RelayDatabase, SqlRow } from './database/database.js'
import { EdgeAdminUnavailableError, edgeLocationHint, type EdgeAdmin } from './edge-admin-client.js'
import {
EDGE_EXPORT_FAILURE_GAP_MS,
DEVICE_COLUMNS,
EDGE_EXPORT_SKIP_AFTER_MS,
EdgeMoveBusyError,
@@ -39,6 +40,7 @@ export class EdgeAssignments {
private seeded = false
// When this director's exports started failing as unavailable; any usable answer clears it.
private exportFailingSince: number | undefined
private exportLastFailureAt: number | undefined
// With `admin` (the edge keeps host data in its objects) every move onto the edge pushes the
// host's devices before the answer, and every move off exports them back first.
@@ -243,8 +245,13 @@ export class EdgeAssignments {
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
const now = this.now()
this.exportFailingSince ??= now
const failingForMs = now - this.exportFailingSince
// Only continuous failures count: a gap longer than EDGE_EXPORT_FAILURE_GAP_MS (desktops retry
// every 2 s) starts the clock again, so one old failure cannot skip an export on its own.
if (this.exportLastFailureAt === undefined || now - this.exportLastFailureAt > EDGE_EXPORT_FAILURE_GAP_MS) {
this.exportFailingSince = now
}
this.exportLastFailureAt = now
const failingForMs = now - (this.exportFailingSince ?? now)
if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() }
// The edge has been unreachable too long: place the host on its Postgres devices (changes
// made on the edge are lost) rather than refusing every ask until the edge is back.
@@ -252,6 +259,7 @@ export class EdgeAssignments {
return true
}
this.exportFailingSince = undefined
this.exportLastFailureAt = undefined
if (exported.outcome === 'busy') { throw new EdgeMoveBusyError() }
// An object with no assignment never held this host's data (or was erased): keep Postgres.
if (!exported.assignment) { return true }