mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
relay: export skip needs continuous failures; evacuation sweep runs soon after start, then every 10 min
Failures more than 30 s apart restart the unavailable clock, so one old failure cannot skip an export. The sweep reads the edge rows (cell_id has no index): first pass 45-75 s after start (a cohort change is a new revision), every 45-75 s while hosts are queued, else every 7.5-12.5 min.
This commit is contained in:
@@ -26,6 +26,8 @@ export type EdgeRelayConfig = {
|
||||
// An edge outage must not strand every leaving host on 503s: after this long of nothing but
|
||||
// failed exports, a leaving host keeps its Postgres devices and is placed anyway.
|
||||
export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000
|
||||
// Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge).
|
||||
export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000
|
||||
|
||||
export function edgeCohortBucket(relayHostId: string): number {
|
||||
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100
|
||||
|
||||
@@ -9,7 +9,8 @@ import {
|
||||
type EdgeAdmin,
|
||||
type EdgeDrainResult
|
||||
} from './edge-admin-client.js'
|
||||
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS,
|
||||
EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js'
|
||||
|
||||
const EDGE_URL = 'https://edge.relay.example.test'
|
||||
@@ -140,14 +141,20 @@ describe('edge evacuation sweep', () => {
|
||||
await h.insert(hostId(1))
|
||||
const sweep = vi.spyOn(h.evacuation, 'sweep')
|
||||
h.evacuation.start()
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - 1)
|
||||
// First pass soon after start, then the slow interval once nothing more is queued.
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS - 1)
|
||||
expect(sweep).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(sweep).toHaveBeenCalledTimes(1)
|
||||
await vi.waitFor(() => expect(h.admin.drained).toEqual([hostId(1)]))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(h.admin.drained).toEqual([hostId(1)])
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(2)
|
||||
h.evacuation.stop()
|
||||
await vi.advanceTimersByTimeAsync(10 * EDGE_EVACUATION_INTERVAL_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(1)
|
||||
expect(sweep).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -7,7 +7,11 @@ import type { RelayDatabase } from './database/database.js'
|
||||
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
|
||||
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
|
||||
|
||||
export const EDGE_EVACUATION_INTERVAL_MS = 60_000
|
||||
// The cohort only changes when a new revision starts, so a pass soon after start and then a slow
|
||||
// background pass are enough; each pass reads the edge rows (cell_id has no index).
|
||||
export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000
|
||||
// First pass after start, and the next pass while more leaving hosts are queued.
|
||||
export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000
|
||||
// Drains sent per tick (per director), so a cohort cut moves desktops back gradually.
|
||||
export const EDGE_EVACUATION_BATCH = 20
|
||||
// Edge rows read per tick; the cursor walks the table across ticks.
|
||||
@@ -21,6 +25,7 @@ export class EdgeEvacuation {
|
||||
private readonly drainedAt = new Map<string, number>()
|
||||
private timer: ReturnType<typeof setTimeout> | undefined
|
||||
private stopped = true
|
||||
private busy = true
|
||||
|
||||
constructor(
|
||||
private readonly database: RelayDatabase,
|
||||
@@ -49,6 +54,7 @@ export class EdgeEvacuation {
|
||||
if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) }
|
||||
}
|
||||
const leaving = await this.leavingHosts()
|
||||
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined
|
||||
let answered = 0
|
||||
for (const identity of leaving) {
|
||||
try {
|
||||
@@ -105,8 +111,9 @@ export class EdgeEvacuation {
|
||||
|
||||
private schedule(): void {
|
||||
if (this.stopped) { return }
|
||||
// Jittered (45-75 s), so directors do not sweep together.
|
||||
const delay = EDGE_EVACUATION_INTERVAL_MS * (0.75 + this.random() / 2)
|
||||
// Jittered (±25%), so directors do not sweep together.
|
||||
const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS
|
||||
const delay = interval * (0.75 + this.random() / 2)
|
||||
this.timer = setTimeout(() => {
|
||||
void this.sweep()
|
||||
.catch((error: unknown) =>
|
||||
|
||||
@@ -179,6 +179,16 @@ async function harness(cohort: 'host' | 'account' = 'host', extra: Partial<EdgeR
|
||||
return { database, fake, clock, assignments, assign, insertDevice, postgresDevices, row, leaveCohort }
|
||||
}
|
||||
|
||||
// Asks every 20 s (desktops retry every 2 s; failures further apart than 30 s restart the clock)
|
||||
// until `ms` after the current time, expecting the 503 each time.
|
||||
async function keepFailing(h: { clock: { now: number }; assign: () => Promise<Response> }, ms: number) {
|
||||
const end = h.clock.now + ms
|
||||
while (h.clock.now < end) {
|
||||
h.clock.now = Math.min(end, h.clock.now + 20_000)
|
||||
if (h.clock.now < end) { expect((await h.assign()).status).toBe(503) }
|
||||
}
|
||||
}
|
||||
|
||||
describe('relay edge phase 2: the director moves host data with the host', () => {
|
||||
it('pushes the host and its Postgres devices before answering with the edge, with a location hint', async () => {
|
||||
const h = await harness()
|
||||
@@ -256,7 +266,7 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
|
||||
h.fake.down = true
|
||||
h.leaveCohort()
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1
|
||||
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
|
||||
const early = await h.assign()
|
||||
expect(early.status).toBe(503)
|
||||
expect(early.headers.get('retry-after')).toBe('2')
|
||||
@@ -289,9 +299,25 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
|
||||
h.fake.connected = true
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
h.fake.down = true
|
||||
h.clock.now += 100_000
|
||||
h.clock.now += 20_000
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1
|
||||
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
h.clock.now += 1
|
||||
expect((await h.assign()).status).toBe(200)
|
||||
})
|
||||
|
||||
it('does not count failures further apart than 30 s as one outage', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.assign()
|
||||
h.leaveCohort()
|
||||
h.fake.down = true
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
// One old failure, then quiet for longer than the skip threshold: the next failure starts over.
|
||||
h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS + 1
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1)
|
||||
expect((await h.assign()).status).toBe(503)
|
||||
h.clock.now += 1
|
||||
expect((await h.assign()).status).toBe(200)
|
||||
|
||||
@@ -5,6 +5,7 @@ import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmissi
|
||||
import type { RelayDatabase, SqlRow } from './database/database.js'
|
||||
import { EdgeAdminUnavailableError, edgeLocationHint, type EdgeAdmin } from './edge-admin-client.js'
|
||||
import {
|
||||
EDGE_EXPORT_FAILURE_GAP_MS,
|
||||
DEVICE_COLUMNS,
|
||||
EDGE_EXPORT_SKIP_AFTER_MS,
|
||||
EdgeMoveBusyError,
|
||||
@@ -39,6 +40,7 @@ export class EdgeAssignments {
|
||||
private seeded = false
|
||||
// When this director's exports started failing as unavailable; any usable answer clears it.
|
||||
private exportFailingSince: number | undefined
|
||||
private exportLastFailureAt: number | undefined
|
||||
|
||||
// With `admin` (the edge keeps host data in its objects) every move onto the edge pushes the
|
||||
// host's devices before the answer, and every move off exports them back first.
|
||||
@@ -243,8 +245,13 @@ export class EdgeAssignments {
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
const now = this.now()
|
||||
this.exportFailingSince ??= now
|
||||
const failingForMs = now - this.exportFailingSince
|
||||
// Only continuous failures count: a gap longer than EDGE_EXPORT_FAILURE_GAP_MS (desktops retry
|
||||
// every 2 s) starts the clock again, so one old failure cannot skip an export on its own.
|
||||
if (this.exportLastFailureAt === undefined || now - this.exportLastFailureAt > EDGE_EXPORT_FAILURE_GAP_MS) {
|
||||
this.exportFailingSince = now
|
||||
}
|
||||
this.exportLastFailureAt = now
|
||||
const failingForMs = now - (this.exportFailingSince ?? now)
|
||||
if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() }
|
||||
// The edge has been unreachable too long: place the host on its Postgres devices (changes
|
||||
// made on the edge are lost) rather than refusing every ask until the edge is back.
|
||||
@@ -252,6 +259,7 @@ export class EdgeAssignments {
|
||||
return true
|
||||
}
|
||||
this.exportFailingSince = undefined
|
||||
this.exportLastFailureAt = undefined
|
||||
if (exported.outcome === 'busy') { throw new EdgeMoveBusyError() }
|
||||
// An object with no assignment never held this host's data (or was erased): keep Postgres.
|
||||
if (!exported.assignment) { return true }
|
||||
|
||||
Reference in New Issue
Block a user