diff --git a/cloud/apps/relay/src/edge-relay-cohort.ts b/cloud/apps/relay/src/edge-relay-cohort.ts index 4f9e626abb9..729e1c99c08 100644 --- a/cloud/apps/relay/src/edge-relay-cohort.ts +++ b/cloud/apps/relay/src/edge-relay-cohort.ts @@ -26,6 +26,8 @@ export type EdgeRelayConfig = { // An edge outage must not strand every leaving host on 503s: after this long of nothing but // failed exports, a leaving host keeps its Postgres devices and is placed anyway. export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000 +// Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge). +export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000 export function edgeCohortBucket(relayHostId: string): number { return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100 diff --git a/cloud/apps/relay/src/edge-relay-evacuation.test.ts b/cloud/apps/relay/src/edge-relay-evacuation.test.ts index 4a521b8daeb..ae668d2617f 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.test.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.test.ts @@ -9,7 +9,8 @@ import { type EdgeAdmin, type EdgeDrainResult } from './edge-admin-client.js' -import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js' +import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS, + EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js' import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js' const EDGE_URL = 'https://edge.relay.example.test' @@ -140,14 +141,20 @@ describe('edge evacuation sweep', () => { await h.insert(hostId(1)) const sweep = vi.spyOn(h.evacuation, 'sweep') h.evacuation.start() - await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - 1) + // First pass soon after start, then the slow interval once nothing more is queued. + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS - 1) expect(sweep).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1) expect(sweep).toHaveBeenCalledTimes(1) - await vi.waitFor(() => expect(h.admin.drained).toEqual([hostId(1)])) + await vi.advanceTimersByTimeAsync(0) + expect(h.admin.drained).toEqual([hostId(1)]) + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(sweep).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_INTERVAL_MS - EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(sweep).toHaveBeenCalledTimes(2) h.evacuation.stop() await vi.advanceTimersByTimeAsync(10 * EDGE_EVACUATION_INTERVAL_MS) - expect(sweep).toHaveBeenCalledTimes(1) + expect(sweep).toHaveBeenCalledTimes(2) }) }) diff --git a/cloud/apps/relay/src/edge-relay-evacuation.ts b/cloud/apps/relay/src/edge-relay-evacuation.ts index 5851acbb3a2..e1419323499 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.ts @@ -7,7 +7,11 @@ import type { RelayDatabase } from './database/database.js' import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js' import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js' -export const EDGE_EVACUATION_INTERVAL_MS = 60_000 +// The cohort only changes when a new revision starts, so a pass soon after start and then a slow +// background pass are enough; each pass reads the edge rows (cell_id has no index). +export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000 +// First pass after start, and the next pass while more leaving hosts are queued. +export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000 // Drains sent per tick (per director), so a cohort cut moves desktops back gradually. export const EDGE_EVACUATION_BATCH = 20 // Edge rows read per tick; the cursor walks the table across ticks. @@ -21,6 +25,7 @@ export class EdgeEvacuation { private readonly drainedAt = new Map() private timer: ReturnType | undefined private stopped = true + private busy = true constructor( private readonly database: RelayDatabase, @@ -49,6 +54,7 @@ export class EdgeEvacuation { if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) } } const leaving = await this.leavingHosts() + this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined let answered = 0 for (const identity of leaving) { try { @@ -105,8 +111,9 @@ export class EdgeEvacuation { private schedule(): void { if (this.stopped) { return } - // Jittered (45-75 s), so directors do not sweep together. - const delay = EDGE_EVACUATION_INTERVAL_MS * (0.75 + this.random() / 2) + // Jittered (±25%), so directors do not sweep together. + const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS + const delay = interval * (0.75 + this.random() / 2) this.timer = setTimeout(() => { void this.sweep() .catch((error: unknown) => diff --git a/cloud/apps/relay/src/edge-relay-phase2.test.ts b/cloud/apps/relay/src/edge-relay-phase2.test.ts index db7025e58b8..ddea6e93a4b 100644 --- a/cloud/apps/relay/src/edge-relay-phase2.test.ts +++ b/cloud/apps/relay/src/edge-relay-phase2.test.ts @@ -179,6 +179,16 @@ async function harness(cohort: 'host' | 'account' = 'host', extra: Partial Promise }, ms: number) { + const end = h.clock.now + ms + while (h.clock.now < end) { + h.clock.now = Math.min(end, h.clock.now + 20_000) + if (h.clock.now < end) { expect((await h.assign()).status).toBe(503) } + } +} + describe('relay edge phase 2: the director moves host data with the host', () => { it('pushes the host and its Postgres devices before answering with the edge, with a location hint', async () => { const h = await harness() @@ -256,7 +266,7 @@ describe('relay edge phase 2: the director moves host data with the host', () => h.fake.down = true h.leaveCohort() expect((await h.assign()).status).toBe(503) - h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1 + await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1) const early = await h.assign() expect(early.status).toBe(503) expect(early.headers.get('retry-after')).toBe('2') @@ -289,9 +299,25 @@ describe('relay edge phase 2: the director moves host data with the host', () => h.fake.connected = true expect((await h.assign()).status).toBe(503) h.fake.down = true - h.clock.now += 100_000 + h.clock.now += 20_000 expect((await h.assign()).status).toBe(503) - h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS - 1 + await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1) + expect((await h.assign()).status).toBe(503) + h.clock.now += 1 + expect((await h.assign()).status).toBe(200) + }) + + it('does not count failures further apart than 30 s as one outage', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness() + await h.assign() + h.leaveCohort() + h.fake.down = true + expect((await h.assign()).status).toBe(503) + // One old failure, then quiet for longer than the skip threshold: the next failure starts over. + h.clock.now += EDGE_EXPORT_SKIP_AFTER_MS + 1 + expect((await h.assign()).status).toBe(503) + await keepFailing(h, EDGE_EXPORT_SKIP_AFTER_MS - 1) expect((await h.assign()).status).toBe(503) h.clock.now += 1 expect((await h.assign()).status).toBe(200) diff --git a/cloud/apps/relay/src/edge-relay.ts b/cloud/apps/relay/src/edge-relay.ts index 09624990adc..266ce08cfd6 100644 --- a/cloud/apps/relay/src/edge-relay.ts +++ b/cloud/apps/relay/src/edge-relay.ts @@ -5,6 +5,7 @@ import { lockedSelector, registerExistingOnlyCell, requireSelectorMatchesAdmissi import type { RelayDatabase, SqlRow } from './database/database.js' import { EdgeAdminUnavailableError, edgeLocationHint, type EdgeAdmin } from './edge-admin-client.js' import { + EDGE_EXPORT_FAILURE_GAP_MS, DEVICE_COLUMNS, EDGE_EXPORT_SKIP_AFTER_MS, EdgeMoveBusyError, @@ -39,6 +40,7 @@ export class EdgeAssignments { private seeded = false // When this director's exports started failing as unavailable; any usable answer clears it. private exportFailingSince: number | undefined + private exportLastFailureAt: number | undefined // With `admin` (the edge keeps host data in its objects) every move onto the edge pushes the // host's devices before the answer, and every move off exports them back first. @@ -243,8 +245,13 @@ export class EdgeAssignments { } catch (error) { if (!(error instanceof EdgeAdminUnavailableError)) { throw error } const now = this.now() - this.exportFailingSince ??= now - const failingForMs = now - this.exportFailingSince + // Only continuous failures count: a gap longer than EDGE_EXPORT_FAILURE_GAP_MS (desktops retry + // every 2 s) starts the clock again, so one old failure cannot skip an export on its own. + if (this.exportLastFailureAt === undefined || now - this.exportLastFailureAt > EDGE_EXPORT_FAILURE_GAP_MS) { + this.exportFailingSince = now + } + this.exportLastFailureAt = now + const failingForMs = now - (this.exportFailingSince ?? now) if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() } // The edge has been unreachable too long: place the host on its Postgres devices (changes // made on the edge are lost) rather than refusing every ask until the edge is back. @@ -252,6 +259,7 @@ export class EdgeAssignments { return true } this.exportFailingSince = undefined + this.exportLastFailureAt = undefined if (exported.outcome === 'busy') { throw new EdgeMoveBusyError() } // An object with no assignment never held this host's data (or was erased): keep Postgres. if (!exported.assignment) { return true }