mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(relay): runtime edge control on the director, safer skipped exports, faster evacuation (#39773)
- Directors with an edge read director/edge.json from the relay-control bucket through the shared flag channel: mode on/off/down and rolloutPermille (0..1000). Boot defaults are mode on and the env percent; every cohort decision reads the applied flags. Off empties the cohort and hosts leave normally; down also skips the export at once (reason mode_down). The permille bucket keeps every host of P percent in at P x 10 permille. A control change re-arms the evacuation sweep within 5 s. - A skipped export (120 s outage clock or mode down) removes the host's relay_devices rows before placing it, so revokes made on the edge are never undone; logs a count only. - Evacuation: 100 drains per tick with 10 in flight, each pass starting at a random point of the keys and wrapping around; pages stay 500 rows and an idle pass reads each edge row once. - Fence retries: 56 attempts (last about 25.5 h after the failure), 50 per sweep with 10 in flight; fence_abandoned now carries outcome, cell and epoch like fence_failed.
This commit is contained in:
@@ -26,6 +26,8 @@ export function startControlFlagChannel<Flags>(input: {
|
||||
defaults: Flags
|
||||
parse: ControlFlagParse<Flags>
|
||||
appliedEvent: string
|
||||
// Called once per applied generation, after it is logged; a throw is logged, never retried.
|
||||
onApplied?: (applied: AppliedControlFlags<Flags>) => void
|
||||
fetch?: typeof fetch
|
||||
random?: () => number
|
||||
pollMs?: number
|
||||
@@ -134,6 +136,17 @@ export function startControlFlagChannel<Flags>(input: {
|
||||
const { flags, ignoredKeys } = parsed
|
||||
applied = ignoredKeys.length > 0 ? { generation, flags, ignoredKeys } : { generation, flags }
|
||||
console.log(JSON.stringify({ event: input.appliedEvent, ...applied }))
|
||||
try {
|
||||
input.onApplied?.(applied)
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_control_flags_listener_failed',
|
||||
object: input.objectName,
|
||||
reason: error instanceof Error ? error.message : 'unknown'
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const poll = (): Promise<void> => {
|
||||
|
||||
@@ -5,13 +5,17 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra
|
||||
import type { SqlRow } from './database/database.js'
|
||||
import { z } from 'zod'
|
||||
import type { EdgeDeviceRow } from './edge-admin-client.js'
|
||||
import { EDGE_CONTROL_DEFAULTS, edgeRolloutPermille, type EdgeControlFlags } from './edge-relay-control.js'
|
||||
|
||||
export type EdgeRelayConfig = {
|
||||
cellUrl: string
|
||||
// Present: phase 2 (host data in the edge's objects); director → edge admin calls use it.
|
||||
adminSigningKey?: Uint8Array
|
||||
// Share of hosts (by hash of relayHostId) in the edge cohort, 0-100.
|
||||
// Share of hosts (by hash of relayHostId) in the edge cohort, 0-100: the boot default, which a
|
||||
// runtime rolloutPermille replaces.
|
||||
rolloutPercent: number
|
||||
// Runtime control (director/edge.json), read on every call; absent means the defaults.
|
||||
control?: () => EdgeControlFlags
|
||||
allowHosts: ReadonlySet<string>
|
||||
// Accounts (relay token `sub`) whose hosts all join, e.g. the team's own accounts.
|
||||
allowUsers?: ReadonlySet<string>
|
||||
@@ -29,22 +33,37 @@ export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000
|
||||
// Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge).
|
||||
export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000
|
||||
|
||||
export function edgeCohortBucket(relayHostId: string): number {
|
||||
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100
|
||||
function edgeCohortHash(relayHostId: string): number {
|
||||
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0)
|
||||
}
|
||||
|
||||
// Deny wins, then host allow, then region scope; within the region, account allow or rollout share.
|
||||
export function edgeCohortBucket(relayHostId: string): number {
|
||||
return edgeCohortHash(relayHostId) % 100
|
||||
}
|
||||
|
||||
// The percent bucket times ten plus a digit from the hash's higher part, so for every percent P a
|
||||
// host is under P × 10 permille exactly when it is under P percent: no host moves when the env
|
||||
// percent becomes the equal permille.
|
||||
export function edgeCohortBucketPermille(relayHostId: string): number {
|
||||
const hash = edgeCohortHash(relayHostId)
|
||||
return (hash % 100) * 10 + (Math.floor(hash / 100) % 10)
|
||||
}
|
||||
|
||||
// Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope;
|
||||
// within the region, account allow or the rollout share.
|
||||
export function inEdgeCohort(
|
||||
config: EdgeRelayConfig,
|
||||
relayHostId: string,
|
||||
region: RelayRegion | undefined,
|
||||
userId?: string
|
||||
): boolean {
|
||||
const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS
|
||||
if (flags.mode !== 'on') { return false }
|
||||
if (config.denyHosts.has(relayHostId)) { return false }
|
||||
if (config.allowHosts.has(relayHostId)) { return true }
|
||||
if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
|
||||
if (userId !== undefined && config.allowUsers?.has(userId)) { return true }
|
||||
return edgeCohortBucket(relayHostId) < config.rolloutPercent
|
||||
return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags)
|
||||
}
|
||||
|
||||
// Another director holds the host's row, or the host is on the edge and cannot leave yet (its
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest'
|
||||
import {
|
||||
EDGE_CONTROL_APPLIED_EVENT,
|
||||
EDGE_CONTROL_DEFAULTS,
|
||||
EDGE_CONTROL_OBJECT_NAME,
|
||||
parseEdgeControl,
|
||||
startEdgeControl,
|
||||
type EdgeControlFlags
|
||||
} from './edge-relay-control.js'
|
||||
import { edgeCohortBucket, edgeCohortBucketPermille, inEdgeCohort, type EdgeRelayConfig } from './edge-relay-cohort.js'
|
||||
|
||||
function edgeConfig(overrides: Partial<EdgeRelayConfig> = {}): EdgeRelayConfig {
|
||||
return {
|
||||
cellUrl: 'https://edge.relay.example.test',
|
||||
rolloutPercent: 0,
|
||||
allowHosts: new Set(),
|
||||
denyHosts: new Set(),
|
||||
regions: new Set(['us-central1']),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function hostId(index: number): string {
|
||||
return `host${String(index).padStart(12, '0')}`
|
||||
}
|
||||
|
||||
// Storage answers the stored object (or 404) by generation, like the cell flag channel's tests.
|
||||
function fakeGoogle() {
|
||||
const state: { object: { generation: number; body: string } | null; paths: string[] } = {
|
||||
object: null,
|
||||
paths: []
|
||||
}
|
||||
const fetchImpl: typeof fetch = async (target) => {
|
||||
const url = new URL(String(target))
|
||||
if (url.hostname === 'metadata.google.internal') {
|
||||
if (url.pathname.endsWith('/project/project-id')) { return new Response('onorca-cloud') }
|
||||
return Response.json({ access_token: 'ya29.secret-access-token', expires_in: 3_600 })
|
||||
}
|
||||
state.paths.push(decodeURIComponent(url.pathname))
|
||||
const object = state.object
|
||||
if (!object) { return new Response(null, { status: 404 }) }
|
||||
if (url.searchParams.get('ifGenerationNotMatch') === String(object.generation)) {
|
||||
return new Response(null, { status: 304 })
|
||||
}
|
||||
return new Response(object.body, { headers: { 'x-goog-generation': String(object.generation) } })
|
||||
}
|
||||
const store = (generation: number, flags: Record<string, unknown>) => {
|
||||
state.object = { generation, body: JSON.stringify({ v: 1, flags }) }
|
||||
}
|
||||
return { state, fetchImpl, store }
|
||||
}
|
||||
|
||||
describe('edge runtime control object', () => {
|
||||
let log: MockInstance<typeof console.log>
|
||||
let warn: MockInstance<typeof console.warn>
|
||||
beforeEach(() => {
|
||||
log = vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
})
|
||||
afterEach(() => {
|
||||
log.mockRestore()
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('parses mode and permille, reports unknown keys, and voids the object on any bad known value', () => {
|
||||
expect(parseEdgeControl({ v: 1, flags: {} })).toEqual({ flags: { mode: 'on' }, ignoredKeys: [] })
|
||||
expect(parseEdgeControl({ v: 1, flags: { mode: 'down', rolloutPermille: 0, cohort: 'x' } })).toEqual({
|
||||
flags: { mode: 'down', rolloutPermille: 0 },
|
||||
ignoredKeys: ['cohort']
|
||||
})
|
||||
expect(parseEdgeControl({ v: 1, flags: { rolloutPermille: 1_000 } })).toEqual({
|
||||
flags: { mode: 'on', rolloutPermille: 1_000 },
|
||||
ignoredKeys: []
|
||||
})
|
||||
for (const flags of [
|
||||
{ mode: 'paused' },
|
||||
{ mode: true },
|
||||
{ rolloutPermille: 1_001 },
|
||||
{ rolloutPermille: -1 },
|
||||
{ rolloutPermille: 2.5 },
|
||||
{ rolloutPermille: '5' },
|
||||
{ mode: 'off', rolloutPermille: null }
|
||||
]) {
|
||||
expect(parseEdgeControl({ v: 1, flags })).toBeNull()
|
||||
}
|
||||
expect(parseEdgeControl({ v: 2, flags: {} })).toBeNull()
|
||||
expect(parseEdgeControl({ flags: {} })).toBeNull()
|
||||
})
|
||||
|
||||
it('reads director/edge.json, logs each applied change with no ids, and tells listeners', async () => {
|
||||
const google = fakeGoogle()
|
||||
const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false })
|
||||
const changes: EdgeControlFlags[] = []
|
||||
control.onChange(() => changes.push(control.flags()))
|
||||
await control.poll()
|
||||
expect(control.flags()).toEqual(EDGE_CONTROL_DEFAULTS)
|
||||
expect(google.state.paths.at(-1)).toBe(`/storage/v1/b/onorca-cloud-relay-control/o/${EDGE_CONTROL_OBJECT_NAME}`)
|
||||
google.store(3, { mode: 'off', rolloutPermille: 25 })
|
||||
await control.poll()
|
||||
// Unchanged object: no second change.
|
||||
await control.poll()
|
||||
google.store(4, { mode: 'down', note: 'incident' })
|
||||
await control.poll()
|
||||
// A bad value voids the object: the last applied flags stay.
|
||||
google.store(5, { mode: 'sideways' })
|
||||
await control.poll()
|
||||
expect(control.flags()).toEqual({ mode: 'down' })
|
||||
expect(changes).toEqual([{ mode: 'off', rolloutPermille: 25 }, { mode: 'down' }])
|
||||
const applied = log.mock.calls
|
||||
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
|
||||
.filter((line) => line.event === EDGE_CONTROL_APPLIED_EVENT)
|
||||
expect(applied).toEqual([
|
||||
{ event: EDGE_CONTROL_APPLIED_EVENT, generation: 3, flags: { mode: 'off', rolloutPermille: 25 } },
|
||||
{ event: EDGE_CONTROL_APPLIED_EVENT, generation: 4, flags: { mode: 'down' }, ignoredKeys: ['note'] }
|
||||
])
|
||||
control.stop()
|
||||
})
|
||||
|
||||
it('logs a throwing listener and still applies the flags', async () => {
|
||||
const google = fakeGoogle()
|
||||
const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false })
|
||||
control.onChange(() => {
|
||||
throw new Error('listener_broke')
|
||||
})
|
||||
google.store(2, { mode: 'off' })
|
||||
await control.poll()
|
||||
expect(control.flags()).toEqual({ mode: 'off' })
|
||||
expect(warn.mock.calls.map((call) => String(call[0])).join('\n')).toContain('orca_relay_control_flags_listener_failed')
|
||||
control.stop()
|
||||
})
|
||||
})
|
||||
|
||||
describe('edge cohort under runtime control', () => {
|
||||
it('reads the applied flags on every call: off and down empty the cohort, even allowed hosts', () => {
|
||||
const flags: EdgeControlFlags = { mode: 'on' }
|
||||
const host = hostId(1)
|
||||
const config = edgeConfig({
|
||||
rolloutPercent: 100,
|
||||
allowHosts: new Set([hostId(2)]),
|
||||
allowUsers: new Set(['user-team']),
|
||||
control: () => flags
|
||||
})
|
||||
expect(inEdgeCohort(config, host, 'us-central1')).toBe(true)
|
||||
for (const mode of ['off', 'down'] as const) {
|
||||
flags.mode = mode
|
||||
expect(inEdgeCohort(config, host, 'us-central1')).toBe(false)
|
||||
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(false)
|
||||
expect(inEdgeCohort(config, host, 'us-central1', 'user-team')).toBe(false)
|
||||
}
|
||||
flags.mode = 'on'
|
||||
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true)
|
||||
// A permille replaces the env percent; deny, allow and region rules still apply around it.
|
||||
flags.rolloutPermille = 0
|
||||
expect(inEdgeCohort(config, host, 'us-central1')).toBe(false)
|
||||
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true)
|
||||
flags.rolloutPermille = 1_000
|
||||
expect(inEdgeCohort(config, host, 'us-central1')).toBe(true)
|
||||
expect(inEdgeCohort(config, host, 'asia-east2')).toBe(false)
|
||||
expect(inEdgeCohort({ ...config, denyHosts: new Set([host]) }, host, 'us-central1')).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps every host of P percent in at P × 10 permille, for every P', () => {
|
||||
// floor(permille bucket / 10) is the percent bucket, so `bucket < P × 10` iff `bucket < P`.
|
||||
const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index))
|
||||
expect(hosts.filter((host) => Math.floor(edgeCohortBucketPermille(host) / 10) !== edgeCohortBucket(host))).toEqual([])
|
||||
const sample = hosts.slice(0, 2_000)
|
||||
const moved: string[] = []
|
||||
for (let percent = 0; percent <= 100; percent += 1) {
|
||||
const byPercent = edgeConfig({ rolloutPercent: percent })
|
||||
const byPermille = edgeConfig({ rolloutPercent: 0, control: () => ({ mode: 'on', rolloutPermille: percent * 10 }) })
|
||||
for (const host of sample) {
|
||||
if (inEdgeCohort(byPercent, host, 'us-central1') !== inEdgeCohort(byPermille, host, 'us-central1')) {
|
||||
moved.push(`${percent}:${host}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(moved).toEqual([])
|
||||
// The env percent alone (boot, no permille flag) answers as before.
|
||||
expect(sample.filter((host) => inEdgeCohort(edgeConfig({ rolloutPercent: 10 }), host, 'us-central1')))
|
||||
.toEqual(sample.filter((host) => edgeCohortBucket(host) < 10))
|
||||
})
|
||||
|
||||
it('steps in 1 permille shares, nested, roughly evenly', () => {
|
||||
const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index))
|
||||
const share = (permille: number) =>
|
||||
hosts.filter((host) => inEdgeCohort(edgeConfig({ control: () => ({ mode: 'on', rolloutPermille: permille }) }), host, 'us-central1'))
|
||||
const one = share(1)
|
||||
const five = share(5)
|
||||
expect(one.length).toBeGreaterThan(5)
|
||||
expect(one.length).toBeLessThan(40)
|
||||
expect(five.length).toBeGreaterThan(70)
|
||||
expect(five.length).toBeLessThan(130)
|
||||
// A larger share keeps every host of a smaller one.
|
||||
expect(one.every((host) => five.includes(host))).toBe(true)
|
||||
const buckets = new Set(hosts.map((host) => edgeCohortBucketPermille(host)))
|
||||
expect(buckets.size).toBe(1_000)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
// Runtime control of the relay edge on the director: director/edge.json in the relay-control
|
||||
// bucket, read by the shared flag channel (cell/control-flags/relay-control-flag-channel.ts), so a
|
||||
// rollout step or a rollback needs no director deploy. Boot uses the defaults (mode on, the env
|
||||
// rollout percent); a failed read keeps the last applied flags.
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
startControlFlagChannel,
|
||||
type ControlFlagParse
|
||||
} from './cell/control-flags/relay-control-flag-channel.js'
|
||||
|
||||
// `off`: no host is in the cohort; hosts on the edge leave normally (export, place, fence) and the
|
||||
// evacuation sweep drains them. `down`: as off, and the edge is known dead, so a leaving host's
|
||||
// export is skipped at once (EdgeAssignments.leaveEdge).
|
||||
export type EdgeControlMode = 'on' | 'off' | 'down'
|
||||
|
||||
export type EdgeControlFlags = {
|
||||
mode: EdgeControlMode
|
||||
// Replaces the env rollout percent when present (1‰ steps).
|
||||
rolloutPermille?: number
|
||||
}
|
||||
|
||||
export const EDGE_CONTROL_DEFAULTS: EdgeControlFlags = { mode: 'on' }
|
||||
export const EDGE_CONTROL_OBJECT_NAME = 'director/edge.json'
|
||||
export const EDGE_CONTROL_APPLIED_EVENT = 'orca_relay_edge_control_applied'
|
||||
|
||||
const EdgeControlFlagsSchema = z.object({
|
||||
mode: z.enum(['on', 'off', 'down']).default(EDGE_CONTROL_DEFAULTS.mode),
|
||||
rolloutPermille: z.number().int().min(0).max(1_000).optional()
|
||||
})
|
||||
const KNOWN_FLAG_KEYS = new Set(Object.keys(EdgeControlFlagsSchema.shape))
|
||||
const EdgeControlObjectSchema = z.object({ v: z.literal(1), flags: EdgeControlFlagsSchema })
|
||||
|
||||
// Unknown keys (a newer writer's, or a typo) are dropped and reported; a known key with a bad value
|
||||
// voids the whole object, so the last applied flags stay.
|
||||
export const parseEdgeControl: ControlFlagParse<EdgeControlFlags> = (body) => {
|
||||
const parsed = EdgeControlObjectSchema.safeParse(body)
|
||||
if (!parsed.success) {
|
||||
return null
|
||||
}
|
||||
const named = body !== null && typeof body === 'object' && 'flags' in body ? body.flags : {}
|
||||
const ignoredKeys =
|
||||
named !== null && typeof named === 'object'
|
||||
? Object.keys(named).filter((key) => !KNOWN_FLAG_KEYS.has(key)).sort()
|
||||
: []
|
||||
const { mode, rolloutPermille } = parsed.data.flags
|
||||
return { flags: rolloutPermille === undefined ? { mode } : { mode, rolloutPermille }, ignoredKeys }
|
||||
}
|
||||
|
||||
export type EdgeControl = {
|
||||
// The flags applied now; callers read this on every decision, never caching it.
|
||||
flags: () => EdgeControlFlags
|
||||
// Called after each newly applied generation.
|
||||
onChange: (listener: () => void) => void
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
export function startEdgeControl(
|
||||
input: { fetch?: typeof fetch; random?: () => number; pollMs?: number; autoStart?: boolean } = {}
|
||||
): EdgeControl & { poll: () => Promise<void> } {
|
||||
const listeners: (() => void)[] = []
|
||||
const channel = startControlFlagChannel({
|
||||
objectName: EDGE_CONTROL_OBJECT_NAME,
|
||||
defaults: EDGE_CONTROL_DEFAULTS,
|
||||
parse: parseEdgeControl,
|
||||
appliedEvent: EDGE_CONTROL_APPLIED_EVENT,
|
||||
onApplied: () => {
|
||||
for (const listener of listeners) {
|
||||
listener()
|
||||
}
|
||||
},
|
||||
...input
|
||||
})
|
||||
return {
|
||||
flags: () => channel.applied().flags,
|
||||
onChange: (listener) => {
|
||||
listeners.push(listener)
|
||||
},
|
||||
poll: channel.poll,
|
||||
stop: channel.stop
|
||||
}
|
||||
}
|
||||
|
||||
// Effective share of hosts on the edge in permille: the applied flag, else the env percent.
|
||||
export function edgeRolloutPermille(rolloutPercent: number, flags: EdgeControlFlags): number {
|
||||
return flags.rolloutPermille ?? rolloutPercent * 10
|
||||
}
|
||||
@@ -9,9 +9,16 @@ import {
|
||||
type EdgeAdmin,
|
||||
type EdgeDrainResult
|
||||
} from './edge-admin-client.js'
|
||||
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS,
|
||||
EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import {
|
||||
EDGE_EVACUATION_BATCH,
|
||||
EDGE_EVACUATION_BUSY_INTERVAL_MS,
|
||||
EDGE_EVACUATION_CONCURRENCY,
|
||||
EDGE_EVACUATION_CONTROL_DELAY_MS,
|
||||
EDGE_EVACUATION_INTERVAL_MS,
|
||||
EdgeEvacuation
|
||||
} from './edge-relay-evacuation.js'
|
||||
import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js'
|
||||
import type { EdgeControlFlags } from './edge-relay-control.js'
|
||||
|
||||
const EDGE_URL = 'https://edge.relay.example.test'
|
||||
const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!')
|
||||
@@ -31,6 +38,13 @@ class FakeAdmin implements EdgeAdmin {
|
||||
drained: string[] = []
|
||||
unavailable = new Set<string>()
|
||||
answer: EdgeDrainResult = { outcome: 'drained', controls: 1 }
|
||||
// Set: each drain waits for release(), so the test can count drains in flight.
|
||||
holding = false
|
||||
readonly held: (() => void)[] = []
|
||||
|
||||
release(): void {
|
||||
for (const resolve of this.held.splice(0)) { resolve() }
|
||||
}
|
||||
|
||||
async assign(): Promise<never> {
|
||||
throw new Error('unexpected assign')
|
||||
@@ -43,11 +57,12 @@ class FakeAdmin implements EdgeAdmin {
|
||||
async drain(relayHostId: string): Promise<EdgeDrainResult> {
|
||||
if (this.unavailable.has(relayHostId)) { throw new EdgeAdminUnavailableError('status_404') }
|
||||
this.drained.push(relayHostId)
|
||||
if (this.holding) { await new Promise<void>((resolve) => this.held.push(resolve)) }
|
||||
return this.answer
|
||||
}
|
||||
}
|
||||
|
||||
async function harness(overrides: Partial<EdgeRelayConfig> = {}) {
|
||||
async function harness(overrides: Partial<EdgeRelayConfig> = {}, random = 0.5) {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const config: EdgeRelayConfig = {
|
||||
@@ -60,7 +75,7 @@ async function harness(overrides: Partial<EdgeRelayConfig> = {}) {
|
||||
}
|
||||
const admin = new FakeAdmin()
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => 0.5)
|
||||
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => random)
|
||||
const insert = async (relayHostId: string, cellId = EDGE_CELL_ID, userId = 'user-1') =>
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignments
|
||||
@@ -107,11 +122,11 @@ describe('edge evacuation sweep', () => {
|
||||
it(`drains at most ${EDGE_EVACUATION_BATCH} per tick, the rest on the next, and not again within the cooldown`, async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
for (let index = 0; index < 25; index += 1) { await h.insert(hostId(index)) }
|
||||
for (let index = 0; index < EDGE_EVACUATION_BATCH + 25; index += 1) { await h.insert(hostId(index)) }
|
||||
h.admin.answer = { outcome: 'idle' }
|
||||
expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH)
|
||||
expect(await h.evacuation.sweep()).toBe(5)
|
||||
expect(new Set(h.admin.drained).size).toBe(25)
|
||||
expect(await h.evacuation.sweep()).toBe(25)
|
||||
expect(new Set(h.admin.drained).size).toBe(EDGE_EVACUATION_BATCH + 25)
|
||||
expect(await h.evacuation.sweep()).toBe(0)
|
||||
h.clock.now += 10 * 60_000
|
||||
expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH)
|
||||
@@ -158,6 +173,99 @@ describe('edge evacuation sweep', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('edge evacuation under runtime control', () => {
|
||||
it('re-arms an idle sweep within 5 s of a control change, so a rollback drains at once', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const flags: EdgeControlFlags = { mode: 'on' }
|
||||
const h = await harness({ rolloutPercent: 100, control: () => flags })
|
||||
await h.insert(hostId(1))
|
||||
const sweep = vi.spyOn(h.evacuation, 'sweep')
|
||||
h.evacuation.start()
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(1)
|
||||
expect(h.admin.drained).toEqual([])
|
||||
// Idle now: the next pass would be ~10 min away.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
flags.mode = 'off'
|
||||
h.evacuation.controlChanged()
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(2)
|
||||
expect(h.admin.drained).toEqual([hostId(1)])
|
||||
h.evacuation.stop()
|
||||
})
|
||||
|
||||
it('re-arms after a pass that was running when the control changed', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 0 }
|
||||
const h = await harness({ control: () => flags })
|
||||
await h.insert(hostId(1))
|
||||
h.admin.holding = true
|
||||
const sweep = vi.spyOn(h.evacuation, 'sweep')
|
||||
h.evacuation.start()
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
|
||||
expect(h.admin.held).toHaveLength(1)
|
||||
flags.mode = 'down'
|
||||
h.evacuation.controlChanged()
|
||||
h.admin.release()
|
||||
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS)
|
||||
expect(sweep).toHaveBeenCalledTimes(2)
|
||||
h.evacuation.stop()
|
||||
})
|
||||
|
||||
it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
for (let index = 0; index < 35; index += 1) { await h.insert(hostId(index)) }
|
||||
h.admin.holding = true
|
||||
let done = false
|
||||
const pass = h.evacuation.sweep().then((answered) => {
|
||||
done = true
|
||||
return answered
|
||||
})
|
||||
let most = 0
|
||||
while (!done) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
most = Math.max(most, h.admin.held.length)
|
||||
h.admin.release()
|
||||
}
|
||||
expect(await pass).toBe(35)
|
||||
expect(most).toBe(EDGE_EVACUATION_CONCURRENCY)
|
||||
})
|
||||
|
||||
it('starts each pass at a random point of the keys and wraps around to it', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const users = ['20000000-a', '60000000-a', 'a0000000-a', 'e0000000-a']
|
||||
const drainedFrom = async (random: number) => {
|
||||
const h = await harness({}, random)
|
||||
for (const [index, user] of users.entries()) { await h.insert(hostId(index), EDGE_CELL_ID, user) }
|
||||
expect(await h.evacuation.sweep()).toBe(4)
|
||||
return h.admin.drained
|
||||
}
|
||||
expect(await drainedFrom(0.5)).toEqual([hostId(2), hostId(3), hostId(0), hostId(1)])
|
||||
expect(await drainedFrom(0.1)).toEqual([hostId(0), hostId(1), hostId(2), hostId(3)])
|
||||
expect(await drainedFrom(0.3)).toEqual([hostId(1), hostId(2), hostId(3), hostId(0)])
|
||||
})
|
||||
|
||||
it('pages an idle pass: one page of edge rows per tick, then idle until the next pass', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness({ rolloutPercent: 100 })
|
||||
for (let index = 0; index < 1_200; index += 1) { await h.insert(hostId(index)) }
|
||||
const query = vi.spyOn(h.database, 'query')
|
||||
const pages = () => query.mock.calls.filter(([sql]) => String(sql).includes('FROM relay_assignments a')).length
|
||||
const busy = () => (h.evacuation as unknown as { busy: boolean }).busy
|
||||
await h.evacuation.sweep()
|
||||
expect([pages(), busy()]).toEqual([1, true])
|
||||
await h.evacuation.sweep()
|
||||
expect([pages(), busy()]).toEqual([2, true])
|
||||
// The short last page wraps to the start of the keys, which holds nothing before the origin.
|
||||
await h.evacuation.sweep()
|
||||
expect([pages(), busy()]).toEqual([4, false])
|
||||
expect(h.admin.drained).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('edge admin drain call', () => {
|
||||
it('posts the signed drain op and reads both answers; anything else is unavailable', async () => {
|
||||
const answers: [number, unknown][] = [
|
||||
|
||||
@@ -7,27 +7,38 @@ import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contr
|
||||
import type { RelayDatabase } from './database/database.js'
|
||||
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
|
||||
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
|
||||
import { retryPendingFences } from './edge-relay-fence.js'
|
||||
import { forEachBounded, retryPendingFences } from './edge-relay-fence.js'
|
||||
|
||||
// The cohort only changes when a new revision starts, so a pass soon after start and then a slow
|
||||
// background pass are enough; each pass reads the edge rows (cell_id has no index).
|
||||
// A cohort change (director/edge.json, or a new revision) re-arms the sweep (controlChanged), so a
|
||||
// soon pass and then a slow background pass are enough; each pass reads the edge rows (cell_id has
|
||||
// no index).
|
||||
export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000
|
||||
// First pass after start, and the next pass while more leaving hosts are queued.
|
||||
export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000
|
||||
// Drains sent per tick (per director), so a cohort cut moves desktops back gradually.
|
||||
export const EDGE_EVACUATION_BATCH = 20
|
||||
// Edge rows read per tick; the cursor walks the table across ticks.
|
||||
// Longest wait for a pass after the runtime control changes, so a rollback starts at once.
|
||||
export const EDGE_EVACUATION_CONTROL_DELAY_MS = 5_000
|
||||
// Drains sent per tick (per director): a full rollback of ~19k hosts over 5 directors takes ~40 min.
|
||||
export const EDGE_EVACUATION_BATCH = 100
|
||||
// Drains in flight at once per director.
|
||||
export const EDGE_EVACUATION_CONCURRENCY = 10
|
||||
// Edge rows read per page; the cursor walks the table across ticks.
|
||||
const EDGE_EVACUATION_PAGE = 500
|
||||
// A host this director drained is not drained again for this long (a desktop that is offline
|
||||
// answers idle, and asks the director on its own when it comes back).
|
||||
const EDGE_EVACUATION_COOLDOWN_MS = 10 * 60_000
|
||||
// A pass starts at a random point of the key space and wraps around to it, so directors sweeping
|
||||
// at once mostly drain different hosts.
|
||||
type Pass = { origin: Identity; cursor: Identity; wrapped: boolean }
|
||||
|
||||
export class EdgeEvacuation {
|
||||
private cursor: Identity | undefined
|
||||
// Undefined between passes.
|
||||
private pass: Pass | undefined
|
||||
private readonly drainedAt = new Map<string, number>()
|
||||
private timer: ReturnType<typeof setTimeout> | undefined
|
||||
private stopped = true
|
||||
private busy = true
|
||||
private sweeping = false
|
||||
private controlChangedWhileSweeping = false
|
||||
|
||||
constructor(
|
||||
private readonly database: RelayDatabase,
|
||||
@@ -49,6 +60,19 @@ export class EdgeEvacuation {
|
||||
this.timer = undefined
|
||||
}
|
||||
|
||||
// The runtime control changed (e.g. mode off for a rollback): the next pass runs within
|
||||
// EDGE_EVACUATION_CONTROL_DELAY_MS instead of waiting out the idle interval.
|
||||
controlChanged(): void {
|
||||
this.busy = true
|
||||
if (this.stopped) { return }
|
||||
if (this.sweeping) {
|
||||
this.controlChangedWhileSweeping = true
|
||||
return
|
||||
}
|
||||
clearTimeout(this.timer)
|
||||
this.schedule(EDGE_EVACUATION_CONTROL_DELAY_MS * this.random())
|
||||
}
|
||||
|
||||
// One tick: returns how many drains the edge answered.
|
||||
async sweep(): Promise<number> {
|
||||
const now = this.now()
|
||||
@@ -56,12 +80,12 @@ export class EdgeEvacuation {
|
||||
if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) }
|
||||
}
|
||||
const leaving = await this.leavingHosts()
|
||||
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined
|
||||
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.pass !== undefined
|
||||
let answered = 0
|
||||
for (const identity of leaving) {
|
||||
await forEachBounded(leaving, EDGE_EVACUATION_CONCURRENCY, async (identity) => {
|
||||
try {
|
||||
const result = await this.admin.drain(identity.relayHostId)
|
||||
this.drainedAt.set(`${identity.userId}\n${identity.relayHostId}`, this.now())
|
||||
this.drainedAt.set(drainKey(identity), this.now())
|
||||
answered += 1
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
@@ -74,51 +98,85 @@ export class EdgeEvacuation {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message }))
|
||||
}
|
||||
}
|
||||
})
|
||||
// Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured.
|
||||
if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true }
|
||||
return answered
|
||||
}
|
||||
|
||||
// Edge rows (one page after the cursor) whose host left the cohort, skipping recent drains.
|
||||
// Edge rows (a page after the cursor; a second when the first ends the pass's first half) whose
|
||||
// host left the cohort, skipping recent drains. An idle pass reads each edge row once.
|
||||
private async leavingHosts(): Promise<Identity[]> {
|
||||
const cursor = this.cursor ?? { userId: '', relayHostId: '' }
|
||||
const rows = await this.database.query(
|
||||
`SELECT a.user_id, a.relay_host_id, p.preferred_region
|
||||
FROM relay_assignments a
|
||||
LEFT JOIN relay_assignment_region_preferences p
|
||||
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
|
||||
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
|
||||
ORDER BY a.user_id, a.relay_host_id
|
||||
LIMIT ?`,
|
||||
[this.edgeCellId, cursor.userId, cursor.userId, cursor.relayHostId, EDGE_EVACUATION_PAGE]
|
||||
)
|
||||
// A short page reached the end: the next tick starts over.
|
||||
this.cursor = undefined
|
||||
const leaving: Identity[] = []
|
||||
for (const [index, row] of rows.entries()) {
|
||||
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
|
||||
if (rows.length === EDGE_EVACUATION_PAGE && index === rows.length - 1) { this.cursor = identity }
|
||||
const stored = RelayRegionSchema.safeParse(row.preferred_region)
|
||||
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
|
||||
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
|
||||
if (this.drainedAt.has(`${identity.userId}\n${identity.relayHostId}`)) { continue }
|
||||
leaving.push(identity)
|
||||
if (leaving.length === EDGE_EVACUATION_BATCH) {
|
||||
this.pass ??= this.newPass()
|
||||
for (let pages = 0; pages < 2 && this.pass !== undefined; pages += 1) {
|
||||
const pass: Pass = this.pass
|
||||
const rows = await this.readPage(pass)
|
||||
let filled = false
|
||||
for (const row of rows) {
|
||||
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
|
||||
pass.cursor = identity
|
||||
const stored = RelayRegionSchema.safeParse(row.preferred_region)
|
||||
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
|
||||
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
|
||||
if (this.drainedAt.has(drainKey(identity))) { continue }
|
||||
leaving.push(identity)
|
||||
// Resume after the last host taken, so the rest of the page is read next tick.
|
||||
this.cursor = identity
|
||||
break
|
||||
if (leaving.length === EDGE_EVACUATION_BATCH) {
|
||||
filled = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (filled || rows.length === EDGE_EVACUATION_PAGE) { break }
|
||||
// A short page ends this half: wrap to the start of the keys, or end the pass.
|
||||
if (pass.wrapped) {
|
||||
this.pass = undefined
|
||||
} else {
|
||||
pass.wrapped = true
|
||||
pass.cursor = { userId: '', relayHostId: '' }
|
||||
}
|
||||
}
|
||||
return leaving
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
// Keys after the cursor; once wrapped, only up to the pass's origin.
|
||||
private async readPage(pass: Pass) {
|
||||
const { cursor, origin, wrapped } = pass
|
||||
return await this.database.query(
|
||||
`SELECT a.user_id, a.relay_host_id, p.preferred_region
|
||||
FROM relay_assignments a
|
||||
LEFT JOIN relay_assignment_region_preferences p
|
||||
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
|
||||
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
|
||||
${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''}
|
||||
ORDER BY a.user_id, a.relay_host_id
|
||||
LIMIT ?`,
|
||||
[
|
||||
this.edgeCellId,
|
||||
cursor.userId,
|
||||
cursor.userId,
|
||||
cursor.relayHostId,
|
||||
...(wrapped ? [origin.userId, origin.userId, origin.relayHostId] : []),
|
||||
EDGE_EVACUATION_PAGE
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
// Account ids are UUIDs, so a random hex prefix lands anywhere in the key space; ids of another
|
||||
// shape are still all swept, from a less spread origin.
|
||||
private newPass(): Pass {
|
||||
const prefix = Math.floor(this.random() * 0x1_0000_0000).toString(16).padStart(8, '0')
|
||||
const origin = { userId: prefix, relayHostId: '' }
|
||||
return { origin, cursor: origin, wrapped: false }
|
||||
}
|
||||
|
||||
private schedule(delayMs?: number): void {
|
||||
if (this.stopped) { return }
|
||||
// Jittered (±25%), so directors do not sweep together.
|
||||
const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS
|
||||
const delay = interval * (0.75 + this.random() / 2)
|
||||
const delay = delayMs ?? interval * (0.75 + this.random() / 2)
|
||||
this.timer = setTimeout(() => {
|
||||
this.sweeping = true
|
||||
void this.sweep()
|
||||
.catch((error: unknown) =>
|
||||
console.warn(
|
||||
@@ -128,8 +186,17 @@ export class EdgeEvacuation {
|
||||
})
|
||||
)
|
||||
)
|
||||
.finally(() => this.schedule())
|
||||
.finally(() => {
|
||||
this.sweeping = false
|
||||
const changed = this.controlChangedWhileSweeping
|
||||
this.controlChangedWhileSweeping = false
|
||||
this.schedule(changed ? EDGE_EVACUATION_CONTROL_DELAY_MS * this.random() : undefined)
|
||||
})
|
||||
}, delay)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
|
||||
function drainKey(identity: Identity): string {
|
||||
return `${identity.userId}\n${identity.relayHostId}`
|
||||
}
|
||||
|
||||
@@ -12,7 +12,15 @@ import {
|
||||
type EdgeAssignResult
|
||||
} from './edge-admin-client.js'
|
||||
import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js'
|
||||
import {
|
||||
EDGE_FENCE_MAX_ATTEMPTS,
|
||||
EDGE_FENCE_RETRY_BASE_MS,
|
||||
EDGE_FENCE_RETRY_BATCH,
|
||||
EDGE_FENCE_RETRY_MAX_MS,
|
||||
edgeFenceRetryDelayMs,
|
||||
recordPendingFence,
|
||||
retryPendingFences
|
||||
} from './edge-relay-fence.js'
|
||||
import {
|
||||
EDGE_CELL_ID,
|
||||
EDGE_LOCKED_CALLS_MAX,
|
||||
@@ -51,6 +59,10 @@ class ScriptedEdge implements EdgeAdmin {
|
||||
return { outcome, assignmentEpoch: outcome === 'stale' ? body.assignmentEpoch + 5 : body.assignmentEpoch }
|
||||
}
|
||||
|
||||
get hanging(): number {
|
||||
return this.waiting.length
|
||||
}
|
||||
|
||||
release(): void {
|
||||
for (const resolve of this.waiting.splice(0)) { resolve() }
|
||||
}
|
||||
@@ -229,12 +241,56 @@ describe('relay edge pending fence retries', () => {
|
||||
retry: true
|
||||
})
|
||||
expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([
|
||||
{ event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS }
|
||||
{
|
||||
event: 'orca_relay_edge_fence_abandoned',
|
||||
outcome: 'stale',
|
||||
cellId: 'us-c1',
|
||||
assignmentEpoch: 9,
|
||||
attempts: EDGE_FENCE_MAX_ATTEMPTS
|
||||
}
|
||||
])
|
||||
expect(edgeFenceRetryDelayMs(1)).toBe(60_000)
|
||||
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000)
|
||||
})
|
||||
|
||||
it('keeps retrying a fence through more than a day of edge outage', () => {
|
||||
// Recorded at 0, first retry one base delay later, then each attempt's backoff.
|
||||
let lastAttemptAt = EDGE_FENCE_RETRY_BASE_MS
|
||||
for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) {
|
||||
lastAttemptAt += edgeFenceRetryDelayMs(attempt)
|
||||
}
|
||||
expect(lastAttemptAt).toBeGreaterThanOrEqual(24 * 60 * 60_000)
|
||||
// Never slower than the cap, so the edge is not left claiming hosts long after it is back.
|
||||
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(EDGE_FENCE_RETRY_MAX_MS)
|
||||
})
|
||||
|
||||
it(`retries up to ${EDGE_FENCE_RETRY_BATCH} due fences per pass, 10 in flight at once`, async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
const hosts = Array.from({ length: EDGE_FENCE_RETRY_BATCH + 10 }, (_, index) => `fencehost${String(index).padStart(7, '0')}`)
|
||||
for (const host of hosts) {
|
||||
await h.placeRow('us-c1', 9, host)
|
||||
await recordPendingFence(h.database, { userId: USER, relayHostId: host }, 'us-c1', 9, h.clock.now)
|
||||
}
|
||||
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
|
||||
h.edge.answers = hosts.map(() => 'hang')
|
||||
let settled = false
|
||||
const pass = h.retry().then((left) => {
|
||||
settled = true
|
||||
return left
|
||||
})
|
||||
let maxInFlight = 0
|
||||
while (!settled) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
maxInFlight = Math.max(maxInFlight, h.edge.hanging)
|
||||
h.edge.release()
|
||||
}
|
||||
expect(await pass).toBe(true)
|
||||
expect(h.edge.calls).toHaveLength(EDGE_FENCE_RETRY_BATCH)
|
||||
expect(maxInFlight).toBe(10)
|
||||
expect(await h.pending()).toHaveLength(10)
|
||||
})
|
||||
|
||||
it('lets only one director claim an attempt', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
|
||||
@@ -7,13 +7,16 @@ import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.j
|
||||
import { integer, text, type Identity } from './edge-relay-cohort.js'
|
||||
|
||||
// Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`,
|
||||
// cannot land until the host moves again, and that move fences anew).
|
||||
export const EDGE_FENCE_MAX_ATTEMPTS = 8
|
||||
// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all.
|
||||
// cannot land until the host moves again, and that move fences anew). Enough to outlast a day-long
|
||||
// edge outage: the last attempt comes about 25.5 h after the failure.
|
||||
export const EDGE_FENCE_MAX_ATTEMPTS = 56
|
||||
// First retry one busy sweep after the failure, doubling up to the cap (31 min to reach it, then
|
||||
// every 30 min).
|
||||
export const EDGE_FENCE_RETRY_BASE_MS = 60_000
|
||||
const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
|
||||
// Pending fences retried per sweep (per director).
|
||||
const EDGE_FENCE_RETRY_BATCH = 20
|
||||
export const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
|
||||
// Pending fences retried per sweep (per director), with at most EDGE_FENCE_RETRY_CONCURRENCY in flight.
|
||||
export const EDGE_FENCE_RETRY_BATCH = 50
|
||||
const EDGE_FENCE_RETRY_CONCURRENCY = 10
|
||||
|
||||
export function edgeFenceRetryDelayMs(attempts: number): number {
|
||||
return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS)
|
||||
@@ -91,7 +94,7 @@ export async function retryPendingFences(
|
||||
WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`,
|
||||
[now(), EDGE_FENCE_RETRY_BATCH]
|
||||
)
|
||||
for (const row of due) {
|
||||
await forEachBounded(due, EDGE_FENCE_RETRY_CONCURRENCY, async (row) => {
|
||||
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
|
||||
const attempts = integer(row, 'attempts') + 1
|
||||
// Claim the attempt: a director that read the same row loses here and skips it.
|
||||
@@ -102,7 +105,7 @@ export async function retryPendingFences(
|
||||
[attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId,
|
||||
attempts - 1, integer(row, 'next_attempt_at')]
|
||||
)
|
||||
if (claimed.length !== 1) { continue }
|
||||
if (claimed.length !== 1) { return }
|
||||
const result = await retryOne(database, admin, edgeCellId, identity, attempts)
|
||||
if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
|
||||
// Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0.
|
||||
@@ -111,10 +114,7 @@ export async function retryPendingFences(
|
||||
[identity.userId, identity.relayHostId, attempts]
|
||||
)
|
||||
}
|
||||
if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
|
||||
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts }))
|
||||
}
|
||||
}
|
||||
})
|
||||
return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0
|
||||
}
|
||||
|
||||
@@ -150,5 +150,31 @@ async function retryOne(
|
||||
console.error(
|
||||
JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true })
|
||||
)
|
||||
if (attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
|
||||
// The record is dropped: the edge keeps claiming the host until it moves again.
|
||||
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', outcome, cellId, assignmentEpoch, attempts }))
|
||||
}
|
||||
return 'failed'
|
||||
}
|
||||
|
||||
// Runs `task` over `items` with at most `limit` in flight. A throw does not stop the other workers;
|
||||
// the first one is rethrown once all have settled.
|
||||
export async function forEachBounded<T>(
|
||||
items: readonly T[],
|
||||
limit: number,
|
||||
task: (item: T) => Promise<void>
|
||||
): Promise<void> {
|
||||
let next = 0
|
||||
const worker = async (): Promise<void> => {
|
||||
while (next < items.length) {
|
||||
const item = items[next]
|
||||
next += 1
|
||||
if (item !== undefined) { await task(item) }
|
||||
}
|
||||
}
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: Math.min(limit, items.length) }, async () => await worker())
|
||||
)
|
||||
const failed = results.find((result): result is PromiseRejectedResult => result.status === 'rejected')
|
||||
if (failed) { throw failed.reason instanceof Error ? failed.reason : new Error('relay_edge_task_failed') }
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type EdgeDeviceRow
|
||||
} from './edge-admin-client.js'
|
||||
import { EDGE_CELL_ID, EDGE_EXPORT_SKIP_AFTER_MS, EdgeAssignments, type EdgeRelayConfig } from './edge-relay.js'
|
||||
import type { EdgeControlFlags } from './edge-relay-control.js'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-1', relayHostId: token }))
|
||||
@@ -276,13 +277,14 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
|
||||
const response = await h.assign()
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 })
|
||||
// Postgres keeps what it had: the edge's revoke is lost, nothing is written back.
|
||||
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }])
|
||||
// The edge's revoke is unknown here: the host's Postgres devices are removed, never resurrected.
|
||||
expect(await h.postgresDevices()).toEqual([])
|
||||
expect(skipped()).toHaveLength(1)
|
||||
expect(JSON.parse(String(skipped()[0]![0]))).toEqual({
|
||||
event: 'orca_relay_edge_export_skipped',
|
||||
reason: 'relay_edge_admin_unavailable:network',
|
||||
failingForMs: EDGE_EXPORT_SKIP_AFTER_MS
|
||||
failingForMs: EDGE_EXPORT_SKIP_AFTER_MS,
|
||||
devicesRemoved: 1
|
||||
})
|
||||
})
|
||||
|
||||
@@ -337,7 +339,7 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
|
||||
const response = await h.assign()
|
||||
expect(response.status).toBe(200)
|
||||
expect(exportCalls).toBe(1)
|
||||
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }])
|
||||
expect(await h.postgresDevices()).toEqual([])
|
||||
expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 })
|
||||
})
|
||||
|
||||
@@ -370,6 +372,57 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
|
||||
expect(h.fake.pushes[0]).toMatchObject({ userId: USER, relayHostId: HOST })
|
||||
})
|
||||
|
||||
it('with mode off or down, sends no new host to the edge', async () => {
|
||||
for (const mode of ['off', 'down'] as const) {
|
||||
const h = await harness('host', { control: () => ({ mode }) })
|
||||
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' })
|
||||
expect(h.fake.pushes).toHaveLength(0)
|
||||
expect(await h.row()).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('with mode off, moves hosts on the edge off normally: export, place, fence', async () => {
|
||||
const flags: EdgeControlFlags = { mode: 'on' }
|
||||
const h = await harness('host', { control: () => flags })
|
||||
await h.insertDevice(device('phone-a'))
|
||||
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: EDGE_URL })
|
||||
h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })]
|
||||
flags.mode = 'off'
|
||||
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' })
|
||||
// The edge's revoke carries over.
|
||||
expect(h.fake.exports).toBe(1)
|
||||
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: 1_800_000_000_000 }])
|
||||
expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 })
|
||||
})
|
||||
|
||||
it('with mode down, skips the export at once, removes the Postgres devices, and fences once', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const flags: EdgeControlFlags = { mode: 'on' }
|
||||
const h = await harness('host', { control: () => flags })
|
||||
await h.insertDevice(device('phone-a'))
|
||||
await h.insertDevice(device('phone-b'))
|
||||
await h.assign()
|
||||
// phone-a was revoked on the edge, which then died; Postgres still holds it live.
|
||||
h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })]
|
||||
h.fake.down = true
|
||||
flags.mode = 'down'
|
||||
const assign = vi.spyOn(h.fake, 'assign')
|
||||
const response = await h.assign()
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 })
|
||||
expect(h.fake.exports).toBe(0)
|
||||
expect(await h.postgresDevices()).toEqual([])
|
||||
const skipped = warn.mock.calls
|
||||
.map(([line]) => JSON.parse(String(line)) as Record<string, unknown>)
|
||||
.filter((line) => line.event === 'orca_relay_edge_export_skipped')
|
||||
expect(skipped).toEqual([{ event: 'orca_relay_edge_export_skipped', reason: 'mode_down', devicesRemoved: 2 }])
|
||||
// One fence try, recorded for the sweep to retry.
|
||||
expect(assign).toHaveBeenCalledTimes(1)
|
||||
const pending = await h.database.query(`SELECT cell_id, assignment_epoch FROM relay_edge_fence_pending`)
|
||||
expect(pending.map((row) => [String(row.cell_id), Number(row.assignment_epoch)])).toEqual([['us-c1', 9]])
|
||||
})
|
||||
|
||||
it('does not touch the edge for hosts that were never on it', async () => {
|
||||
const h = await harness()
|
||||
h.leaveCohort()
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
} from './edge-relay.js'
|
||||
import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js'
|
||||
import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js'
|
||||
import { EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -236,4 +237,38 @@ describePostgres('relay edge on PostgreSQL', () => {
|
||||
expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([])
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('mode down: a leaving host loses its Postgres devices, and the sweep drains wrapping around its origin', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const flags = { mode: 'on' as 'on' | 'down' }
|
||||
const config = { ...edgeConfig, control: () => flags }
|
||||
const drained: string[] = []
|
||||
const admin = {
|
||||
assign: async (body: EdgeAssignBody) => ({ outcome: 'applied' as const, assignmentEpoch: body.assignmentEpoch }),
|
||||
exportHost: async (): Promise<never> => {
|
||||
throw new Error('unexpected export')
|
||||
},
|
||||
drain: async (relayHostId: string) => {
|
||||
drained.push(relayHostId)
|
||||
return { outcome: 'idle' as const }
|
||||
}
|
||||
}
|
||||
const edge = new EdgeAssignments(database, config, Date.now, admin)
|
||||
await edge.ensureEdgeCell()
|
||||
const downHosts = ['edgepgdown000001', 'edgepgdown000002'].map((relayHostId) => ({ userId: 'edge-pg-user', relayHostId }))
|
||||
for (const host of downHosts) { await edge.moveToEdge(host, 0) }
|
||||
await database.query(
|
||||
`INSERT INTO relay_devices (user_id, relay_host_id, relay_device_id, current_hash, current_version,
|
||||
current_expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
['edge-pg-user', downHosts[0]!.relayHostId, 'phone-a', 'hash-phone-a', 2, 1_900_000_000_000, 1_800_000_000_000]
|
||||
)
|
||||
flags.mode = 'down'
|
||||
// Origin 'f3333333' sorts after 'edge-pg-user': the pass finds both hosts only once wrapped.
|
||||
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, Date.now, () => 0.95)
|
||||
await evacuation.sweep()
|
||||
expect(drained.filter((id) => id.startsWith('edgepgdown'))).toEqual(downHosts.map((host) => host.relayHostId))
|
||||
expect(await edge.leaveEdge(downHosts[0]!)).toBe(true)
|
||||
expect(await database.query(`SELECT 1 FROM relay_devices WHERE relay_host_id = ?`, [downHosts[0]!.relayHostId])).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -264,6 +264,8 @@ export class EdgeAssignments {
|
||||
)
|
||||
)[0]
|
||||
if (!row || text(row, 'cell_id') !== EDGE_CELL_ID) { return false }
|
||||
// The edge is known dead (director/edge.json): no export to wait for.
|
||||
if (this.config.control?.().mode === 'down') { return await this.skipExport(identity, 'mode_down') }
|
||||
let exported
|
||||
try {
|
||||
exported = await this.admin.exportHost(identity.relayHostId)
|
||||
@@ -278,10 +280,9 @@ export class EdgeAssignments {
|
||||
this.exportLastFailureAt = now
|
||||
const failingForMs = now - (this.exportFailingSince ?? now)
|
||||
if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() }
|
||||
// The edge has been unreachable too long: place the host on its Postgres devices (changes
|
||||
// made on the edge are lost) rather than refusing every ask until the edge is back.
|
||||
console.warn(JSON.stringify({ event: 'orca_relay_edge_export_skipped', reason: error.message, failingForMs }))
|
||||
return true
|
||||
// The edge has been unreachable too long: place the host anyway rather than refusing every
|
||||
// ask until the edge is back.
|
||||
return await this.skipExport(identity, error.message, failingForMs)
|
||||
}
|
||||
this.exportFailingSince = undefined
|
||||
this.exportLastFailureAt = undefined
|
||||
@@ -313,13 +314,34 @@ export class EdgeAssignments {
|
||||
return true
|
||||
}
|
||||
|
||||
// The host leaves without the edge's copy of its devices. The edge was their source of truth and
|
||||
// revokes made there are unknown, so the Postgres rows are removed rather than resurrected: its
|
||||
// phones pair again. Logs a count only.
|
||||
private async skipExport(identity: Identity, reason: string, failingForMs?: number): Promise<true> {
|
||||
const removed = await this.database.query(
|
||||
`DELETE FROM relay_devices WHERE user_id = ? AND relay_host_id = ? RETURNING relay_device_id`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_edge_export_skipped',
|
||||
reason,
|
||||
...(failingForMs === undefined ? {} : { failingForMs }),
|
||||
devicesRemoved: removed.length
|
||||
})
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
// After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve)
|
||||
// rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land
|
||||
// is recorded for the evacuation sweep to retry (edge-relay-fence.ts).
|
||||
async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise<boolean> {
|
||||
if (!this.admin) { return true }
|
||||
// An edge whose exports are failing gets one try, so a skipped export is not slowed further.
|
||||
const attempts = this.exportFailingSince === undefined ? 3 : 1
|
||||
// An edge whose exports are failing (or that is marked down) gets one try, so a skipped export
|
||||
// is not slowed further.
|
||||
const down = this.exportFailingSince !== undefined || this.config.control?.().mode === 'down'
|
||||
const attempts = down ? 1 : 3
|
||||
return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now())
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
cellFlagParser
|
||||
} from './cell/control-flags/cell-flags.js'
|
||||
import { startControlFlagChannel } from './cell/control-flags/relay-control-flag-channel.js'
|
||||
import { startEdgeControl } from './edge-relay-control.js'
|
||||
import { readCloudRunInstanceZone } from './process-lifecycle/cloud-run-instance-zone.js'
|
||||
import { readSandboxTcpCounters } from './observability/sandbox-tcp-counters.js'
|
||||
import {
|
||||
@@ -68,6 +69,8 @@ const cellFlagChannel =
|
||||
})
|
||||
: null
|
||||
cellFlagsApplied = cellFlagChannel ? () => cellFlagChannel.applied().flags : null
|
||||
// Directors with an edge read director/edge.json; until it lands, mode on and the env rollout.
|
||||
const edgeControl = config.role === 'director' && config.edge ? startEdgeControl() : undefined
|
||||
const {
|
||||
server,
|
||||
sessions,
|
||||
@@ -79,7 +82,7 @@ const {
|
||||
ready,
|
||||
cellIncarnation,
|
||||
shadowSeatPoller
|
||||
} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied })
|
||||
} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied, edgeControl })
|
||||
// Cloud Run sets K_SERVICE; GCE cells and local runs skip the metadata read.
|
||||
if (process.env.K_SERVICE) {
|
||||
void readCloudRunInstanceZone().then((zone) => observability.setInstanceZone(zone))
|
||||
@@ -229,6 +232,7 @@ const shutdown = (): void => {
|
||||
regionalRehomeWorker?.stop()
|
||||
shadowSeatPoller?.stop()
|
||||
cellFlagChannel?.stop()
|
||||
edgeControl?.stop()
|
||||
sessions.drain(0)
|
||||
server.close(() => void closeDatabaseAndExit(shutdownStartedAt).catch(() => undefined))
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import { createRelayApp } from './app.js'
|
||||
import { EDGE_CELL_ID, EdgeAssignments } from './edge-relay.js'
|
||||
import { EdgeEvacuation } from './edge-relay-evacuation.js'
|
||||
import type { EdgeControl } from './edge-relay-control.js'
|
||||
import { createEdgeAdmin } from './edge-admin-client.js'
|
||||
import { classifyAssignmentLease } from './assignment/assignment-lease.js'
|
||||
import { defaultIntakePerSec, type CellFlags } from './cell/control-flags/cell-flags.js'
|
||||
@@ -138,6 +139,8 @@ export function createRelayServer(
|
||||
connectionLedgerLimits?: { hardCap: number; controlReserve: number }
|
||||
cellIncarnation?: string
|
||||
cellFlags?: () => AppliedControlFlags<CellFlags>
|
||||
// Runtime edge control (director/edge.json); absent, the edge runs on its boot config.
|
||||
edgeControl?: EdgeControl
|
||||
} = {}
|
||||
) {
|
||||
const cellIncarnation = options.cellIncarnation ?? randomUUID()
|
||||
@@ -179,16 +182,20 @@ export function createRelayServer(
|
||||
issuer: config.publicUrl
|
||||
})
|
||||
: undefined
|
||||
const edgeConfig =
|
||||
config.edge && options.edgeControl ? { ...config.edge, control: options.edgeControl.flags } : config.edge
|
||||
const edgeAssignments =
|
||||
config.role === 'director' && config.edge
|
||||
? new EdgeAssignments(observedDatabase, config.edge, options.now, edgeAdmin)
|
||||
config.role === 'director' && edgeConfig
|
||||
? new EdgeAssignments(observedDatabase, edgeConfig, options.now, edgeAdmin)
|
||||
: undefined
|
||||
// Hosts that left the cohort are drained off the edge, rather than waiting out their lease.
|
||||
const edgeEvacuation =
|
||||
config.role === 'director' && config.edge && edgeAdmin
|
||||
? new EdgeEvacuation(observedDatabase, config.edge, edgeAdmin, EDGE_CELL_ID, options.now, options.random)
|
||||
config.role === 'director' && edgeConfig && edgeAdmin
|
||||
? new EdgeEvacuation(observedDatabase, edgeConfig, edgeAdmin, EDGE_CELL_ID, options.now, options.random)
|
||||
: undefined
|
||||
edgeEvacuation?.start()
|
||||
// A rollback (mode off/down, a smaller share) starts draining within seconds.
|
||||
options.edgeControl?.onChange(() => edgeEvacuation?.controlChanged())
|
||||
if (edgeAssignments) {
|
||||
void edgeAssignments.ensureEdgeCell().then(
|
||||
() => console.warn(JSON.stringify({ event: 'orca_relay_edge_cell_ready' })),
|
||||
|
||||
Reference in New Issue
Block a user