fix(relay): runtime edge control on the director, safer skipped exports, faster evacuation (#39773)

- Directors with an edge read director/edge.json from the relay-control bucket through the shared
  flag channel: mode on/off/down and rolloutPermille (0..1000). Boot defaults are mode on and the
  env percent; every cohort decision reads the applied flags. Off empties the cohort and hosts
  leave normally; down also skips the export at once (reason mode_down). The permille bucket keeps
  every host of P percent in at P x 10 permille. A control change re-arms the evacuation sweep
  within 5 s.
- A skipped export (120 s outage clock or mode down) removes the host's relay_devices rows before
  placing it, so revokes made on the edge are never undone; logs a count only.
- Evacuation: 100 drains per tick with 10 in flight, each pass starting at a random point of the
  keys and wrapping around; pages stay 500 rows and an idle pass reads each edge row once.
- Fence retries: 56 attempts (last about 25.5 h after the failure), 50 per sweep with 10 in
  flight; fence_abandoned now carries outcome, cell and epoch like fence_failed.
This commit is contained in:
Neil
2026-11-07 23:38:33 -08:00
committed by GitHub
parent c362cb183e
commit 5df115db15
13 changed files with 774 additions and 80 deletions
@@ -26,6 +26,8 @@ export function startControlFlagChannel<Flags>(input: {
defaults: Flags
parse: ControlFlagParse<Flags>
appliedEvent: string
// Called once per applied generation, after it is logged; a throw is logged, never retried.
onApplied?: (applied: AppliedControlFlags<Flags>) => void
fetch?: typeof fetch
random?: () => number
pollMs?: number
@@ -134,6 +136,17 @@ export function startControlFlagChannel<Flags>(input: {
const { flags, ignoredKeys } = parsed
applied = ignoredKeys.length > 0 ? { generation, flags, ignoredKeys } : { generation, flags }
console.log(JSON.stringify({ event: input.appliedEvent, ...applied }))
try {
input.onApplied?.(applied)
} catch (error) {
console.warn(
JSON.stringify({
event: 'orca_relay_control_flags_listener_failed',
object: input.objectName,
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
}
const poll = (): Promise<void> => {
+24 -5
View File
@@ -5,13 +5,17 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra
import type { SqlRow } from './database/database.js'
import { z } from 'zod'
import type { EdgeDeviceRow } from './edge-admin-client.js'
import { EDGE_CONTROL_DEFAULTS, edgeRolloutPermille, type EdgeControlFlags } from './edge-relay-control.js'
export type EdgeRelayConfig = {
cellUrl: string
// Present: phase 2 (host data in the edge's objects); director → edge admin calls use it.
adminSigningKey?: Uint8Array
// Share of hosts (by hash of relayHostId) in the edge cohort, 0-100.
// Share of hosts (by hash of relayHostId) in the edge cohort, 0-100: the boot default, which a
// runtime rolloutPermille replaces.
rolloutPercent: number
// Runtime control (director/edge.json), read on every call; absent means the defaults.
control?: () => EdgeControlFlags
allowHosts: ReadonlySet<string>
// Accounts (relay token `sub`) whose hosts all join, e.g. the team's own accounts.
allowUsers?: ReadonlySet<string>
@@ -29,22 +33,37 @@ export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000
// Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge).
export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000
export function edgeCohortBucket(relayHostId: string): number {
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100
function edgeCohortHash(relayHostId: string): number {
return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0)
}
// Deny wins, then host allow, then region scope; within the region, account allow or rollout share.
export function edgeCohortBucket(relayHostId: string): number {
return edgeCohortHash(relayHostId) % 100
}
// The percent bucket times ten plus a digit from the hash's higher part, so for every percent P a
// host is under P × 10 permille exactly when it is under P percent: no host moves when the env
// percent becomes the equal permille.
export function edgeCohortBucketPermille(relayHostId: string): number {
const hash = edgeCohortHash(relayHostId)
return (hash % 100) * 10 + (Math.floor(hash / 100) % 10)
}
// Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope;
// within the region, account allow or the rollout share.
export function inEdgeCohort(
config: EdgeRelayConfig,
relayHostId: string,
region: RelayRegion | undefined,
userId?: string
): boolean {
const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS
if (flags.mode !== 'on') { return false }
if (config.denyHosts.has(relayHostId)) { return false }
if (config.allowHosts.has(relayHostId)) { return true }
if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
if (userId !== undefined && config.allowUsers?.has(userId)) { return true }
return edgeCohortBucket(relayHostId) < config.rolloutPercent
return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags)
}
// Another director holds the host's row, or the host is on the edge and cannot leave yet (its
@@ -0,0 +1,198 @@
import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest'
import {
EDGE_CONTROL_APPLIED_EVENT,
EDGE_CONTROL_DEFAULTS,
EDGE_CONTROL_OBJECT_NAME,
parseEdgeControl,
startEdgeControl,
type EdgeControlFlags
} from './edge-relay-control.js'
import { edgeCohortBucket, edgeCohortBucketPermille, inEdgeCohort, type EdgeRelayConfig } from './edge-relay-cohort.js'
function edgeConfig(overrides: Partial<EdgeRelayConfig> = {}): EdgeRelayConfig {
return {
cellUrl: 'https://edge.relay.example.test',
rolloutPercent: 0,
allowHosts: new Set(),
denyHosts: new Set(),
regions: new Set(['us-central1']),
...overrides
}
}
function hostId(index: number): string {
return `host${String(index).padStart(12, '0')}`
}
// Storage answers the stored object (or 404) by generation, like the cell flag channel's tests.
function fakeGoogle() {
const state: { object: { generation: number; body: string } | null; paths: string[] } = {
object: null,
paths: []
}
const fetchImpl: typeof fetch = async (target) => {
const url = new URL(String(target))
if (url.hostname === 'metadata.google.internal') {
if (url.pathname.endsWith('/project/project-id')) { return new Response('onorca-cloud') }
return Response.json({ access_token: 'ya29.secret-access-token', expires_in: 3_600 })
}
state.paths.push(decodeURIComponent(url.pathname))
const object = state.object
if (!object) { return new Response(null, { status: 404 }) }
if (url.searchParams.get('ifGenerationNotMatch') === String(object.generation)) {
return new Response(null, { status: 304 })
}
return new Response(object.body, { headers: { 'x-goog-generation': String(object.generation) } })
}
const store = (generation: number, flags: Record<string, unknown>) => {
state.object = { generation, body: JSON.stringify({ v: 1, flags }) }
}
return { state, fetchImpl, store }
}
describe('edge runtime control object', () => {
let log: MockInstance<typeof console.log>
let warn: MockInstance<typeof console.warn>
beforeEach(() => {
log = vi.spyOn(console, 'log').mockImplementation(() => {})
warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
})
afterEach(() => {
log.mockRestore()
warn.mockRestore()
})
it('parses mode and permille, reports unknown keys, and voids the object on any bad known value', () => {
expect(parseEdgeControl({ v: 1, flags: {} })).toEqual({ flags: { mode: 'on' }, ignoredKeys: [] })
expect(parseEdgeControl({ v: 1, flags: { mode: 'down', rolloutPermille: 0, cohort: 'x' } })).toEqual({
flags: { mode: 'down', rolloutPermille: 0 },
ignoredKeys: ['cohort']
})
expect(parseEdgeControl({ v: 1, flags: { rolloutPermille: 1_000 } })).toEqual({
flags: { mode: 'on', rolloutPermille: 1_000 },
ignoredKeys: []
})
for (const flags of [
{ mode: 'paused' },
{ mode: true },
{ rolloutPermille: 1_001 },
{ rolloutPermille: -1 },
{ rolloutPermille: 2.5 },
{ rolloutPermille: '5' },
{ mode: 'off', rolloutPermille: null }
]) {
expect(parseEdgeControl({ v: 1, flags })).toBeNull()
}
expect(parseEdgeControl({ v: 2, flags: {} })).toBeNull()
expect(parseEdgeControl({ flags: {} })).toBeNull()
})
it('reads director/edge.json, logs each applied change with no ids, and tells listeners', async () => {
const google = fakeGoogle()
const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false })
const changes: EdgeControlFlags[] = []
control.onChange(() => changes.push(control.flags()))
await control.poll()
expect(control.flags()).toEqual(EDGE_CONTROL_DEFAULTS)
expect(google.state.paths.at(-1)).toBe(`/storage/v1/b/onorca-cloud-relay-control/o/${EDGE_CONTROL_OBJECT_NAME}`)
google.store(3, { mode: 'off', rolloutPermille: 25 })
await control.poll()
// Unchanged object: no second change.
await control.poll()
google.store(4, { mode: 'down', note: 'incident' })
await control.poll()
// A bad value voids the object: the last applied flags stay.
google.store(5, { mode: 'sideways' })
await control.poll()
expect(control.flags()).toEqual({ mode: 'down' })
expect(changes).toEqual([{ mode: 'off', rolloutPermille: 25 }, { mode: 'down' }])
const applied = log.mock.calls
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
.filter((line) => line.event === EDGE_CONTROL_APPLIED_EVENT)
expect(applied).toEqual([
{ event: EDGE_CONTROL_APPLIED_EVENT, generation: 3, flags: { mode: 'off', rolloutPermille: 25 } },
{ event: EDGE_CONTROL_APPLIED_EVENT, generation: 4, flags: { mode: 'down' }, ignoredKeys: ['note'] }
])
control.stop()
})
it('logs a throwing listener and still applies the flags', async () => {
const google = fakeGoogle()
const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false })
control.onChange(() => {
throw new Error('listener_broke')
})
google.store(2, { mode: 'off' })
await control.poll()
expect(control.flags()).toEqual({ mode: 'off' })
expect(warn.mock.calls.map((call) => String(call[0])).join('\n')).toContain('orca_relay_control_flags_listener_failed')
control.stop()
})
})
describe('edge cohort under runtime control', () => {
it('reads the applied flags on every call: off and down empty the cohort, even allowed hosts', () => {
const flags: EdgeControlFlags = { mode: 'on' }
const host = hostId(1)
const config = edgeConfig({
rolloutPercent: 100,
allowHosts: new Set([hostId(2)]),
allowUsers: new Set(['user-team']),
control: () => flags
})
expect(inEdgeCohort(config, host, 'us-central1')).toBe(true)
for (const mode of ['off', 'down'] as const) {
flags.mode = mode
expect(inEdgeCohort(config, host, 'us-central1')).toBe(false)
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(false)
expect(inEdgeCohort(config, host, 'us-central1', 'user-team')).toBe(false)
}
flags.mode = 'on'
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true)
// A permille replaces the env percent; deny, allow and region rules still apply around it.
flags.rolloutPermille = 0
expect(inEdgeCohort(config, host, 'us-central1')).toBe(false)
expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true)
flags.rolloutPermille = 1_000
expect(inEdgeCohort(config, host, 'us-central1')).toBe(true)
expect(inEdgeCohort(config, host, 'asia-east2')).toBe(false)
expect(inEdgeCohort({ ...config, denyHosts: new Set([host]) }, host, 'us-central1')).toBe(false)
})
it('keeps every host of P percent in at P × 10 permille, for every P', () => {
// floor(permille bucket / 10) is the percent bucket, so `bucket < P × 10` iff `bucket < P`.
const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index))
expect(hosts.filter((host) => Math.floor(edgeCohortBucketPermille(host) / 10) !== edgeCohortBucket(host))).toEqual([])
const sample = hosts.slice(0, 2_000)
const moved: string[] = []
for (let percent = 0; percent <= 100; percent += 1) {
const byPercent = edgeConfig({ rolloutPercent: percent })
const byPermille = edgeConfig({ rolloutPercent: 0, control: () => ({ mode: 'on', rolloutPermille: percent * 10 }) })
for (const host of sample) {
if (inEdgeCohort(byPercent, host, 'us-central1') !== inEdgeCohort(byPermille, host, 'us-central1')) {
moved.push(`${percent}:${host}`)
}
}
}
expect(moved).toEqual([])
// The env percent alone (boot, no permille flag) answers as before.
expect(sample.filter((host) => inEdgeCohort(edgeConfig({ rolloutPercent: 10 }), host, 'us-central1')))
.toEqual(sample.filter((host) => edgeCohortBucket(host) < 10))
})
it('steps in 1 permille shares, nested, roughly evenly', () => {
const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index))
const share = (permille: number) =>
hosts.filter((host) => inEdgeCohort(edgeConfig({ control: () => ({ mode: 'on', rolloutPermille: permille }) }), host, 'us-central1'))
const one = share(1)
const five = share(5)
expect(one.length).toBeGreaterThan(5)
expect(one.length).toBeLessThan(40)
expect(five.length).toBeGreaterThan(70)
expect(five.length).toBeLessThan(130)
// A larger share keeps every host of a smaller one.
expect(one.every((host) => five.includes(host))).toBe(true)
const buckets = new Set(hosts.map((host) => edgeCohortBucketPermille(host)))
expect(buckets.size).toBe(1_000)
})
})
@@ -0,0 +1,86 @@
// Runtime control of the relay edge on the director: director/edge.json in the relay-control
// bucket, read by the shared flag channel (cell/control-flags/relay-control-flag-channel.ts), so a
// rollout step or a rollback needs no director deploy. Boot uses the defaults (mode on, the env
// rollout percent); a failed read keeps the last applied flags.
import { z } from 'zod'
import {
startControlFlagChannel,
type ControlFlagParse
} from './cell/control-flags/relay-control-flag-channel.js'
// `off`: no host is in the cohort; hosts on the edge leave normally (export, place, fence) and the
// evacuation sweep drains them. `down`: as off, and the edge is known dead, so a leaving host's
// export is skipped at once (EdgeAssignments.leaveEdge).
export type EdgeControlMode = 'on' | 'off' | 'down'
export type EdgeControlFlags = {
mode: EdgeControlMode
// Replaces the env rollout percent when present (1‰ steps).
rolloutPermille?: number
}
export const EDGE_CONTROL_DEFAULTS: EdgeControlFlags = { mode: 'on' }
export const EDGE_CONTROL_OBJECT_NAME = 'director/edge.json'
export const EDGE_CONTROL_APPLIED_EVENT = 'orca_relay_edge_control_applied'
const EdgeControlFlagsSchema = z.object({
mode: z.enum(['on', 'off', 'down']).default(EDGE_CONTROL_DEFAULTS.mode),
rolloutPermille: z.number().int().min(0).max(1_000).optional()
})
const KNOWN_FLAG_KEYS = new Set(Object.keys(EdgeControlFlagsSchema.shape))
const EdgeControlObjectSchema = z.object({ v: z.literal(1), flags: EdgeControlFlagsSchema })
// Unknown keys (a newer writer's, or a typo) are dropped and reported; a known key with a bad value
// voids the whole object, so the last applied flags stay.
export const parseEdgeControl: ControlFlagParse<EdgeControlFlags> = (body) => {
const parsed = EdgeControlObjectSchema.safeParse(body)
if (!parsed.success) {
return null
}
const named = body !== null && typeof body === 'object' && 'flags' in body ? body.flags : {}
const ignoredKeys =
named !== null && typeof named === 'object'
? Object.keys(named).filter((key) => !KNOWN_FLAG_KEYS.has(key)).sort()
: []
const { mode, rolloutPermille } = parsed.data.flags
return { flags: rolloutPermille === undefined ? { mode } : { mode, rolloutPermille }, ignoredKeys }
}
export type EdgeControl = {
// The flags applied now; callers read this on every decision, never caching it.
flags: () => EdgeControlFlags
// Called after each newly applied generation.
onChange: (listener: () => void) => void
stop: () => void
}
export function startEdgeControl(
input: { fetch?: typeof fetch; random?: () => number; pollMs?: number; autoStart?: boolean } = {}
): EdgeControl & { poll: () => Promise<void> } {
const listeners: (() => void)[] = []
const channel = startControlFlagChannel({
objectName: EDGE_CONTROL_OBJECT_NAME,
defaults: EDGE_CONTROL_DEFAULTS,
parse: parseEdgeControl,
appliedEvent: EDGE_CONTROL_APPLIED_EVENT,
onApplied: () => {
for (const listener of listeners) {
listener()
}
},
...input
})
return {
flags: () => channel.applied().flags,
onChange: (listener) => {
listeners.push(listener)
},
poll: channel.poll,
stop: channel.stop
}
}
// Effective share of hosts on the edge in permille: the applied flag, else the env percent.
export function edgeRolloutPermille(rolloutPercent: number, flags: EdgeControlFlags): number {
return flags.rolloutPermille ?? rolloutPercent * 10
}
@@ -9,9 +9,16 @@ import {
type EdgeAdmin,
type EdgeDrainResult
} from './edge-admin-client.js'
import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS,
EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
import {
EDGE_EVACUATION_BATCH,
EDGE_EVACUATION_BUSY_INTERVAL_MS,
EDGE_EVACUATION_CONCURRENCY,
EDGE_EVACUATION_CONTROL_DELAY_MS,
EDGE_EVACUATION_INTERVAL_MS,
EdgeEvacuation
} from './edge-relay-evacuation.js'
import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js'
import type { EdgeControlFlags } from './edge-relay-control.js'
const EDGE_URL = 'https://edge.relay.example.test'
const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!')
@@ -31,6 +38,13 @@ class FakeAdmin implements EdgeAdmin {
drained: string[] = []
unavailable = new Set<string>()
answer: EdgeDrainResult = { outcome: 'drained', controls: 1 }
// Set: each drain waits for release(), so the test can count drains in flight.
holding = false
readonly held: (() => void)[] = []
release(): void {
for (const resolve of this.held.splice(0)) { resolve() }
}
async assign(): Promise<never> {
throw new Error('unexpected assign')
@@ -43,11 +57,12 @@ class FakeAdmin implements EdgeAdmin {
async drain(relayHostId: string): Promise<EdgeDrainResult> {
if (this.unavailable.has(relayHostId)) { throw new EdgeAdminUnavailableError('status_404') }
this.drained.push(relayHostId)
if (this.holding) { await new Promise<void>((resolve) => this.held.push(resolve)) }
return this.answer
}
}
async function harness(overrides: Partial<EdgeRelayConfig> = {}) {
async function harness(overrides: Partial<EdgeRelayConfig> = {}, random = 0.5) {
const database = await openInMemoryRelayDatabase()
databases.push(database)
const config: EdgeRelayConfig = {
@@ -60,7 +75,7 @@ async function harness(overrides: Partial<EdgeRelayConfig> = {}) {
}
const admin = new FakeAdmin()
const clock = { now: 1_700_000_000_000 }
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => 0.5)
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => random)
const insert = async (relayHostId: string, cellId = EDGE_CELL_ID, userId = 'user-1') =>
await database.query(
`INSERT INTO relay_assignments
@@ -107,11 +122,11 @@ describe('edge evacuation sweep', () => {
it(`drains at most ${EDGE_EVACUATION_BATCH} per tick, the rest on the next, and not again within the cooldown`, async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
for (let index = 0; index < 25; index += 1) { await h.insert(hostId(index)) }
for (let index = 0; index < EDGE_EVACUATION_BATCH + 25; index += 1) { await h.insert(hostId(index)) }
h.admin.answer = { outcome: 'idle' }
expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH)
expect(await h.evacuation.sweep()).toBe(5)
expect(new Set(h.admin.drained).size).toBe(25)
expect(await h.evacuation.sweep()).toBe(25)
expect(new Set(h.admin.drained).size).toBe(EDGE_EVACUATION_BATCH + 25)
expect(await h.evacuation.sweep()).toBe(0)
h.clock.now += 10 * 60_000
expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH)
@@ -158,6 +173,99 @@ describe('edge evacuation sweep', () => {
})
})
describe('edge evacuation under runtime control', () => {
it('re-arms an idle sweep within 5 s of a control change, so a rollback drains at once', async () => {
vi.useFakeTimers()
vi.spyOn(console, 'warn').mockImplementation(() => {})
const flags: EdgeControlFlags = { mode: 'on' }
const h = await harness({ rolloutPercent: 100, control: () => flags })
await h.insert(hostId(1))
const sweep = vi.spyOn(h.evacuation, 'sweep')
h.evacuation.start()
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(sweep).toHaveBeenCalledTimes(1)
expect(h.admin.drained).toEqual([])
// Idle now: the next pass would be ~10 min away.
await vi.advanceTimersByTimeAsync(60_000)
flags.mode = 'off'
h.evacuation.controlChanged()
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS)
expect(sweep).toHaveBeenCalledTimes(2)
expect(h.admin.drained).toEqual([hostId(1)])
h.evacuation.stop()
})
it('re-arms after a pass that was running when the control changed', async () => {
vi.useFakeTimers()
vi.spyOn(console, 'warn').mockImplementation(() => {})
const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 0 }
const h = await harness({ control: () => flags })
await h.insert(hostId(1))
h.admin.holding = true
const sweep = vi.spyOn(h.evacuation, 'sweep')
h.evacuation.start()
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS)
expect(h.admin.held).toHaveLength(1)
flags.mode = 'down'
h.evacuation.controlChanged()
h.admin.release()
await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS)
expect(sweep).toHaveBeenCalledTimes(2)
h.evacuation.stop()
})
it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
for (let index = 0; index < 35; index += 1) { await h.insert(hostId(index)) }
h.admin.holding = true
let done = false
const pass = h.evacuation.sweep().then((answered) => {
done = true
return answered
})
let most = 0
while (!done) {
await new Promise((resolve) => setTimeout(resolve, 0))
most = Math.max(most, h.admin.held.length)
h.admin.release()
}
expect(await pass).toBe(35)
expect(most).toBe(EDGE_EVACUATION_CONCURRENCY)
})
it('starts each pass at a random point of the keys and wraps around to it', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const users = ['20000000-a', '60000000-a', 'a0000000-a', 'e0000000-a']
const drainedFrom = async (random: number) => {
const h = await harness({}, random)
for (const [index, user] of users.entries()) { await h.insert(hostId(index), EDGE_CELL_ID, user) }
expect(await h.evacuation.sweep()).toBe(4)
return h.admin.drained
}
expect(await drainedFrom(0.5)).toEqual([hostId(2), hostId(3), hostId(0), hostId(1)])
expect(await drainedFrom(0.1)).toEqual([hostId(0), hostId(1), hostId(2), hostId(3)])
expect(await drainedFrom(0.3)).toEqual([hostId(1), hostId(2), hostId(3), hostId(0)])
})
it('pages an idle pass: one page of edge rows per tick, then idle until the next pass', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness({ rolloutPercent: 100 })
for (let index = 0; index < 1_200; index += 1) { await h.insert(hostId(index)) }
const query = vi.spyOn(h.database, 'query')
const pages = () => query.mock.calls.filter(([sql]) => String(sql).includes('FROM relay_assignments a')).length
const busy = () => (h.evacuation as unknown as { busy: boolean }).busy
await h.evacuation.sweep()
expect([pages(), busy()]).toEqual([1, true])
await h.evacuation.sweep()
expect([pages(), busy()]).toEqual([2, true])
// The short last page wraps to the start of the keys, which holds nothing before the origin.
await h.evacuation.sweep()
expect([pages(), busy()]).toEqual([4, false])
expect(h.admin.drained).toEqual([])
})
})
describe('edge admin drain call', () => {
it('posts the signed drain op and reads both answers; anything else is unavailable', async () => {
const answers: [number, unknown][] = [
+106 -39
View File
@@ -7,27 +7,38 @@ import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contr
import type { RelayDatabase } from './database/database.js'
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
import { retryPendingFences } from './edge-relay-fence.js'
import { forEachBounded, retryPendingFences } from './edge-relay-fence.js'
// The cohort only changes when a new revision starts, so a pass soon after start and then a slow
// background pass are enough; each pass reads the edge rows (cell_id has no index).
// A cohort change (director/edge.json, or a new revision) re-arms the sweep (controlChanged), so a
// soon pass and then a slow background pass are enough; each pass reads the edge rows (cell_id has
// no index).
export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000
// First pass after start, and the next pass while more leaving hosts are queued.
export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000
// Drains sent per tick (per director), so a cohort cut moves desktops back gradually.
export const EDGE_EVACUATION_BATCH = 20
// Edge rows read per tick; the cursor walks the table across ticks.
// Longest wait for a pass after the runtime control changes, so a rollback starts at once.
export const EDGE_EVACUATION_CONTROL_DELAY_MS = 5_000
// Drains sent per tick (per director): a full rollback of ~19k hosts over 5 directors takes ~40 min.
export const EDGE_EVACUATION_BATCH = 100
// Drains in flight at once per director.
export const EDGE_EVACUATION_CONCURRENCY = 10
// Edge rows read per page; the cursor walks the table across ticks.
const EDGE_EVACUATION_PAGE = 500
// A host this director drained is not drained again for this long (a desktop that is offline
// answers idle, and asks the director on its own when it comes back).
const EDGE_EVACUATION_COOLDOWN_MS = 10 * 60_000
// A pass starts at a random point of the key space and wraps around to it, so directors sweeping
// at once mostly drain different hosts.
type Pass = { origin: Identity; cursor: Identity; wrapped: boolean }
export class EdgeEvacuation {
private cursor: Identity | undefined
// Undefined between passes.
private pass: Pass | undefined
private readonly drainedAt = new Map<string, number>()
private timer: ReturnType<typeof setTimeout> | undefined
private stopped = true
private busy = true
private sweeping = false
private controlChangedWhileSweeping = false
constructor(
private readonly database: RelayDatabase,
@@ -49,6 +60,19 @@ export class EdgeEvacuation {
this.timer = undefined
}
// The runtime control changed (e.g. mode off for a rollback): the next pass runs within
// EDGE_EVACUATION_CONTROL_DELAY_MS instead of waiting out the idle interval.
controlChanged(): void {
this.busy = true
if (this.stopped) { return }
if (this.sweeping) {
this.controlChangedWhileSweeping = true
return
}
clearTimeout(this.timer)
this.schedule(EDGE_EVACUATION_CONTROL_DELAY_MS * this.random())
}
// One tick: returns how many drains the edge answered.
async sweep(): Promise<number> {
const now = this.now()
@@ -56,12 +80,12 @@ export class EdgeEvacuation {
if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) }
}
const leaving = await this.leavingHosts()
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined
this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.pass !== undefined
let answered = 0
for (const identity of leaving) {
await forEachBounded(leaving, EDGE_EVACUATION_CONCURRENCY, async (identity) => {
try {
const result = await this.admin.drain(identity.relayHostId)
this.drainedAt.set(`${identity.userId}\n${identity.relayHostId}`, this.now())
this.drainedAt.set(drainKey(identity), this.now())
answered += 1
console.warn(
JSON.stringify({
@@ -74,51 +98,85 @@ export class EdgeEvacuation {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message }))
}
}
})
// Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured.
if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true }
return answered
}
// Edge rows (one page after the cursor) whose host left the cohort, skipping recent drains.
// Edge rows (a page after the cursor; a second when the first ends the pass's first half) whose
// host left the cohort, skipping recent drains. An idle pass reads each edge row once.
private async leavingHosts(): Promise<Identity[]> {
const cursor = this.cursor ?? { userId: '', relayHostId: '' }
const rows = await this.database.query(
`SELECT a.user_id, a.relay_host_id, p.preferred_region
FROM relay_assignments a
LEFT JOIN relay_assignment_region_preferences p
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
ORDER BY a.user_id, a.relay_host_id
LIMIT ?`,
[this.edgeCellId, cursor.userId, cursor.userId, cursor.relayHostId, EDGE_EVACUATION_PAGE]
)
// A short page reached the end: the next tick starts over.
this.cursor = undefined
const leaving: Identity[] = []
for (const [index, row] of rows.entries()) {
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
if (rows.length === EDGE_EVACUATION_PAGE && index === rows.length - 1) { this.cursor = identity }
const stored = RelayRegionSchema.safeParse(row.preferred_region)
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
if (this.drainedAt.has(`${identity.userId}\n${identity.relayHostId}`)) { continue }
leaving.push(identity)
if (leaving.length === EDGE_EVACUATION_BATCH) {
this.pass ??= this.newPass()
for (let pages = 0; pages < 2 && this.pass !== undefined; pages += 1) {
const pass: Pass = this.pass
const rows = await this.readPage(pass)
let filled = false
for (const row of rows) {
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
pass.cursor = identity
const stored = RelayRegionSchema.safeParse(row.preferred_region)
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
if (this.drainedAt.has(drainKey(identity))) { continue }
leaving.push(identity)
// Resume after the last host taken, so the rest of the page is read next tick.
this.cursor = identity
break
if (leaving.length === EDGE_EVACUATION_BATCH) {
filled = true
break
}
}
if (filled || rows.length === EDGE_EVACUATION_PAGE) { break }
// A short page ends this half: wrap to the start of the keys, or end the pass.
if (pass.wrapped) {
this.pass = undefined
} else {
pass.wrapped = true
pass.cursor = { userId: '', relayHostId: '' }
}
}
return leaving
}
private schedule(): void {
// Keys after the cursor; once wrapped, only up to the pass's origin.
private async readPage(pass: Pass) {
const { cursor, origin, wrapped } = pass
return await this.database.query(
`SELECT a.user_id, a.relay_host_id, p.preferred_region
FROM relay_assignments a
LEFT JOIN relay_assignment_region_preferences p
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''}
ORDER BY a.user_id, a.relay_host_id
LIMIT ?`,
[
this.edgeCellId,
cursor.userId,
cursor.userId,
cursor.relayHostId,
...(wrapped ? [origin.userId, origin.userId, origin.relayHostId] : []),
EDGE_EVACUATION_PAGE
]
)
}
// Account ids are UUIDs, so a random hex prefix lands anywhere in the key space; ids of another
// shape are still all swept, from a less spread origin.
private newPass(): Pass {
const prefix = Math.floor(this.random() * 0x1_0000_0000).toString(16).padStart(8, '0')
const origin = { userId: prefix, relayHostId: '' }
return { origin, cursor: origin, wrapped: false }
}
private schedule(delayMs?: number): void {
if (this.stopped) { return }
// Jittered (±25%), so directors do not sweep together.
const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS
const delay = interval * (0.75 + this.random() / 2)
const delay = delayMs ?? interval * (0.75 + this.random() / 2)
this.timer = setTimeout(() => {
this.sweeping = true
void this.sweep()
.catch((error: unknown) =>
console.warn(
@@ -128,8 +186,17 @@ export class EdgeEvacuation {
})
)
)
.finally(() => this.schedule())
.finally(() => {
this.sweeping = false
const changed = this.controlChangedWhileSweeping
this.controlChangedWhileSweeping = false
this.schedule(changed ? EDGE_EVACUATION_CONTROL_DELAY_MS * this.random() : undefined)
})
}, delay)
this.timer.unref?.()
}
}
function drainKey(identity: Identity): string {
return `${identity.userId}\n${identity.relayHostId}`
}
+58 -2
View File
@@ -12,7 +12,15 @@ import {
type EdgeAssignResult
} from './edge-admin-client.js'
import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js'
import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js'
import {
EDGE_FENCE_MAX_ATTEMPTS,
EDGE_FENCE_RETRY_BASE_MS,
EDGE_FENCE_RETRY_BATCH,
EDGE_FENCE_RETRY_MAX_MS,
edgeFenceRetryDelayMs,
recordPendingFence,
retryPendingFences
} from './edge-relay-fence.js'
import {
EDGE_CELL_ID,
EDGE_LOCKED_CALLS_MAX,
@@ -51,6 +59,10 @@ class ScriptedEdge implements EdgeAdmin {
return { outcome, assignmentEpoch: outcome === 'stale' ? body.assignmentEpoch + 5 : body.assignmentEpoch }
}
get hanging(): number {
return this.waiting.length
}
release(): void {
for (const resolve of this.waiting.splice(0)) { resolve() }
}
@@ -229,12 +241,56 @@ describe('relay edge pending fence retries', () => {
retry: true
})
expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([
{ event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS }
{
event: 'orca_relay_edge_fence_abandoned',
outcome: 'stale',
cellId: 'us-c1',
assignmentEpoch: 9,
attempts: EDGE_FENCE_MAX_ATTEMPTS
}
])
expect(edgeFenceRetryDelayMs(1)).toBe(60_000)
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000)
})
it('keeps retrying a fence through more than a day of edge outage', () => {
// Recorded at 0, first retry one base delay later, then each attempt's backoff.
let lastAttemptAt = EDGE_FENCE_RETRY_BASE_MS
for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) {
lastAttemptAt += edgeFenceRetryDelayMs(attempt)
}
expect(lastAttemptAt).toBeGreaterThanOrEqual(24 * 60 * 60_000)
// Never slower than the cap, so the edge is not left claiming hosts long after it is back.
expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(EDGE_FENCE_RETRY_MAX_MS)
})
it(`retries up to ${EDGE_FENCE_RETRY_BATCH} due fences per pass, 10 in flight at once`, async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
const hosts = Array.from({ length: EDGE_FENCE_RETRY_BATCH + 10 }, (_, index) => `fencehost${String(index).padStart(7, '0')}`)
for (const host of hosts) {
await h.placeRow('us-c1', 9, host)
await recordPendingFence(h.database, { userId: USER, relayHostId: host }, 'us-c1', 9, h.clock.now)
}
h.clock.now += EDGE_FENCE_RETRY_BASE_MS
h.edge.answers = hosts.map(() => 'hang')
let settled = false
const pass = h.retry().then((left) => {
settled = true
return left
})
let maxInFlight = 0
while (!settled) {
await new Promise((resolve) => setTimeout(resolve, 0))
maxInFlight = Math.max(maxInFlight, h.edge.hanging)
h.edge.release()
}
expect(await pass).toBe(true)
expect(h.edge.calls).toHaveLength(EDGE_FENCE_RETRY_BATCH)
expect(maxInFlight).toBe(10)
expect(await h.pending()).toHaveLength(10)
})
it('lets only one director claim an attempt', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
+38 -12
View File
@@ -7,13 +7,16 @@ import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.j
import { integer, text, type Identity } from './edge-relay-cohort.js'
// Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`,
// cannot land until the host moves again, and that move fences anew).
export const EDGE_FENCE_MAX_ATTEMPTS = 8
// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all.
// cannot land until the host moves again, and that move fences anew). Enough to outlast a day-long
// edge outage: the last attempt comes about 25.5 h after the failure.
export const EDGE_FENCE_MAX_ATTEMPTS = 56
// First retry one busy sweep after the failure, doubling up to the cap (31 min to reach it, then
// every 30 min).
export const EDGE_FENCE_RETRY_BASE_MS = 60_000
const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
// Pending fences retried per sweep (per director).
const EDGE_FENCE_RETRY_BATCH = 20
export const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000
// Pending fences retried per sweep (per director), with at most EDGE_FENCE_RETRY_CONCURRENCY in flight.
export const EDGE_FENCE_RETRY_BATCH = 50
const EDGE_FENCE_RETRY_CONCURRENCY = 10
export function edgeFenceRetryDelayMs(attempts: number): number {
return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS)
@@ -91,7 +94,7 @@ export async function retryPendingFences(
WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`,
[now(), EDGE_FENCE_RETRY_BATCH]
)
for (const row of due) {
await forEachBounded(due, EDGE_FENCE_RETRY_CONCURRENCY, async (row) => {
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
const attempts = integer(row, 'attempts') + 1
// Claim the attempt: a director that read the same row loses here and skips it.
@@ -102,7 +105,7 @@ export async function retryPendingFences(
[attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId,
attempts - 1, integer(row, 'next_attempt_at')]
)
if (claimed.length !== 1) { continue }
if (claimed.length !== 1) { return }
const result = await retryOne(database, admin, edgeCellId, identity, attempts)
if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
// Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0.
@@ -111,10 +114,7 @@ export async function retryPendingFences(
[identity.userId, identity.relayHostId, attempts]
)
}
if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts }))
}
}
})
return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0
}
@@ -150,5 +150,31 @@ async function retryOne(
console.error(
JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true })
)
if (attempts >= EDGE_FENCE_MAX_ATTEMPTS) {
// The record is dropped: the edge keeps claiming the host until it moves again.
console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', outcome, cellId, assignmentEpoch, attempts }))
}
return 'failed'
}
// Runs `task` over `items` with at most `limit` in flight. A throw does not stop the other workers;
// the first one is rethrown once all have settled.
export async function forEachBounded<T>(
items: readonly T[],
limit: number,
task: (item: T) => Promise<void>
): Promise<void> {
let next = 0
const worker = async (): Promise<void> => {
while (next < items.length) {
const item = items[next]
next += 1
if (item !== undefined) { await task(item) }
}
}
const results = await Promise.allSettled(
Array.from({ length: Math.min(limit, items.length) }, async () => await worker())
)
const failed = results.find((result): result is PromiseRejectedResult => result.status === 'rejected')
if (failed) { throw failed.reason instanceof Error ? failed.reason : new Error('relay_edge_task_failed') }
}
+57 -4
View File
@@ -8,6 +8,7 @@ import {
type EdgeDeviceRow
} from './edge-admin-client.js'
import { EDGE_CELL_ID, EDGE_EXPORT_SKIP_AFTER_MS, EdgeAssignments, type EdgeRelayConfig } from './edge-relay.js'
import type { EdgeControlFlags } from './edge-relay-control.js'
const fakes = vi.hoisted(() => ({
verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-1', relayHostId: token }))
@@ -276,13 +277,14 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
const response = await h.assign()
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 })
// Postgres keeps what it had: the edge's revoke is lost, nothing is written back.
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }])
// The edge's revoke is unknown here: the host's Postgres devices are removed, never resurrected.
expect(await h.postgresDevices()).toEqual([])
expect(skipped()).toHaveLength(1)
expect(JSON.parse(String(skipped()[0]![0]))).toEqual({
event: 'orca_relay_edge_export_skipped',
reason: 'relay_edge_admin_unavailable:network',
failingForMs: EDGE_EXPORT_SKIP_AFTER_MS
failingForMs: EDGE_EXPORT_SKIP_AFTER_MS,
devicesRemoved: 1
})
})
@@ -337,7 +339,7 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
const response = await h.assign()
expect(response.status).toBe(200)
expect(exportCalls).toBe(1)
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }])
expect(await h.postgresDevices()).toEqual([])
expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 })
})
@@ -370,6 +372,57 @@ describe('relay edge phase 2: the director moves host data with the host', () =>
expect(h.fake.pushes[0]).toMatchObject({ userId: USER, relayHostId: HOST })
})
it('with mode off or down, sends no new host to the edge', async () => {
for (const mode of ['off', 'down'] as const) {
const h = await harness('host', { control: () => ({ mode }) })
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' })
expect(h.fake.pushes).toHaveLength(0)
expect(await h.row()).toBeUndefined()
}
})
it('with mode off, moves hosts on the edge off normally: export, place, fence', async () => {
const flags: EdgeControlFlags = { mode: 'on' }
const h = await harness('host', { control: () => flags })
await h.insertDevice(device('phone-a'))
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: EDGE_URL })
h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })]
flags.mode = 'off'
expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' })
// The edge's revoke carries over.
expect(h.fake.exports).toBe(1)
expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: 1_800_000_000_000 }])
expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 })
})
it('with mode down, skips the export at once, removes the Postgres devices, and fences once', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
const flags: EdgeControlFlags = { mode: 'on' }
const h = await harness('host', { control: () => flags })
await h.insertDevice(device('phone-a'))
await h.insertDevice(device('phone-b'))
await h.assign()
// phone-a was revoked on the edge, which then died; Postgres still holds it live.
h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })]
h.fake.down = true
flags.mode = 'down'
const assign = vi.spyOn(h.fake, 'assign')
const response = await h.assign()
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 })
expect(h.fake.exports).toBe(0)
expect(await h.postgresDevices()).toEqual([])
const skipped = warn.mock.calls
.map(([line]) => JSON.parse(String(line)) as Record<string, unknown>)
.filter((line) => line.event === 'orca_relay_edge_export_skipped')
expect(skipped).toEqual([{ event: 'orca_relay_edge_export_skipped', reason: 'mode_down', devicesRemoved: 2 }])
// One fence try, recorded for the sweep to retry.
expect(assign).toHaveBeenCalledTimes(1)
const pending = await h.database.query(`SELECT cell_id, assignment_epoch FROM relay_edge_fence_pending`)
expect(pending.map((row) => [String(row.cell_id), Number(row.assignment_epoch)])).toEqual([['us-c1', 9]])
})
it('does not touch the edge for hosts that were never on it', async () => {
const h = await harness()
h.leaveCohort()
@@ -11,6 +11,7 @@ import {
} from './edge-relay.js'
import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js'
import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js'
import { EdgeEvacuation } from './edge-relay-evacuation.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
@@ -236,4 +237,38 @@ describePostgres('relay edge on PostgreSQL', () => {
expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([])
vi.restoreAllMocks()
})
it('mode down: a leaving host loses its Postgres devices, and the sweep drains wrapping around its origin', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
const flags = { mode: 'on' as 'on' | 'down' }
const config = { ...edgeConfig, control: () => flags }
const drained: string[] = []
const admin = {
assign: async (body: EdgeAssignBody) => ({ outcome: 'applied' as const, assignmentEpoch: body.assignmentEpoch }),
exportHost: async (): Promise<never> => {
throw new Error('unexpected export')
},
drain: async (relayHostId: string) => {
drained.push(relayHostId)
return { outcome: 'idle' as const }
}
}
const edge = new EdgeAssignments(database, config, Date.now, admin)
await edge.ensureEdgeCell()
const downHosts = ['edgepgdown000001', 'edgepgdown000002'].map((relayHostId) => ({ userId: 'edge-pg-user', relayHostId }))
for (const host of downHosts) { await edge.moveToEdge(host, 0) }
await database.query(
`INSERT INTO relay_devices (user_id, relay_host_id, relay_device_id, current_hash, current_version,
current_expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)`,
['edge-pg-user', downHosts[0]!.relayHostId, 'phone-a', 'hash-phone-a', 2, 1_900_000_000_000, 1_800_000_000_000]
)
flags.mode = 'down'
// Origin 'f3333333' sorts after 'edge-pg-user': the pass finds both hosts only once wrapped.
const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, Date.now, () => 0.95)
await evacuation.sweep()
expect(drained.filter((id) => id.startsWith('edgepgdown'))).toEqual(downHosts.map((host) => host.relayHostId))
expect(await edge.leaveEdge(downHosts[0]!)).toBe(true)
expect(await database.query(`SELECT 1 FROM relay_devices WHERE relay_host_id = ?`, [downHosts[0]!.relayHostId])).toEqual([])
})
})
+28 -6
View File
@@ -264,6 +264,8 @@ export class EdgeAssignments {
)
)[0]
if (!row || text(row, 'cell_id') !== EDGE_CELL_ID) { return false }
// The edge is known dead (director/edge.json): no export to wait for.
if (this.config.control?.().mode === 'down') { return await this.skipExport(identity, 'mode_down') }
let exported
try {
exported = await this.admin.exportHost(identity.relayHostId)
@@ -278,10 +280,9 @@ export class EdgeAssignments {
this.exportLastFailureAt = now
const failingForMs = now - (this.exportFailingSince ?? now)
if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() }
// The edge has been unreachable too long: place the host on its Postgres devices (changes
// made on the edge are lost) rather than refusing every ask until the edge is back.
console.warn(JSON.stringify({ event: 'orca_relay_edge_export_skipped', reason: error.message, failingForMs }))
return true
// The edge has been unreachable too long: place the host anyway rather than refusing every
// ask until the edge is back.
return await this.skipExport(identity, error.message, failingForMs)
}
this.exportFailingSince = undefined
this.exportLastFailureAt = undefined
@@ -313,13 +314,34 @@ export class EdgeAssignments {
return true
}
// The host leaves without the edge's copy of its devices. The edge was their source of truth and
// revokes made there are unknown, so the Postgres rows are removed rather than resurrected: its
// phones pair again. Logs a count only.
private async skipExport(identity: Identity, reason: string, failingForMs?: number): Promise<true> {
const removed = await this.database.query(
`DELETE FROM relay_devices WHERE user_id = ? AND relay_host_id = ? RETURNING relay_device_id`,
[identity.userId, identity.relayHostId]
)
console.warn(
JSON.stringify({
event: 'orca_relay_edge_export_skipped',
reason,
...(failingForMs === undefined ? {} : { failingForMs }),
devicesRemoved: removed.length
})
)
return true
}
// After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve)
// rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land
// is recorded for the evacuation sweep to retry (edge-relay-fence.ts).
async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise<boolean> {
if (!this.admin) { return true }
// An edge whose exports are failing gets one try, so a skipped export is not slowed further.
const attempts = this.exportFailingSince === undefined ? 3 : 1
// An edge whose exports are failing (or that is marked down) gets one try, so a skipped export
// is not slowed further.
const down = this.exportFailingSince !== undefined || this.config.control?.().mode === 'down'
const attempts = down ? 1 : 3
return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now())
}
+5 -1
View File
@@ -15,6 +15,7 @@ import {
cellFlagParser
} from './cell/control-flags/cell-flags.js'
import { startControlFlagChannel } from './cell/control-flags/relay-control-flag-channel.js'
import { startEdgeControl } from './edge-relay-control.js'
import { readCloudRunInstanceZone } from './process-lifecycle/cloud-run-instance-zone.js'
import { readSandboxTcpCounters } from './observability/sandbox-tcp-counters.js'
import {
@@ -68,6 +69,8 @@ const cellFlagChannel =
})
: null
cellFlagsApplied = cellFlagChannel ? () => cellFlagChannel.applied().flags : null
// Directors with an edge read director/edge.json; until it lands, mode on and the env rollout.
const edgeControl = config.role === 'director' && config.edge ? startEdgeControl() : undefined
const {
server,
sessions,
@@ -79,7 +82,7 @@ const {
ready,
cellIncarnation,
shadowSeatPoller
} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied })
} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied, edgeControl })
// Cloud Run sets K_SERVICE; GCE cells and local runs skip the metadata read.
if (process.env.K_SERVICE) {
void readCloudRunInstanceZone().then((zone) => observability.setInstanceZone(zone))
@@ -229,6 +232,7 @@ const shutdown = (): void => {
regionalRehomeWorker?.stop()
shadowSeatPoller?.stop()
cellFlagChannel?.stop()
edgeControl?.stop()
sessions.drain(0)
server.close(() => void closeDatabaseAndExit(shutdownStartedAt).catch(() => undefined))
}
+11 -4
View File
@@ -20,6 +20,7 @@ import {
import { createRelayApp } from './app.js'
import { EDGE_CELL_ID, EdgeAssignments } from './edge-relay.js'
import { EdgeEvacuation } from './edge-relay-evacuation.js'
import type { EdgeControl } from './edge-relay-control.js'
import { createEdgeAdmin } from './edge-admin-client.js'
import { classifyAssignmentLease } from './assignment/assignment-lease.js'
import { defaultIntakePerSec, type CellFlags } from './cell/control-flags/cell-flags.js'
@@ -138,6 +139,8 @@ export function createRelayServer(
connectionLedgerLimits?: { hardCap: number; controlReserve: number }
cellIncarnation?: string
cellFlags?: () => AppliedControlFlags<CellFlags>
// Runtime edge control (director/edge.json); absent, the edge runs on its boot config.
edgeControl?: EdgeControl
} = {}
) {
const cellIncarnation = options.cellIncarnation ?? randomUUID()
@@ -179,16 +182,20 @@ export function createRelayServer(
issuer: config.publicUrl
})
: undefined
const edgeConfig =
config.edge && options.edgeControl ? { ...config.edge, control: options.edgeControl.flags } : config.edge
const edgeAssignments =
config.role === 'director' && config.edge
? new EdgeAssignments(observedDatabase, config.edge, options.now, edgeAdmin)
config.role === 'director' && edgeConfig
? new EdgeAssignments(observedDatabase, edgeConfig, options.now, edgeAdmin)
: undefined
// Hosts that left the cohort are drained off the edge, rather than waiting out their lease.
const edgeEvacuation =
config.role === 'director' && config.edge && edgeAdmin
? new EdgeEvacuation(observedDatabase, config.edge, edgeAdmin, EDGE_CELL_ID, options.now, options.random)
config.role === 'director' && edgeConfig && edgeAdmin
? new EdgeEvacuation(observedDatabase, edgeConfig, edgeAdmin, EDGE_CELL_ID, options.now, options.random)
: undefined
edgeEvacuation?.start()
// A rollback (mode off/down, a smaller share) starts draining within seconds.
options.edgeControl?.onChange(() => edgeEvacuation?.controlChanged())
if (edgeAssignments) {
void edgeAssignments.ensureEdgeCell().then(
() => console.warn(JSON.stringify({ event: 'orca_relay_edge_cell_ready' })),