diff --git a/cloud/apps/relay/src/cell/control-flags/relay-control-flag-channel.ts b/cloud/apps/relay/src/cell/control-flags/relay-control-flag-channel.ts index 1c2ac775844..0a789d18037 100644 --- a/cloud/apps/relay/src/cell/control-flags/relay-control-flag-channel.ts +++ b/cloud/apps/relay/src/cell/control-flags/relay-control-flag-channel.ts @@ -26,6 +26,8 @@ export function startControlFlagChannel(input: { defaults: Flags parse: ControlFlagParse appliedEvent: string + // Called once per applied generation, after it is logged; a throw is logged, never retried. + onApplied?: (applied: AppliedControlFlags) => void fetch?: typeof fetch random?: () => number pollMs?: number @@ -134,6 +136,17 @@ export function startControlFlagChannel(input: { const { flags, ignoredKeys } = parsed applied = ignoredKeys.length > 0 ? { generation, flags, ignoredKeys } : { generation, flags } console.log(JSON.stringify({ event: input.appliedEvent, ...applied })) + try { + input.onApplied?.(applied) + } catch (error) { + console.warn( + JSON.stringify({ + event: 'orca_relay_control_flags_listener_failed', + object: input.objectName, + reason: error instanceof Error ? error.message : 'unknown' + }) + ) + } } const poll = (): Promise => { diff --git a/cloud/apps/relay/src/edge-relay-cohort.ts b/cloud/apps/relay/src/edge-relay-cohort.ts index 842208f1592..5227e8f9ff4 100644 --- a/cloud/apps/relay/src/edge-relay-cohort.ts +++ b/cloud/apps/relay/src/edge-relay-cohort.ts @@ -5,13 +5,17 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contra import type { SqlRow } from './database/database.js' import { z } from 'zod' import type { EdgeDeviceRow } from './edge-admin-client.js' +import { EDGE_CONTROL_DEFAULTS, edgeRolloutPermille, type EdgeControlFlags } from './edge-relay-control.js' export type EdgeRelayConfig = { cellUrl: string // Present: phase 2 (host data in the edge's objects); director → edge admin calls use it. adminSigningKey?: Uint8Array - // Share of hosts (by hash of relayHostId) in the edge cohort, 0-100. + // Share of hosts (by hash of relayHostId) in the edge cohort, 0-100: the boot default, which a + // runtime rolloutPermille replaces. rolloutPercent: number + // Runtime control (director/edge.json), read on every call; absent means the defaults. + control?: () => EdgeControlFlags allowHosts: ReadonlySet // Accounts (relay token `sub`) whose hosts all join, e.g. the team's own accounts. allowUsers?: ReadonlySet @@ -29,22 +33,37 @@ export const EDGE_EXPORT_SKIP_AFTER_MS = 120_000 // Export failures further apart than this are not one outage (see EdgeAssignments.leaveEdge). export const EDGE_EXPORT_FAILURE_GAP_MS = 30_000 -export function edgeCohortBucket(relayHostId: string): number { - return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) % 100 +function edgeCohortHash(relayHostId: string): number { + return createHash('sha256').update(`edge-cohort:${relayHostId}`).digest().readUInt32BE(0) } -// Deny wins, then host allow, then region scope; within the region, account allow or rollout share. +export function edgeCohortBucket(relayHostId: string): number { + return edgeCohortHash(relayHostId) % 100 +} + +// The percent bucket times ten plus a digit from the hash's higher part, so for every percent P a +// host is under P × 10 permille exactly when it is under P percent: no host moves when the env +// percent becomes the equal permille. +export function edgeCohortBucketPermille(relayHostId: string): number { + const hash = edgeCohortHash(relayHostId) + return (hash % 100) * 10 + (Math.floor(hash / 100) % 10) +} + +// Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope; +// within the region, account allow or the rollout share. export function inEdgeCohort( config: EdgeRelayConfig, relayHostId: string, region: RelayRegion | undefined, userId?: string ): boolean { + const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS + if (flags.mode !== 'on') { return false } if (config.denyHosts.has(relayHostId)) { return false } if (config.allowHosts.has(relayHostId)) { return true } if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false } if (userId !== undefined && config.allowUsers?.has(userId)) { return true } - return edgeCohortBucket(relayHostId) < config.rolloutPercent + return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags) } // Another director holds the host's row, or the host is on the edge and cannot leave yet (its diff --git a/cloud/apps/relay/src/edge-relay-control.test.ts b/cloud/apps/relay/src/edge-relay-control.test.ts new file mode 100644 index 00000000000..9ca7f196adc --- /dev/null +++ b/cloud/apps/relay/src/edge-relay-control.test.ts @@ -0,0 +1,198 @@ +import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest' +import { + EDGE_CONTROL_APPLIED_EVENT, + EDGE_CONTROL_DEFAULTS, + EDGE_CONTROL_OBJECT_NAME, + parseEdgeControl, + startEdgeControl, + type EdgeControlFlags +} from './edge-relay-control.js' +import { edgeCohortBucket, edgeCohortBucketPermille, inEdgeCohort, type EdgeRelayConfig } from './edge-relay-cohort.js' + +function edgeConfig(overrides: Partial = {}): EdgeRelayConfig { + return { + cellUrl: 'https://edge.relay.example.test', + rolloutPercent: 0, + allowHosts: new Set(), + denyHosts: new Set(), + regions: new Set(['us-central1']), + ...overrides + } +} + +function hostId(index: number): string { + return `host${String(index).padStart(12, '0')}` +} + +// Storage answers the stored object (or 404) by generation, like the cell flag channel's tests. +function fakeGoogle() { + const state: { object: { generation: number; body: string } | null; paths: string[] } = { + object: null, + paths: [] + } + const fetchImpl: typeof fetch = async (target) => { + const url = new URL(String(target)) + if (url.hostname === 'metadata.google.internal') { + if (url.pathname.endsWith('/project/project-id')) { return new Response('onorca-cloud') } + return Response.json({ access_token: 'ya29.secret-access-token', expires_in: 3_600 }) + } + state.paths.push(decodeURIComponent(url.pathname)) + const object = state.object + if (!object) { return new Response(null, { status: 404 }) } + if (url.searchParams.get('ifGenerationNotMatch') === String(object.generation)) { + return new Response(null, { status: 304 }) + } + return new Response(object.body, { headers: { 'x-goog-generation': String(object.generation) } }) + } + const store = (generation: number, flags: Record) => { + state.object = { generation, body: JSON.stringify({ v: 1, flags }) } + } + return { state, fetchImpl, store } +} + +describe('edge runtime control object', () => { + let log: MockInstance + let warn: MockInstance + beforeEach(() => { + log = vi.spyOn(console, 'log').mockImplementation(() => {}) + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + }) + afterEach(() => { + log.mockRestore() + warn.mockRestore() + }) + + it('parses mode and permille, reports unknown keys, and voids the object on any bad known value', () => { + expect(parseEdgeControl({ v: 1, flags: {} })).toEqual({ flags: { mode: 'on' }, ignoredKeys: [] }) + expect(parseEdgeControl({ v: 1, flags: { mode: 'down', rolloutPermille: 0, cohort: 'x' } })).toEqual({ + flags: { mode: 'down', rolloutPermille: 0 }, + ignoredKeys: ['cohort'] + }) + expect(parseEdgeControl({ v: 1, flags: { rolloutPermille: 1_000 } })).toEqual({ + flags: { mode: 'on', rolloutPermille: 1_000 }, + ignoredKeys: [] + }) + for (const flags of [ + { mode: 'paused' }, + { mode: true }, + { rolloutPermille: 1_001 }, + { rolloutPermille: -1 }, + { rolloutPermille: 2.5 }, + { rolloutPermille: '5' }, + { mode: 'off', rolloutPermille: null } + ]) { + expect(parseEdgeControl({ v: 1, flags })).toBeNull() + } + expect(parseEdgeControl({ v: 2, flags: {} })).toBeNull() + expect(parseEdgeControl({ flags: {} })).toBeNull() + }) + + it('reads director/edge.json, logs each applied change with no ids, and tells listeners', async () => { + const google = fakeGoogle() + const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false }) + const changes: EdgeControlFlags[] = [] + control.onChange(() => changes.push(control.flags())) + await control.poll() + expect(control.flags()).toEqual(EDGE_CONTROL_DEFAULTS) + expect(google.state.paths.at(-1)).toBe(`/storage/v1/b/onorca-cloud-relay-control/o/${EDGE_CONTROL_OBJECT_NAME}`) + google.store(3, { mode: 'off', rolloutPermille: 25 }) + await control.poll() + // Unchanged object: no second change. + await control.poll() + google.store(4, { mode: 'down', note: 'incident' }) + await control.poll() + // A bad value voids the object: the last applied flags stay. + google.store(5, { mode: 'sideways' }) + await control.poll() + expect(control.flags()).toEqual({ mode: 'down' }) + expect(changes).toEqual([{ mode: 'off', rolloutPermille: 25 }, { mode: 'down' }]) + const applied = log.mock.calls + .map((call) => JSON.parse(String(call[0])) as Record) + .filter((line) => line.event === EDGE_CONTROL_APPLIED_EVENT) + expect(applied).toEqual([ + { event: EDGE_CONTROL_APPLIED_EVENT, generation: 3, flags: { mode: 'off', rolloutPermille: 25 } }, + { event: EDGE_CONTROL_APPLIED_EVENT, generation: 4, flags: { mode: 'down' }, ignoredKeys: ['note'] } + ]) + control.stop() + }) + + it('logs a throwing listener and still applies the flags', async () => { + const google = fakeGoogle() + const control = startEdgeControl({ fetch: google.fetchImpl, autoStart: false }) + control.onChange(() => { + throw new Error('listener_broke') + }) + google.store(2, { mode: 'off' }) + await control.poll() + expect(control.flags()).toEqual({ mode: 'off' }) + expect(warn.mock.calls.map((call) => String(call[0])).join('\n')).toContain('orca_relay_control_flags_listener_failed') + control.stop() + }) +}) + +describe('edge cohort under runtime control', () => { + it('reads the applied flags on every call: off and down empty the cohort, even allowed hosts', () => { + const flags: EdgeControlFlags = { mode: 'on' } + const host = hostId(1) + const config = edgeConfig({ + rolloutPercent: 100, + allowHosts: new Set([hostId(2)]), + allowUsers: new Set(['user-team']), + control: () => flags + }) + expect(inEdgeCohort(config, host, 'us-central1')).toBe(true) + for (const mode of ['off', 'down'] as const) { + flags.mode = mode + expect(inEdgeCohort(config, host, 'us-central1')).toBe(false) + expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(false) + expect(inEdgeCohort(config, host, 'us-central1', 'user-team')).toBe(false) + } + flags.mode = 'on' + expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true) + // A permille replaces the env percent; deny, allow and region rules still apply around it. + flags.rolloutPermille = 0 + expect(inEdgeCohort(config, host, 'us-central1')).toBe(false) + expect(inEdgeCohort(config, hostId(2), 'us-central1')).toBe(true) + flags.rolloutPermille = 1_000 + expect(inEdgeCohort(config, host, 'us-central1')).toBe(true) + expect(inEdgeCohort(config, host, 'asia-east2')).toBe(false) + expect(inEdgeCohort({ ...config, denyHosts: new Set([host]) }, host, 'us-central1')).toBe(false) + }) + + it('keeps every host of P percent in at P × 10 permille, for every P', () => { + // floor(permille bucket / 10) is the percent bucket, so `bucket < P × 10` iff `bucket < P`. + const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index)) + expect(hosts.filter((host) => Math.floor(edgeCohortBucketPermille(host) / 10) !== edgeCohortBucket(host))).toEqual([]) + const sample = hosts.slice(0, 2_000) + const moved: string[] = [] + for (let percent = 0; percent <= 100; percent += 1) { + const byPercent = edgeConfig({ rolloutPercent: percent }) + const byPermille = edgeConfig({ rolloutPercent: 0, control: () => ({ mode: 'on', rolloutPermille: percent * 10 }) }) + for (const host of sample) { + if (inEdgeCohort(byPercent, host, 'us-central1') !== inEdgeCohort(byPermille, host, 'us-central1')) { + moved.push(`${percent}:${host}`) + } + } + } + expect(moved).toEqual([]) + // The env percent alone (boot, no permille flag) answers as before. + expect(sample.filter((host) => inEdgeCohort(edgeConfig({ rolloutPercent: 10 }), host, 'us-central1'))) + .toEqual(sample.filter((host) => edgeCohortBucket(host) < 10)) + }) + + it('steps in 1 permille shares, nested, roughly evenly', () => { + const hosts = Array.from({ length: 20_000 }, (_, index) => hostId(index)) + const share = (permille: number) => + hosts.filter((host) => inEdgeCohort(edgeConfig({ control: () => ({ mode: 'on', rolloutPermille: permille }) }), host, 'us-central1')) + const one = share(1) + const five = share(5) + expect(one.length).toBeGreaterThan(5) + expect(one.length).toBeLessThan(40) + expect(five.length).toBeGreaterThan(70) + expect(five.length).toBeLessThan(130) + // A larger share keeps every host of a smaller one. + expect(one.every((host) => five.includes(host))).toBe(true) + const buckets = new Set(hosts.map((host) => edgeCohortBucketPermille(host))) + expect(buckets.size).toBe(1_000) + }) +}) diff --git a/cloud/apps/relay/src/edge-relay-control.ts b/cloud/apps/relay/src/edge-relay-control.ts new file mode 100644 index 00000000000..b59eff1cece --- /dev/null +++ b/cloud/apps/relay/src/edge-relay-control.ts @@ -0,0 +1,86 @@ +// Runtime control of the relay edge on the director: director/edge.json in the relay-control +// bucket, read by the shared flag channel (cell/control-flags/relay-control-flag-channel.ts), so a +// rollout step or a rollback needs no director deploy. Boot uses the defaults (mode on, the env +// rollout percent); a failed read keeps the last applied flags. +import { z } from 'zod' +import { + startControlFlagChannel, + type ControlFlagParse +} from './cell/control-flags/relay-control-flag-channel.js' + +// `off`: no host is in the cohort; hosts on the edge leave normally (export, place, fence) and the +// evacuation sweep drains them. `down`: as off, and the edge is known dead, so a leaving host's +// export is skipped at once (EdgeAssignments.leaveEdge). +export type EdgeControlMode = 'on' | 'off' | 'down' + +export type EdgeControlFlags = { + mode: EdgeControlMode + // Replaces the env rollout percent when present (1‰ steps). + rolloutPermille?: number +} + +export const EDGE_CONTROL_DEFAULTS: EdgeControlFlags = { mode: 'on' } +export const EDGE_CONTROL_OBJECT_NAME = 'director/edge.json' +export const EDGE_CONTROL_APPLIED_EVENT = 'orca_relay_edge_control_applied' + +const EdgeControlFlagsSchema = z.object({ + mode: z.enum(['on', 'off', 'down']).default(EDGE_CONTROL_DEFAULTS.mode), + rolloutPermille: z.number().int().min(0).max(1_000).optional() +}) +const KNOWN_FLAG_KEYS = new Set(Object.keys(EdgeControlFlagsSchema.shape)) +const EdgeControlObjectSchema = z.object({ v: z.literal(1), flags: EdgeControlFlagsSchema }) + +// Unknown keys (a newer writer's, or a typo) are dropped and reported; a known key with a bad value +// voids the whole object, so the last applied flags stay. +export const parseEdgeControl: ControlFlagParse = (body) => { + const parsed = EdgeControlObjectSchema.safeParse(body) + if (!parsed.success) { + return null + } + const named = body !== null && typeof body === 'object' && 'flags' in body ? body.flags : {} + const ignoredKeys = + named !== null && typeof named === 'object' + ? Object.keys(named).filter((key) => !KNOWN_FLAG_KEYS.has(key)).sort() + : [] + const { mode, rolloutPermille } = parsed.data.flags + return { flags: rolloutPermille === undefined ? { mode } : { mode, rolloutPermille }, ignoredKeys } +} + +export type EdgeControl = { + // The flags applied now; callers read this on every decision, never caching it. + flags: () => EdgeControlFlags + // Called after each newly applied generation. + onChange: (listener: () => void) => void + stop: () => void +} + +export function startEdgeControl( + input: { fetch?: typeof fetch; random?: () => number; pollMs?: number; autoStart?: boolean } = {} +): EdgeControl & { poll: () => Promise } { + const listeners: (() => void)[] = [] + const channel = startControlFlagChannel({ + objectName: EDGE_CONTROL_OBJECT_NAME, + defaults: EDGE_CONTROL_DEFAULTS, + parse: parseEdgeControl, + appliedEvent: EDGE_CONTROL_APPLIED_EVENT, + onApplied: () => { + for (const listener of listeners) { + listener() + } + }, + ...input + }) + return { + flags: () => channel.applied().flags, + onChange: (listener) => { + listeners.push(listener) + }, + poll: channel.poll, + stop: channel.stop + } +} + +// Effective share of hosts on the edge in permille: the applied flag, else the env percent. +export function edgeRolloutPermille(rolloutPercent: number, flags: EdgeControlFlags): number { + return flags.rolloutPermille ?? rolloutPercent * 10 +} diff --git a/cloud/apps/relay/src/edge-relay-evacuation.test.ts b/cloud/apps/relay/src/edge-relay-evacuation.test.ts index ae668d2617f..20e503bbce5 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.test.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.test.ts @@ -9,9 +9,16 @@ import { type EdgeAdmin, type EdgeDrainResult } from './edge-admin-client.js' -import { EDGE_EVACUATION_BATCH, EDGE_EVACUATION_BUSY_INTERVAL_MS, - EDGE_EVACUATION_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js' +import { + EDGE_EVACUATION_BATCH, + EDGE_EVACUATION_BUSY_INTERVAL_MS, + EDGE_EVACUATION_CONCURRENCY, + EDGE_EVACUATION_CONTROL_DELAY_MS, + EDGE_EVACUATION_INTERVAL_MS, + EdgeEvacuation +} from './edge-relay-evacuation.js' import { EDGE_CELL_ID, type EdgeRelayConfig } from './edge-relay.js' +import type { EdgeControlFlags } from './edge-relay-control.js' const EDGE_URL = 'https://edge.relay.example.test' const SIGNING_KEY = new TextEncoder().encode('edge-admin-key-with-at-least-32-bytes!!') @@ -31,6 +38,13 @@ class FakeAdmin implements EdgeAdmin { drained: string[] = [] unavailable = new Set() answer: EdgeDrainResult = { outcome: 'drained', controls: 1 } + // Set: each drain waits for release(), so the test can count drains in flight. + holding = false + readonly held: (() => void)[] = [] + + release(): void { + for (const resolve of this.held.splice(0)) { resolve() } + } async assign(): Promise { throw new Error('unexpected assign') @@ -43,11 +57,12 @@ class FakeAdmin implements EdgeAdmin { async drain(relayHostId: string): Promise { if (this.unavailable.has(relayHostId)) { throw new EdgeAdminUnavailableError('status_404') } this.drained.push(relayHostId) + if (this.holding) { await new Promise((resolve) => this.held.push(resolve)) } return this.answer } } -async function harness(overrides: Partial = {}) { +async function harness(overrides: Partial = {}, random = 0.5) { const database = await openInMemoryRelayDatabase() databases.push(database) const config: EdgeRelayConfig = { @@ -60,7 +75,7 @@ async function harness(overrides: Partial = {}) { } const admin = new FakeAdmin() const clock = { now: 1_700_000_000_000 } - const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => 0.5) + const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, () => clock.now, () => random) const insert = async (relayHostId: string, cellId = EDGE_CELL_ID, userId = 'user-1') => await database.query( `INSERT INTO relay_assignments @@ -107,11 +122,11 @@ describe('edge evacuation sweep', () => { it(`drains at most ${EDGE_EVACUATION_BATCH} per tick, the rest on the next, and not again within the cooldown`, async () => { vi.spyOn(console, 'warn').mockImplementation(() => {}) const h = await harness() - for (let index = 0; index < 25; index += 1) { await h.insert(hostId(index)) } + for (let index = 0; index < EDGE_EVACUATION_BATCH + 25; index += 1) { await h.insert(hostId(index)) } h.admin.answer = { outcome: 'idle' } expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH) - expect(await h.evacuation.sweep()).toBe(5) - expect(new Set(h.admin.drained).size).toBe(25) + expect(await h.evacuation.sweep()).toBe(25) + expect(new Set(h.admin.drained).size).toBe(EDGE_EVACUATION_BATCH + 25) expect(await h.evacuation.sweep()).toBe(0) h.clock.now += 10 * 60_000 expect(await h.evacuation.sweep()).toBe(EDGE_EVACUATION_BATCH) @@ -158,6 +173,99 @@ describe('edge evacuation sweep', () => { }) }) +describe('edge evacuation under runtime control', () => { + it('re-arms an idle sweep within 5 s of a control change, so a rollback drains at once', async () => { + vi.useFakeTimers() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const flags: EdgeControlFlags = { mode: 'on' } + const h = await harness({ rolloutPercent: 100, control: () => flags }) + await h.insert(hostId(1)) + const sweep = vi.spyOn(h.evacuation, 'sweep') + h.evacuation.start() + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(sweep).toHaveBeenCalledTimes(1) + expect(h.admin.drained).toEqual([]) + // Idle now: the next pass would be ~10 min away. + await vi.advanceTimersByTimeAsync(60_000) + flags.mode = 'off' + h.evacuation.controlChanged() + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS) + expect(sweep).toHaveBeenCalledTimes(2) + expect(h.admin.drained).toEqual([hostId(1)]) + h.evacuation.stop() + }) + + it('re-arms after a pass that was running when the control changed', async () => { + vi.useFakeTimers() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 0 } + const h = await harness({ control: () => flags }) + await h.insert(hostId(1)) + h.admin.holding = true + const sweep = vi.spyOn(h.evacuation, 'sweep') + h.evacuation.start() + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_BUSY_INTERVAL_MS) + expect(h.admin.held).toHaveLength(1) + flags.mode = 'down' + h.evacuation.controlChanged() + h.admin.release() + await vi.advanceTimersByTimeAsync(EDGE_EVACUATION_CONTROL_DELAY_MS) + expect(sweep).toHaveBeenCalledTimes(2) + h.evacuation.stop() + }) + + it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness() + for (let index = 0; index < 35; index += 1) { await h.insert(hostId(index)) } + h.admin.holding = true + let done = false + const pass = h.evacuation.sweep().then((answered) => { + done = true + return answered + }) + let most = 0 + while (!done) { + await new Promise((resolve) => setTimeout(resolve, 0)) + most = Math.max(most, h.admin.held.length) + h.admin.release() + } + expect(await pass).toBe(35) + expect(most).toBe(EDGE_EVACUATION_CONCURRENCY) + }) + + it('starts each pass at a random point of the keys and wraps around to it', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const users = ['20000000-a', '60000000-a', 'a0000000-a', 'e0000000-a'] + const drainedFrom = async (random: number) => { + const h = await harness({}, random) + for (const [index, user] of users.entries()) { await h.insert(hostId(index), EDGE_CELL_ID, user) } + expect(await h.evacuation.sweep()).toBe(4) + return h.admin.drained + } + expect(await drainedFrom(0.5)).toEqual([hostId(2), hostId(3), hostId(0), hostId(1)]) + expect(await drainedFrom(0.1)).toEqual([hostId(0), hostId(1), hostId(2), hostId(3)]) + expect(await drainedFrom(0.3)).toEqual([hostId(1), hostId(2), hostId(3), hostId(0)]) + }) + + it('pages an idle pass: one page of edge rows per tick, then idle until the next pass', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness({ rolloutPercent: 100 }) + for (let index = 0; index < 1_200; index += 1) { await h.insert(hostId(index)) } + const query = vi.spyOn(h.database, 'query') + const pages = () => query.mock.calls.filter(([sql]) => String(sql).includes('FROM relay_assignments a')).length + const busy = () => (h.evacuation as unknown as { busy: boolean }).busy + await h.evacuation.sweep() + expect([pages(), busy()]).toEqual([1, true]) + await h.evacuation.sweep() + expect([pages(), busy()]).toEqual([2, true]) + // The short last page wraps to the start of the keys, which holds nothing before the origin. + await h.evacuation.sweep() + expect([pages(), busy()]).toEqual([4, false]) + expect(h.admin.drained).toEqual([]) + }) +}) + describe('edge admin drain call', () => { it('posts the signed drain op and reads both answers; anything else is unavailable', async () => { const answers: [number, unknown][] = [ diff --git a/cloud/apps/relay/src/edge-relay-evacuation.ts b/cloud/apps/relay/src/edge-relay-evacuation.ts index 0248d5cbee9..ae1c31f5056 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.ts @@ -7,27 +7,38 @@ import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contr import type { RelayDatabase } from './database/database.js' import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js' import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js' -import { retryPendingFences } from './edge-relay-fence.js' +import { forEachBounded, retryPendingFences } from './edge-relay-fence.js' -// The cohort only changes when a new revision starts, so a pass soon after start and then a slow -// background pass are enough; each pass reads the edge rows (cell_id has no index). +// A cohort change (director/edge.json, or a new revision) re-arms the sweep (controlChanged), so a +// soon pass and then a slow background pass are enough; each pass reads the edge rows (cell_id has +// no index). export const EDGE_EVACUATION_INTERVAL_MS = 10 * 60_000 // First pass after start, and the next pass while more leaving hosts are queued. export const EDGE_EVACUATION_BUSY_INTERVAL_MS = 60_000 -// Drains sent per tick (per director), so a cohort cut moves desktops back gradually. -export const EDGE_EVACUATION_BATCH = 20 -// Edge rows read per tick; the cursor walks the table across ticks. +// Longest wait for a pass after the runtime control changes, so a rollback starts at once. +export const EDGE_EVACUATION_CONTROL_DELAY_MS = 5_000 +// Drains sent per tick (per director): a full rollback of ~19k hosts over 5 directors takes ~40 min. +export const EDGE_EVACUATION_BATCH = 100 +// Drains in flight at once per director. +export const EDGE_EVACUATION_CONCURRENCY = 10 +// Edge rows read per page; the cursor walks the table across ticks. const EDGE_EVACUATION_PAGE = 500 // A host this director drained is not drained again for this long (a desktop that is offline // answers idle, and asks the director on its own when it comes back). const EDGE_EVACUATION_COOLDOWN_MS = 10 * 60_000 +// A pass starts at a random point of the key space and wraps around to it, so directors sweeping +// at once mostly drain different hosts. +type Pass = { origin: Identity; cursor: Identity; wrapped: boolean } export class EdgeEvacuation { - private cursor: Identity | undefined + // Undefined between passes. + private pass: Pass | undefined private readonly drainedAt = new Map() private timer: ReturnType | undefined private stopped = true private busy = true + private sweeping = false + private controlChangedWhileSweeping = false constructor( private readonly database: RelayDatabase, @@ -49,6 +60,19 @@ export class EdgeEvacuation { this.timer = undefined } + // The runtime control changed (e.g. mode off for a rollback): the next pass runs within + // EDGE_EVACUATION_CONTROL_DELAY_MS instead of waiting out the idle interval. + controlChanged(): void { + this.busy = true + if (this.stopped) { return } + if (this.sweeping) { + this.controlChangedWhileSweeping = true + return + } + clearTimeout(this.timer) + this.schedule(EDGE_EVACUATION_CONTROL_DELAY_MS * this.random()) + } + // One tick: returns how many drains the edge answered. async sweep(): Promise { const now = this.now() @@ -56,12 +80,12 @@ export class EdgeEvacuation { if (now - sentAt >= EDGE_EVACUATION_COOLDOWN_MS) { this.drainedAt.delete(key) } } const leaving = await this.leavingHosts() - this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.cursor !== undefined + this.busy = leaving.length >= EDGE_EVACUATION_BATCH || this.pass !== undefined let answered = 0 - for (const identity of leaving) { + await forEachBounded(leaving, EDGE_EVACUATION_CONCURRENCY, async (identity) => { try { const result = await this.admin.drain(identity.relayHostId) - this.drainedAt.set(`${identity.userId}\n${identity.relayHostId}`, this.now()) + this.drainedAt.set(drainKey(identity), this.now()) answered += 1 console.warn( JSON.stringify({ @@ -74,51 +98,85 @@ export class EdgeEvacuation { if (!(error instanceof EdgeAdminUnavailableError)) { throw error } console.warn(JSON.stringify({ event: 'orca_relay_edge_drain_failed', reason: error.message })) } - } + }) // Pending fences keep the busy cadence, so their backoff (from 60 s) is honoured. if (await retryPendingFences(this.database, this.admin, this.edgeCellId, this.now)) { this.busy = true } return answered } - // Edge rows (one page after the cursor) whose host left the cohort, skipping recent drains. + // Edge rows (a page after the cursor; a second when the first ends the pass's first half) whose + // host left the cohort, skipping recent drains. An idle pass reads each edge row once. private async leavingHosts(): Promise { - const cursor = this.cursor ?? { userId: '', relayHostId: '' } - const rows = await this.database.query( - `SELECT a.user_id, a.relay_host_id, p.preferred_region - FROM relay_assignments a - LEFT JOIN relay_assignment_region_preferences p - ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id - WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?)) - ORDER BY a.user_id, a.relay_host_id - LIMIT ?`, - [this.edgeCellId, cursor.userId, cursor.userId, cursor.relayHostId, EDGE_EVACUATION_PAGE] - ) - // A short page reached the end: the next tick starts over. - this.cursor = undefined const leaving: Identity[] = [] - for (const [index, row] of rows.entries()) { - const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') } - if (rows.length === EDGE_EVACUATION_PAGE && index === rows.length - 1) { this.cursor = identity } - const stored = RelayRegionSchema.safeParse(row.preferred_region) - const region = stored.success ? stored.data : RELAY_DEFAULT_REGION - if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue } - if (this.drainedAt.has(`${identity.userId}\n${identity.relayHostId}`)) { continue } - leaving.push(identity) - if (leaving.length === EDGE_EVACUATION_BATCH) { + this.pass ??= this.newPass() + for (let pages = 0; pages < 2 && this.pass !== undefined; pages += 1) { + const pass: Pass = this.pass + const rows = await this.readPage(pass) + let filled = false + for (const row of rows) { + const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') } + pass.cursor = identity + const stored = RelayRegionSchema.safeParse(row.preferred_region) + const region = stored.success ? stored.data : RELAY_DEFAULT_REGION + if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue } + if (this.drainedAt.has(drainKey(identity))) { continue } + leaving.push(identity) // Resume after the last host taken, so the rest of the page is read next tick. - this.cursor = identity - break + if (leaving.length === EDGE_EVACUATION_BATCH) { + filled = true + break + } + } + if (filled || rows.length === EDGE_EVACUATION_PAGE) { break } + // A short page ends this half: wrap to the start of the keys, or end the pass. + if (pass.wrapped) { + this.pass = undefined + } else { + pass.wrapped = true + pass.cursor = { userId: '', relayHostId: '' } } } return leaving } - private schedule(): void { + // Keys after the cursor; once wrapped, only up to the pass's origin. + private async readPage(pass: Pass) { + const { cursor, origin, wrapped } = pass + return await this.database.query( + `SELECT a.user_id, a.relay_host_id, p.preferred_region + FROM relay_assignments a + LEFT JOIN relay_assignment_region_preferences p + ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id + WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?)) + ${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''} + ORDER BY a.user_id, a.relay_host_id + LIMIT ?`, + [ + this.edgeCellId, + cursor.userId, + cursor.userId, + cursor.relayHostId, + ...(wrapped ? [origin.userId, origin.userId, origin.relayHostId] : []), + EDGE_EVACUATION_PAGE + ] + ) + } + + // Account ids are UUIDs, so a random hex prefix lands anywhere in the key space; ids of another + // shape are still all swept, from a less spread origin. + private newPass(): Pass { + const prefix = Math.floor(this.random() * 0x1_0000_0000).toString(16).padStart(8, '0') + const origin = { userId: prefix, relayHostId: '' } + return { origin, cursor: origin, wrapped: false } + } + + private schedule(delayMs?: number): void { if (this.stopped) { return } // Jittered (±25%), so directors do not sweep together. const interval = this.busy ? EDGE_EVACUATION_BUSY_INTERVAL_MS : EDGE_EVACUATION_INTERVAL_MS - const delay = interval * (0.75 + this.random() / 2) + const delay = delayMs ?? interval * (0.75 + this.random() / 2) this.timer = setTimeout(() => { + this.sweeping = true void this.sweep() .catch((error: unknown) => console.warn( @@ -128,8 +186,17 @@ export class EdgeEvacuation { }) ) ) - .finally(() => this.schedule()) + .finally(() => { + this.sweeping = false + const changed = this.controlChangedWhileSweeping + this.controlChangedWhileSweeping = false + this.schedule(changed ? EDGE_EVACUATION_CONTROL_DELAY_MS * this.random() : undefined) + }) }, delay) this.timer.unref?.() } } + +function drainKey(identity: Identity): string { + return `${identity.userId}\n${identity.relayHostId}` +} diff --git a/cloud/apps/relay/src/edge-relay-fence.test.ts b/cloud/apps/relay/src/edge-relay-fence.test.ts index fec6f5d456d..3fa7bbc0cb1 100644 --- a/cloud/apps/relay/src/edge-relay-fence.test.ts +++ b/cloud/apps/relay/src/edge-relay-fence.test.ts @@ -12,7 +12,15 @@ import { type EdgeAssignResult } from './edge-admin-client.js' import { EDGE_EVACUATION_BUSY_INTERVAL_MS, EdgeEvacuation } from './edge-relay-evacuation.js' -import { EDGE_FENCE_MAX_ATTEMPTS, EDGE_FENCE_RETRY_BASE_MS, edgeFenceRetryDelayMs, retryPendingFences } from './edge-relay-fence.js' +import { + EDGE_FENCE_MAX_ATTEMPTS, + EDGE_FENCE_RETRY_BASE_MS, + EDGE_FENCE_RETRY_BATCH, + EDGE_FENCE_RETRY_MAX_MS, + edgeFenceRetryDelayMs, + recordPendingFence, + retryPendingFences +} from './edge-relay-fence.js' import { EDGE_CELL_ID, EDGE_LOCKED_CALLS_MAX, @@ -51,6 +59,10 @@ class ScriptedEdge implements EdgeAdmin { return { outcome, assignmentEpoch: outcome === 'stale' ? body.assignmentEpoch + 5 : body.assignmentEpoch } } + get hanging(): number { + return this.waiting.length + } + release(): void { for (const resolve of this.waiting.splice(0)) { resolve() } } @@ -229,12 +241,56 @@ describe('relay edge pending fence retries', () => { retry: true }) expect(logged(error, 'orca_relay_edge_fence_abandoned')).toEqual([ - { event: 'orca_relay_edge_fence_abandoned', attempts: EDGE_FENCE_MAX_ATTEMPTS } + { + event: 'orca_relay_edge_fence_abandoned', + outcome: 'stale', + cellId: 'us-c1', + assignmentEpoch: 9, + attempts: EDGE_FENCE_MAX_ATTEMPTS + } ]) expect(edgeFenceRetryDelayMs(1)).toBe(60_000) expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(30 * 60_000) }) + it('keeps retrying a fence through more than a day of edge outage', () => { + // Recorded at 0, first retry one base delay later, then each attempt's backoff. + let lastAttemptAt = EDGE_FENCE_RETRY_BASE_MS + for (let attempt = 1; attempt < EDGE_FENCE_MAX_ATTEMPTS; attempt += 1) { + lastAttemptAt += edgeFenceRetryDelayMs(attempt) + } + expect(lastAttemptAt).toBeGreaterThanOrEqual(24 * 60 * 60_000) + // Never slower than the cap, so the edge is not left claiming hosts long after it is back. + expect(edgeFenceRetryDelayMs(EDGE_FENCE_MAX_ATTEMPTS)).toBe(EDGE_FENCE_RETRY_MAX_MS) + }) + + it(`retries up to ${EDGE_FENCE_RETRY_BATCH} due fences per pass, 10 in flight at once`, async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness() + const hosts = Array.from({ length: EDGE_FENCE_RETRY_BATCH + 10 }, (_, index) => `fencehost${String(index).padStart(7, '0')}`) + for (const host of hosts) { + await h.placeRow('us-c1', 9, host) + await recordPendingFence(h.database, { userId: USER, relayHostId: host }, 'us-c1', 9, h.clock.now) + } + h.clock.now += EDGE_FENCE_RETRY_BASE_MS + h.edge.answers = hosts.map(() => 'hang') + let settled = false + const pass = h.retry().then((left) => { + settled = true + return left + }) + let maxInFlight = 0 + while (!settled) { + await new Promise((resolve) => setTimeout(resolve, 0)) + maxInFlight = Math.max(maxInFlight, h.edge.hanging) + h.edge.release() + } + expect(await pass).toBe(true) + expect(h.edge.calls).toHaveLength(EDGE_FENCE_RETRY_BATCH) + expect(maxInFlight).toBe(10) + expect(await h.pending()).toHaveLength(10) + }) + it('lets only one director claim an attempt', async () => { vi.spyOn(console, 'warn').mockImplementation(() => {}) vi.spyOn(console, 'error').mockImplementation(() => {}) diff --git a/cloud/apps/relay/src/edge-relay-fence.ts b/cloud/apps/relay/src/edge-relay-fence.ts index 6034e57a8e1..3d01795ddaa 100644 --- a/cloud/apps/relay/src/edge-relay-fence.ts +++ b/cloud/apps/relay/src/edge-relay-fence.ts @@ -7,13 +7,16 @@ import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.j import { integer, text, type Identity } from './edge-relay-cohort.js' // Retries per pending fence before it is dropped (a fence the edge keeps refusing, e.g. `stale`, -// cannot land until the host moves again, and that move fences anew). -export const EDGE_FENCE_MAX_ATTEMPTS = 8 -// First retry one busy sweep after the failure, doubling up to the cap: about 1.5 h in all. +// cannot land until the host moves again, and that move fences anew). Enough to outlast a day-long +// edge outage: the last attempt comes about 25.5 h after the failure. +export const EDGE_FENCE_MAX_ATTEMPTS = 56 +// First retry one busy sweep after the failure, doubling up to the cap (31 min to reach it, then +// every 30 min). export const EDGE_FENCE_RETRY_BASE_MS = 60_000 -const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000 -// Pending fences retried per sweep (per director). -const EDGE_FENCE_RETRY_BATCH = 20 +export const EDGE_FENCE_RETRY_MAX_MS = 30 * 60_000 +// Pending fences retried per sweep (per director), with at most EDGE_FENCE_RETRY_CONCURRENCY in flight. +export const EDGE_FENCE_RETRY_BATCH = 50 +const EDGE_FENCE_RETRY_CONCURRENCY = 10 export function edgeFenceRetryDelayMs(attempts: number): number { return Math.min(EDGE_FENCE_RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), EDGE_FENCE_RETRY_MAX_MS) @@ -91,7 +94,7 @@ export async function retryPendingFences( WHERE next_attempt_at <= ? ORDER BY next_attempt_at LIMIT ?`, [now(), EDGE_FENCE_RETRY_BATCH] ) - for (const row of due) { + await forEachBounded(due, EDGE_FENCE_RETRY_CONCURRENCY, async (row) => { const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') } const attempts = integer(row, 'attempts') + 1 // Claim the attempt: a director that read the same row loses here and skips it. @@ -102,7 +105,7 @@ export async function retryPendingFences( [attempts, now() + edgeFenceRetryDelayMs(attempts), identity.userId, identity.relayHostId, attempts - 1, integer(row, 'next_attempt_at')] ) - if (claimed.length !== 1) { continue } + if (claimed.length !== 1) { return } const result = await retryOne(database, admin, edgeCellId, identity, attempts) if (result !== 'failed' || attempts >= EDGE_FENCE_MAX_ATTEMPTS) { // Only the claimed record: a newer failure recorded meanwhile restarted at attempts = 0. @@ -111,10 +114,7 @@ export async function retryPendingFences( [identity.userId, identity.relayHostId, attempts] ) } - if (result === 'failed' && attempts >= EDGE_FENCE_MAX_ATTEMPTS) { - console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', attempts })) - } - } + }) return (await database.query(`SELECT attempts FROM relay_edge_fence_pending LIMIT 1`)).length > 0 } @@ -150,5 +150,31 @@ async function retryOne( console.error( JSON.stringify({ event: 'orca_relay_edge_fence_failed', outcome, cellId, assignmentEpoch, attempts, retry: true }) ) + if (attempts >= EDGE_FENCE_MAX_ATTEMPTS) { + // The record is dropped: the edge keeps claiming the host until it moves again. + console.error(JSON.stringify({ event: 'orca_relay_edge_fence_abandoned', outcome, cellId, assignmentEpoch, attempts })) + } return 'failed' } + +// Runs `task` over `items` with at most `limit` in flight. A throw does not stop the other workers; +// the first one is rethrown once all have settled. +export async function forEachBounded( + items: readonly T[], + limit: number, + task: (item: T) => Promise +): Promise { + let next = 0 + const worker = async (): Promise => { + while (next < items.length) { + const item = items[next] + next += 1 + if (item !== undefined) { await task(item) } + } + } + const results = await Promise.allSettled( + Array.from({ length: Math.min(limit, items.length) }, async () => await worker()) + ) + const failed = results.find((result): result is PromiseRejectedResult => result.status === 'rejected') + if (failed) { throw failed.reason instanceof Error ? failed.reason : new Error('relay_edge_task_failed') } +} diff --git a/cloud/apps/relay/src/edge-relay-phase2.test.ts b/cloud/apps/relay/src/edge-relay-phase2.test.ts index ddea6e93a4b..455027e74be 100644 --- a/cloud/apps/relay/src/edge-relay-phase2.test.ts +++ b/cloud/apps/relay/src/edge-relay-phase2.test.ts @@ -8,6 +8,7 @@ import { type EdgeDeviceRow } from './edge-admin-client.js' import { EDGE_CELL_ID, EDGE_EXPORT_SKIP_AFTER_MS, EdgeAssignments, type EdgeRelayConfig } from './edge-relay.js' +import type { EdgeControlFlags } from './edge-relay-control.js' const fakes = vi.hoisted(() => ({ verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-1', relayHostId: token })) @@ -276,13 +277,14 @@ describe('relay edge phase 2: the director moves host data with the host', () => const response = await h.assign() expect(response.status).toBe(200) expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 }) - // Postgres keeps what it had: the edge's revoke is lost, nothing is written back. - expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }]) + // The edge's revoke is unknown here: the host's Postgres devices are removed, never resurrected. + expect(await h.postgresDevices()).toEqual([]) expect(skipped()).toHaveLength(1) expect(JSON.parse(String(skipped()[0]![0]))).toEqual({ event: 'orca_relay_edge_export_skipped', reason: 'relay_edge_admin_unavailable:network', - failingForMs: EDGE_EXPORT_SKIP_AFTER_MS + failingForMs: EDGE_EXPORT_SKIP_AFTER_MS, + devicesRemoved: 1 }) }) @@ -337,7 +339,7 @@ describe('relay edge phase 2: the director moves host data with the host', () => const response = await h.assign() expect(response.status).toBe(200) expect(exportCalls).toBe(1) - expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: null }]) + expect(await h.postgresDevices()).toEqual([]) expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 }) }) @@ -370,6 +372,57 @@ describe('relay edge phase 2: the director moves host data with the host', () => expect(h.fake.pushes[0]).toMatchObject({ userId: USER, relayHostId: HOST }) }) + it('with mode off or down, sends no new host to the edge', async () => { + for (const mode of ['off', 'down'] as const) { + const h = await harness('host', { control: () => ({ mode }) }) + expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' }) + expect(h.fake.pushes).toHaveLength(0) + expect(await h.row()).toBeUndefined() + } + }) + + it('with mode off, moves hosts on the edge off normally: export, place, fence', async () => { + const flags: EdgeControlFlags = { mode: 'on' } + const h = await harness('host', { control: () => flags }) + await h.insertDevice(device('phone-a')) + expect(await (await h.assign()).json()).toMatchObject({ cellUrl: EDGE_URL }) + h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })] + flags.mode = 'off' + expect(await (await h.assign()).json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test' }) + // The edge's revoke carries over. + expect(h.fake.exports).toBe(1) + expect(await h.postgresDevices()).toEqual([{ id: 'phone-a', revoked: 1_800_000_000_000 }]) + expect(h.fake.assignment).toMatchObject({ cellId: 'us-c1', assignmentEpoch: 9 }) + }) + + it('with mode down, skips the export at once, removes the Postgres devices, and fences once', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + const flags: EdgeControlFlags = { mode: 'on' } + const h = await harness('host', { control: () => flags }) + await h.insertDevice(device('phone-a')) + await h.insertDevice(device('phone-b')) + await h.assign() + // phone-a was revoked on the edge, which then died; Postgres still holds it live. + h.fake.devices = [device('phone-a', { revoked_at: 1_800_000_000_000 })] + h.fake.down = true + flags.mode = 'down' + const assign = vi.spyOn(h.fake, 'assign') + const response = await h.assign() + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ cellUrl: 'https://us-c1.relay.example.test', assignmentEpoch: 9 }) + expect(h.fake.exports).toBe(0) + expect(await h.postgresDevices()).toEqual([]) + const skipped = warn.mock.calls + .map(([line]) => JSON.parse(String(line)) as Record) + .filter((line) => line.event === 'orca_relay_edge_export_skipped') + expect(skipped).toEqual([{ event: 'orca_relay_edge_export_skipped', reason: 'mode_down', devicesRemoved: 2 }]) + // One fence try, recorded for the sweep to retry. + expect(assign).toHaveBeenCalledTimes(1) + const pending = await h.database.query(`SELECT cell_id, assignment_epoch FROM relay_edge_fence_pending`) + expect(pending.map((row) => [String(row.cell_id), Number(row.assignment_epoch)])).toEqual([['us-c1', 9]]) + }) + it('does not touch the edge for hosts that were never on it', async () => { const h = await harness() h.leaveCohort() diff --git a/cloud/apps/relay/src/edge-relay-postgres.test.ts b/cloud/apps/relay/src/edge-relay-postgres.test.ts index bfffacf8191..aec056d715a 100644 --- a/cloud/apps/relay/src/edge-relay-postgres.test.ts +++ b/cloud/apps/relay/src/edge-relay-postgres.test.ts @@ -11,6 +11,7 @@ import { } from './edge-relay.js' import type { EdgeAssignBody, EdgeDeviceRow } from './edge-admin-client.js' import { EDGE_FENCE_RETRY_BASE_MS, retryPendingFences } from './edge-relay-fence.js' +import { EdgeEvacuation } from './edge-relay-evacuation.js' const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL const describePostgres = databaseUrl ? describe : describe.skip @@ -236,4 +237,38 @@ describePostgres('relay edge on PostgreSQL', () => { expect(await database.query(`SELECT 1 FROM relay_edge_fence_pending WHERE user_id = ?`, [host.userId])).toEqual([]) vi.restoreAllMocks() }) + + it('mode down: a leaving host loses its Postgres devices, and the sweep drains wrapping around its origin', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + const flags = { mode: 'on' as 'on' | 'down' } + const config = { ...edgeConfig, control: () => flags } + const drained: string[] = [] + const admin = { + assign: async (body: EdgeAssignBody) => ({ outcome: 'applied' as const, assignmentEpoch: body.assignmentEpoch }), + exportHost: async (): Promise => { + throw new Error('unexpected export') + }, + drain: async (relayHostId: string) => { + drained.push(relayHostId) + return { outcome: 'idle' as const } + } + } + const edge = new EdgeAssignments(database, config, Date.now, admin) + await edge.ensureEdgeCell() + const downHosts = ['edgepgdown000001', 'edgepgdown000002'].map((relayHostId) => ({ userId: 'edge-pg-user', relayHostId })) + for (const host of downHosts) { await edge.moveToEdge(host, 0) } + await database.query( + `INSERT INTO relay_devices (user_id, relay_host_id, relay_device_id, current_hash, current_version, + current_expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ['edge-pg-user', downHosts[0]!.relayHostId, 'phone-a', 'hash-phone-a', 2, 1_900_000_000_000, 1_800_000_000_000] + ) + flags.mode = 'down' + // Origin 'f3333333' sorts after 'edge-pg-user': the pass finds both hosts only once wrapped. + const evacuation = new EdgeEvacuation(database, config, admin, EDGE_CELL_ID, Date.now, () => 0.95) + await evacuation.sweep() + expect(drained.filter((id) => id.startsWith('edgepgdown'))).toEqual(downHosts.map((host) => host.relayHostId)) + expect(await edge.leaveEdge(downHosts[0]!)).toBe(true) + expect(await database.query(`SELECT 1 FROM relay_devices WHERE relay_host_id = ?`, [downHosts[0]!.relayHostId])).toEqual([]) + }) }) diff --git a/cloud/apps/relay/src/edge-relay.ts b/cloud/apps/relay/src/edge-relay.ts index 08a59a69ae6..a4d4952ac16 100644 --- a/cloud/apps/relay/src/edge-relay.ts +++ b/cloud/apps/relay/src/edge-relay.ts @@ -264,6 +264,8 @@ export class EdgeAssignments { ) )[0] if (!row || text(row, 'cell_id') !== EDGE_CELL_ID) { return false } + // The edge is known dead (director/edge.json): no export to wait for. + if (this.config.control?.().mode === 'down') { return await this.skipExport(identity, 'mode_down') } let exported try { exported = await this.admin.exportHost(identity.relayHostId) @@ -278,10 +280,9 @@ export class EdgeAssignments { this.exportLastFailureAt = now const failingForMs = now - (this.exportFailingSince ?? now) if (failingForMs < (this.config.exportSkipAfterMs ?? EDGE_EXPORT_SKIP_AFTER_MS)) { throw new EdgeMoveBusyError() } - // The edge has been unreachable too long: place the host on its Postgres devices (changes - // made on the edge are lost) rather than refusing every ask until the edge is back. - console.warn(JSON.stringify({ event: 'orca_relay_edge_export_skipped', reason: error.message, failingForMs })) - return true + // The edge has been unreachable too long: place the host anyway rather than refusing every + // ask until the edge is back. + return await this.skipExport(identity, error.message, failingForMs) } this.exportFailingSince = undefined this.exportLastFailureAt = undefined @@ -313,13 +314,34 @@ export class EdgeAssignments { return true } + // The host leaves without the edge's copy of its devices. The edge was their source of truth and + // revokes made there are unknown, so the Postgres rows are removed rather than resurrected: its + // phones pair again. Logs a count only. + private async skipExport(identity: Identity, reason: string, failingForMs?: number): Promise { + const removed = await this.database.query( + `DELETE FROM relay_devices WHERE user_id = ? AND relay_host_id = ? RETURNING relay_device_id`, + [identity.userId, identity.relayHostId] + ) + console.warn( + JSON.stringify({ + event: 'orca_relay_edge_export_skipped', + reason, + ...(failingForMs === undefined ? {} : { failingForMs }), + devicesRemoved: removed.length + }) + ) + return true + } + // After the host left: tell the edge where it went, so its phones get WRONG_CELL (re-resolve) // rather than HOST_OFFLINE. Until it lands the edge's freeze does the same; one that does not land // is recorded for the evacuation sweep to retry (edge-relay-fence.ts). async fenceAway(identity: Identity, cellId: string, assignmentEpoch: number): Promise { if (!this.admin) { return true } - // An edge whose exports are failing gets one try, so a skipped export is not slowed further. - const attempts = this.exportFailingSince === undefined ? 3 : 1 + // An edge whose exports are failing (or that is marked down) gets one try, so a skipped export + // is not slowed further. + const down = this.exportFailingSince !== undefined || this.config.control?.().mode === 'down' + const attempts = down ? 1 : 3 return await fenceEdge(this.database, this.admin, { ...identity, cellId, assignmentEpoch }, attempts, this.now()) } diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index 85fb33550c4..a4e10d398b9 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -15,6 +15,7 @@ import { cellFlagParser } from './cell/control-flags/cell-flags.js' import { startControlFlagChannel } from './cell/control-flags/relay-control-flag-channel.js' +import { startEdgeControl } from './edge-relay-control.js' import { readCloudRunInstanceZone } from './process-lifecycle/cloud-run-instance-zone.js' import { readSandboxTcpCounters } from './observability/sandbox-tcp-counters.js' import { @@ -68,6 +69,8 @@ const cellFlagChannel = }) : null cellFlagsApplied = cellFlagChannel ? () => cellFlagChannel.applied().flags : null +// Directors with an edge read director/edge.json; until it lands, mode on and the env rollout. +const edgeControl = config.role === 'director' && config.edge ? startEdgeControl() : undefined const { server, sessions, @@ -79,7 +82,7 @@ const { ready, cellIncarnation, shadowSeatPoller -} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied }) +} = createRelayServer(config, database, { cellFlags: cellFlagChannel?.applied, edgeControl }) // Cloud Run sets K_SERVICE; GCE cells and local runs skip the metadata read. if (process.env.K_SERVICE) { void readCloudRunInstanceZone().then((zone) => observability.setInstanceZone(zone)) @@ -229,6 +232,7 @@ const shutdown = (): void => { regionalRehomeWorker?.stop() shadowSeatPoller?.stop() cellFlagChannel?.stop() + edgeControl?.stop() sessions.drain(0) server.close(() => void closeDatabaseAndExit(shutdownStartedAt).catch(() => undefined)) } diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 5dabc81c75e..bdc212e2d18 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -20,6 +20,7 @@ import { import { createRelayApp } from './app.js' import { EDGE_CELL_ID, EdgeAssignments } from './edge-relay.js' import { EdgeEvacuation } from './edge-relay-evacuation.js' +import type { EdgeControl } from './edge-relay-control.js' import { createEdgeAdmin } from './edge-admin-client.js' import { classifyAssignmentLease } from './assignment/assignment-lease.js' import { defaultIntakePerSec, type CellFlags } from './cell/control-flags/cell-flags.js' @@ -138,6 +139,8 @@ export function createRelayServer( connectionLedgerLimits?: { hardCap: number; controlReserve: number } cellIncarnation?: string cellFlags?: () => AppliedControlFlags + // Runtime edge control (director/edge.json); absent, the edge runs on its boot config. + edgeControl?: EdgeControl } = {} ) { const cellIncarnation = options.cellIncarnation ?? randomUUID() @@ -179,16 +182,20 @@ export function createRelayServer( issuer: config.publicUrl }) : undefined + const edgeConfig = + config.edge && options.edgeControl ? { ...config.edge, control: options.edgeControl.flags } : config.edge const edgeAssignments = - config.role === 'director' && config.edge - ? new EdgeAssignments(observedDatabase, config.edge, options.now, edgeAdmin) + config.role === 'director' && edgeConfig + ? new EdgeAssignments(observedDatabase, edgeConfig, options.now, edgeAdmin) : undefined // Hosts that left the cohort are drained off the edge, rather than waiting out their lease. const edgeEvacuation = - config.role === 'director' && config.edge && edgeAdmin - ? new EdgeEvacuation(observedDatabase, config.edge, edgeAdmin, EDGE_CELL_ID, options.now, options.random) + config.role === 'director' && edgeConfig && edgeAdmin + ? new EdgeEvacuation(observedDatabase, edgeConfig, edgeAdmin, EDGE_CELL_ID, options.now, options.random) : undefined edgeEvacuation?.start() + // A rollback (mode off/down, a smaller share) starts draining within seconds. + options.edgeControl?.onChange(() => edgeEvacuation?.controlChanged()) if (edgeAssignments) { void edgeAssignments.ensureEdgeCell().then( () => console.warn(JSON.stringify({ event: 'orca_relay_edge_cell_ready' })),