From 1ff06145db0df91bceb8df49a5dda380504055eb Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 14:50:05 -0800 Subject: [PATCH] relay: the edge evacuation sweep leaves region scope to the assign path The assign path judges edge cohort membership with the region the desktop asks for; the sweep used the stored region preference, which can miss or lag it. A host whose two disagreed was placed on the edge, drained by the sweep, and placed back on its next ask: at 1% this drained 12 of ~31 moved hosts within 10 minutes, and each director would repeat it every cooldown. The sweep now ignores region scope (it still drains hosts that left the cohort by mode, deny or share); a host that asks for a region outside the scope leaves on that ask. --- cloud/apps/relay/src/edge-relay-cohort.ts | 6 +++--- .../apps/relay/src/edge-relay-evacuation.test.ts | 16 ++++++++++++++++ cloud/apps/relay/src/edge-relay-evacuation.ts | 13 ++++++------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/cloud/apps/relay/src/edge-relay-cohort.ts b/cloud/apps/relay/src/edge-relay-cohort.ts index 5227e8f9ff4..d6a8818c55d 100644 --- a/cloud/apps/relay/src/edge-relay-cohort.ts +++ b/cloud/apps/relay/src/edge-relay-cohort.ts @@ -50,18 +50,18 @@ export function edgeCohortBucketPermille(relayHostId: string): number { } // Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope; -// within the region, account allow or the rollout share. +// within the region, account allow or the rollout share. Region 'any' skips the region scope. export function inEdgeCohort( config: EdgeRelayConfig, relayHostId: string, - region: RelayRegion | undefined, + region: RelayRegion | undefined | 'any', userId?: string ): boolean { const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS if (flags.mode !== 'on') { return false } if (config.denyHosts.has(relayHostId)) { return false } if (config.allowHosts.has(relayHostId)) { return true } - if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false } + if (region !== 'any' && !config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false } if (userId !== undefined && config.allowUsers?.has(userId)) { return true } return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags) } diff --git a/cloud/apps/relay/src/edge-relay-evacuation.test.ts b/cloud/apps/relay/src/edge-relay-evacuation.test.ts index 9b671710f28..855eb2ea55f 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.test.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.test.ts @@ -108,6 +108,22 @@ describe('edge evacuation under runtime control', () => { h.evacuation.stop() }) + it('leaves region scope to the assign path: a stored preference outside it drains nobody', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 1000 } + const h = await harness({ control: () => flags }) + // On the edge by its ask (no region, or us-central1); its stored preference says otherwise. + await h.insert(hostId(1)) + await h.prefer(hostId(1), 'asia-east2') + await h.insert(hostId(2)) + expect(await h.evacuation.sweep()).toBe(0) + expect(h.admin.drained).toEqual([]) + // Leaving the cohort still drains, whatever the stored region. + flags.rolloutPermille = 0 + expect(await h.evacuation.sweep()).toBe(2) + expect(h.admin.drained.sort()).toEqual([hostId(1), hostId(2)]) + }) + it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => { vi.spyOn(console, 'warn').mockImplementation(() => {}) const h = await harness() diff --git a/cloud/apps/relay/src/edge-relay-evacuation.ts b/cloud/apps/relay/src/edge-relay-evacuation.ts index ae1c31f5056..db103f67c55 100644 --- a/cloud/apps/relay/src/edge-relay-evacuation.ts +++ b/cloud/apps/relay/src/edge-relay-evacuation.ts @@ -3,7 +3,6 @@ // rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then // re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also // retries fences that did not land after a host left (edge-relay-fence.ts). -import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions' import type { RelayDatabase } from './database/database.js' import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js' import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js' @@ -116,9 +115,11 @@ export class EdgeEvacuation { for (const row of rows) { const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') } pass.cursor = identity - const stored = RelayRegionSchema.safeParse(row.preferred_region) - const region = stored.success ? stored.data : RELAY_DEFAULT_REGION - if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue } + // Region scope is the assign path's call alone: it decides with the region the desktop + // asks for, which the stored preference can miss or lag. Judging it here too drained + // hosts that the next ask put straight back (a drain every few minutes). A host that asks + // for a region outside the scope leaves on that ask. + if (inEdgeCohort(this.config, identity.relayHostId, 'any', identity.userId)) { continue } if (this.drainedAt.has(drainKey(identity))) { continue } leaving.push(identity) // Resume after the last host taken, so the rest of the page is read next tick. @@ -143,10 +144,8 @@ export class EdgeEvacuation { private async readPage(pass: Pass) { const { cursor, origin, wrapped } = pass return await this.database.query( - `SELECT a.user_id, a.relay_host_id, p.preferred_region + `SELECT a.user_id, a.relay_host_id FROM relay_assignments a - LEFT JOIN relay_assignment_region_preferences p - ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?)) ${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''} ORDER BY a.user_id, a.relay_host_id