relay: the edge evacuation sweep leaves region scope to the assign path

The assign path judges edge cohort membership with the region the desktop asks for; the sweep used
the stored region preference, which can miss or lag it. A host whose two disagreed was placed on the
edge, drained by the sweep, and placed back on its next ask: at 1% this drained 12 of ~31 moved
hosts within 10 minutes, and each director would repeat it every cooldown. The sweep now ignores
region scope (it still drains hosts that left the cohort by mode, deny or share); a host that asks
for a region outside the scope leaves on that ask.
This commit is contained in:
Neil
2026-11-10 14:50:05 -08:00
parent c2e6ebbf0b
commit 1ff06145db
3 changed files with 25 additions and 10 deletions
+3 -3
View File
@@ -50,18 +50,18 @@ export function edgeCohortBucketPermille(relayHostId: string): number {
}
// Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope;
// within the region, account allow or the rollout share.
// within the region, account allow or the rollout share. Region 'any' skips the region scope.
export function inEdgeCohort(
config: EdgeRelayConfig,
relayHostId: string,
region: RelayRegion | undefined,
region: RelayRegion | undefined | 'any',
userId?: string
): boolean {
const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS
if (flags.mode !== 'on') { return false }
if (config.denyHosts.has(relayHostId)) { return false }
if (config.allowHosts.has(relayHostId)) { return true }
if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
if (region !== 'any' && !config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
if (userId !== undefined && config.allowUsers?.has(userId)) { return true }
return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags)
}
@@ -108,6 +108,22 @@ describe('edge evacuation under runtime control', () => {
h.evacuation.stop()
})
it('leaves region scope to the assign path: a stored preference outside it drains nobody', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 1000 }
const h = await harness({ control: () => flags })
// On the edge by its ask (no region, or us-central1); its stored preference says otherwise.
await h.insert(hostId(1))
await h.prefer(hostId(1), 'asia-east2')
await h.insert(hostId(2))
expect(await h.evacuation.sweep()).toBe(0)
expect(h.admin.drained).toEqual([])
// Leaving the cohort still drains, whatever the stored region.
flags.rolloutPermille = 0
expect(await h.evacuation.sweep()).toBe(2)
expect(h.admin.drained.sort()).toEqual([hostId(1), hostId(2)])
})
it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
@@ -3,7 +3,6 @@
// rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also
// retries fences that did not land after a host left (edge-relay-fence.ts).
import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions'
import type { RelayDatabase } from './database/database.js'
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
@@ -116,9 +115,11 @@ export class EdgeEvacuation {
for (const row of rows) {
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
pass.cursor = identity
const stored = RelayRegionSchema.safeParse(row.preferred_region)
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
// Region scope is the assign path's call alone: it decides with the region the desktop
// asks for, which the stored preference can miss or lag. Judging it here too drained
// hosts that the next ask put straight back (a drain every few minutes). A host that asks
// for a region outside the scope leaves on that ask.
if (inEdgeCohort(this.config, identity.relayHostId, 'any', identity.userId)) { continue }
if (this.drainedAt.has(drainKey(identity))) { continue }
leaving.push(identity)
// Resume after the last host taken, so the rest of the page is read next tick.
@@ -143,10 +144,8 @@ export class EdgeEvacuation {
private async readPage(pass: Pass) {
const { cursor, origin, wrapped } = pass
return await this.database.query(
`SELECT a.user_id, a.relay_host_id, p.preferred_region
`SELECT a.user_id, a.relay_host_id
FROM relay_assignments a
LEFT JOIN relay_assignment_region_preferences p
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''}
ORDER BY a.user_id, a.relay_host_id