mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
relay: the edge evacuation sweep leaves region scope to the assign path
The assign path judges edge cohort membership with the region the desktop asks for; the sweep used the stored region preference, which can miss or lag it. A host whose two disagreed was placed on the edge, drained by the sweep, and placed back on its next ask: at 1% this drained 12 of ~31 moved hosts within 10 minutes, and each director would repeat it every cooldown. The sweep now ignores region scope (it still drains hosts that left the cohort by mode, deny or share); a host that asks for a region outside the scope leaves on that ask.
This commit is contained in:
@@ -50,18 +50,18 @@ export function edgeCohortBucketPermille(relayHostId: string): number {
|
||||
}
|
||||
|
||||
// Mode off or down empties the cohort. Otherwise deny wins, then host allow, then region scope;
|
||||
// within the region, account allow or the rollout share.
|
||||
// within the region, account allow or the rollout share. Region 'any' skips the region scope.
|
||||
export function inEdgeCohort(
|
||||
config: EdgeRelayConfig,
|
||||
relayHostId: string,
|
||||
region: RelayRegion | undefined,
|
||||
region: RelayRegion | undefined | 'any',
|
||||
userId?: string
|
||||
): boolean {
|
||||
const flags = config.control?.() ?? EDGE_CONTROL_DEFAULTS
|
||||
if (flags.mode !== 'on') { return false }
|
||||
if (config.denyHosts.has(relayHostId)) { return false }
|
||||
if (config.allowHosts.has(relayHostId)) { return true }
|
||||
if (!config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
|
||||
if (region !== 'any' && !config.regions.has(region ?? RELAY_DEFAULT_REGION)) { return false }
|
||||
if (userId !== undefined && config.allowUsers?.has(userId)) { return true }
|
||||
return edgeCohortBucketPermille(relayHostId) < edgeRolloutPermille(config.rolloutPercent, flags)
|
||||
}
|
||||
|
||||
@@ -108,6 +108,22 @@ describe('edge evacuation under runtime control', () => {
|
||||
h.evacuation.stop()
|
||||
})
|
||||
|
||||
it('leaves region scope to the assign path: a stored preference outside it drains nobody', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const flags: EdgeControlFlags = { mode: 'on', rolloutPermille: 1000 }
|
||||
const h = await harness({ control: () => flags })
|
||||
// On the edge by its ask (no region, or us-central1); its stored preference says otherwise.
|
||||
await h.insert(hostId(1))
|
||||
await h.prefer(hostId(1), 'asia-east2')
|
||||
await h.insert(hostId(2))
|
||||
expect(await h.evacuation.sweep()).toBe(0)
|
||||
expect(h.admin.drained).toEqual([])
|
||||
// Leaving the cohort still drains, whatever the stored region.
|
||||
flags.rolloutPermille = 0
|
||||
expect(await h.evacuation.sweep()).toBe(2)
|
||||
expect(h.admin.drained.sort()).toEqual([hostId(1), hostId(2)])
|
||||
})
|
||||
|
||||
it(`sends at most ${EDGE_EVACUATION_CONCURRENCY} drains at once`, async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
// rows whose host is no longer in the cohort and asks the edge to drain them; the desktop then
|
||||
// re-asks /v1/assign, and that ask exports its data and places it on a cell (app.ts). Each pass also
|
||||
// retries fences that did not land after a host left (edge-relay-fence.ts).
|
||||
import { RELAY_DEFAULT_REGION, RelayRegionSchema } from '@orca-cloud/relay-contract/relay-regions'
|
||||
import type { RelayDatabase } from './database/database.js'
|
||||
import { EdgeAdminUnavailableError, type EdgeAdmin } from './edge-admin-client.js'
|
||||
import { inEdgeCohort, text, type EdgeRelayConfig, type Identity } from './edge-relay-cohort.js'
|
||||
@@ -116,9 +115,11 @@ export class EdgeEvacuation {
|
||||
for (const row of rows) {
|
||||
const identity = { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }
|
||||
pass.cursor = identity
|
||||
const stored = RelayRegionSchema.safeParse(row.preferred_region)
|
||||
const region = stored.success ? stored.data : RELAY_DEFAULT_REGION
|
||||
if (inEdgeCohort(this.config, identity.relayHostId, region, identity.userId)) { continue }
|
||||
// Region scope is the assign path's call alone: it decides with the region the desktop
|
||||
// asks for, which the stored preference can miss or lag. Judging it here too drained
|
||||
// hosts that the next ask put straight back (a drain every few minutes). A host that asks
|
||||
// for a region outside the scope leaves on that ask.
|
||||
if (inEdgeCohort(this.config, identity.relayHostId, 'any', identity.userId)) { continue }
|
||||
if (this.drainedAt.has(drainKey(identity))) { continue }
|
||||
leaving.push(identity)
|
||||
// Resume after the last host taken, so the rest of the page is read next tick.
|
||||
@@ -143,10 +144,8 @@ export class EdgeEvacuation {
|
||||
private async readPage(pass: Pass) {
|
||||
const { cursor, origin, wrapped } = pass
|
||||
return await this.database.query(
|
||||
`SELECT a.user_id, a.relay_host_id, p.preferred_region
|
||||
`SELECT a.user_id, a.relay_host_id
|
||||
FROM relay_assignments a
|
||||
LEFT JOIN relay_assignment_region_preferences p
|
||||
ON p.user_id = a.user_id AND p.relay_host_id = a.relay_host_id
|
||||
WHERE a.cell_id = ? AND (a.user_id > ? OR (a.user_id = ? AND a.relay_host_id > ?))
|
||||
${wrapped ? 'AND (a.user_id < ? OR (a.user_id = ? AND a.relay_host_id <= ?))' : ''}
|
||||
ORDER BY a.user_id, a.relay_host_id
|
||||
|
||||
Reference in New Issue
Block a user