fix(relay): harden regional preference recovery

This commit is contained in:
Jinwoo-H
2026-08-25 15:25:29 -04:00
parent bc98655a39
commit 6fae9ea163
7 changed files with 383 additions and 109 deletions
+5 -2
View File
@@ -7,8 +7,11 @@ for 24 hours. A cached region changes only when the alternative is materially fa
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
a region preference. A rolled-back director that rejects the new field is retried once without
only that field while preserving reconnect behavior.
a region preference, with a short in-memory backoff so recovery does not repeatedly repay failed
probes. The preference is advisory: if that region has no healthy capacity, the director falls back
to any healthy general cell. A rolled-back director that rejects the new field is retried once
without only that field while preserving reconnect behavior. This client ignores catalog regions
it does not recognize and continues measuring the regions it supports.
The selection measures the desktop network path. Folder workspaces and SSH workspaces share the
same local broker and do not run probes on remote hosts. The phone continues to connect to the
@@ -24,7 +24,8 @@ const AUDITED_GLOBAL_FETCH_LINES = new Map<string, number>([
['main/orca-profiles/profile-cloud-org-members-client.ts', 1],
['main/rate-limits/codex-fetcher.ts', 3],
['main/runtime/relay/relay-http-client.ts', 2],
['main/runtime/relay/relay-region-preference.ts', 3],
['main/runtime/relay/relay-region-catalog.ts', 1],
['main/runtime/relay/relay-region-preference.ts', 2],
['main/source-control/hosted-review-api-request.ts', 1],
['main/speech/openai-transcription-client.ts', 1],
// Main HTTP port: one type declaration plus the Node fallback call. The fallback
@@ -117,6 +117,37 @@ describe('relay HTTP client', () => {
})
})
it('bounds the combined region and reconnect compatibility fallbacks', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
.mockResolvedValueOnce(Response.json({ error: 'invalid_request' }, { status: 400 }))
.mockResolvedValueOnce(Response.json({ error: 'invalid_request' }, { status: 400 }))
.mockResolvedValueOnce(
Response.json({
v: 1,
cellUrl: 'https://relay-c1.example',
assignmentEpoch: 4,
lease: 'lease-jwt'
})
)
await expect(
requestRelayAssignment({
directorUrl: 'https://relay.example',
relayToken: 'scoped-token',
relayHostId: 'AbCdEf0123_-xyZ9',
reconnect: true,
preferredRegion: 'asia-east2',
fetch
})
).resolves.toMatchObject({ assignmentEpoch: 4 })
expect(fetch).toHaveBeenCalledTimes(3)
expect(JSON.parse(String(fetch.mock.calls[2]?.[1]?.body))).toEqual({
v: 1,
relayHostId: 'AbCdEf0123_-xyZ9'
})
})
it('retries once unhinted when a rolled-back director rejects the hint', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
@@ -0,0 +1,130 @@
import { z } from 'zod'
import { readFetchResponseJsonWithinLimit } from '../../../shared/fetch-response-body'
import { cancelUnreadResponseBody } from '../../lib/unread-response-body'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export type RelayRegion = (typeof RELAY_REGIONS)[number]
export type RelayRegionCatalog = {
v: 1
regions: { region: RelayRegion; probeOrigins: string[] }[]
}
const CATALOG_MAX_BYTES = 16 * 1024
const RelayRegionSchema = z.enum(RELAY_REGIONS)
const RelayProbeOriginSchema = z.string().max(2_048).refine(isCanonicalRelayProbeOrigin)
const RelayRegionCatalogSchema = z
.object({
v: z.literal(1),
regions: z.array(z.unknown())
})
.passthrough()
const RelayRegionCatalogEntrySchema = z.object({ region: z.string().max(128) }).passthrough()
const KnownRelayRegionCatalogEntrySchema = z
.object({ region: RelayRegionSchema, probeOrigins: z.array(z.unknown()).min(1) })
.passthrough()
export function parseRelayRegion(value: unknown): RelayRegion | undefined {
const parsed = RelayRegionSchema.safeParse(value)
return parsed.success ? parsed.data : undefined
}
export async function fetchRelayRegionCatalog(
directorUrl: string,
fetch: typeof globalThis.fetch,
timeoutMs: number
): Promise<RelayRegionCatalog> {
if (!isCanonicalDirectorOrigin(directorUrl)) {
throw new Error('invalid relay director origin')
}
const response = await fetch(`${directorUrl}/v1/regions`, {
method: 'GET',
cache: 'no-store',
redirect: 'error',
signal: AbortSignal.timeout(timeoutMs)
})
if (!response.ok) {
await cancelUnreadResponseBody(response)
throw new Error(`relay region catalog failed (${response.status})`)
}
const body = await readFetchResponseJsonWithinLimit<unknown>(response, CATALOG_MAX_BYTES, {
structuralTokens: 256,
nestingDepth: 8
})
return parseRelayRegionCatalog(body, directorUrl)
}
export function isCanonicalRelayProbeOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
function parseRelayRegionCatalog(body: unknown, directorUrl: string): RelayRegionCatalog {
const parsed = RelayRegionCatalogSchema.safeParse(body)
if (!parsed.success) {
throw new Error('invalid relay region catalog')
}
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
const known: RelayRegionCatalog['regions'] = []
for (const rawEntry of parsed.data.regions) {
const entry = RelayRegionCatalogEntrySchema.safeParse(rawEntry)
if (!entry.success) {
continue
}
const region = parseRelayRegion(entry.data.region)
if (!region) {
continue
}
const knownEntry = KnownRelayRegionCatalogEntrySchema.safeParse(rawEntry)
if (!knownEntry.success) {
throw new Error('invalid relay region entry')
}
if (regions.has(region)) {
throw new Error('duplicate relay region')
}
regions.add(region)
const probeOrigins = parseProbeOrigins(knownEntry.data.probeOrigins, origins, directorUrl)
known.push({ region, probeOrigins })
}
return { v: 1, regions: known }
}
function parseProbeOrigins(
rawOrigins: unknown[],
origins: Set<string>,
directorUrl: string
): string[] {
const parsed: string[] = []
for (const rawOrigin of rawOrigins.slice(0, 2)) {
const origin = RelayProbeOriginSchema.safeParse(rawOrigin)
if (!origin.success || !isProbeOriginForDirector(origin.data, directorUrl)) {
throw new Error('invalid relay probe origin')
}
if (origins.has(origin.data)) {
throw new Error('duplicate relay probe origin')
}
origins.add(origin.data)
parsed.push(origin.data)
}
return parsed
}
function isCanonicalDirectorOrigin(value: string): boolean {
try {
const url = new URL(value)
const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)
return (
url.origin === value && (url.protocol === 'https:' || (url.protocol === 'http:' && loopback))
)
} catch {
return false
}
}
function isProbeOriginForDirector(origin: string, directorUrl: string): boolean {
return new URL(origin).hostname.endsWith(`.${new URL(directorUrl).hostname}`)
}
@@ -0,0 +1,50 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayRegionPreferenceResolver } from './relay-region-preference'
const DIRECTOR = 'https://relay.example.test'
const ASIA = 'https://asia-c1.relay.example.test'
const tempPaths: string[] = []
afterEach(() => {
for (const path of tempPaths.splice(0)) {
rmSync(path, { recursive: true, force: true })
}
})
describe('Relay region preference resilience', () => {
it('shares one catalog and probe cycle across concurrent resolutions', async () => {
const path = mkdtempSync(join(tmpdir(), 'orca-relay-region-concurrency-'))
tempPaths.push(path)
let releaseFirstProbe: ((latency: number) => void) | undefined
const firstProbe = new Promise<number>((resolve) => {
releaseFirstProbe = resolve
})
let probeCount = 0
const probe = vi.fn(async () => {
probeCount += 1
return probeCount === 1 ? await firstProbe : 20
})
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({ v: 1, regions: [{ region: 'asia-east2', probeOrigins: [ASIA] }] })
)
const resolver = new RelayRegionPreferenceResolver({
directorUrl: DIRECTOR,
userDataPath: path,
fetch,
probe,
now: () => 1_000
})
const first = resolver.resolve()
const second = resolver.resolve()
await vi.waitFor(() => expect(probe).toHaveBeenCalledOnce())
releaseFirstProbe?.(20)
await expect(Promise.all([first, second])).resolves.toEqual(['asia-east2', 'asia-east2'])
expect(fetch).toHaveBeenCalledOnce()
expect(probe).toHaveBeenCalledTimes(3)
})
})
@@ -9,9 +9,13 @@ const DIRECTOR = 'https://relay.example.test'
const US = 'https://us-c1.relay.example.test'
const US_SECONDARY = 'https://us-c2.relay.example.test'
const ASIA = 'https://asia-c1.relay.example.test'
const ASIA_SECONDARY = 'https://asia-c2.relay.example.test'
const ASIA_TERTIARY = 'https://asia-c3.relay.example.test'
const tempPaths: string[] = []
afterEach(() => {
vi.unstubAllEnvs()
vi.restoreAllMocks()
for (const path of tempPaths.splice(0)) {
rmSync(path, { recursive: true, force: true })
}
@@ -143,6 +147,10 @@ describe('Relay region preference', () => {
{ region: 'us-central1', probeOrigins: [US] },
{ region: 'asia-east2', probeOrigins: [US] }
],
[
{ region: 'us-central1', probeOrigins: [US] },
{ region: 'us-central1', probeOrigins: [US_SECONDARY] }
],
[{ region: 'us-central1', probeOrigins: ['http://us.relay.example.test'] }],
[{ region: 'us-central1', probeOrigins: ['https://external.example.test'] }]
]
@@ -170,6 +178,33 @@ describe('Relay region preference', () => {
).resolves.toBeUndefined()
})
it('ignores future regions and metadata while bounding known-region probes', async () => {
const { calls, probe } = sampledProbe({
[ASIA]: [30, 32, 34],
[ASIA_SECONDARY]: [28, 30, 32],
[ASIA_TERTIARY]: [1, 1, 1]
})
const resolver = new RelayRegionPreferenceResolver({
directorUrl: DIRECTOR,
userDataPath: userDataPath(),
fetch: catalogFetch([
...Array.from({ length: 17 }, (_, index) => ({ futureMetadata: index })),
{ region: 'europe-west1', futureShape: true },
{
region: 'asia-east2',
probeOrigins: [ASIA, ASIA_SECONDARY, ASIA_TERTIARY],
metadata: true
}
]),
probe,
now: () => 1_000
})
await expect(resolver.resolve()).resolves.toBe('asia-east2')
expect(calls).not.toContain(ASIA_TERTIARY)
expect(calls).toHaveLength(6)
})
it('recovers from corrupt cache and cancels an old directors error response', async () => {
const path = userDataPath()
writeFileSync(cachePath(path), '{not-json')
@@ -186,6 +221,7 @@ describe('Relay region preference', () => {
rmSync(cachePath(path), { force: true })
let cancelled = 0
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const oldDirector = vi.fn<typeof globalThis.fetch>(async () =>
cancelTrackingResponse(404, () => {
cancelled += 1
@@ -200,6 +236,7 @@ describe('Relay region preference', () => {
}).resolve()
).resolves.toBeUndefined()
expect(cancelled).toBe(1)
expect(warn).not.toHaveBeenCalled()
})
it('rejects a cache expiry beyond the 24-hour bound', async () => {
@@ -241,6 +278,69 @@ describe('Relay region preference', () => {
expect(fetch).not.toHaveBeenCalled()
})
it('uses the documented environment override without network access', async () => {
vi.stubEnv('ORCA_RELAY_REGION_OVERRIDE', 'asia-east2')
const fetch = vi.fn<typeof globalThis.fetch>()
const resolver = new RelayRegionPreferenceResolver({
directorUrl: DIRECTOR,
userDataPath: userDataPath(),
fetch
})
await expect(resolver.resolve()).resolves.toBe('asia-east2')
expect(fetch).not.toHaveBeenCalled()
})
it('briefly caches failed probe cycles and logs once per failure streak', async () => {
let now = 1_000
const path = userDataPath()
const fetch = catalogFetch([{ region: 'asia-east2', probeOrigins: [ASIA] }])
let healthy = false
const probe = vi.fn(async () => (healthy ? 20 : null))
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const resolver = new RelayRegionPreferenceResolver({
directorUrl: DIRECTOR,
userDataPath: path,
fetch,
probe,
now: () => now
})
await expect(resolver.resolve()).resolves.toBeUndefined()
await expect(resolver.resolve()).resolves.toBeUndefined()
expect(fetch).toHaveBeenCalledOnce()
expect(probe).toHaveBeenCalledOnce()
now += 2 * 60_000 + 1
healthy = true
await expect(resolver.resolve()).resolves.toBe('asia-east2')
rmSync(cachePath(path), { force: true })
expect(fetch).toHaveBeenCalledTimes(2)
expect(probe).toHaveBeenCalledTimes(4)
expect(warn).toHaveBeenCalledOnce()
now += 1
healthy = false
await expect(resolver.resolve()).resolves.toBeUndefined()
expect(fetch).toHaveBeenCalledTimes(3)
expect(probe).toHaveBeenCalledTimes(5)
expect(warn).toHaveBeenCalledTimes(2)
})
it('logs a bounded reason instead of an arbitrary request error', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const resolver = new RelayRegionPreferenceResolver({
directorUrl: DIRECTOR,
userDataPath: userDataPath(),
fetch: vi.fn(async () => {
throw new Error('private upstream detail')
})
})
await expect(resolver.resolve()).resolves.toBeUndefined()
expect(warn).toHaveBeenCalledWith('[relay] region preference unavailable:', 'request-failed')
})
it('bounds an offline catalog request and returns no preference', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(
async (_url, init) =>
+65 -106
View File
@@ -1,64 +1,30 @@
import { existsSync, readFileSync, statSync } from 'node:fs'
import { randomUUID } from 'node:crypto'
import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { performance } from 'node:perf_hooks'
import { z } from 'zod'
import { cancelUnreadResponseBody } from '../../lib/unread-response-body'
import { readFetchResponseJsonWithinLimit } from '../../../shared/fetch-response-body'
import { hardenExistingSecureFile, writeSecureJsonFile } from '../../../shared/secure-file'
import {
fetchRelayRegionCatalog,
isCanonicalRelayProbeOrigin,
parseRelayRegion,
RELAY_REGIONS,
type RelayRegion,
type RelayRegionCatalog
} from './relay-region-catalog'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export type RelayRegion = (typeof RELAY_REGIONS)[number]
export { RELAY_REGIONS, type RelayRegion } from './relay-region-catalog'
const RELAY_REGION_CACHE_FILENAME = 'orca-relay-region-preference.json'
const CACHE_MAX_BYTES = 8 * 1024
const CATALOG_MAX_BYTES = 16 * 1024
const CACHE_TTL_MS = 24 * 60 * 60_000
const FAILURE_RETRY_MS = 2 * 60_000
const PROBE_SAMPLES = 3
const PROBE_TIMEOUT_MS = 1_500
const SWITCH_MINIMUM_MS = 25
const SWITCH_RATIO = 0.8
const RelayRegionSchema = z.enum(RELAY_REGIONS)
const RelayProbeOriginSchema = z.string().max(2_048).refine(isCanonicalHttpsOrigin)
const RelayRegionCatalogSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
const RelayRegionCacheSchema = z
.object({
@@ -70,7 +36,6 @@ const RelayRegionCacheSchema = z
})
.strict()
type RelayRegionCatalog = z.infer<typeof RelayRegionCatalogSchema>
type RelayRegionCache = z.infer<typeof RelayRegionCacheSchema>
type RegionMeasurement = { region: RelayRegion; latencyMs: number }
@@ -88,17 +53,19 @@ type RelayRegionPreferenceOptions = {
export class RelayRegionPreferenceResolver {
private readonly options: RelayRegionPreferenceOptions
private pending: Promise<RelayRegion | undefined> | null = null
private retryAfter = 0
private failureLogged = false
constructor(options: RelayRegionPreferenceOptions) {
this.options = options
}
async resolve(): Promise<RelayRegion | undefined> {
const override = RelayRegionSchema.safeParse(
const override = parseRelayRegion(
this.options.diagnosticOverride ?? process.env.ORCA_RELAY_REGION_OVERRIDE
)
if (override.success) {
return override.data
if (override) {
return override
}
const now = (this.options.now ?? Date.now)()
@@ -106,11 +73,29 @@ export class RelayRegionPreferenceResolver {
if (cache && cache.expiresAt > now) {
return cache.region
}
if (now < this.retryAfter) {
return undefined
}
if (this.pending) {
return await this.pending
}
this.pending = this.refresh(cache, now).catch(() => undefined)
this.pending = this.refresh(cache, now).then(
(region) => {
this.retryAfter = 0
this.failureLogged = false
return region
},
(error) => {
this.retryAfter = (this.options.now ?? Date.now)() + FAILURE_RETRY_MS
const reason = relayRegionFailureReason(error)
if (!this.failureLogged && reason !== 'catalog-http-404') {
console.warn('[relay] region preference unavailable:', reason)
this.failureLogged = true
}
return undefined
}
)
try {
return await this.pending
} finally {
@@ -142,11 +127,11 @@ export class RelayRegionPreferenceResolver {
).filter((measurement): measurement is RegionMeasurement => measurement !== null)
const selected = selectRegionMeasurement(measurements, previous)
if (!selected) {
return undefined
throw new Error('relay region probes unavailable')
}
try {
writeSecureJsonFile(join(this.options.userDataPath, RELAY_REGION_CACHE_FILENAME), {
writeRelayRegionCache(join(this.options.userDataPath, RELAY_REGION_CACHE_FILENAME), {
v: 1,
directorUrl: this.options.directorUrl,
region: selected.region,
@@ -171,46 +156,13 @@ export function createRelayRegionPreferenceReader(input: {
return () => resolver.resolve()
}
async function fetchRelayRegionCatalog(
directorUrl: string,
fetch: typeof globalThis.fetch,
timeoutMs: number
): Promise<RelayRegionCatalog> {
if (!isCanonicalDirectorOrigin(directorUrl)) {
throw new Error('invalid relay director origin')
}
const response = await fetch(`${directorUrl}/v1/regions`, {
method: 'GET',
cache: 'no-store',
redirect: 'error',
signal: AbortSignal.timeout(timeoutMs)
})
if (!response.ok) {
await cancelUnreadResponseBody(response)
throw new Error(`relay region catalog failed (${response.status})`)
}
const body = await readFetchResponseJsonWithinLimit<unknown>(response, CATALOG_MAX_BYTES, {
structuralTokens: 64,
nestingDepth: 8
})
const catalog = RelayRegionCatalogSchema.parse(body)
if (
catalog.regions.some((entry) =>
entry.probeOrigins.some((origin) => !isProbeOriginForDirector(origin, directorUrl))
)
) {
throw new Error('relay probe origin does not belong to the director')
}
return catalog
}
export async function probeRelayOrigin(
origin: string,
fetch: typeof globalThis.fetch,
now = () => performance.now(),
timeoutMs = PROBE_TIMEOUT_MS
): Promise<number | null> {
if (!RelayProbeOriginSchema.safeParse(origin).success) {
if (!isCanonicalRelayProbeOrigin(origin)) {
return null
}
const startedAt = now()
@@ -281,7 +233,6 @@ function readRelayRegionCache(userDataPath: string, directorUrl: string, now: nu
if (!existsSync(path)) {
return null
}
hardenExistingSecureFile(path)
if (statSync(path).size > CACHE_MAX_BYTES) {
return null
}
@@ -296,27 +247,35 @@ function readRelayRegionCache(userDataPath: string, directorUrl: string, now: nu
}
}
function isCanonicalHttpsOrigin(value: string): boolean {
function writeRelayRegionCache(path: string, cache: RelayRegionCache): void {
// This cache is non-secret; avoid synchronous Windows ACL subprocesses on the main thread.
const temporaryPath = `${path}.${process.pid}.${randomUUID()}.tmp`
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
writeFileSync(temporaryPath, JSON.stringify(cache), { encoding: 'utf8', mode: 0o600 })
renameSync(temporaryPath, path)
} catch (error) {
rmSync(temporaryPath, { force: true })
throw error
}
}
function isCanonicalDirectorOrigin(value: string): boolean {
try {
const url = new URL(value)
const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)
return (
url.origin === value && (url.protocol === 'https:' || (url.protocol === 'http:' && loopback))
)
} catch {
return false
function relayRegionFailureReason(error: unknown): string {
if (!(error instanceof Error)) {
return 'request-failed'
}
}
function isProbeOriginForDirector(origin: string, directorUrl: string): boolean {
return new URL(origin).hostname.endsWith(`.${new URL(directorUrl).hostname}`)
if (error.message === 'relay region probes unavailable') {
return 'probes-unavailable'
}
if (
error.message === 'invalid relay director origin' ||
error.message === 'invalid relay region catalog' ||
error.message === 'invalid relay region entry' ||
error.message === 'duplicate relay region' ||
error.message === 'invalid relay probe origin' ||
error.message === 'duplicate relay probe origin'
) {
return 'invalid-catalog'
}
const status = /^relay region catalog failed \(([1-5]\d\d)\)$/.exec(error.message)?.[1]
return status ? `catalog-http-${status}` : 'request-failed'
}