mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 16:03:49 +00:00
fix(relay): harden regional preference recovery
This commit is contained in:
@@ -7,8 +7,11 @@ for 24 hours. A cached region changes only when the alternative is materially fa
|
||||
|
||||
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
|
||||
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
|
||||
a region preference. A rolled-back director that rejects the new field is retried once without
|
||||
only that field while preserving reconnect behavior.
|
||||
a region preference, with a short in-memory backoff so recovery does not repeatedly repay failed
|
||||
probes. The preference is advisory: if that region has no healthy capacity, the director falls back
|
||||
to any healthy general cell. A rolled-back director that rejects the new field is retried once
|
||||
without only that field while preserving reconnect behavior. This client ignores catalog regions
|
||||
it does not recognize and continues measuring the regions it supports.
|
||||
|
||||
The selection measures the desktop network path. Folder workspaces and SSH workspaces share the
|
||||
same local broker and do not run probes on remote hosts. The phone continues to connect to the
|
||||
|
||||
@@ -24,7 +24,8 @@ const AUDITED_GLOBAL_FETCH_LINES = new Map<string, number>([
|
||||
['main/orca-profiles/profile-cloud-org-members-client.ts', 1],
|
||||
['main/rate-limits/codex-fetcher.ts', 3],
|
||||
['main/runtime/relay/relay-http-client.ts', 2],
|
||||
['main/runtime/relay/relay-region-preference.ts', 3],
|
||||
['main/runtime/relay/relay-region-catalog.ts', 1],
|
||||
['main/runtime/relay/relay-region-preference.ts', 2],
|
||||
['main/source-control/hosted-review-api-request.ts', 1],
|
||||
['main/speech/openai-transcription-client.ts', 1],
|
||||
// Main HTTP port: one type declaration plus the Node fallback call. The fallback
|
||||
|
||||
@@ -117,6 +117,37 @@ describe('relay HTTP client', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('bounds the combined region and reconnect compatibility fallbacks', async () => {
|
||||
const fetch = vi
|
||||
.fn<typeof globalThis.fetch>()
|
||||
.mockResolvedValueOnce(Response.json({ error: 'invalid_request' }, { status: 400 }))
|
||||
.mockResolvedValueOnce(Response.json({ error: 'invalid_request' }, { status: 400 }))
|
||||
.mockResolvedValueOnce(
|
||||
Response.json({
|
||||
v: 1,
|
||||
cellUrl: 'https://relay-c1.example',
|
||||
assignmentEpoch: 4,
|
||||
lease: 'lease-jwt'
|
||||
})
|
||||
)
|
||||
|
||||
await expect(
|
||||
requestRelayAssignment({
|
||||
directorUrl: 'https://relay.example',
|
||||
relayToken: 'scoped-token',
|
||||
relayHostId: 'AbCdEf0123_-xyZ9',
|
||||
reconnect: true,
|
||||
preferredRegion: 'asia-east2',
|
||||
fetch
|
||||
})
|
||||
).resolves.toMatchObject({ assignmentEpoch: 4 })
|
||||
expect(fetch).toHaveBeenCalledTimes(3)
|
||||
expect(JSON.parse(String(fetch.mock.calls[2]?.[1]?.body))).toEqual({
|
||||
v: 1,
|
||||
relayHostId: 'AbCdEf0123_-xyZ9'
|
||||
})
|
||||
})
|
||||
|
||||
it('retries once unhinted when a rolled-back director rejects the hint', async () => {
|
||||
const fetch = vi
|
||||
.fn<typeof globalThis.fetch>()
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { z } from 'zod'
|
||||
import { readFetchResponseJsonWithinLimit } from '../../../shared/fetch-response-body'
|
||||
import { cancelUnreadResponseBody } from '../../lib/unread-response-body'
|
||||
|
||||
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
|
||||
export type RelayRegion = (typeof RELAY_REGIONS)[number]
|
||||
export type RelayRegionCatalog = {
|
||||
v: 1
|
||||
regions: { region: RelayRegion; probeOrigins: string[] }[]
|
||||
}
|
||||
|
||||
const CATALOG_MAX_BYTES = 16 * 1024
|
||||
const RelayRegionSchema = z.enum(RELAY_REGIONS)
|
||||
const RelayProbeOriginSchema = z.string().max(2_048).refine(isCanonicalRelayProbeOrigin)
|
||||
const RelayRegionCatalogSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
regions: z.array(z.unknown())
|
||||
})
|
||||
.passthrough()
|
||||
const RelayRegionCatalogEntrySchema = z.object({ region: z.string().max(128) }).passthrough()
|
||||
const KnownRelayRegionCatalogEntrySchema = z
|
||||
.object({ region: RelayRegionSchema, probeOrigins: z.array(z.unknown()).min(1) })
|
||||
.passthrough()
|
||||
|
||||
export function parseRelayRegion(value: unknown): RelayRegion | undefined {
|
||||
const parsed = RelayRegionSchema.safeParse(value)
|
||||
return parsed.success ? parsed.data : undefined
|
||||
}
|
||||
|
||||
export async function fetchRelayRegionCatalog(
|
||||
directorUrl: string,
|
||||
fetch: typeof globalThis.fetch,
|
||||
timeoutMs: number
|
||||
): Promise<RelayRegionCatalog> {
|
||||
if (!isCanonicalDirectorOrigin(directorUrl)) {
|
||||
throw new Error('invalid relay director origin')
|
||||
}
|
||||
const response = await fetch(`${directorUrl}/v1/regions`, {
|
||||
method: 'GET',
|
||||
cache: 'no-store',
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
})
|
||||
if (!response.ok) {
|
||||
await cancelUnreadResponseBody(response)
|
||||
throw new Error(`relay region catalog failed (${response.status})`)
|
||||
}
|
||||
const body = await readFetchResponseJsonWithinLimit<unknown>(response, CATALOG_MAX_BYTES, {
|
||||
structuralTokens: 256,
|
||||
nestingDepth: 8
|
||||
})
|
||||
return parseRelayRegionCatalog(body, directorUrl)
|
||||
}
|
||||
|
||||
export function isCanonicalRelayProbeOrigin(value: string): boolean {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
return url.protocol === 'https:' && url.origin === value
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function parseRelayRegionCatalog(body: unknown, directorUrl: string): RelayRegionCatalog {
|
||||
const parsed = RelayRegionCatalogSchema.safeParse(body)
|
||||
if (!parsed.success) {
|
||||
throw new Error('invalid relay region catalog')
|
||||
}
|
||||
const regions = new Set<RelayRegion>()
|
||||
const origins = new Set<string>()
|
||||
const known: RelayRegionCatalog['regions'] = []
|
||||
for (const rawEntry of parsed.data.regions) {
|
||||
const entry = RelayRegionCatalogEntrySchema.safeParse(rawEntry)
|
||||
if (!entry.success) {
|
||||
continue
|
||||
}
|
||||
const region = parseRelayRegion(entry.data.region)
|
||||
if (!region) {
|
||||
continue
|
||||
}
|
||||
const knownEntry = KnownRelayRegionCatalogEntrySchema.safeParse(rawEntry)
|
||||
if (!knownEntry.success) {
|
||||
throw new Error('invalid relay region entry')
|
||||
}
|
||||
if (regions.has(region)) {
|
||||
throw new Error('duplicate relay region')
|
||||
}
|
||||
regions.add(region)
|
||||
const probeOrigins = parseProbeOrigins(knownEntry.data.probeOrigins, origins, directorUrl)
|
||||
known.push({ region, probeOrigins })
|
||||
}
|
||||
return { v: 1, regions: known }
|
||||
}
|
||||
|
||||
function parseProbeOrigins(
|
||||
rawOrigins: unknown[],
|
||||
origins: Set<string>,
|
||||
directorUrl: string
|
||||
): string[] {
|
||||
const parsed: string[] = []
|
||||
for (const rawOrigin of rawOrigins.slice(0, 2)) {
|
||||
const origin = RelayProbeOriginSchema.safeParse(rawOrigin)
|
||||
if (!origin.success || !isProbeOriginForDirector(origin.data, directorUrl)) {
|
||||
throw new Error('invalid relay probe origin')
|
||||
}
|
||||
if (origins.has(origin.data)) {
|
||||
throw new Error('duplicate relay probe origin')
|
||||
}
|
||||
origins.add(origin.data)
|
||||
parsed.push(origin.data)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function isCanonicalDirectorOrigin(value: string): boolean {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)
|
||||
return (
|
||||
url.origin === value && (url.protocol === 'https:' || (url.protocol === 'http:' && loopback))
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function isProbeOriginForDirector(origin: string, directorUrl: string): boolean {
|
||||
return new URL(origin).hostname.endsWith(`.${new URL(directorUrl).hostname}`)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { RelayRegionPreferenceResolver } from './relay-region-preference'
|
||||
|
||||
const DIRECTOR = 'https://relay.example.test'
|
||||
const ASIA = 'https://asia-c1.relay.example.test'
|
||||
const tempPaths: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const path of tempPaths.splice(0)) {
|
||||
rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
describe('Relay region preference resilience', () => {
|
||||
it('shares one catalog and probe cycle across concurrent resolutions', async () => {
|
||||
const path = mkdtempSync(join(tmpdir(), 'orca-relay-region-concurrency-'))
|
||||
tempPaths.push(path)
|
||||
let releaseFirstProbe: ((latency: number) => void) | undefined
|
||||
const firstProbe = new Promise<number>((resolve) => {
|
||||
releaseFirstProbe = resolve
|
||||
})
|
||||
let probeCount = 0
|
||||
const probe = vi.fn(async () => {
|
||||
probeCount += 1
|
||||
return probeCount === 1 ? await firstProbe : 20
|
||||
})
|
||||
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
|
||||
Response.json({ v: 1, regions: [{ region: 'asia-east2', probeOrigins: [ASIA] }] })
|
||||
)
|
||||
const resolver = new RelayRegionPreferenceResolver({
|
||||
directorUrl: DIRECTOR,
|
||||
userDataPath: path,
|
||||
fetch,
|
||||
probe,
|
||||
now: () => 1_000
|
||||
})
|
||||
|
||||
const first = resolver.resolve()
|
||||
const second = resolver.resolve()
|
||||
await vi.waitFor(() => expect(probe).toHaveBeenCalledOnce())
|
||||
releaseFirstProbe?.(20)
|
||||
|
||||
await expect(Promise.all([first, second])).resolves.toEqual(['asia-east2', 'asia-east2'])
|
||||
expect(fetch).toHaveBeenCalledOnce()
|
||||
expect(probe).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
})
|
||||
@@ -9,9 +9,13 @@ const DIRECTOR = 'https://relay.example.test'
|
||||
const US = 'https://us-c1.relay.example.test'
|
||||
const US_SECONDARY = 'https://us-c2.relay.example.test'
|
||||
const ASIA = 'https://asia-c1.relay.example.test'
|
||||
const ASIA_SECONDARY = 'https://asia-c2.relay.example.test'
|
||||
const ASIA_TERTIARY = 'https://asia-c3.relay.example.test'
|
||||
const tempPaths: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs()
|
||||
vi.restoreAllMocks()
|
||||
for (const path of tempPaths.splice(0)) {
|
||||
rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
@@ -143,6 +147,10 @@ describe('Relay region preference', () => {
|
||||
{ region: 'us-central1', probeOrigins: [US] },
|
||||
{ region: 'asia-east2', probeOrigins: [US] }
|
||||
],
|
||||
[
|
||||
{ region: 'us-central1', probeOrigins: [US] },
|
||||
{ region: 'us-central1', probeOrigins: [US_SECONDARY] }
|
||||
],
|
||||
[{ region: 'us-central1', probeOrigins: ['http://us.relay.example.test'] }],
|
||||
[{ region: 'us-central1', probeOrigins: ['https://external.example.test'] }]
|
||||
]
|
||||
@@ -170,6 +178,33 @@ describe('Relay region preference', () => {
|
||||
).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('ignores future regions and metadata while bounding known-region probes', async () => {
|
||||
const { calls, probe } = sampledProbe({
|
||||
[ASIA]: [30, 32, 34],
|
||||
[ASIA_SECONDARY]: [28, 30, 32],
|
||||
[ASIA_TERTIARY]: [1, 1, 1]
|
||||
})
|
||||
const resolver = new RelayRegionPreferenceResolver({
|
||||
directorUrl: DIRECTOR,
|
||||
userDataPath: userDataPath(),
|
||||
fetch: catalogFetch([
|
||||
...Array.from({ length: 17 }, (_, index) => ({ futureMetadata: index })),
|
||||
{ region: 'europe-west1', futureShape: true },
|
||||
{
|
||||
region: 'asia-east2',
|
||||
probeOrigins: [ASIA, ASIA_SECONDARY, ASIA_TERTIARY],
|
||||
metadata: true
|
||||
}
|
||||
]),
|
||||
probe,
|
||||
now: () => 1_000
|
||||
})
|
||||
|
||||
await expect(resolver.resolve()).resolves.toBe('asia-east2')
|
||||
expect(calls).not.toContain(ASIA_TERTIARY)
|
||||
expect(calls).toHaveLength(6)
|
||||
})
|
||||
|
||||
it('recovers from corrupt cache and cancels an old directors error response', async () => {
|
||||
const path = userDataPath()
|
||||
writeFileSync(cachePath(path), '{not-json')
|
||||
@@ -186,6 +221,7 @@ describe('Relay region preference', () => {
|
||||
|
||||
rmSync(cachePath(path), { force: true })
|
||||
let cancelled = 0
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const oldDirector = vi.fn<typeof globalThis.fetch>(async () =>
|
||||
cancelTrackingResponse(404, () => {
|
||||
cancelled += 1
|
||||
@@ -200,6 +236,7 @@ describe('Relay region preference', () => {
|
||||
}).resolve()
|
||||
).resolves.toBeUndefined()
|
||||
expect(cancelled).toBe(1)
|
||||
expect(warn).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a cache expiry beyond the 24-hour bound', async () => {
|
||||
@@ -241,6 +278,69 @@ describe('Relay region preference', () => {
|
||||
expect(fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses the documented environment override without network access', async () => {
|
||||
vi.stubEnv('ORCA_RELAY_REGION_OVERRIDE', 'asia-east2')
|
||||
const fetch = vi.fn<typeof globalThis.fetch>()
|
||||
const resolver = new RelayRegionPreferenceResolver({
|
||||
directorUrl: DIRECTOR,
|
||||
userDataPath: userDataPath(),
|
||||
fetch
|
||||
})
|
||||
|
||||
await expect(resolver.resolve()).resolves.toBe('asia-east2')
|
||||
expect(fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('briefly caches failed probe cycles and logs once per failure streak', async () => {
|
||||
let now = 1_000
|
||||
const path = userDataPath()
|
||||
const fetch = catalogFetch([{ region: 'asia-east2', probeOrigins: [ASIA] }])
|
||||
let healthy = false
|
||||
const probe = vi.fn(async () => (healthy ? 20 : null))
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const resolver = new RelayRegionPreferenceResolver({
|
||||
directorUrl: DIRECTOR,
|
||||
userDataPath: path,
|
||||
fetch,
|
||||
probe,
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await expect(resolver.resolve()).resolves.toBeUndefined()
|
||||
await expect(resolver.resolve()).resolves.toBeUndefined()
|
||||
expect(fetch).toHaveBeenCalledOnce()
|
||||
expect(probe).toHaveBeenCalledOnce()
|
||||
|
||||
now += 2 * 60_000 + 1
|
||||
healthy = true
|
||||
await expect(resolver.resolve()).resolves.toBe('asia-east2')
|
||||
rmSync(cachePath(path), { force: true })
|
||||
expect(fetch).toHaveBeenCalledTimes(2)
|
||||
expect(probe).toHaveBeenCalledTimes(4)
|
||||
expect(warn).toHaveBeenCalledOnce()
|
||||
|
||||
now += 1
|
||||
healthy = false
|
||||
await expect(resolver.resolve()).resolves.toBeUndefined()
|
||||
expect(fetch).toHaveBeenCalledTimes(3)
|
||||
expect(probe).toHaveBeenCalledTimes(5)
|
||||
expect(warn).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('logs a bounded reason instead of an arbitrary request error', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const resolver = new RelayRegionPreferenceResolver({
|
||||
directorUrl: DIRECTOR,
|
||||
userDataPath: userDataPath(),
|
||||
fetch: vi.fn(async () => {
|
||||
throw new Error('private upstream detail')
|
||||
})
|
||||
})
|
||||
|
||||
await expect(resolver.resolve()).resolves.toBeUndefined()
|
||||
expect(warn).toHaveBeenCalledWith('[relay] region preference unavailable:', 'request-failed')
|
||||
})
|
||||
|
||||
it('bounds an offline catalog request and returns no preference', async () => {
|
||||
const fetch = vi.fn<typeof globalThis.fetch>(
|
||||
async (_url, init) =>
|
||||
|
||||
@@ -1,64 +1,30 @@
|
||||
import { existsSync, readFileSync, statSync } from 'node:fs'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { z } from 'zod'
|
||||
import { cancelUnreadResponseBody } from '../../lib/unread-response-body'
|
||||
import { readFetchResponseJsonWithinLimit } from '../../../shared/fetch-response-body'
|
||||
import { hardenExistingSecureFile, writeSecureJsonFile } from '../../../shared/secure-file'
|
||||
import {
|
||||
fetchRelayRegionCatalog,
|
||||
isCanonicalRelayProbeOrigin,
|
||||
parseRelayRegion,
|
||||
RELAY_REGIONS,
|
||||
type RelayRegion,
|
||||
type RelayRegionCatalog
|
||||
} from './relay-region-catalog'
|
||||
|
||||
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
|
||||
export type RelayRegion = (typeof RELAY_REGIONS)[number]
|
||||
export { RELAY_REGIONS, type RelayRegion } from './relay-region-catalog'
|
||||
|
||||
const RELAY_REGION_CACHE_FILENAME = 'orca-relay-region-preference.json'
|
||||
const CACHE_MAX_BYTES = 8 * 1024
|
||||
const CATALOG_MAX_BYTES = 16 * 1024
|
||||
const CACHE_TTL_MS = 24 * 60 * 60_000
|
||||
const FAILURE_RETRY_MS = 2 * 60_000
|
||||
const PROBE_SAMPLES = 3
|
||||
const PROBE_TIMEOUT_MS = 1_500
|
||||
const SWITCH_MINIMUM_MS = 25
|
||||
const SWITCH_RATIO = 0.8
|
||||
|
||||
const RelayRegionSchema = z.enum(RELAY_REGIONS)
|
||||
const RelayProbeOriginSchema = z.string().max(2_048).refine(isCanonicalHttpsOrigin)
|
||||
const RelayRegionCatalogSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
regions: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
region: RelayRegionSchema,
|
||||
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
|
||||
})
|
||||
.strict()
|
||||
)
|
||||
.max(RELAY_REGIONS.length)
|
||||
})
|
||||
.strict()
|
||||
.superRefine((catalog, context) => {
|
||||
const regions = new Set<RelayRegion>()
|
||||
const origins = new Set<string>()
|
||||
for (const [regionIndex, entry] of catalog.regions.entries()) {
|
||||
if (regions.has(entry.region)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'duplicate relay region',
|
||||
path: ['regions', regionIndex, 'region']
|
||||
})
|
||||
}
|
||||
regions.add(entry.region)
|
||||
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
|
||||
if (origins.has(origin)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'duplicate relay probe origin',
|
||||
path: ['regions', regionIndex, 'probeOrigins', originIndex]
|
||||
})
|
||||
}
|
||||
origins.add(origin)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const RelayRegionCacheSchema = z
|
||||
.object({
|
||||
@@ -70,7 +36,6 @@ const RelayRegionCacheSchema = z
|
||||
})
|
||||
.strict()
|
||||
|
||||
type RelayRegionCatalog = z.infer<typeof RelayRegionCatalogSchema>
|
||||
type RelayRegionCache = z.infer<typeof RelayRegionCacheSchema>
|
||||
type RegionMeasurement = { region: RelayRegion; latencyMs: number }
|
||||
|
||||
@@ -88,17 +53,19 @@ type RelayRegionPreferenceOptions = {
|
||||
export class RelayRegionPreferenceResolver {
|
||||
private readonly options: RelayRegionPreferenceOptions
|
||||
private pending: Promise<RelayRegion | undefined> | null = null
|
||||
private retryAfter = 0
|
||||
private failureLogged = false
|
||||
|
||||
constructor(options: RelayRegionPreferenceOptions) {
|
||||
this.options = options
|
||||
}
|
||||
|
||||
async resolve(): Promise<RelayRegion | undefined> {
|
||||
const override = RelayRegionSchema.safeParse(
|
||||
const override = parseRelayRegion(
|
||||
this.options.diagnosticOverride ?? process.env.ORCA_RELAY_REGION_OVERRIDE
|
||||
)
|
||||
if (override.success) {
|
||||
return override.data
|
||||
if (override) {
|
||||
return override
|
||||
}
|
||||
|
||||
const now = (this.options.now ?? Date.now)()
|
||||
@@ -106,11 +73,29 @@ export class RelayRegionPreferenceResolver {
|
||||
if (cache && cache.expiresAt > now) {
|
||||
return cache.region
|
||||
}
|
||||
if (now < this.retryAfter) {
|
||||
return undefined
|
||||
}
|
||||
if (this.pending) {
|
||||
return await this.pending
|
||||
}
|
||||
|
||||
this.pending = this.refresh(cache, now).catch(() => undefined)
|
||||
this.pending = this.refresh(cache, now).then(
|
||||
(region) => {
|
||||
this.retryAfter = 0
|
||||
this.failureLogged = false
|
||||
return region
|
||||
},
|
||||
(error) => {
|
||||
this.retryAfter = (this.options.now ?? Date.now)() + FAILURE_RETRY_MS
|
||||
const reason = relayRegionFailureReason(error)
|
||||
if (!this.failureLogged && reason !== 'catalog-http-404') {
|
||||
console.warn('[relay] region preference unavailable:', reason)
|
||||
this.failureLogged = true
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
)
|
||||
try {
|
||||
return await this.pending
|
||||
} finally {
|
||||
@@ -142,11 +127,11 @@ export class RelayRegionPreferenceResolver {
|
||||
).filter((measurement): measurement is RegionMeasurement => measurement !== null)
|
||||
const selected = selectRegionMeasurement(measurements, previous)
|
||||
if (!selected) {
|
||||
return undefined
|
||||
throw new Error('relay region probes unavailable')
|
||||
}
|
||||
|
||||
try {
|
||||
writeSecureJsonFile(join(this.options.userDataPath, RELAY_REGION_CACHE_FILENAME), {
|
||||
writeRelayRegionCache(join(this.options.userDataPath, RELAY_REGION_CACHE_FILENAME), {
|
||||
v: 1,
|
||||
directorUrl: this.options.directorUrl,
|
||||
region: selected.region,
|
||||
@@ -171,46 +156,13 @@ export function createRelayRegionPreferenceReader(input: {
|
||||
return () => resolver.resolve()
|
||||
}
|
||||
|
||||
async function fetchRelayRegionCatalog(
|
||||
directorUrl: string,
|
||||
fetch: typeof globalThis.fetch,
|
||||
timeoutMs: number
|
||||
): Promise<RelayRegionCatalog> {
|
||||
if (!isCanonicalDirectorOrigin(directorUrl)) {
|
||||
throw new Error('invalid relay director origin')
|
||||
}
|
||||
const response = await fetch(`${directorUrl}/v1/regions`, {
|
||||
method: 'GET',
|
||||
cache: 'no-store',
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
})
|
||||
if (!response.ok) {
|
||||
await cancelUnreadResponseBody(response)
|
||||
throw new Error(`relay region catalog failed (${response.status})`)
|
||||
}
|
||||
const body = await readFetchResponseJsonWithinLimit<unknown>(response, CATALOG_MAX_BYTES, {
|
||||
structuralTokens: 64,
|
||||
nestingDepth: 8
|
||||
})
|
||||
const catalog = RelayRegionCatalogSchema.parse(body)
|
||||
if (
|
||||
catalog.regions.some((entry) =>
|
||||
entry.probeOrigins.some((origin) => !isProbeOriginForDirector(origin, directorUrl))
|
||||
)
|
||||
) {
|
||||
throw new Error('relay probe origin does not belong to the director')
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
export async function probeRelayOrigin(
|
||||
origin: string,
|
||||
fetch: typeof globalThis.fetch,
|
||||
now = () => performance.now(),
|
||||
timeoutMs = PROBE_TIMEOUT_MS
|
||||
): Promise<number | null> {
|
||||
if (!RelayProbeOriginSchema.safeParse(origin).success) {
|
||||
if (!isCanonicalRelayProbeOrigin(origin)) {
|
||||
return null
|
||||
}
|
||||
const startedAt = now()
|
||||
@@ -281,7 +233,6 @@ function readRelayRegionCache(userDataPath: string, directorUrl: string, now: nu
|
||||
if (!existsSync(path)) {
|
||||
return null
|
||||
}
|
||||
hardenExistingSecureFile(path)
|
||||
if (statSync(path).size > CACHE_MAX_BYTES) {
|
||||
return null
|
||||
}
|
||||
@@ -296,27 +247,35 @@ function readRelayRegionCache(userDataPath: string, directorUrl: string, now: nu
|
||||
}
|
||||
}
|
||||
|
||||
function isCanonicalHttpsOrigin(value: string): boolean {
|
||||
function writeRelayRegionCache(path: string, cache: RelayRegionCache): void {
|
||||
// This cache is non-secret; avoid synchronous Windows ACL subprocesses on the main thread.
|
||||
const temporaryPath = `${path}.${process.pid}.${randomUUID()}.tmp`
|
||||
try {
|
||||
const url = new URL(value)
|
||||
return url.protocol === 'https:' && url.origin === value
|
||||
} catch {
|
||||
return false
|
||||
writeFileSync(temporaryPath, JSON.stringify(cache), { encoding: 'utf8', mode: 0o600 })
|
||||
renameSync(temporaryPath, path)
|
||||
} catch (error) {
|
||||
rmSync(temporaryPath, { force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function isCanonicalDirectorOrigin(value: string): boolean {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)
|
||||
return (
|
||||
url.origin === value && (url.protocol === 'https:' || (url.protocol === 'http:' && loopback))
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
function relayRegionFailureReason(error: unknown): string {
|
||||
if (!(error instanceof Error)) {
|
||||
return 'request-failed'
|
||||
}
|
||||
}
|
||||
|
||||
function isProbeOriginForDirector(origin: string, directorUrl: string): boolean {
|
||||
return new URL(origin).hostname.endsWith(`.${new URL(directorUrl).hostname}`)
|
||||
if (error.message === 'relay region probes unavailable') {
|
||||
return 'probes-unavailable'
|
||||
}
|
||||
if (
|
||||
error.message === 'invalid relay director origin' ||
|
||||
error.message === 'invalid relay region catalog' ||
|
||||
error.message === 'invalid relay region entry' ||
|
||||
error.message === 'duplicate relay region' ||
|
||||
error.message === 'invalid relay probe origin' ||
|
||||
error.message === 'duplicate relay probe origin'
|
||||
) {
|
||||
return 'invalid-catalog'
|
||||
}
|
||||
const status = /^relay region catalog failed \(([1-5]\d\d)\)$/.exec(error.message)?.[1]
|
||||
return status ? `catalog-http-${status}` : 'request-failed'
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user