Files
orca/cloud/apps/relay-edge/src/preauth.ts
Neil c362cb183e relay-edge: keep valid desktops through JWKS outages, deep object probe, cohort-scaled alerts (#39772)
* relay-edge: keep valid desktops on JWKS outages, deep object probe, cohort-scaled alerts

- auth: tell 'token invalid' from 'keys unavailable'. An auth-refresh that cannot get keys keeps
  the control and its session unchanged (metric auth_refresh:keys_unavailable); the exp+grace
  close still applies. A control upgrade without keys gets 503 instead of 401 (shipped desktops
  treat both as a retried connect failure). The last good JWKS is kept in the colo's Cache API
  (24 h max age) as a fallback for cold isolates; a stored set lacking the token's key never
  reads as invalid.
- admin export ignores closing controls, like drain, so a just-drained host is not busy.
- GET /__health/object: canary HostRelay reads its SQLite (SELECT 1), no writes; per-source
  limited; 503 on failure or after 3 s. The alerts Worker probes it (object-probe, > 2 s alerts).
- alerts: MIN_VOLUME stays configurable, reconnect-surge rule (control hellos >= max(3x previous,
  RECONNECT_SURGE_MIN)), optional cloudflare-errors rule from the GraphQL Analytics API
  (CF_GRAPHQL_TOKEN). Rules moved to alerts-rules.ts.
- metrics: heartbeat events carry their 1-in-10 sampling factor as their last double.

* relay-edge: persisted JWKS trusted for 30 min, as the cells' stale window

* relay-edge alerts: opt-in reuse of the Analytics Engine read token for the GraphQL error rule
2026-11-07 23:38:28 -08:00

63 lines
2.9 KiB
TypeScript

// Limits on sockets that have not authenticated yet (phone connects, host data sockets), and on
// the unauthenticated object health probe (health.ts). Desktops
// authenticate with a relay JWT before the upgrade; phones and data sockets prove a credential in
// their first frame, so until then they are only bounded here.
import { RELAY_CLOSE_CODE } from '../../../packages/relay-contract/src/close-codes'
import { RELAY_PROTOCOL_LIMITS } from '../../../packages/relay-contract/src/protocol-limits'
import type { Env } from './env'
import { metric, requestPlace } from './metrics'
// The caller's address as Cloudflare saw it (clients cannot set this header). IPv6 callers are
// keyed by their /64, which one subscriber usually holds whole.
export function sourceKey(request: Request): string {
const ip = request.headers.get('cf-connecting-ip')?.trim() ?? ''
if (!ip.includes(':')) { return ip || 'unknown' }
const [head, tail = ''] = ip.split('::')
const left = head ? head.split(':') : []
const right = ip.includes('::') && tail ? tail.split(':') : []
const groups = ip.includes('::') ? [...left, ...new Array(Math.max(0, 8 - left.length - right.length)).fill('0'), ...right] : left
return `${groups
.slice(0, 4)
.map((g) => g.toLowerCase().replace(/^0+(?=.)/, ''))
.join(':') }::/64`
}
// Counted per source per colo by the Rate Limiting binding. Fails open: a limiter outage must not
// refuse live users. Without a binding (local spike runs) nothing is limited.
export async function admitPreAuth(env: Env, request: Request, kind: 'phone' | 'data' | 'health'): Promise<boolean> {
if (!env.PREAUTH_LIMITER) { return true }
try {
const { success } = await env.PREAUTH_LIMITER.limit({ key: `${kind}:${sourceKey(request)}` })
if (!success) { metric(env, 'preauth_limited', 1, kind, [], '', requestPlace(request)) }
return success
} catch {
return true
}
}
// A phone dialled the edge for a host the director never pushed (or one already wiped): answer
// from the Worker without creating any object storage. WRONG_CELL sends the phone back to the
// director, as a cell does for a host it does not hold; it never reads as a revoked credential.
export function notOnEdge(): Response {
const [client, server] = Object.values(new WebSocketPair()) as [WebSocket, WebSocket]
server.accept()
const timer = setTimeout(() => {
try {
server.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'first frame timeout')
} catch {}
}, RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs)
server.addEventListener(
'message',
() => {
clearTimeout(timer)
try {
server.send(JSON.stringify({ type: 'relay-hello', ok: false, code: RELAY_CLOSE_CODE.WRONG_CELL }))
server.close(RELAY_CLOSE_CODE.WRONG_CELL, 'host not on this cell')
} catch {}
},
{ once: true }
)
server.addEventListener('close', () => clearTimeout(timer))
return new Response(null, { status: 101, webSocket: client })
}