mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
* relay-edge: keep valid desktops on JWKS outages, deep object probe, cohort-scaled alerts - auth: tell 'token invalid' from 'keys unavailable'. An auth-refresh that cannot get keys keeps the control and its session unchanged (metric auth_refresh:keys_unavailable); the exp+grace close still applies. A control upgrade without keys gets 503 instead of 401 (shipped desktops treat both as a retried connect failure). The last good JWKS is kept in the colo's Cache API (24 h max age) as a fallback for cold isolates; a stored set lacking the token's key never reads as invalid. - admin export ignores closing controls, like drain, so a just-drained host is not busy. - GET /__health/object: canary HostRelay reads its SQLite (SELECT 1), no writes; per-source limited; 503 on failure or after 3 s. The alerts Worker probes it (object-probe, > 2 s alerts). - alerts: MIN_VOLUME stays configurable, reconnect-surge rule (control hellos >= max(3x previous, RECONNECT_SURGE_MIN)), optional cloudflare-errors rule from the GraphQL Analytics API (CF_GRAPHQL_TOKEN). Rules moved to alerts-rules.ts. - metrics: heartbeat events carry their 1-in-10 sampling factor as their last double. * relay-edge: persisted JWKS trusted for 30 min, as the cells' stale window * relay-edge alerts: opt-in reuse of the Analytics Engine read token for the GraphQL error rule
63 lines
2.9 KiB
TypeScript
63 lines
2.9 KiB
TypeScript
// Limits on sockets that have not authenticated yet (phone connects, host data sockets), and on
|
|
// the unauthenticated object health probe (health.ts). Desktops
|
|
// authenticate with a relay JWT before the upgrade; phones and data sockets prove a credential in
|
|
// their first frame, so until then they are only bounded here.
|
|
import { RELAY_CLOSE_CODE } from '../../../packages/relay-contract/src/close-codes'
|
|
import { RELAY_PROTOCOL_LIMITS } from '../../../packages/relay-contract/src/protocol-limits'
|
|
import type { Env } from './env'
|
|
import { metric, requestPlace } from './metrics'
|
|
|
|
// The caller's address as Cloudflare saw it (clients cannot set this header). IPv6 callers are
|
|
// keyed by their /64, which one subscriber usually holds whole.
|
|
export function sourceKey(request: Request): string {
|
|
const ip = request.headers.get('cf-connecting-ip')?.trim() ?? ''
|
|
if (!ip.includes(':')) { return ip || 'unknown' }
|
|
const [head, tail = ''] = ip.split('::')
|
|
const left = head ? head.split(':') : []
|
|
const right = ip.includes('::') && tail ? tail.split(':') : []
|
|
const groups = ip.includes('::') ? [...left, ...new Array(Math.max(0, 8 - left.length - right.length)).fill('0'), ...right] : left
|
|
return `${groups
|
|
.slice(0, 4)
|
|
.map((g) => g.toLowerCase().replace(/^0+(?=.)/, ''))
|
|
.join(':') }::/64`
|
|
}
|
|
|
|
// Counted per source per colo by the Rate Limiting binding. Fails open: a limiter outage must not
|
|
// refuse live users. Without a binding (local spike runs) nothing is limited.
|
|
export async function admitPreAuth(env: Env, request: Request, kind: 'phone' | 'data' | 'health'): Promise<boolean> {
|
|
if (!env.PREAUTH_LIMITER) { return true }
|
|
try {
|
|
const { success } = await env.PREAUTH_LIMITER.limit({ key: `${kind}:${sourceKey(request)}` })
|
|
if (!success) { metric(env, 'preauth_limited', 1, kind, [], '', requestPlace(request)) }
|
|
return success
|
|
} catch {
|
|
return true
|
|
}
|
|
}
|
|
|
|
// A phone dialled the edge for a host the director never pushed (or one already wiped): answer
|
|
// from the Worker without creating any object storage. WRONG_CELL sends the phone back to the
|
|
// director, as a cell does for a host it does not hold; it never reads as a revoked credential.
|
|
export function notOnEdge(): Response {
|
|
const [client, server] = Object.values(new WebSocketPair()) as [WebSocket, WebSocket]
|
|
server.accept()
|
|
const timer = setTimeout(() => {
|
|
try {
|
|
server.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'first frame timeout')
|
|
} catch {}
|
|
}, RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs)
|
|
server.addEventListener(
|
|
'message',
|
|
() => {
|
|
clearTimeout(timer)
|
|
try {
|
|
server.send(JSON.stringify({ type: 'relay-hello', ok: false, code: RELAY_CLOSE_CODE.WRONG_CELL }))
|
|
server.close(RELAY_CLOSE_CODE.WRONG_CELL, 'host not on this cell')
|
|
} catch {}
|
|
},
|
|
{ once: true }
|
|
)
|
|
server.addEventListener('close', () => clearTimeout(timer))
|
|
return new Response(null, { status: 101, webSocket: client })
|
|
}
|