mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
- Directors with an edge read director/edge.json from the relay-control bucket through the shared flag channel: mode on/off/down and rolloutPermille (0..1000). Boot defaults are mode on and the env percent; every cohort decision reads the applied flags. Off empties the cohort and hosts leave normally; down also skips the export at once (reason mode_down). The permille bucket keeps every host of P percent in at P x 10 permille. A control change re-arms the evacuation sweep within 5 s. - A skipped export (120 s outage clock or mode down) removes the host's relay_devices rows before placing it, so revokes made on the edge are never undone; logs a count only. - Evacuation: 100 drains per tick with 10 in flight, each pass starting at a random point of the keys and wrapping around; pages stay 500 rows and an idle pass reads each edge row once. - Fence retries: 56 attempts (last about 25.5 h after the failure), 50 per sweep with 10 in flight; fence_abandoned now carries outcome, cell and epoch like fence_failed.
969 lines
38 KiB
TypeScript
969 lines
38 KiB
TypeScript
import { createAdaptorServer } from '@hono/node-server'
|
||
import { hasAdmissionCapacity, RELAY_ADMISSION_BUDGETS, cellPlacementCeiling } from '@orca-cloud/relay-contract/admission-budgets'
|
||
import { HostDataAuthSchema, parseRelayHostCapabilities, RELAY_ASSIGNMENT_LEASE_HEADER, RELAY_HOST_CAPABILITIES_HEADER } from '@orca-cloud/relay-contract/control-messages'
|
||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract/relay-regions'
|
||
import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract/protocol-limits'
|
||
import { RelayAuthSchema } from '@orca-cloud/relay-contract/credential-messages'
|
||
import type { IncomingMessage } from 'node:http'
|
||
import { randomUUID } from 'node:crypto'
|
||
import { performance } from 'node:perf_hooks'
|
||
import { WebSocketServer } from 'ws'
|
||
import type WebSocket from 'ws'
|
||
import type { RawData } from 'ws'
|
||
import { CellAdmitEffectiveWriter } from './cell/reservations/cell-admit-effective-writer.js'
|
||
import {
|
||
RESERVE_DEAD_MAN_BOOT_GRACE_MS,
|
||
ReserveDeadMan,
|
||
reserveDeadManWindowMs
|
||
} from './cell/reservations/cell-reserve-dead-man.js'
|
||
import { createRelayApp } from './app.js'
|
||
import { EDGE_CELL_ID, EdgeAssignments } from './edge-relay.js'
|
||
import { EdgeEvacuation } from './edge-relay-evacuation.js'
|
||
import type { EdgeControl } from './edge-relay-control.js'
|
||
import { createEdgeAdmin } from './edge-admin-client.js'
|
||
import { classifyAssignmentLease } from './assignment/assignment-lease.js'
|
||
import { defaultIntakePerSec, type CellFlags } from './cell/control-flags/cell-flags.js'
|
||
import { CellReserveBook } from './cell/reservations/cell-reserve-book.js'
|
||
import type { DemoteRequest, ReserveRequest } from './cell/reservations/cell-reserve-contract.js'
|
||
import { AssignmentLeaseShadow } from './assignment/assignment-lease-shadow.js'
|
||
import { RelayAssignmentStore } from './assignment/assignment-store.js'
|
||
import type { RelayConfig } from './config.js'
|
||
import { RelayCredentialStore } from './credentials/credential-store.js'
|
||
import {
|
||
readRelayDatabasePoolOldestWaitMs,
|
||
readRelayDatabasePoolPressure,
|
||
type RelayDatabase
|
||
} from './database/database.js'
|
||
import { HostSessionRegistry } from './connections/host-session-registry.js'
|
||
import { observeRelayDatabase } from './database/observed-relay-database.js'
|
||
import { RelayObservability } from './observability/relay-observability.js'
|
||
import { combineRegionalRehomeSafety } from './assignment/regional-rehome/regional-rehome-safety.js'
|
||
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './connections/relay-connection-ledger.js'
|
||
import { PlacementLoadBand } from './assignment/reserve-placement/placement-load-band.js'
|
||
import type { AppliedControlFlags } from './cell/control-flags/relay-control-flag-channel.js'
|
||
import { createRelayLocalReadiness } from './process-lifecycle/relay-local-readiness.js'
|
||
import { createRelayReadiness } from './process-lifecycle/relay-readiness.js'
|
||
import { createRelayTokenVerifier, readBearer } from './authentication/relay-token-verifier.js'
|
||
import { createStaleIfErrorJWKSet } from './authentication/stale-if-error-jwks.js'
|
||
import { closeRelayWebSocket } from './connections/relay-websocket-close.js'
|
||
import { ShadowDirectoryCompare } from './cell/seats/shadow-directory-compare.js'
|
||
import { type ShadowSeatPoller, startShadowSeatPoller } from './cell/seats/shadow-seat-directory.js'
|
||
import { CellReserveClient } from './cell/reservations/cell-reserve-client.js'
|
||
import { googleMetadataIdentityToken, reusedIdentityToken } from './authentication/google-metadata-identity-token.js'
|
||
import { ReserveAssignment } from './assignment/reserve-placement/reserve-assignment.js'
|
||
import { ReservePlacer } from './assignment/reserve-placement/reserve-placement.js'
|
||
import { ReserveRehome, ReserveRehomeSwitch } from './assignment/reserve-placement/reserve-rehome.js'
|
||
import {
|
||
reconcileReserveLedger,
|
||
RESERVE_LEDGER_RECONCILE_MS,
|
||
ReserveLedgerWriter
|
||
} from './assignment/reserve-placement/reserve-ledger-writer.js'
|
||
import { ProcessQueuedByteBudget } from './connections/splice-forwarder.js'
|
||
|
||
// Malformed upgrade paths must return client errors; an uncaught listener URIError kills the process.
|
||
function decodePathSegment(value: string): string | null {
|
||
try {
|
||
return decodeURIComponent(value)
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
// Healthy cells peak at 2 waiters; a pool’s worth queued past half the 2s timeout means hellos already fail. Count floor ignores lone slow waiters.
|
||
export const HOST_HELLO_SHED_OLDEST_WAIT_MS = 1_000
|
||
const HOST_HELLO_SHED_RETRY_AFTER_SECONDS = 2
|
||
// Well inside the flag channel's 5 s read, so a flip back starts re-registering within ~6 s.
|
||
export const RESERVE_MODE_WATCH_MS = 1_000
|
||
// Shipped desktops ignore it; it is for the ones that learn to read it.
|
||
const CELL_FULL_RETRY_AFTER_SECONDS = 2
|
||
|
||
function rejectUpgrade(
|
||
socket: NodeJS.WritableStream,
|
||
status: number,
|
||
message: string,
|
||
retryAfterSeconds?: number
|
||
): void {
|
||
const retryAfter = retryAfterSeconds === undefined ? '' : `Retry-After: ${retryAfterSeconds}\r\n`
|
||
socket.write(
|
||
`HTTP/1.1 ${status} ${message}\r\n${retryAfter}Connection: close\r\nContent-Length: 0\r\n\r\n`
|
||
)
|
||
if ('destroy' in socket && typeof socket.destroy === 'function') socket.destroy()
|
||
}
|
||
|
||
function noDelay(socket: WebSocket): void {
|
||
const transport = (socket as WebSocket & { _socket?: { setNoDelay: (enabled: boolean) => void } })
|
||
._socket
|
||
transport?.setNoDelay(true)
|
||
}
|
||
|
||
// Handle ws receiver errors or they crash the process; ws already closes the socket.
|
||
function guardSocketErrors(socket: WebSocket, kind: string): void {
|
||
socket.on('error', (error) => {
|
||
console.warn(`[orca-relay] ${kind} socket error: ${error.message}`)
|
||
})
|
||
}
|
||
|
||
function firstHeader(value: string | string[] | undefined): string | undefined {
|
||
return Array.isArray(value) ? value[0] : value
|
||
}
|
||
|
||
function admissionSource(request: IncomingMessage): string {
|
||
const forwarded = request.headers['x-forwarded-for']
|
||
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
|
||
.split(',')
|
||
.map((entry) => entry.trim())
|
||
.filter(Boolean)
|
||
// Google Front End appends client and load-balancer addresses after any
|
||
// caller-supplied values, so only the penultimate hop is trustworthy.
|
||
return chain.length >= 2 ? chain.at(-2)! : (request.socket.remoteAddress ?? 'unknown')
|
||
}
|
||
|
||
function firstPayload(raw: RawData, expectedType: string): unknown {
|
||
try {
|
||
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
|
||
if (parsed.type !== expectedType) return null
|
||
const { type: _type, ...rest } = parsed
|
||
return rest
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
export function createRelayServer(
|
||
config: RelayConfig,
|
||
database: RelayDatabase,
|
||
options: {
|
||
now?: () => number
|
||
random?: () => number
|
||
connectionLedgerLimits?: { hardCap: number; controlReserve: number }
|
||
cellIncarnation?: string
|
||
cellFlags?: () => AppliedControlFlags<CellFlags>
|
||
// Runtime edge control (director/edge.json); absent, the edge runs on its boot config.
|
||
edgeControl?: EdgeControl
|
||
} = {}
|
||
) {
|
||
const cellIncarnation = options.cellIncarnation ?? randomUUID()
|
||
const observability = new RelayObservability({
|
||
role: config.role,
|
||
cellId: config.cellId,
|
||
region: config.region ?? RELAY_DEFAULT_REGION
|
||
})
|
||
const observedDatabase = observeRelayDatabase(database, observability)
|
||
const controls = new WebSocketServer({
|
||
noServer: true,
|
||
clientTracking: false,
|
||
perMessageDeflate: false,
|
||
maxPayload: 1024 * 1024
|
||
})
|
||
const relayJwks = createStaleIfErrorJWKSet(new URL(config.jwksUrl))
|
||
const verifyRelayToken = createRelayTokenVerifier(config, relayJwks)
|
||
const store = new RelayCredentialStore(observedDatabase, options.now)
|
||
// Bound once the poller below exists; without one, no poll ever reads as dead.
|
||
const pollEvidence: { deadCellVerdict?: ShadowSeatPoller['deadCellVerdict'] } = {}
|
||
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
|
||
requireLiveCells: config.role === 'director',
|
||
placementLoadBand: config.role === 'director' ? new PlacementLoadBand(options.random) : undefined,
|
||
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
|
||
// The director runs in the database's region; only its rehome readers use this.
|
||
regionalRehomeDirectorRegion:
|
||
config.role === 'director' ? (config.region ?? RELAY_DEFAULT_REGION) : undefined,
|
||
recordControlRenewal: (durationMs, outcome) =>
|
||
observability.recordControlRenewal?.(durationMs, outcome),
|
||
deadCellVerdict: (cellId, now) => pollEvidence.deadCellVerdict?.(cellId, now) ?? null
|
||
})
|
||
// Relay edge: directors with an edge URL seed the edge cell and then answer the cohort there.
|
||
// Until the seed lands (or if it fails) the override answers nothing and today's path runs.
|
||
const edgeAdmin =
|
||
config.role === 'director' && config.edge?.adminSigningKey
|
||
? createEdgeAdmin({
|
||
cellUrl: config.edge.cellUrl,
|
||
signingKey: config.edge.adminSigningKey,
|
||
issuer: config.publicUrl
|
||
})
|
||
: undefined
|
||
const edgeConfig =
|
||
config.edge && options.edgeControl ? { ...config.edge, control: options.edgeControl.flags } : config.edge
|
||
const edgeAssignments =
|
||
config.role === 'director' && edgeConfig
|
||
? new EdgeAssignments(observedDatabase, edgeConfig, options.now, edgeAdmin)
|
||
: undefined
|
||
// Hosts that left the cohort are drained off the edge, rather than waiting out their lease.
|
||
const edgeEvacuation =
|
||
config.role === 'director' && edgeConfig && edgeAdmin
|
||
? new EdgeEvacuation(observedDatabase, edgeConfig, edgeAdmin, EDGE_CELL_ID, options.now, options.random)
|
||
: undefined
|
||
edgeEvacuation?.start()
|
||
// A rollback (mode off/down, a smaller share) starts draining within seconds.
|
||
options.edgeControl?.onChange(() => edgeEvacuation?.controlChanged())
|
||
if (edgeAssignments) {
|
||
void edgeAssignments.ensureEdgeCell().then(
|
||
() => console.warn(JSON.stringify({ event: 'orca_relay_edge_cell_ready' })),
|
||
(error: unknown) =>
|
||
console.error(
|
||
JSON.stringify({
|
||
event: 'orca_relay_edge_cell_seed_failed',
|
||
reason: error instanceof Error ? error.message : 'unknown'
|
||
})
|
||
)
|
||
)
|
||
}
|
||
const readiness = createRelayReadiness(observedDatabase, config.jwksUrl, {
|
||
jwksGraceMs: config.readinessJwksGraceMs,
|
||
sqlGraceMs: config.readinessSqlGraceMs,
|
||
observe: (observation) => observability.recordReadiness(observation),
|
||
observeGrace: (event) => observability.recordReadinessGrace(event)
|
||
})
|
||
const ready = readiness.check
|
||
// Only actual booking activity may feed reserve dead-man; configured-on directors missing map/credentials book nothing.
|
||
const placing = { on: false }
|
||
const shadowSeatPoller = startShadowSeatPoller(config, {
|
||
listCells: () => assignments.seatFeedCells(),
|
||
reserver: () => placing.on
|
||
})
|
||
pollEvidence.deadCellVerdict = shadowSeatPoller?.deadCellVerdict
|
||
const shadowCompare = shadowSeatPoller
|
||
? new ShadowDirectoryCompare(shadowSeatPoller.directory, options.now)
|
||
: undefined
|
||
const directorId = randomUUID()
|
||
const rehomeAudience = config.rehomeAudience
|
||
// Only a director that books writes rows, and only for the joins it booked.
|
||
const reserveLedger =
|
||
config.role === 'director' && shadowSeatPoller && config.reservePlacement === 'on'
|
||
? new ReserveLedgerWriter(observedDatabase, { now: options.now })
|
||
: undefined
|
||
const rehomePercent = config.reserveRehomePercent ?? 0
|
||
const rehomeSwitch =
|
||
config.role === 'director' && shadowSeatPoller && rehomeAudience && config.reservePlacement === 'on' && rehomePercent > 0
|
||
? new ReserveRehomeSwitch(async () => await assignments.regionRehomeSwitch(), options.now)
|
||
: undefined
|
||
rehomeSwitch?.start()
|
||
const reservePlacement =
|
||
config.role === 'director' &&
|
||
shadowSeatPoller &&
|
||
rehomeAudience &&
|
||
(config.reservePlacement ?? 'off') !== 'off'
|
||
? new ReserveAssignment({
|
||
mode: config.reservePlacement ?? 'off',
|
||
directory: shadowSeatPoller.directory,
|
||
cells: shadowSeatPoller.cells,
|
||
startedAt: shadowSeatPoller.startedAt,
|
||
placer: new ReservePlacer(options.now, options.random),
|
||
client: new CellReserveClient({
|
||
directorId,
|
||
identityToken: reusedIdentityToken(
|
||
() => googleMetadataIdentityToken(rehomeAudience),
|
||
options.now
|
||
)
|
||
}),
|
||
readRow: async (identity) => await assignments.hostWhereabouts(identity),
|
||
databaseBusy: () => readRelayDatabasePoolPressure(database).databasePoolWaiting > 0,
|
||
...(rehomeSwitch
|
||
? {
|
||
rehome: new ReserveRehome({
|
||
percent: rehomePercent,
|
||
switchOpen: (now) => rehomeSwitch.isOpen(now)
|
||
})
|
||
}
|
||
: {}),
|
||
now: options.now,
|
||
random: options.random
|
||
})
|
||
: undefined
|
||
placing.on = reservePlacement?.placementMode === 'on'
|
||
// Warn until dead-man flips back: non-placing directors otherwise strand reserve-cell hosts in memory.
|
||
const offWithReserveTimer =
|
||
config.role === 'director' && shadowSeatPoller && !placing.on
|
||
? setInterval(() => {
|
||
const { directory } = shadowSeatPoller
|
||
const cells = directory.cellIds().filter((cellId) => directory.admitModeOf(cellId) === 'reserve')
|
||
if (cells.length === 0) return
|
||
console.error(
|
||
JSON.stringify({
|
||
event: 'orca_relay_reserve_placement_off_with_reserve_cells',
|
||
cells,
|
||
placement: reservePlacement?.placementMode ?? 'off',
|
||
configured: config.reservePlacement ?? 'off'
|
||
})
|
||
)
|
||
}, 60_000)
|
||
: null
|
||
offWithReserveTimer?.unref()
|
||
if (reserveLedger && shadowSeatPoller) {
|
||
const directory = shadowSeatPoller.directory
|
||
directory.onBookedJoin = (join) => {
|
||
if (join.reservedBy !== directorId) return
|
||
reserveLedger.enqueue({
|
||
userId: join.userId,
|
||
relayHostId: join.relayHostId,
|
||
cellId: join.cellId,
|
||
epoch: join.epoch
|
||
})
|
||
}
|
||
directory.onOutranked = (seat) => reservePlacement?.supersedeOutranked(seat)
|
||
reserveLedger.start()
|
||
const reconcile = (): void => {
|
||
const now = (options.now ?? Date.now)()
|
||
void reconcileReserveLedger({
|
||
writer: reserveLedger,
|
||
seats: () =>
|
||
directory
|
||
.cellIds()
|
||
.filter((cellId) => directory.admitModeOf(cellId) === 'reserve')
|
||
.flatMap((cellId) =>
|
||
directory.seatsOnCell(cellId).map(({ userId, relayHostId, seat }) => ({
|
||
userId,
|
||
relayHostId,
|
||
cellId,
|
||
epoch: seat.epoch,
|
||
joinedAt: seat.joinedAt
|
||
}))
|
||
),
|
||
demote: (seat, row) => {
|
||
reservePlacement?.demoteSeat(seat, 'reconcile')
|
||
// The re-assign mints above the row, so the mirror can take it.
|
||
reservePlacement?.raiseEpochFloor(seat, row.epoch)
|
||
console.warn(
|
||
JSON.stringify({
|
||
event: 'orca_relay_reserve_ledger_seat_behind_row',
|
||
cellId: seat.cellId,
|
||
epoch: seat.epoch,
|
||
rowEpoch: row.epoch
|
||
})
|
||
)
|
||
},
|
||
now
|
||
})
|
||
.catch(() => console.warn('[orca-relay] reserve ledger reconcile deferred'))
|
||
.finally(schedule)
|
||
}
|
||
// Jittered, so five directors do not reconcile together.
|
||
const schedule = (): void => {
|
||
const random = options.random ?? Math.random
|
||
setTimeout(reconcile, RESERVE_LEDGER_RECONCILE_MS + random() * 60_000).unref?.()
|
||
}
|
||
// A restart loses the queue: the first run goes as soon as the map is complete.
|
||
const clock = options.now ?? Date.now
|
||
const bootedAt = clock()
|
||
const firstRun = setInterval(() => {
|
||
if (!directory.isComplete() && clock() - bootedAt < RESERVE_LEDGER_RECONCILE_MS) return
|
||
clearInterval(firstRun)
|
||
reconcile()
|
||
}, 1_000)
|
||
firstRun.unref?.()
|
||
}
|
||
const configuredConnectionLimits =
|
||
config.connectionHardCap === undefined
|
||
? null
|
||
: (options.connectionLedgerLimits ?? {
|
||
hardCap: config.connectionHardCap,
|
||
controlReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls
|
||
})
|
||
const connectionLedger =
|
||
configuredConnectionLimits === null
|
||
? null
|
||
: new RelayConnectionLedger(
|
||
configuredConnectionLimits.hardCap,
|
||
configuredConnectionLimits.controlReserve
|
||
)
|
||
const placementCeiling =
|
||
config.connectionHardCap === undefined || config.connectionUnobservedBound === undefined
|
||
? null
|
||
: cellPlacementCeiling(config.connectionHardCap, config.connectionUnobservedBound)
|
||
const reserveBook =
|
||
config.role === 'cell' && connectionLedger && placementCeiling !== null
|
||
? new CellReserveBook(
|
||
{
|
||
tryReserve: () => connectionLedger.tryReserveBooking(placementCeiling),
|
||
canReserve: () => connectionLedger.canReserveBooking(placementCeiling)
|
||
},
|
||
() =>
|
||
options.cellFlags?.().flags.intakePerSec ?? defaultIntakePerSec(config.region),
|
||
options.now
|
||
)
|
||
: null
|
||
const databaseShedding = (): boolean =>
|
||
readRelayDatabasePoolPressure(database).databasePoolWaiting >= config.databasePoolMax &&
|
||
readRelayDatabasePoolOldestWaitMs(database) >= HOST_HELLO_SHED_OLDEST_WAIT_MS
|
||
const reserveDeadMan = reserveBook
|
||
? new ReserveDeadMan(
|
||
config.cellId,
|
||
options.now,
|
||
reserveDeadManWindowMs(config.cellId),
|
||
undefined,
|
||
RESERVE_DEAD_MAN_BOOT_GRACE_MS
|
||
)
|
||
: null
|
||
const effectiveAdmitMode = (): 'db' | 'reserve' => {
|
||
const applied = options.cellFlags?.()
|
||
if (!applied) return 'db'
|
||
return reserveDeadMan ? reserveDeadMan.mode(applied) : applied.flags.admitMode
|
||
}
|
||
const queuedBytes = new ProcessQueuedByteBudget()
|
||
const sessions = new HostSessionRegistry(
|
||
config,
|
||
verifyRelayToken,
|
||
store,
|
||
assignments,
|
||
queuedBytes,
|
||
observability,
|
||
options.now,
|
||
options.random,
|
||
cellIncarnation,
|
||
new AssignmentLeaseShadow({
|
||
enabled: () => options.cellFlags?.().flags.ticketCheck === 'shadow',
|
||
key: config.assignmentSigningKey,
|
||
cellId: config.cellId
|
||
}),
|
||
reserveBook
|
||
? {
|
||
mode: effectiveAdmitMode,
|
||
directorContact: () => reserveDeadMan?.contact(),
|
||
reregisterInFlight: () => options.cellFlags?.().flags.reregisterInFlight,
|
||
phonePath: () => options.cellFlags?.().flags.phonePath ?? 'db',
|
||
ticketEnforce: () => options.cellFlags?.().flags.ticketCheck === 'enforce',
|
||
dryRunEnabled: () => options.cellFlags?.().flags.reserveDryRun === true,
|
||
databaseShedding: () => databaseShedding(),
|
||
book: reserveBook,
|
||
verifyLease: (input) =>
|
||
classifyAssignmentLease({
|
||
...input,
|
||
key: config.assignmentSigningKey,
|
||
cellId: config.cellId
|
||
})
|
||
}
|
||
: undefined
|
||
)
|
||
// Expire booking units even with no reserving directors; leaving reserve also re-registers memory controls.
|
||
let appliedAdmitMode = effectiveAdmitMode()
|
||
const admitEffective =
|
||
config.role === 'cell' && reserveBook
|
||
? new CellAdmitEffectiveWriter({
|
||
write: async (mode) =>
|
||
await assignments.recordCellAdmitEffective({ cellId: config.cellId, cellIncarnation, mode }),
|
||
claimTrip: async (slots) =>
|
||
await assignments.claimDeadManTripSlot({ cellId: config.cellId, cellIncarnation, slots }),
|
||
databaseBusy: () => readRelayDatabasePoolPressure(database).databasePoolWaiting > 0,
|
||
now: options.now ?? Date.now
|
||
})
|
||
: undefined
|
||
const reserveSweepTimer = reserveBook
|
||
? setInterval(() => {
|
||
reserveBook.sweep()
|
||
const admitMode = effectiveAdmitMode()
|
||
if (appliedAdmitMode === 'reserve' && admitMode !== 'reserve') {
|
||
reserveBook.clear()
|
||
sessions.reregisterMemoryControls()
|
||
}
|
||
appliedAdmitMode = admitMode
|
||
// A booted cell runs db defaults until its first flag read, so it writes nothing until then.
|
||
const held = reserveDeadMan?.pendingTrip()
|
||
admitEffective?.tick(
|
||
admitMode,
|
||
options.cellFlags?.().flags.admitMode === 'reserve',
|
||
held ? { slots: held.slots, permit: () => reserveDeadMan?.permit(held.generation) } : null
|
||
)
|
||
}, RESERVE_MODE_WATCH_MS)
|
||
: null
|
||
reserveSweepTimer?.unref()
|
||
const app = createRelayApp(config, {
|
||
store,
|
||
assignments,
|
||
drain: (graceMs, options) => sessions.drain(graceMs, options ?? {}),
|
||
drainHost: (input) => sessions.drainHost(input),
|
||
idleRehome: (input) => {
|
||
const now = (options.now ?? Date.now)()
|
||
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
|
||
return Promise.resolve({ outcome: 'deferred', reason: 'director-safety-stale' })
|
||
}
|
||
return sessions.idleRehome(input,
|
||
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
|
||
input.directorSafety,
|
||
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
|
||
), input.cohortPercent),
|
||
() => assignments.reconcileIdleRegionalRehome(input)
|
||
)
|
||
},
|
||
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
|
||
cellIncarnation,
|
||
cellSeatFeed: (sinceSeq) => sessions.seatFeed(sinceSeq),
|
||
cellFlags: options.cellFlags,
|
||
...(reserveBook && placementCeiling !== null
|
||
? {
|
||
cellReserve: (request: ReserveRequest) => sessions.reserve(request),
|
||
cellReserverPoll: () => reserveDeadMan?.contact(),
|
||
// Report reserve until every memory control has a lease; flag workflow must wait before recording db and resuming sweeps.
|
||
cellAdmitModeRaw: effectiveAdmitMode,
|
||
cellAdmitModeEffective: (): 'db' | 'reserve' =>
|
||
effectiveAdmitMode() === 'reserve' || sessions.reregistrationPending() > 0
|
||
? 'reserve'
|
||
: 'db',
|
||
cellPhonePathEffective: () => sessions.phonePathEffective(),
|
||
cellDemote: (request: DemoteRequest) => sessions.demote(request),
|
||
cellReserveCounts: () => sessions.reserveCounts(),
|
||
cellPlacementCeiling: placementCeiling
|
||
}
|
||
: {}),
|
||
isDraining: () => sessions.isDraining(),
|
||
runtimeCounts: () => runtimeCounts(),
|
||
regionalRehomeSafetySnapshot: () => ({
|
||
...observability.regionalRehomeRuntimeSafety(),
|
||
...readRelayDatabasePoolPressure(database)
|
||
}),
|
||
ready,
|
||
readinessDegradation: () => readiness.degradedDependencies(),
|
||
readinessLocal: () => options.cellFlags?.().flags.readinessLocal ?? false,
|
||
relayJwks,
|
||
localReadiness: createRelayLocalReadiness({
|
||
listening: () => server.listening,
|
||
keys: relayJwks,
|
||
failingDependencies: () => readiness.failingDependencies()
|
||
}),
|
||
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
|
||
recordAssignmentRejectionReason: (lane, reason) =>
|
||
observability.recordAssignmentRejectionReason?.(lane, reason),
|
||
recordDrainReturnRetryAfter: (seconds) => observability.recordDrainReturnRetryAfter?.(seconds),
|
||
recordAdmissionServiceMs: (lane, durationMs) =>
|
||
observability.recordAdmissionServiceMs?.(lane, durationMs),
|
||
recordStickySlot: (sample) => observability.recordStickySlot?.(sample),
|
||
recordAssignmentUnavailable: (cause) => observability.recordAssignmentUnavailable?.(cause),
|
||
recordRegionRequest: (region) => observability.recordRegionRequest?.(region),
|
||
shadowSeats: shadowSeatPoller?.directory,
|
||
reservePlacement,
|
||
edgeAssignments,
|
||
compareShadowSeats: shadowCompare
|
||
? (route, identity, answer) => {
|
||
shadowCompare.compare(route, identity, answer)
|
||
}
|
||
: undefined,
|
||
recordRegionSelection: (input) => observability.recordRegionSelection?.(input)
|
||
})
|
||
const observedFetch: typeof app.fetch = async (...args) => {
|
||
const startedAt = performance.now()
|
||
try {
|
||
return await app.fetch(...args)
|
||
} finally {
|
||
observability.recordHttp(performance.now() - startedAt)
|
||
}
|
||
}
|
||
const server = createAdaptorServer({ fetch: observedFetch })
|
||
const clients = new WebSocketServer({
|
||
noServer: true,
|
||
clientTracking: false,
|
||
perMessageDeflate: false,
|
||
maxPayload: RELAY_PROTOCOL_LIMITS.maxFrameBytes
|
||
})
|
||
const dataSockets = new WebSocketServer({
|
||
noServer: true,
|
||
clientTracking: false,
|
||
perMessageDeflate: false,
|
||
maxPayload: RELAY_PROTOCOL_LIMITS.maxFrameBytes
|
||
})
|
||
let preAuthConnections = 0
|
||
let totalConnections = 0
|
||
const preAuthBySource = new Map<string, number>()
|
||
const preAuthAttemptsBySource = new Map<string, { windowStartedAt: number; count: number }>()
|
||
|
||
const admit = (source: string): boolean => {
|
||
const now = Date.now()
|
||
const currentWindow = preAuthAttemptsBySource.get(source)
|
||
const attempts =
|
||
!currentWindow || now - currentWindow.windowStartedAt >= 60_000
|
||
? { windowStartedAt: now, count: 0 }
|
||
: currentWindow
|
||
const sourceCount = preAuthBySource.get(source) ?? 0
|
||
if (
|
||
!hasAdmissionCapacity({
|
||
totalRequests:
|
||
connectionLedger?.counts().physicalConnections ?? totalConnections,
|
||
preAuthConnections,
|
||
sourcePreAuthConnections: sourceCount,
|
||
totalRequestCeiling:
|
||
configuredConnectionLimits === null
|
||
? undefined
|
||
: configuredConnectionLimits.hardCap - configuredConnectionLimits.controlReserve
|
||
}) ||
|
||
attempts.count >= RELAY_ADMISSION_BUDGETS.maxPreAuthAttemptsPerSourcePerMinute
|
||
) {
|
||
return false
|
||
}
|
||
attempts.count++
|
||
preAuthAttemptsBySource.delete(source)
|
||
preAuthAttemptsBySource.set(source, attempts)
|
||
// A bounded LRU keeps source churn from becoming its own memory attack.
|
||
if (preAuthAttemptsBySource.size > 4_096) {
|
||
preAuthAttemptsBySource.delete(preAuthAttemptsBySource.keys().next().value!)
|
||
}
|
||
preAuthConnections++
|
||
preAuthBySource.set(source, sourceCount + 1)
|
||
return true
|
||
}
|
||
const authenticated = (source: string): void => {
|
||
preAuthConnections = Math.max(0, preAuthConnections - 1)
|
||
const next = (preAuthBySource.get(source) ?? 1) - 1
|
||
if (next <= 0) preAuthBySource.delete(source)
|
||
else preAuthBySource.set(source, next)
|
||
}
|
||
|
||
const trackConnection = (
|
||
socket: WebSocket,
|
||
upgrade: RelayConnectionUpgrade | null
|
||
): void => {
|
||
if (upgrade) {
|
||
upgrade.promote(socket)
|
||
return
|
||
}
|
||
totalConnections++
|
||
socket.once('close', () => {
|
||
totalConnections = Math.max(0, totalConnections - 1)
|
||
})
|
||
}
|
||
|
||
const awaitFirstFrame = (
|
||
socket: WebSocket,
|
||
source: string,
|
||
callback: (raw: RawData) => Promise<void>
|
||
): void => {
|
||
let finished = false
|
||
const timer = setTimeout(() => {
|
||
if (finished) return
|
||
finished = true
|
||
authenticated(source)
|
||
observability.recordAuth(false)
|
||
closeRelayWebSocket(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'first frame timeout')
|
||
}, RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs)
|
||
socket.once('message', (raw, binary) => {
|
||
if (finished) return
|
||
finished = true
|
||
clearTimeout(timer)
|
||
authenticated(source)
|
||
if (binary) {
|
||
observability.recordAuth(false)
|
||
closeRelayWebSocket(
|
||
socket,
|
||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||
'first frame must be text'
|
||
)
|
||
return
|
||
}
|
||
void callback(raw).catch((error: unknown) => {
|
||
console.warn(
|
||
'[orca-relay] first frame handler failed',
|
||
error instanceof Error ? error.message : ''
|
||
)
|
||
closeRelayWebSocket(
|
||
socket,
|
||
RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
|
||
'relay temporarily unavailable'
|
||
)
|
||
})
|
||
})
|
||
socket.once('close', () => {
|
||
if (!finished) {
|
||
finished = true
|
||
clearTimeout(timer)
|
||
authenticated(source)
|
||
}
|
||
})
|
||
}
|
||
|
||
server.on('upgrade', (request, socket, head) => {
|
||
const url = new URL(request.url ?? '/', config.publicUrl)
|
||
const source = admissionSource(request)
|
||
if (url.search) {
|
||
rejectUpgrade(socket, 400, 'Bad Request')
|
||
return
|
||
}
|
||
if (url.pathname.startsWith('/v1/connect/')) {
|
||
const hostId = decodePathSegment(url.pathname.slice('/v1/connect/'.length))
|
||
if (hostId === null || !/^[A-Za-z0-9_-]{16}$/.test(hostId)) {
|
||
rejectUpgrade(socket, 429, 'Too Many Requests')
|
||
return
|
||
}
|
||
const phoneAdmission = connectionLedger?.tryReservePhone() ?? null
|
||
if (connectionLedger && !phoneAdmission) {
|
||
rejectUpgrade(socket, 503, 'Service Unavailable')
|
||
return
|
||
}
|
||
if (!admit(source)) {
|
||
phoneAdmission?.upgrade.release()
|
||
phoneAdmission?.hostData.release()
|
||
rejectUpgrade(socket, 429, 'Too Many Requests')
|
||
return
|
||
}
|
||
const releasePhoneUpgrade = (): void => {
|
||
authenticated(source)
|
||
phoneAdmission?.upgrade.release()
|
||
phoneAdmission?.hostData.release()
|
||
}
|
||
socket.once('close', releasePhoneUpgrade)
|
||
try {
|
||
clients.handleUpgrade(request, socket, head, (webSocket) => {
|
||
socket.off('close', releasePhoneUpgrade)
|
||
trackConnection(webSocket, phoneAdmission?.upgrade ?? null)
|
||
guardSocketErrors(webSocket, 'client')
|
||
if (phoneAdmission) {
|
||
webSocket.once('close', () => phoneAdmission.hostData.release())
|
||
}
|
||
noDelay(webSocket)
|
||
awaitFirstFrame(webSocket, source, async (raw) => {
|
||
const auth = RelayAuthSchema.safeParse(firstPayload(raw, 'relay-auth'))
|
||
if (!auth.success) {
|
||
phoneAdmission?.hostData.release()
|
||
observability.recordAuth(false)
|
||
webSocket.send(
|
||
JSON.stringify({ type: 'relay-hello', ok: false, code: RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL })
|
||
)
|
||
closeRelayWebSocket(
|
||
webSocket,
|
||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||
'invalid relay auth'
|
||
)
|
||
return
|
||
}
|
||
if (config.role === 'director') {
|
||
const invite = await store.resolveInviteForMove(hostId, auth.data.credential)
|
||
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
|
||
// Released combined-service invites gain their first durable cell assignment here.
|
||
// A host on the relay edge is answered there; the liveness rule would re-place it.
|
||
const assignment = identity
|
||
? ((await edgeAssignments?.edgeAssignment(identity)) ??
|
||
(await assignments.resolve(identity)) ??
|
||
(await assignments.assign(identity)))
|
||
: null
|
||
if (!invite || !assignment) {
|
||
phoneAdmission?.hostData.release()
|
||
observability.recordAuth(false)
|
||
webSocket.send(
|
||
JSON.stringify({
|
||
type: 'relay-hello',
|
||
ok: false,
|
||
code: RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL
|
||
})
|
||
)
|
||
closeRelayWebSocket(
|
||
webSocket,
|
||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||
'invalid invite'
|
||
)
|
||
return
|
||
}
|
||
phoneAdmission?.hostData.release()
|
||
observability.recordAuth(true)
|
||
webSocket.send(
|
||
JSON.stringify({
|
||
type: 'relay-moved',
|
||
v: 1,
|
||
cellUrl: assignment.cellUrl,
|
||
assignmentEpoch: assignment.assignmentEpoch
|
||
})
|
||
)
|
||
closeRelayWebSocket(webSocket, RELAY_CLOSE_CODE.DRAINING, 'connect to assigned cell')
|
||
return
|
||
}
|
||
await sessions.acceptClient(
|
||
webSocket,
|
||
hostId,
|
||
auth.data.credential,
|
||
phoneAdmission?.hostData
|
||
)
|
||
})
|
||
})
|
||
} catch {
|
||
socket.off('close', releasePhoneUpgrade)
|
||
releasePhoneUpgrade()
|
||
socket.destroy()
|
||
}
|
||
return
|
||
}
|
||
if (url.pathname.startsWith('/v1/host/data/')) {
|
||
if (config.role === 'director') {
|
||
rejectUpgrade(socket, 404, 'Not Found')
|
||
return
|
||
}
|
||
const connId = decodePathSegment(url.pathname.slice('/v1/host/data/'.length))
|
||
if (!connId || connId.length > 128) {
|
||
rejectUpgrade(socket, 429, 'Too Many Requests')
|
||
return
|
||
}
|
||
const dataUpgrade = connectionLedger?.tryReserveHostData(connId) ?? null
|
||
if (connectionLedger && !dataUpgrade) {
|
||
rejectUpgrade(socket, 503, 'Service Unavailable')
|
||
return
|
||
}
|
||
if (!admit(source)) {
|
||
dataUpgrade?.release()
|
||
rejectUpgrade(socket, 429, 'Too Many Requests')
|
||
return
|
||
}
|
||
const releaseDataUpgrade = (): void => {
|
||
authenticated(source)
|
||
dataUpgrade?.release()
|
||
}
|
||
socket.once('close', releaseDataUpgrade)
|
||
try {
|
||
dataSockets.handleUpgrade(request, socket, head, (webSocket) => {
|
||
socket.off('close', releaseDataUpgrade)
|
||
trackConnection(webSocket, dataUpgrade)
|
||
guardSocketErrors(webSocket, 'host-data')
|
||
noDelay(webSocket)
|
||
awaitFirstFrame(webSocket, source, async (raw) => {
|
||
const auth = HostDataAuthSchema.safeParse(firstPayload(raw, 'host-data-auth'))
|
||
if (!auth.success) {
|
||
observability.recordAuth(false)
|
||
closeRelayWebSocket(
|
||
webSocket,
|
||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||
'invalid host data auth'
|
||
)
|
||
return
|
||
}
|
||
const accepted = await sessions.acceptHostData(
|
||
webSocket,
|
||
connId,
|
||
auth.data.connTicket,
|
||
auth.data.generation
|
||
)
|
||
if (accepted) dataUpgrade?.commitHostData()
|
||
})
|
||
})
|
||
} catch {
|
||
socket.off('close', releaseDataUpgrade)
|
||
releaseDataUpgrade()
|
||
socket.destroy()
|
||
}
|
||
return
|
||
}
|
||
if (url.pathname !== '/v1/host/control') {
|
||
rejectUpgrade(socket, 404, 'Not Found')
|
||
return
|
||
}
|
||
if (config.role === 'director') {
|
||
rejectUpgrade(socket, 404, 'Not Found')
|
||
return
|
||
}
|
||
const bearer = readBearer(request.headers.authorization)
|
||
if (!bearer) {
|
||
observability.recordAuth(false)
|
||
rejectUpgrade(socket, 401, 'Unauthorized')
|
||
return
|
||
}
|
||
void verifyRelayToken(bearer).then((identity) => {
|
||
if (socket.destroyed) return
|
||
if (!identity) {
|
||
observability.recordAuth(false)
|
||
rejectUpgrade(socket, 401, 'Unauthorized')
|
||
return
|
||
}
|
||
const isRebind = sessions.hasActiveControl({
|
||
userId: identity.sub,
|
||
relayHostId: identity.relayHostId
|
||
})
|
||
const reserveAdmission = sessions.reserveAdmissionFor({
|
||
userId: identity.sub,
|
||
relayHostId: identity.relayHostId
|
||
})
|
||
// Shed before several 2s pool waits; shipped desktops use ordinary connect-error backoff and ignore Retry-After.
|
||
// Preserve live-control rebinds: they rotate leases rather than reconnect.
|
||
if (!isRebind && reserveAdmission === null && databaseShedding()) {
|
||
observability.recordHostHelloShed()
|
||
rejectUpgrade(socket, 503, 'Service Unavailable', HOST_HELLO_SHED_RETRY_AFTER_SECONDS)
|
||
return
|
||
}
|
||
// Booked upgrades can reach hard cap; return booking unit only after upgrade reserves its own, counting host once.
|
||
const controlUpgrade =
|
||
connectionLedger?.tryReserveControl(isRebind || reserveAdmission === 'booked') ?? null
|
||
if (controlUpgrade && reserveAdmission === 'booked') {
|
||
reserveBook?.handOff(identity.sub, identity.relayHostId)
|
||
}
|
||
if (
|
||
(connectionLedger && !controlUpgrade) ||
|
||
(!connectionLedger && totalConnections >= RELAY_ADMISSION_BUDGETS.cloudRunConcurrency)
|
||
) {
|
||
rejectUpgrade(
|
||
socket,
|
||
connectionLedger ? 503 : 429,
|
||
connectionLedger ? 'Service Unavailable' : 'Too Many Requests',
|
||
connectionLedger && sessions.inReserveMode() ? CELL_FULL_RETRY_AFTER_SECONDS : undefined
|
||
)
|
||
return
|
||
}
|
||
// Register release before other work can throw; outer-catch socket destruction must return the connection unit.
|
||
const releaseControlUpgrade = (): void => controlUpgrade?.release()
|
||
socket.once('close', releaseControlUpgrade)
|
||
observability.recordAuth(true)
|
||
try {
|
||
controls.handleUpgrade(request, socket, head, (webSocket) => {
|
||
socket.off('close', releaseControlUpgrade)
|
||
trackConnection(webSocket, controlUpgrade)
|
||
guardSocketErrors(webSocket, 'control')
|
||
noDelay(webSocket)
|
||
sessions.acceptControl(
|
||
webSocket,
|
||
identity,
|
||
controlUpgrade?.inclusionWatermark,
|
||
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER]),
|
||
firstHeader(request.headers[RELAY_ASSIGNMENT_LEASE_HEADER])
|
||
)
|
||
})
|
||
} catch {
|
||
socket.off('close', releaseControlUpgrade)
|
||
releaseControlUpgrade()
|
||
socket.destroy()
|
||
}
|
||
}).catch((error: unknown) => {
|
||
// A throw in the upgrade handling above must cost this socket, not the process.
|
||
console.warn(
|
||
`[orca-relay] control upgrade failed: ${error instanceof Error ? error.message : 'unknown'}`
|
||
)
|
||
socket.destroy()
|
||
})
|
||
})
|
||
|
||
server.on('close', () => {
|
||
controls.close()
|
||
clients.close()
|
||
dataSockets.close()
|
||
store.resumeAudit.stop()
|
||
rehomeSwitch?.stop()
|
||
edgeEvacuation?.stop()
|
||
})
|
||
const resumeAuditDropped = (): number => {
|
||
const drops = store.resumeAudit.drops()
|
||
return drops.overflow + drops.aged
|
||
}
|
||
const runtimeCounts = () => {
|
||
const ledgerCounts = connectionLedger?.counts()
|
||
return {
|
||
totalConnections: ledgerCounts?.physicalConnections ?? totalConnections,
|
||
preAuthConnections,
|
||
...sessions.runtimeCounts(),
|
||
queuedBytes: queuedBytes.current(),
|
||
...(config.role === 'cell' ? { resumeAuditDropped: resumeAuditDropped() } : {}),
|
||
...(ledgerCounts
|
||
? {
|
||
inFlightConnections: ledgerCounts.inFlightConnections,
|
||
reservedConnectionUnits: ledgerCounts.reservedConnectionUnits,
|
||
enforcedConnectionUnits: ledgerCounts.enforcedConnectionUnits
|
||
}
|
||
: {})
|
||
}
|
||
}
|
||
const connectionSnapshot = () => connectionLedger?.snapshot()
|
||
return {
|
||
server,
|
||
sessions,
|
||
store,
|
||
assignments,
|
||
queuedBytes,
|
||
observability,
|
||
runtimeCounts,
|
||
connectionSnapshot,
|
||
ready,
|
||
cellIncarnation,
|
||
shadowSeatPoller,
|
||
directorId,
|
||
reservePlacement
|
||
}
|
||
}
|