Files
archived-orca/cloud/apps/relay/src/app.ts
Neil 568567f7ad relay: director edge cohort for the relay edge (dark by default) (#39362)
* feat(relay): director edge override for moving hosts to the relay edge (dark by default)

Directors with ORCA_RELAY_EDGE_CELL_URL seed an existing-only 'edge' cell (registered
with the admission selector so deploy workflows carry it) and answer a host cohort
(hash bucket, allow/deny lists, region scope) on the edge before the reserve and
database lanes. The epoch rises only on a cell change; a host still holding units on
its old cell is told to retry. Phone resolve and the invite move answer edge rows
without moving anything; old cells answer WRONG_CELL to phones of moved hosts.
Unset URL keeps today's behaviour.

* feat(relay): director pushes host data to the relay edge on a move and takes it back before any move away

With ORCA_RELAY_EDGE_ADMIN_SIGNING_KEY set (its own key, never the assignment lease key), a move
onto the edge pushes the host's relay_devices and a location hint under the row lock and commits
only after the edge accepts; an edge holding a newer epoch is re-minted above once. Before any
lane may re-place an edge row, the director exports (refused while the desktop is connected; the
edge freezes the host), replaces the host's Postgres devices, places it as today, then fences the
edge with the new cell and epoch. Idle regional rehome skips edge rows. Unset key: unchanged.

* fix(relay): a deferred edge move keeps the host on today's path instead of a 503 loop

A host still holding old-cell units (a pairing invite holds one for its 10 min TTL) or whose push
the edge did not take now stays where it is and moves on a later ask. 503 stays for a held row
lock and for a desktop still connected to the edge on the way out.

* relay: director refuses an edge URL without the edge admin key

The deployed edge keeps host data in its objects; without the key the director could
not push it and hosts would land on an edge holding none of their devices.

* relay: edge cohort by account (ORCA_RELAY_EDGE_ALLOW_USERS)

Every host of a listed account (relay token sub) joins the edge cohort, in any region;
a denied host still stays off. For the team's own accounts before any share rollout.

* relay: edge modules follow main's relay module moves

* relay: satisfy the changed-lines lint gate in the edge cohort code

Braces on every if, T[] arrays, type aliases, one ternary, and a justified max-lines header on
edge-relay.ts. No behaviour change: relay unit 845 passed.

* relay: move edge cohort config and row helpers to edge-relay-cohort.ts

Keeps edge-relay.ts under the 300-line limit without a max-lines disable. Code moved verbatim;
edge-relay.ts re-exports what other modules import. Relay unit 845 passed.
2026-11-07 18:40:06 -08:00

2559 lines
103 KiB
TypeScript

import { AssignmentRequestSchema, ResolveRequestSchema } from '@orca-cloud/relay-contract/director-messages'
import { IdleRegionalRehomeRequestSchema, type IdleRegionalRehomeRequest, type IdleRegionalRehomeResult } from '@orca-cloud/relay-contract/idle-regional-rehome'
import type { RegionCorrectionResponse } from '@orca-cloud/relay-contract/region-correction'
import { isRelayCellConnectionHardCap, RELAY_ADMISSION_BUDGETS, RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND, cellPlacementCeiling, relayCellAdmissionBounds, type RelayCellConnectionHardCap } from '@orca-cloud/relay-contract/admission-budgets'
import { RELAY_DEFAULT_REGION, RelayRegionSchema, type RelayRegion } from '@orca-cloud/relay-contract/relay-regions'
import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract/protocol-limits'
import { Hono, type Context } from 'hono'
import { bodyLimit } from 'hono/body-limit'
import { SignJWT } from 'jose'
import { z } from 'zod'
import {
createAdminTokenVerifier,
createReadOnlyAdminTokenVerifier,
createRegionalRehomeControlApplyTokenVerifier,
createRegionalRehomeRuntimeTokenVerifier,
createRegionalRehomeTokenVerifier,
createRuntimeTokenVerifier
} from './authentication/admin-token-verifier.js'
import {
RelayAssignmentRowBusyError,
RelayCellInReserveModeError,
RelayHomeCellUnavailableError,
type AssignedRelayAssignment,
type AssignOptions,
type CellFenceAttemptEvidence,
type RelayAssignment,
type RelayAssignmentStore,
type ResolvedRelayAssignment
} from './assignment/assignment-store.js'
import { ASSIGNMENT_LEASE_AUDIENCE, assignmentLeaseKeyId } from './assignment/assignment-lease.js'
import { AssignmentRejectionLogWindow } from './assignment/assignment-rejection-log-window.js'
import { CELL_ADMISSION_STATES } from './cell/admission/cell-admission-selector.js'
import { registerCellSeatFeedRoute } from './cell/seats/cell-seat-feed-route.js'
import { ReservePaceError, type ReserveAssignment, type ReservePlan } from './assignment/reserve-placement/reserve-assignment.js'
import { CELL_RESERVE_ROUTE_PATHS, registerCellReserveRoutes } from './cell/reservations/cell-reserve-routes.js'
import type { DemoteRequest, ReserveOutcome, ReserveRequest } from './cell/reservations/cell-reserve-contract.js'
import type { CellSeatFeedPage } from './cell/seats/cell-seat-log.js'
import { type CellFlags, supportedCellFlags } from './cell/control-flags/cell-flags.js'
import type { AppliedControlFlags } from './cell/control-flags/relay-control-flag-channel.js'
import { RELAY_MAX_CELL_CAPACITY_REQUESTS, type RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credentials/credential-store.js'
import { isRelayDatabaseTransientError } from './database/database.js'
import {
EDGE_CELL_ID,
EDGE_MOVE_BUSY_RETRY_AFTER_SECONDS,
EdgeMoveDeferredError,
EdgeMoveBusyError,
inEdgeCohort,
mapEpochFloor,
type EdgeAssignments
} from './edge-relay.js'
import {
DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS,
RelayDrainReturnAdmission
} from './assignment/public-admission/drain-return-admission.js'
import { googleMetadataIdentityToken } from './authentication/google-metadata-identity-token.js'
import {
RelayPublicAssignmentAdmission,
type AssignmentAdmissionRejection
} from './assignment/public-admission/public-assignment-admission.js'
import { relayHostLogDigest } from './observability/relay-host-log-digest.js'
import type { RelayLocalReadinessVerdict } from './process-lifecycle/relay-local-readiness.js'
import {
classifyShadowSeat,
type ShadowCompareDatabaseAnswer,
type ShadowCompareRoute
} from './cell/seats/shadow-directory-compare.js'
import type { ShadowSeatDirectory } from './cell/seats/shadow-seat-directory.js'
import type { RelayReadinessDependency } from './process-lifecycle/relay-readiness.js'
import type {
AssignmentAdmissionLane,
AssignmentAdmissionOutcome,
AssignmentUnavailableCause,
RegionalRehomeSafetySnapshot,
RelayRuntimeCounts
} from './observability/relay-observability.js'
import {
isRegionalRehomeTrustProbe,
probeRegionalRehomeTrust
} from './assignment/regional-rehome/regional-rehome-trust-probe.js'
import { createRelayTokenVerifier, readBearer } from './authentication/relay-token-verifier.js'
import type { RelayJWKSet } from './authentication/stale-if-error-jwks.js'
import { stagingAsiaProofMembership } from './cell/admission/staging-asia-proof-admission.js'
import {
runInStickySlot,
StickySlotStatements,
type StickySlotSample
} from './database/sticky-slot-statements.js'
const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
isRelayCellConnectionHardCap
)
const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
// A release holds the row for about one lock timeout, so one second is enough.
const ASSIGNMENT_ROW_BUSY_RETRY_AFTER_SECONDS = 1
const REGION_CATALOG_CACHE_MS = 30_000
// Drain pace must fit the roll-step timeout, including its slowest same-cap wave.
const DRAIN_PACE_WINDOW_MAX_MS = 20 * 60 * 1_000
type AdmissionRejectionLogEntry = {
route: 'assign' | 'resolve'
lane: AssignmentAdmissionLane
hinted: boolean
relayHostId: string
reason: AssignmentAdmissionRejection
retryAfterSeconds?: number
}
export function createRelayApp(
config: RelayConfig,
operations: {
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number, options?: { paceWindowMs?: number }) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<IdleRegionalRehomeResult>
drainHost?: (input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceCellIncarnation: string
graceMs: number
}) =>
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
regionalRehomeFetch?: typeof fetch
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
cellIncarnation?: string
cellSeatFeed?: (sinceSeq: number | null) => CellSeatFeedPage
cellFlags?: () => AppliedControlFlags<CellFlags>
// Step 5 on directors; absent keeps today's path for every request.
reservePlacement?: ReserveAssignment
// Relay edge on directors; absent (no edge URL) keeps today's path for every request.
edgeAssignments?: EdgeAssignments
cellReserve?: (request: ReserveRequest) => ReserveOutcome[]
cellDemote?: (request: DemoteRequest) => string
cellReserverPoll?: () => void
cellAdmitModeEffective?: () => 'db' | 'reserve'
cellAdmitModeRaw?: () => 'db' | 'reserve'
cellPhonePathEffective?: () => 'db' | 'memory'
cellReserveCounts?: () => {
bookings: number
intake: { perSec: number; burst: number; tokens: number }
} | null
cellPlacementCeiling?: number
isDraining?: () => boolean
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
readinessDegradation?: () => RelayReadinessDependency[]
// Both only on cells; the switch is the per-cell `readinessLocal` flag, off by default.
readinessLocal?: () => boolean
localReadiness?: () => RelayLocalReadinessVerdict
relayJwks?: RelayJWKSet
recordAssignmentAdmission?: (outcome: AssignmentAdmissionOutcome) => void
recordAssignmentRejectionReason?: (
lane: AssignmentAdmissionLane,
reason: AssignmentAdmissionRejection
) => void
recordDrainReturnRetryAfter?: (seconds: number) => void
recordAdmissionServiceMs?: (lane: 'sticky' | 'drain-return', durationMs: number) => void
recordStickySlot?: (sample: StickySlotSample) => void
recordAssignmentUnavailable?: (cause: AssignmentUnavailableCause) => void
recordRegionRequest?: (region: RelayRegion | undefined) => void
// Step 3 shadow directory; absent unless the director polls seat feeds.
shadowSeats?: ShadowSeatDirectory
compareShadowSeats?: (
route: ShadowCompareRoute,
identity: { userId: string; relayHostId: string },
answer: ShadowCompareDatabaseAnswer
) => void
recordRegionSelection?: (input: {
targetRegion: RelayRegion
selectedRegion?: RelayRegion
fallback: boolean
}) => void
}
): Hono {
const app = new Hono()
let regionCatalogCache:
| { expiresAt: number; value: Awaited<ReturnType<RelayAssignmentStore['regionCatalog']>> }
| undefined
let regionCatalogRefresh:
| Promise<Awaited<ReturnType<RelayAssignmentStore['regionCatalog']>>>
| undefined
const regionCatalog = async () => {
const now = Date.now()
if (regionCatalogCache && regionCatalogCache.expiresAt > now) {
return regionCatalogCache.value
}
regionCatalogRefresh ??= operations.assignments.regionCatalog().then((value) => {
regionCatalogCache = { expiresAt: Date.now() + REGION_CATALOG_CACHE_MS, value }
return value
})
try {
return await regionCatalogRefresh
} finally {
regionCatalogRefresh = undefined
}
}
// Shared with the host-control upgrade and local readiness: one cached key set per process.
const verifyRelayToken = createRelayTokenVerifier(config, operations.relayJwks)
const assignmentLeaseKid = assignmentLeaseKeyId(config.assignmentSigningKey)
const verifyAdminToken = createAdminTokenVerifier(config)
const verifyReadOnlyAdminToken = createReadOnlyAdminTokenVerifier(config)
const verifyRegionalRehomeControlApplyToken =
createRegionalRehomeControlApplyTokenVerifier(config)
const verifyRuntimeToken = createRuntimeTokenVerifier(config)
const verifyRegionalRehomeToken = createRegionalRehomeTokenVerifier(config)
const verifyRegionalRehomeRuntimeToken =
createRegionalRehomeRuntimeTokenVerifier(config)
const regionalRehomeFetch = operations.regionalRehomeFetch ?? fetch
const regionalRehomeIdentityToken =
operations.regionalRehomeIdentityToken ??
((audience: string) => googleMetadataIdentityToken(audience, regionalRehomeFetch))
// Drain returns borrow placement permits and always leave placement one, so
// placement + sticky still bounds the director's database pool.
const drainReturnConcurrency = Math.min(
config.drainReturnConcurrency ?? 1,
config.publicAssignmentConcurrency - 1
)
const publicAssignmentAdmission = new RelayPublicAssignmentAdmission({
maxConcurrent: config.publicAssignmentConcurrency,
maxQueued: config.publicAssignmentQueueMax,
waitMs: config.publicAssignmentWaitMs,
maxReservedConcurrent: config.publicResolveConcurrency,
reservedWaitMs: config.publicResolveWaitMs,
maxDrainReturnConcurrent: drainReturnConcurrency,
maxDrainReturnQueued: config.drainReturnQueueMax ?? 4,
drainReturnWaitMs: config.drainReturnWaitMs ?? 3_000,
drainReturnMinIntervalMs: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS * 1_000,
minIntervalMs: config.publicAssignmentRetryAfterSeconds * 1_000,
onRejected: (reason) => operations.recordAssignmentRejectionReason?.('placement', reason)
})
// Runs after the database answer and never changes or fails the response.
const compareShadowSeats: NonNullable<typeof operations.compareShadowSeats> = (
route,
identity,
answer
) => {
try {
operations.compareShadowSeats?.(route, identity, answer)
} catch (error) {
console.warn(
JSON.stringify({
event: 'orca_relay_shadow_compare_failed',
route,
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
}
const rejectPublicAssignment = (context: Context): Response => {
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
}
// Verify hinted reconnects against durable assignments in a separate bounded lane so placement backlogs cannot starve recovery.
const stickyRetryAfterSeconds = config.publicStickyRetryAfterSeconds ?? 2
const stickyAssignmentAdmission = new RelayPublicAssignmentAdmission({
maxConcurrent: config.publicStickyConcurrency ?? 1,
maxQueued: config.publicStickyQueueMax ?? 64,
waitMs: config.publicStickyWaitMs ?? 2_000,
minIntervalMs: stickyRetryAfterSeconds * 1_000,
onRejected: (reason) => operations.recordAssignmentRejectionReason?.('sticky', reason)
})
const rejectStickyAssignment = (context: Context): Response => {
context.header('Retry-After', String(stickyRetryAfterSeconds))
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
}
// The slot's hold, not the request's latency: herd recovery is slots / this.
const timeStickySlot = (
lease: { release(): void },
queueWaitMs: number
): { release(): void; measure<T>(work: () => Promise<T>): Promise<T> } => {
const startedAt = performance.now()
const statements = new StickySlotStatements()
let released = false
return {
release: () => {
if (released) return
released = true
try {
operations.recordAdmissionServiceMs?.('sticky', performance.now() - startedAt)
operations.recordStickySlot?.({ queueWaitMs, ...statements.close() })
} finally {
lease.release()
}
},
measure: (work) => runInStickySlot(statements, work)
}
}
// Classified server-side from the host's own assignment row, never from a
// client claim: only a host whose home is roll-isolated reaches this lane.
const drainReturnAdmission = new RelayDrainReturnAdmission(publicAssignmentAdmission, {
maxConcurrent: Math.max(1, drainReturnConcurrency),
maxRetryAfterSeconds: config.drainReturnMaxRetryAfterSeconds ?? 300,
onServiceMs: (durationMs) => operations.recordAdmissionServiceMs?.('drain-return', durationMs)
})
const deferDrainReturn = (context: Context, retryAfterSeconds: number): Response => {
context.header('Retry-After', String(retryAfterSeconds))
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
}
// Rollout tooling retries only 503; classify transient admin DB failures like public routes while preserving other route mappings.
const rejectAdminOperation = (
context: Context,
error: unknown,
status: 404 | 409
): Response => {
// The reserve-mode set could not be read: retry, as for a database blip.
const unknownReserveSet = error instanceof RelayCellInReserveModeError && error.cellId === null
if (!isRelayDatabaseTransientError(error) && !unknownReserveSet) {
return context.json({ error: operationError(error) }, status)
}
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
// Log host/reason to distinguish isolated failures; bound windows by route:lane:reason to stay bounded across hosts.
const rejectionLogWindow = new AssignmentRejectionLogWindow<AdmissionRejectionLogEntry>({
windowMs: ASSIGNMENT_REJECTION_LOG_WINDOW_MS,
onWindowClosed: ({ suppressed, sample }) => logAssignmentRejection({ ...sample, suppressed })
})
const logAdmissionRejection = (input: {
route: 'assign' | 'resolve'
lane: AssignmentAdmissionLane
hinted: boolean
relayHostId: string
reason: AssignmentAdmissionRejection | undefined
retryAfterSeconds?: number
}): void => {
if (!input.reason) return
const entry: AdmissionRejectionLogEntry = { ...input, reason: input.reason }
if (!rejectionLogWindow.admit(`${entry.route}:${entry.lane}:${entry.reason}`, entry)) return
logAssignmentRejection(entry)
}
// Content-Length checks alone miss chunked bodies, which req.json would buffer whole.
const limitBody = bodyLimit({
maxSize: RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes,
onError: (context) => context.json({ error: 'request_too_large' }, 413)
})
app.use('*', (context, next) =>
CELL_RESERVE_ROUTE_PATHS.has(context.req.path) ? next() : limitBody(context, next)
)
app.use('/v1/admin/*', async (context, next) => {
if (
context.req.path === '/v1/admin/cell-heartbeat' ||
context.req.path === '/v1/admin/cell-rehome-status' ||
// Its own route checks the rehome identity; an admin check first would verify each poll twice.
context.req.path === '/v1/admin/cell-seats'
) {
return await next()
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer))) return await next()
if (!(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
return await next()
})
registerCellSeatFeedRoute(app, config, {
verifyRegionalRehomeToken,
cellIncarnation: operations.cellIncarnation,
seatFeed: operations.cellSeatFeed,
isDraining: operations.isDraining,
runtimeCounts: operations.runtimeCounts,
cellFlags: operations.cellFlags,
onReserverPoll: operations.cellReserverPoll,
admitModeEffective: operations.cellAdmitModeEffective,
admitModeRaw: operations.cellAdmitModeRaw,
reserveCounts: operations.cellReserveCounts,
placementCeiling: operations.cellPlacementCeiling
})
registerCellReserveRoutes(app, config, {
verifyRegionalRehomeToken,
reserve: operations.cellReserve,
demote: operations.cellDemote
})
// Google Front End reserves /healthz. Report drain cap before isolation so an unsupported pace refuses without touching the cell.
app.get('/health', (context) =>
context.json({
ok: true,
connectionCapacityProtocol: 2,
// Placement takes min(configured, cell-reported) cap, so a fleet raise may precede the rolls.
...(config.role === 'cell' ? {} : { capacityReportedCapMin: true }),
drainPaceWindowMaxMs: DRAIN_PACE_WINDOW_MAX_MS
})
)
app.get('/ready', async (context) => {
const ready = await operations.ready()
if (operations.localReadiness && operations.readinessLocal?.()) {
// The probes still ran above, so their failures are still reported, but never decide.
const local = operations.localReadiness()
if (!local.ready) return context.json({ error: local.reason }, 503)
if (local.failing.length === 0) return context.json({ ok: true })
return context.json({ ok: true, degraded: true, dependency: local.failing })
}
if (!ready) return context.json({ error: 'dependency_unavailable' }, 503)
const dependency = operations.readinessDegradation?.() ?? []
// Still the 200 the load balancer needs, with the marker that says the answer is remembered.
if (dependency.length === 0) return context.json({ ok: true })
return context.json({ ok: true, degraded: true, dependency })
})
app.get('/v1/regions', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
try {
return context.json({ v: 1, regions: await regionCatalog() })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
// Same contract as the assignment routes: a database that is briefly out
// of reach is a retry, not a director fault.
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'region_catalog_temporarily_unavailable' }, 503)
}
})
app.post('/v1/assign', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) {
operations.recordAssignmentUnavailable?.('disabled')
return rejectPublicAssignment(context)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer) return context.json({ error: 'invalid_token' }, 401)
const claims = await verifyRelayToken(bearer)
if (!claims) return context.json({ error: 'invalid_token' }, 401)
if (Number(context.req.header('content-length') ?? 0) > 4 * 1024) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AssignmentRequestSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (body.data.relayHostId !== claims.relayHostId) {
return context.json({ error: 'host_identity_mismatch' }, 403)
}
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
const requestedRegion =
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
const targetRegion =
config.regionalPlacementEnabled !== false && requestedRegion
? requestedRegion
: RELAY_DEFAULT_REGION
operations.recordRegionRequest?.(requestedRegion)
// Only an answer this call minted can be a fallback: a kept assignment was placed earlier.
const recordSelection = (assignment: RelayAssignment, placed: boolean) =>
operations.recordRegionSelection?.({
targetRegion,
selectedRegion: assignment.region,
fallback: placed && assignment.region !== targetRegion
})
// Set when this ask took the host's data back from the relay edge: the first answer elsewhere
// fences the edge (its phones then get WRONG_CELL).
let leftEdge = false
const signAssignment = async (assignment: RelayAssignment) => {
if (leftEdge && assignment.cellId !== EDGE_CELL_ID) {
leftEdge = false
await operations.edgeAssignments?.fenceAway(identity, assignment.cellId, assignment.assignmentEpoch)
}
return await signLease(assignment)
}
const signLease = async (assignment: RelayAssignment) =>
await new SignJWT({
purpose: 'cell-assignment',
cellId: assignment.cellId,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
relayHostId: claims.relayHostId
})
.setProtectedHeader({ alg: 'HS256', kid: assignmentLeaseKid })
.setIssuer(config.publicUrl)
.setAudience(ASSIGNMENT_LEASE_AUDIENCE)
.setSubject(claims.sub)
.setIssuedAt()
.setExpirationTime('5m')
.sign(config.assignmentSigningKey)
// Relay edge: a host in the cohort is moved (or kept) there before any other lane, since the
// sticky and reserve lanes would read the edge row as not live and re-place the host.
const edge = operations.edgeAssignments
const toEdge =
edge?.ready === true &&
body.data.regionCorrection?.action !== 'report' &&
inEdgeCohort(edge.config, claims.relayHostId, requestedRegion, claims.sub)
if (edge?.pushes && !toEdge) {
// Any lane below may re-place a host whose row is on the edge; its devices come back first.
try {
leftEdge = await edge.leaveEdge(identity)
} catch (error) {
if (!(error instanceof EdgeMoveBusyError) && !isRelayDatabaseTransientError(error)) {
throw error
}
logAssignmentRejection({
route: 'assign',
lane: 'none',
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error)
})
context.header('Retry-After', String(EDGE_MOVE_BUSY_RETRY_AFTER_SECONDS))
return context.json({ error: 'assignment_temporarily_unavailable' }, 503)
}
}
if (edge && toEdge) {
let moved: RelayAssignment | null
try {
moved = await edge.moveToEdge(
identity,
Math.max(
mapEpochFloor(operations.shadowSeats, identity, Date.now()),
operations.reservePlacement?.rowEpochFloor(identity) ?? 0
),
requestedRegion
)
} catch (error) {
if (error instanceof EdgeMoveDeferredError) {
// Nothing moved (old-cell units still held, or the edge did not take the push): today's
// path below keeps the host where it is, and a later ask moves it. Never a 503 loop.
console.warn(JSON.stringify({ event: 'orca_relay_edge_move_deferred', reason: error.message }))
moved = null
} else {
if (!(error instanceof EdgeMoveBusyError) && !isRelayDatabaseTransientError(error)) {
throw error
}
logAssignmentRejection({
route: 'assign',
lane: 'none',
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error)
})
context.header('Retry-After', String(EDGE_MOVE_BUSY_RETRY_AFTER_SECONDS))
return context.json({ error: 'assignment_temporarily_unavailable' }, 503)
}
}
if (moved) {
// Any later re-placement off the edge (rollback) mints above the edge epoch.
operations.reservePlacement?.raiseEpochFloor(identity, moved.assignmentEpoch)
return context.json({
v: 1,
cellUrl: moved.cellUrl,
assignmentEpoch: moved.assignmentEpoch,
lease: await signAssignment(moved)
})
}
}
// Step 5: switched-on cells answer from the map; anything else keeps today's path below.
let step5: Pick<AssignOptions, 'epochFloor' | 'placeFresh' | 'regionReserve' | 'reserveSpill'> = {}
if (operations.reservePlacement && body.data.regionCorrection?.action !== 'report') {
let plan: ReservePlan
try {
plan = await operations.reservePlacement.plan(identity, {
reconnect: Boolean(body.data.reconnect),
region: targetRegion,
regionCorrection: body.data.regionCorrection !== undefined,
preferred: config.regionalPlacementEnabled !== false && requestedRegion !== undefined
})
} catch (error) {
if (!(error instanceof ReservePaceError)) throw error
plan = {
kind: 'retry',
retryAfterSeconds: error.retryAfterSeconds,
reason: error.drained ? 'paced-drained' : 'paced'
}
}
if (plan.kind === 'answer') {
operations.recordAssignmentAdmission?.(plan.lane === 'sticky' ? 'sticky' : 'placement')
recordSelection(plan.assignment, plan.lane !== 'sticky')
return context.json({
v: 1,
cellUrl: plan.assignment.cellUrl,
assignmentEpoch: plan.assignment.assignmentEpoch,
lease: await signAssignment(plan.assignment)
})
}
if (plan.kind === 'retry') {
operations.recordRegionSelection?.({ targetRegion, fallback: false })
operations.recordAssignmentUnavailable?.(RESERVE_RETRY_CAUSE[plan.reason])
context.header('Retry-After', String(plan.retryAfterSeconds))
return context.json({ error: 'assignment_temporarily_unavailable' }, 503)
}
step5 = {
epochFloor: plan.epochFloor,
placeFresh: plan.placeFresh,
regionReserve: plan.regionReserve,
reserveSpill: plan.reserveSpill
}
}
let admission: { release(): void } | null = null
let lane: AssignmentAdmissionLane = 'placement'
// Charges the sticky slot's SQL to its request; any other lane runs work as is.
let measureHeld = <T>(work: () => Promise<T>): Promise<T> => work()
if (body.data.reconnect) {
let rejection: AssignmentAdmissionRejection | undefined
const queuedAt = performance.now()
const stickySlot = await stickyAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
rejection = reason
})
if (!stickySlot) {
operations.recordAssignmentAdmission?.('sticky-rejected')
operations.recordAssignmentUnavailable?.('sticky-lane')
logAdmissionRejection({
route: 'assign',
lane: 'sticky',
hinted: true,
relayHostId: claims.relayHostId,
reason: rejection
})
return rejectStickyAssignment(context)
}
const fastLane = timeStickySlot(stickySlot, performance.now() - queuedAt)
let verified: ResolvedRelayAssignment | null = null
try {
verified = await fastLane.measure(() =>
operations.assignments.resolve(identity, { classifyHomeRollIsolation: true })
)
} catch (error) {
fastLane.release()
operations.recordAssignmentAdmission?.('sticky-rejected')
if (isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'assign-verify',
lane: 'none',
hinted: true,
relayHostId: claims.relayHostId,
reason: operationError(error)
})
operations.recordAssignmentUnavailable?.('sticky-verify-database')
return rejectStickyAssignment(context)
}
throw error
}
compareShadowSeats('sticky-verify', identity, verified)
if (verified?.homeCellRollIsolated && drainReturnConcurrency > 0) {
// The sticky slot covered only the verification read; the re-placement
// that follows is the drain lane's work, not the sticky lane's.
fastLane.release()
lane = 'drain-return'
const drainReturn = await drainReturnAdmission.acquire(claims.relayHostId)
if (drainReturn.kind === 'deferred') {
operations.recordAssignmentAdmission?.('drain-return-deferred')
operations.recordAssignmentRejectionReason?.('drain-return', drainReturn.reason)
operations.recordDrainReturnRetryAfter?.(drainReturn.retryAfterSeconds)
operations.recordAssignmentUnavailable?.('drain-return-deferred')
logAdmissionRejection({
route: 'assign',
lane,
hinted: true,
relayHostId: claims.relayHostId,
reason: drainReturn.reason,
retryAfterSeconds: drainReturn.retryAfterSeconds
})
return deferDrainReturn(context, drainReturn.retryAfterSeconds)
}
admission = drainReturn.lease
operations.recordAssignmentAdmission?.('drain-return')
} else if (verified) {
lane = 'sticky'
admission = fastLane
measureHeld = fastLane.measure
operations.recordAssignmentAdmission?.('sticky')
} else {
// An unverified hint joins the placement lane with today's semantics.
fastLane.release()
}
}
if (!admission) {
let rejection: AssignmentAdmissionRejection | undefined
admission = await publicAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
rejection = reason
})
operations.recordAssignmentAdmission?.(admission ? 'placement' : 'placement-rejected')
if (!admission) {
logAdmissionRejection({
route: 'assign',
lane: 'placement',
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: rejection
})
operations.recordAssignmentUnavailable?.('placement-lane')
return rejectPublicAssignment(context)
}
}
let assignment: AssignedRelayAssignment
let regionCorrection: RegionCorrectionResponse | undefined
try {
if (body.data.regionCorrection?.action === 'report') {
const current = await measureHeld(() => operations.assignments.resolve(identity))
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
assignment = current
} else {
// The host asked: it may leave a reserve cell the director proves dead.
assignment = await measureHeld(() =>
operations.assignments.assign(
identity,
requestedRegion,
requestedRegion ? targetRegion : undefined,
undefined,
{ ...step5, hostRequested: true }
)
)
}
if (body.data.regionCorrection) {
try {
const correction = body.data.regionCorrection
const epoch = assignment.assignmentEpoch
regionCorrection = await measureHeld(() =>
operations.assignments.exchangeRegionCorrection(identity, correction, epoch)
)
} catch (error) {
if (body.data.regionCorrection.action === 'report') throw error
// Optional measurement setup must not discard an otherwise valid placement.
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
}
}
} catch (error) {
if (error instanceof ReservePaceError) {
operations.recordRegionSelection?.({ targetRegion, fallback: false })
operations.recordAssignmentUnavailable?.(error.drained ? 'reserve-paced-drained' : 'reserve-paced')
context.header('Retry-After', String(error.retryAfterSeconds))
return context.json({ error: 'assignment_temporarily_unavailable' }, 503)
}
if (error instanceof RelayAssignmentRowBusyError) {
logAssignmentRejection({
route: 'assign',
lane,
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error)
})
// The host's own release is settling; its next dial finds the row free.
context.header('Retry-After', String(ASSIGNMENT_ROW_BUSY_RETRY_AFTER_SECONDS))
operations.recordAssignmentUnavailable?.('relay_assignment_row_busy')
return context.json({ error: 'assignment_row_busy' }, 503)
}
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'assign',
lane,
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
const unavailable = assignmentUnavailableCause(error)
if (unavailable) {
if (lane === 'placement') {
operations.recordRegionSelection?.({ targetRegion, fallback: false })
}
operations.recordAssignmentUnavailable?.(unavailable)
return context.json({ error: operationError(error) }, 503)
}
if (isRelayDatabaseTransientError(error)) {
operations.recordAssignmentUnavailable?.('database')
return lane === 'placement'
? rejectPublicAssignment(context)
: rejectStickyAssignment(context)
}
throw error
} finally {
admission.release()
}
recordSelection(assignment, assignment.placed === true)
// Reconnect grant logs locate hosts; placement-lane grants prove where failed sticky verifications eventually landed.
if (body.data.reconnect) {
console.warn(
`[orca-relay] assignment granted lane=${lane} hinted=true` +
` host=${relayHostLogDigest(claims.relayHostId)} cell=${assignment.cellId}`
)
}
const lease = await signAssignment(assignment)
return context.json({
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
lease,
...(regionCorrection ? { regionCorrection } : {})
})
})
app.post('/v1/resolve', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
if (
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
let rejection: AssignmentAdmissionRejection | undefined
const admission = await publicAssignmentAdmission.acquireReserved(
body.data.relayHostId,
(reason) => {
rejection = reason
}
)
if (!admission) {
logAdmissionRejection({
route: 'resolve',
lane: 'placement',
hinted: false,
relayHostId: body.data.relayHostId,
reason: rejection
})
return rejectPublicAssignment(context)
}
try {
const resolved = await operations.store.resolveResume(
body.data.relayHostId,
body.data.resumeToken
)
if (!resolved) return context.json({ error: 'invalid_credential' }, 401)
const identity = {
userId: resolved.userId,
relayHostId: body.data.relayHostId
}
// A host on the relay edge: the liveness rule below would read its row as no answer.
const onEdge = await operations.edgeAssignments?.edgeAssignment(identity)
if (onEdge) {
return context.json({
v: 1,
cellUrl: onEdge.cellUrl,
assignmentEpoch: onEdge.assignmentEpoch,
leaseExpiresAt: onEdge.leaseExpiresAt
})
}
// This also migrates credentials created by the staging-only combined service.
const resolvedAssignment = await operations.assignments.resolve(identity)
compareShadowSeats('resolve', identity, resolvedAssignment)
const assignment = resolvedAssignment ?? (await operations.assignments.assign(identity))
return context.json({
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
leaseExpiresAt: assignment.leaseExpiresAt
})
} catch (error) {
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'resolve',
lane: 'none',
hinted: false,
relayHostId: body.data.relayHostId,
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
if (isRelayAssignmentUnavailableError(error)) {
if (error instanceof RelayAssignmentRowBusyError) {
context.header('Retry-After', String(ASSIGNMENT_ROW_BUSY_RETRY_AFTER_SECONDS))
}
return context.json({ error: operationError(error) }, 503)
}
if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context)
throw error
} finally {
admission.release()
}
})
app.post('/v1/admin/drain', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
const body = z
.object({
v: z.literal(1),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
// Spreads the drain sends, and so the re-dials, over this window.
paceWindowMs: z.number().int().nonnegative().max(DRAIN_PACE_WINDOW_MAX_MS).optional()
})
.strict()
.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
const paceWindowMs = body.data.paceWindowMs ?? 0
operations.drain(body.data.graceMs, { paceWindowMs })
return context.json({ ok: true, paceWindowMs })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = IdleRegionalRehomeCommandSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
if (config.role !== 'cell' || !operations.drainHost) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = RegionalHostDrainSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
const trustProbe = isRegionalRehomeTrustProbe(body.data)
let sharedRuntimeIdentityRejected: true | undefined
if (trustProbe) {
const runtimeToken = await regionalRehomeIdentityToken(config.rehomeAudience!)
if (
!(await verifyRegionalRehomeRuntimeToken(runtimeToken)) ||
(await verifyRegionalRehomeToken(runtimeToken))
) {
throw new Error('regional_rehome_shared_runtime_rejection_not_proven')
}
if (
!operations.regionalRehomeTrustProbeHostExists ||
operations.regionalRehomeTrustProbeHostExists(body.data)
) {
throw new Error('regional_rehome_trust_probe_host_not_absent')
}
sharedRuntimeIdentityRejected = true
}
const outcome = await operations.drainHost(body.data)
return context.json({
v: 1,
outcome,
...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {})
})
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/runtime-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = RuntimeStatusSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
return context.json({
v: 1,
role: config.role,
cellId: config.cellId,
cellUrl: config.cellUrl,
region: config.region ?? RELAY_DEFAULT_REGION,
imageDigest: config.imageDigest ?? null,
draining: operations.isDraining?.() ?? false,
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
// The flag workflow's read-back: applied switches, never the desired object.
...(operations.cellFlags
? {
flagsApplied: operations.cellFlags(),
supportedFlags: supportedCellFlags(),
// The flag tool paces a flip back's wait from it.
databasePoolMax: config.databasePoolMax
}
: {}),
...(operations.cellAdmitModeEffective
? { admitModeEffective: operations.cellAdmitModeEffective() }
: {}),
...(operations.cellPhonePathEffective
? { phonePathEffective: operations.cellPhonePathEffective() }
: {}),
// Placement as built, not as configured: `on` only when this director books reserve cells.
...(config.role === 'director'
? { reservePlacement: operations.reservePlacement?.placementMode ?? 'off' }
: {}),
connectionCapacity:
config.connectionHardCap === undefined
? null
: {
hardCap: config.connectionHardCap,
controlRebindReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls,
ordinaryConnectionLimit: relayCellAdmissionBounds(config.connectionHardCap)
.socketAdmissionCeiling,
unobservedBound: config.connectionUnobservedBound!,
normalAdmissionPause: cellPlacementCeiling(
config.connectionHardCap,
config.connectionUnobservedBound!
)
},
runtime: operations.runtimeCounts?.() ?? null
})
})
app.post('/v1/admin/cell-heartbeat', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRuntimeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = CellHeartbeatSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.recordCellHeartbeat(body.data)
return context.json({ ok: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-rehome-status', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRuntimeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = CellRegionalRehomeStatusSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.recordCellRegionalRehomeStatus(body.data)
return context.json({ ok: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
const preview = await operations.assignments.previewRegionalRehomeEligibility(
operations.regionalRehomeSafetySnapshot?.()
)
const outcomes = await operations.assignments.regionCorrectionOutcomes()
return context.json({ v: 1, preview, outcomes })
})
app.post('/v1/admin/regional-rehome-control', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = RegionalRehomeControlSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (body.data.action === 'inspect') {
const control = await operations.assignments.inspectRegionalRehomeControl()
return context.json({ v: 1, control })
}
if (!(await verifyRegionalRehomeControlApplyToken(bearer))) {
return context.json({ error: 'insufficient_permission' }, 403)
}
try {
const control = await operations.assignments.applyRegionalRehomeControl(body.data)
return context.json({ v: 1, control })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/regional-rehome-trust-probe', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = RegionalRehomeTrustProbeSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (!config.rehomeAudience || !config.rehomeDirectorServiceAccount) {
return context.json({ error: 'regional_rehome_trust_not_configured' }, 409)
}
try {
// Runtime rows only, which a reserve-mode cell heartbeats as a database one does.
const { status: source } = await operations.assignments.reserveAwareCellDeploymentStatus(
body.data.sourceCellId
)
// Any cell that can be drained can be a rehome source, in either
// direction, so the probe is gated on the protocol and not on a region.
if (
!source.runtime ||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
!source.runtime.ready ||
!source.runtime.heartbeatFresh ||
source.runtime.regionalRehomeProtocol < 1
) {
throw new Error('regional_rehome_trust_probe_source_unavailable')
}
const result = await probeRegionalRehomeTrust({
sourceCellUrl: source.cellUrl,
sourceCellId: body.data.sourceCellId,
sourceCellIncarnation: body.data.sourceCellIncarnation,
audience: config.rehomeAudience,
identityToken: regionalRehomeIdentityToken,
fetch: regionalRehomeFetch
})
return context.json(result)
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const migration = await operations.assignments.startEvacuation(
{ userId: body.data.userId, relayHostId: body.data.relayHostId },
body.data.targetCellId
)
return context.json({ v: 1, migration })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-complete', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminMigrationCompleteSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.completeEvacuation(
{ userId: body.data.userId, relayHostId: body.data.relayHostId },
body.data.assignmentEpoch
)
return context.json({ ok: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-supersede-cell', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminRegisteredCellMigrationSupersedeSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const superseded = await operations.assignments.supersedeRegisteredCellEvacuations(
body.data.sourceCellId,
body.data.currentTargetCellId,
body.data.replacementTargetCellId,
body.data.limit
)
return context.json({ v: 1, superseded })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/rebalance-dormant', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const assignment = await operations.assignments.rebalanceDormant(
{ userId: body.data.userId, relayHostId: body.data.relayHostId },
body.data.targetCellId
)
return context.json({ v: 1, assignment })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminAdmissionSelectorApplySchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const result = await operations.assignments.applyCellAdmissionSelector(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminStagingAsiaProofAdmissionApplySchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const current = await operations.assignments.inspectCellAdmissionSelector()
const result = await operations.assignments.applyCellAdmissionSelector({
attemptId: body.data.attemptId,
expectedGeneration: body.data.expectedGeneration,
membership: stagingAsiaProofMembership(current.selector.membership, body.data.state)
})
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminAdmissionSelectorStatusSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const result = await operations.assignments.inspectCellAdmissionSelector(
body.data.attemptId
)
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminAdmissionSelectorAddMigrationCellsSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const result = await operations.assignments.addMigrationCells({
attemptId: body.data.attemptId,
expectedGeneration: body.data.expectedGeneration,
cells: body.data.cells.map((cell) => ({
id: cell.cellId,
url: cell.cellUrl,
capacityRequests: cell.capacityRequests,
region: cell.region,
connectionHardCap: cell.connectionHardCap,
connectionUnobservedBound: cell.connectionUnobservedBound
}))
})
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-state', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellStateSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.setCellAdmissionState(
body.data.cellId,
'state' in body.data
? body.data.state
: body.data.enabled
? 'general'
: 'existing-only'
)
return context.json({ ok: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
// Read without admitMode; writes feed every sweep and refuse reserve during open drain/migration/rehome flows.
app.post('/v1/admin/cell-admit-mode', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellAdmitModeSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
if (body.data.admitMode) {
const before = await operations.assignments.cellAdmitMode(body.data.cellId)
await operations.assignments.setCellAdmitMode(body.data.cellId, body.data.admitMode)
// The sweeps' switch: every write is on the record, the break-glass flip included.
console.warn(
JSON.stringify({
event: 'orca_relay_cell_admit_mode_recorded',
cellId: body.data.cellId,
from: before.admitMode,
to: body.data.admitMode
})
)
}
return context.json({
v: 1,
cellId: body.data.cellId,
...(await operations.assignments.cellAdmitMode(body.data.cellId)),
// The break-glass flip refuses a cell that is still heartbeating.
heartbeatFresh: await operations.assignments.cellHeartbeatFresh(body.data.cellId)
})
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceLegacyAdoptionSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const expiresAt = await operations.assignments.adoptLegacyCellFence(
body.data.cellId,
body.data.cellIncarnation
)
return context.json({ v: 1, cellId: body.data.cellId, expiresAt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceLegacyAdoptionCommitSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.commitLegacyCellFenceAdoption(
body.data.cellId,
body.data.cellIncarnation
)
return context.json({ v: 1, cellId: body.data.cellId, committed: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attest', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceAttestSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const result = await operations.assignments.attestCellFenceAttempt(
evidence,
body.data.gceOperation
)
return context.json({
v: 1,
cellId: body.data.cellId,
expiresAt: result.expiresAt,
attempt: result.attempt
})
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceAttemptPrepareSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const attempt = await operations.assignments.prepareCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-start', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceAttemptUpdateSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const result = await operations.assignments.startCellFenceApply(
evidence,
body.data.invocationId,
body.data.invocationRequestReason
)
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-plan', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFencePlanSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const attempt = await operations.assignments.bindCellFencePlanGeneration(
evidence,
body.data.planObjectGeneration
)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-operation', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceOperationSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const result = await operations.assignments.recordCellFenceOperation(
evidence,
body.data.invocationId,
body.data.invocationRequestReason,
body.data.gceOperation
)
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceAttemptStatusSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-abort', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellFenceAttemptAbortSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const evidence = cellFenceAttemptEvidence(body.data)
const attempt = await operations.assignments.abortCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-prepare', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminDrainAttemptPrepareSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const result = await operations.assignments.prepareCellDrainAttempt({
attemptId: body.data.attemptId,
cellId: body.data.cellId,
cellIncarnation: body.data.cellIncarnation,
traceValue: body.data.traceValue,
plannedGraceMs: body.data.graceMs
})
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-send', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminDrainAttemptMutationSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const attempt = await operations.assignments.beginCellDrainSend(body.data)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-receipt', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminDrainAttemptReceiptSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const attempt = await operations.assignments.recordCellDrainApplicationReceipt(
body.data
)
return context.json({ v: 1, attempt })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-recover-forward', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminDrainAttemptRecoverSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const result = await operations.assignments.prepareCellDrainRecovery(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-config', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellConfigSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
await operations.assignments.configureCell(
{
id: body.data.cellId,
url: body.data.cellUrl,
capacityRequests: body.data.capacityRequests,
connectionHardCap: body.data.connectionHardCap,
connectionUnobservedBound: body.data.connectionUnobservedBound
},
'state' in body.data ? body.data.state : body.data.enabled
)
return context.json({ ok: true })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate-cell', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellEvacuateSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const started = await operations.assignments.startActiveCellEvacuations(
body.data.sourceCellId,
body.data.targetCellId,
body.data.limit
)
return context.json({ v: 1, started })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-capacity', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellEvacuationCapacitySchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
const capacity = await operations.assignments.cellEvacuationCapacity(
body.data.sourceCellId,
body.data.targetCellId
)
return context.json({ v: 1, ...capacity })
} catch (error) {
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellEvacuationStatusSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) {
return context.json({ error: 'insufficient_permission' }, 403)
}
try {
const status = await operations.assignments.cellEvacuationStatus(
body.data.sourceCellId,
body.data.targetCellId,
body.data.completeReady
)
return context.json({ v: 1, ...status })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
})
// Per director instance: tooling may ask more than one to see every map.
app.post('/v1/admin/host-whereabouts', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminHostWhereaboutsSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
const { relayHostId } = body.data
const shadow = operations.shadowSeats
const now = Date.now()
// Without a user id only the map can name one: relay_assignments has no host-id index.
const userIds = body.data.userId ? [body.data.userId] : (shadow?.userIdsOf(relayHostId) ?? [])
try {
const cellIds = new Set<string>()
const users = []
for (const userId of userIds) {
const identity = { userId, relayHostId }
const database = await operations.assignments.hostWhereabouts(identity)
const seats = shadow?.seatsOf(userId, relayHostId) ?? []
const recentlyLeft = shadow?.recentlyLeftOf(userId, relayHostId, now) ?? []
if (database) cellIds.add(database.cellId)
if (database?.openMigration) {
cellIds.add(database.openMigration.sourceCellId)
cellIds.add(database.openMigration.targetCellId)
}
for (const entry of [...seats, ...recentlyLeft]) cellIds.add(entry.cellId)
users.push({
userId,
database,
seats,
recentlyLeft,
// As /v1/resolve sees it: a row on a cell that is not live reads as no answer.
verdict: shadow
? classifyShadowSeat(
shadow,
identity,
database && shadow.cellLiveness(database.cellId, now) !== 'unlive'
? database
: null,
now
)
: null
})
}
const cells = Object.fromEntries(
[...cellIds].sort().map((cellId) => {
const state = shadow?.cellState(cellId)
return [
cellId,
state
? {
status: state.status,
pollAgeMs: state.lastLiveAt === undefined ? null : now - state.lastLiveAt,
lastFailure: state.lastFailure ?? null,
draining: state.draining ?? null,
flagsApplied: state.flagsApplied ?? null
}
: null
]
})
)
return context.json({
v: 1,
directorIncarnation: operations.cellIncarnation ?? null,
relayHostId,
shadow: shadow ? { state: 'on', complete: shadow.isComplete() } : { state: 'off' },
users,
cells
})
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
})
app.post('/v1/admin/cell-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
if (!bearer || !(await verifyAdminToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = AdminCellStatusSchema.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
try {
// Without the opt-in a reserve-mode cell stays a 404, so an older script never reads its
// mirror rows as host counts.
if (body.data.view !== 'reserve-aware') {
const status = await operations.assignments.cellDeploymentStatus(body.data.cellId)
return context.json({ v: 1, status })
}
const result = await operations.assignments.reserveAwareCellDeploymentStatus(body.data.cellId)
if (result.admitMode === 'db') return context.json({ v: 1, status: { ...result.status, reserve: null } })
const view = operations.reservePlacement?.cellStatus(
body.data.cellId,
result.status.runtime?.cellIncarnation
) ?? { ok: false, reason: 'placement-off' }
if (!view.ok) {
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'reserve_status_unavailable', reason: view.reason }, 503)
}
return context.json({ v: 1, status: { ...result.status, perHost: null, reserve: view.reserve } })
} catch (error) {
return rejectAdminOperation(context, error, 404)
}
})
return app
}
const RESERVE_RETRY_CAUSE = {
'map-incomplete': 'reserve-map-incomplete',
database: 'reserve-database',
paced: 'reserve-paced',
'paced-drained': 'reserve-paced-drained'
} as const satisfies Record<(ReservePlan & { kind: 'retry' })['reason'], AssignmentUnavailableCause>
const AdminHostWhereaboutsSchema = z
.object({
v: z.literal(1),
relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
userId: z.string().min(1).max(256).optional()
})
.strict()
const AdminAssignmentMoveSchema = z
.object({
v: z.literal(1),
userId: z.string().min(1).max(256),
relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
targetCellId: z.string().min(1).max(128)
})
.strict()
const RuntimeStatusSchema = z.object({ v: z.literal(1) }).strict()
const RegionalRehomeSafetySchema = z
.object({
observedAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
sqlFailures: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
reconnects: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
controlActivityRecoveryFailures: z
.number()
.int()
.nonnegative()
.max(Number.MAX_SAFE_INTEGER),
databasePoolWaiting: z.number().int().nonnegative().max(100),
databasePoolWaitersMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
databasePoolWaitMsMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
// Accept cell pool gauges or strict validation rejects every rehome-status heartbeat, preventing protocol-1 registration.
databasePoolTotal: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
databasePoolIdle: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
databasePoolOldestWaitMs: z
.number()
.int()
.nonnegative()
.max(Number.MAX_SAFE_INTEGER)
.optional()
})
.strict()
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
cohortPercent: z.number().int().min(0).max(100),
directorSafety: RegionalRehomeSafetySchema
})
const CellHeartbeatSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
region: RelayRegionSchema.default(RELAY_DEFAULT_REGION),
cellIncarnation: z.string().uuid(),
startedAt: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
ready: z.boolean(),
observedRequests: z.number().int().nonnegative().max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
totalConnections: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
.optional(),
inFlightConnections: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
.optional(),
reservedConnectionUnits: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
.optional(),
enforcedConnectionUnits: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
.optional(),
connectionInclusionWatermark: z
.number()
.int()
.nonnegative()
.max(Number.MAX_SAFE_INTEGER)
.optional(),
connectionHardCap: RelayCellConnectionHardCapSchema.optional(),
connectionUnobservedBound: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
.optional()
})
.strict()
.superRefine((value, context) => {
const values = [
value.totalConnections,
value.inFlightConnections,
value.reservedConnectionUnits,
value.enforcedConnectionUnits,
value.connectionHardCap,
value.connectionUnobservedBound
]
if (
values.some((candidate) => candidate !== undefined) &&
values.some((candidate) => candidate === undefined)
) {
context.addIssue({
code: 'custom',
message: 'connection telemetry must be complete'
})
}
if (
value.enforcedConnectionUnits !== undefined &&
value.totalConnections !== undefined &&
value.inFlightConnections !== undefined &&
value.reservedConnectionUnits !== undefined &&
value.enforcedConnectionUnits !==
value.totalConnections +
value.inFlightConnections +
value.reservedConnectionUnits
) {
context.addIssue({
code: 'custom',
message: 'connection telemetry sum does not match'
})
}
if (
value.connectionHardCap !== undefined &&
value.connectionUnobservedBound! >
relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound
) {
context.addIssue({
code: 'custom',
message: 'connection unobserved bound must leave ordinary admission capacity'
})
}
})
const CellRegionalRehomeStatusSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
regionalRehomeProtocol: z.number().int().min(0).max(3),
safety: RegionalRehomeSafetySchema
})
.strict()
const RegionalRehomeControlSchema = z
.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z
.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z
.number()
.int()
.min(60_000)
.max(60 * 60_000),
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
})
.strict()
])
.superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeTrustProbeSchema = z
.object({
v: z.literal(1),
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid()
})
.strict()
const AdminMigrationCompleteSchema = z
.object({
v: z.literal(1),
userId: z.string().min(1).max(256),
relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
assignmentEpoch: z.number().int().positive().max(Number.MAX_SAFE_INTEGER)
})
.strict()
const AdminRegisteredCellMigrationSupersedeSchema = z
.object({
v: z.literal(1),
sourceCellId: z.string().min(1).max(128),
currentTargetCellId: z.string().min(1).max(128),
replacementTargetCellId: z.string().min(1).max(128),
limit: z.number().int().min(1).max(100),
confirmation: z.literal('SUPERSEDE_REGISTERED_CELL_MIGRATIONS')
})
.strict()
.refine(
(value) =>
new Set([
value.sourceCellId,
value.currentTargetCellId,
value.replacementTargetCellId
]).size === 3
)
const CellIdSchema = z.string().min(1).max(128)
const CellAdmissionStateSchema = z.enum(CELL_ADMISSION_STATES)
const AdmissionSelectorAttemptIdSchema = z.string().regex(/^[A-Za-z0-9_-]{8,128}$/)
const AdmissionSelectorMembershipSchema = z
.object({
existingOnly: z.array(CellIdSchema).max(256),
migrationOnly: z.array(CellIdSchema).max(256),
general: z.array(CellIdSchema).max(256)
})
.strict()
const AdminAdmissionSelectorApplySchema = z
.object({
v: z.literal(1),
attemptId: AdmissionSelectorAttemptIdSchema,
expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
expectedMembershipSha256: z.string().regex(/^[a-f0-9]{64}$/).optional(),
membership: AdmissionSelectorMembershipSchema,
// Deploy director before callers: old callers omit this safely, but old strict directors reject it with 400 before isolation/MUTATION_STARTED, leaving hosts pinned.
rollIsolatedCells: z.array(CellIdSchema).max(256).optional()
})
.strict()
.refine(
(value) => value.expectedGeneration > 0 || value.expectedMembershipSha256 !== undefined
)
const AdminStagingAsiaProofAdmissionApplySchema = z
.object({
v: z.literal(1),
attemptId: AdmissionSelectorAttemptIdSchema,
expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
state: z.enum(['general', 'migration-only'])
})
.strict()
const AdminAdmissionSelectorStatusSchema = z
.object({ v: z.literal(1), attemptId: AdmissionSelectorAttemptIdSchema.optional() })
.strict()
const AdminAdmissionSelectorAddMigrationCellsSchema = z
.object({
v: z.literal(1),
attemptId: AdmissionSelectorAttemptIdSchema,
expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
cells: z
.array(
z
.object({
cellId: CellIdSchema,
cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
capacityRequests: z
.number()
.int()
.positive()
.max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
region: RelayRegionSchema.default(RELAY_DEFAULT_REGION),
connectionHardCap: RelayCellConnectionHardCapSchema,
connectionUnobservedBound: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
})
.strict()
.superRefine((value, context) => {
if (
value.connectionUnobservedBound >
relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound
) {
context.addIssue({
code: 'custom',
path: ['connectionUnobservedBound'],
message: 'connection unobserved bound must leave ordinary admission capacity'
})
}
})
)
.min(1)
.max(128)
})
.strict()
.refine(
({ cells }) =>
new Set(cells.map(({ cellId }) => cellId)).size === cells.length &&
new Set(cells.map(({ cellUrl }) => cellUrl)).size === cells.length
)
const AdminCellAdmitModeSchema = z
.object({ v: z.literal(1), cellId: CellIdSchema, admitMode: z.enum(['db', 'reserve']).optional() })
.strict()
const AdminCellStateSchema = z.union([
z.object({ v: z.literal(1), cellId: CellIdSchema, enabled: z.boolean() }).strict(),
z.object({ v: z.literal(1), cellId: CellIdSchema, state: CellAdmissionStateSchema }).strict()
])
const CellConfigShape = {
v: z.literal(1),
cellId: CellIdSchema,
cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
capacityRequests: z.number().int().positive().max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
connectionHardCap: RelayCellConnectionHardCapSchema.optional(),
connectionUnobservedBound: z
.number()
.int()
.nonnegative()
.max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
.optional()
} as const
const AdminCellConfigSchema = z
.union([
z
.object({
...CellConfigShape,
enabled: z.boolean()
})
.strict(),
z
.object({
...CellConfigShape,
state: CellAdmissionStateSchema
})
.strict()
])
.refine(
(value) =>
(value.connectionHardCap === undefined) ===
(value.connectionUnobservedBound === undefined),
{ message: 'connection hard cap and unobserved bound must be configured together' }
)
.refine(
(value) =>
value.connectionHardCap === undefined ||
value.connectionUnobservedBound! <=
relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound,
{ message: 'connection unobserved bound must leave ordinary admission capacity' }
)
const CellPairShape = {
v: z.literal(1),
sourceCellId: z.string().min(1).max(128),
targetCellId: z.string().min(1).max(128)
} as const
const AdminCellEvacuateSchema = z
.object({ ...CellPairShape, limit: z.number().int().min(1).max(100) })
.strict()
.refine((value) => value.sourceCellId !== value.targetCellId)
const AdminCellEvacuationCapacitySchema = z
.object(CellPairShape)
.strict()
.refine((value) => value.sourceCellId !== value.targetCellId)
const AdminCellEvacuationStatusSchema = z
.object({ ...CellPairShape, completeReady: z.boolean().default(false) })
.strict()
.refine((value) => value.sourceCellId !== value.targetCellId)
const TerraformStateLineageSchema = z
.string()
.regex(/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i)
const CellFenceAttemptBaseShape = {
attemptId: z.string().uuid(),
environment: z.enum(['staging', 'production']),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
migName: z.string().min(1).max(128),
instanceGroup: z.string().url().max(2048),
generationIdentity: z.string().url().max(2048),
fenceCommit: z.string().regex(/^[a-f0-9]{40}$/),
planSha256: z.string().regex(/^[a-f0-9]{64}$/),
planObjectName: z
.string()
.regex(/^terraform\/state\/relay-fence-plans\/(?:staging|production)\/[0-9a-f-]{36}\.tfplan$/),
varFileSha256: z.string().regex(/^[a-f0-9]{64}$/),
terraformStateLineage: TerraformStateLineageSchema,
terraformStateSerial: z.number().int().nonnegative().safe(),
terraformStateObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/),
terraformStateObjectSha256: z.string().regex(/^[a-f0-9]{64}$/),
requestReason: z
.string()
.regex(/^orca-relay-fence\/[0-9a-f]{8}-[0-9a-f-]{27}$/)
} as const
const CellFenceAttemptEvidenceShape = {
...CellFenceAttemptBaseShape,
planObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/)
} as const
const AdminCellFenceAttemptPrepareSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptBaseShape,
confirmation: z.literal('PREPARE_TERRAFORM_CELL_FENCE')
})
.strict()
const AdminCellFencePlanSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptEvidenceShape,
confirmation: z.literal('BIND_TERRAFORM_CELL_FENCE_PLAN')
})
.strict()
const AdminCellFenceAttemptUpdateSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptEvidenceShape,
invocationId: z.string().uuid(),
invocationRequestReason: z.string().min(1).max(256),
confirmation: z.literal('START_TERRAFORM_CELL_FENCE')
})
.strict()
const AdminCellFenceOperationSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptEvidenceShape,
invocationId: z.string().uuid(),
invocationRequestReason: z.string().min(1).max(256),
gceOperation: z.string().min(1).max(256),
confirmation: z.literal('RECORD_TERRAFORM_CELL_FENCE_OPERATION')
})
.strict()
const AdminCellFenceAttemptStatusSchema = z
.object({ v: z.literal(1), cellId: z.string().min(1).max(128) })
.strict()
const AdminCellFenceLegacyAdoptionSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
confirmation: z.literal('ADOPT_LEGACY_TERRAFORM_CELL_FENCE')
})
.strict()
const AdminCellFenceLegacyAdoptionCommitSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
confirmation: z.literal('COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION')
})
.strict()
const AdminCellFenceAttemptAbortSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptBaseShape,
confirmation: z.literal('ABORT_UNSTARTED_TERRAFORM_CELL_FENCE')
})
.strict()
const AdminCellFenceAttestSchema = z
.object({
v: z.literal(1),
...CellFenceAttemptEvidenceShape,
gceOperation: z.string().min(1).max(256),
confirmation: z.literal('ATTEST_TERRAFORM_FENCED_CELL')
})
.strict()
const AdminDrainAttemptPrepareSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
traceValue: z.string().uuid(),
graceMs: z.literal(120_000),
confirmation: z.literal('PREPARE_LEGACY_DRAIN')
})
.strict()
const AdminDrainAttemptMutationSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid()
})
.strict()
const AdminDrainAttemptReceiptSchema = AdminDrainAttemptMutationSchema.extend({
traceValue: z.string().uuid(),
backendStatus: z.number().int().min(200).max(299),
backendInstance: z.string().min(1).max(256).optional()
}).strict()
const AdminDrainAttemptRecoverSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
confirmation: z.literal('RECOVER_LEGACY_DRAIN')
})
.strict()
const RegionalHostDrainSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
userId: z.string().min(1).max(256),
relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: z.number().int().positive(),
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000)
})
.strict()
const AdminCellStatusSchema = z
.object({
v: z.literal(1),
cellId: z.string().min(1).max(128),
// Opt-in: a director without it answers 400, which tells a script to keep today's refusal.
view: z.literal('reserve-aware').optional()
})
.strict()
function cellFenceAttemptEvidence(
value: CellFenceAttemptEvidence
): CellFenceAttemptEvidence {
return {
attemptId: value.attemptId,
environment: value.environment,
cellId: value.cellId,
cellIncarnation: value.cellIncarnation,
migName: value.migName,
instanceGroup: value.instanceGroup,
generationIdentity: value.generationIdentity,
fenceCommit: value.fenceCommit,
planSha256: value.planSha256,
planObjectName: value.planObjectName,
planObjectGeneration: value.planObjectGeneration,
varFileSha256: value.varFileSha256,
terraformStateLineage: value.terraformStateLineage,
terraformStateSerial: value.terraformStateSerial,
terraformStateObjectGeneration: value.terraformStateObjectGeneration,
terraformStateObjectSha256: value.terraformStateObjectSha256,
requestReason: value.requestReason
}
}
function operationError(error: unknown): string {
return error instanceof Error ? error.message : 'operation_failed'
}
// Only window-close logs carry suppressed, excluding the rejection already logged at window open.
function logAssignmentRejection(input: {
route: 'assign' | 'assign-verify' | 'resolve'
lane: AssignmentAdmissionLane | 'none'
hinted: boolean
relayHostId: string
reason: string
cause?: string
cell?: string
suppressed?: number
retryAfterSeconds?: number
}): void {
console.warn(
`[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` +
` hinted=${input.hinted} reason=${input.reason}` +
` host=${relayHostLogDigest(input.relayHostId)}` +
(input.retryAfterSeconds === undefined ? '' : ` retryAfter=${input.retryAfterSeconds}`) +
(input.cause === undefined ? '' : ` cause=${input.cause}`) +
(input.cell === undefined ? '' : ` cell=${input.cell}`) +
(input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`)
)
}
// The home-cell reason is not capacity, but it is the same answer to the client:
// retry, the director cannot place you right now.
const RELAY_ASSIGNMENT_UNAVAILABLE_ERRORS = [
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted',
'relay_home_cell_unavailable',
'relay_assignment_row_busy'
] as const satisfies readonly AssignmentUnavailableCause[]
function assignmentUnavailableCause(
error: unknown
): (typeof RELAY_ASSIGNMENT_UNAVAILABLE_ERRORS)[number] | undefined {
if (!(error instanceof Error)) return undefined
return RELAY_ASSIGNMENT_UNAVAILABLE_ERRORS.find((message) => message === error.message)
}
function isRelayAssignmentUnavailableError(error: unknown): boolean {
return assignmentUnavailableCause(error) !== undefined
}
function homeCellRejectionDetail(
error: unknown
): { cause: string; cell: string } | Record<string, never> {
return error instanceof RelayHomeCellUnavailableError
? { cause: error.unavailableCause, cell: error.cellId }
: {}
}
function isCanonicalRelayOrigin(value: string): boolean {
const url = new URL(value)
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
return (
url.origin === value &&
url.pathname === '/' &&
(url.protocol === 'https:' || (loopback && url.protocol === 'http:'))
)
}
function requestTooLarge(contentLength: string | undefined): boolean {
return Number(contentLength ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
}