Files
orca/cloud/apps/relay/src/assignment-store.ts
T

8375 lines
317 KiB
TypeScript

import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import { IDLE_REHOME_PAGE_SIZE, selectIdleRegionalRehomes } from './idle-regional-rehome-selection.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import {
previewRegionalRehomeEligibility,
type RegionCorrectionPreview
} from './region-correction-preview.js'
import {
exchangeRegionCorrection,
previewRegionCorrection,
REGIONAL_REHOME_CONCURRENT_LIMIT
} from './region-correction-state.js'
import { randomUUID } from 'node:crypto'
import { performance } from 'node:perf_hooks'
import {
ASSIGNMENT_LIMITS,
mayNormallyReassign,
RELAY_ADMISSION_BUDGETS,
RELAY_DEFAULT_REGION,
RELAY_REGIONS,
RELAY_PROTOCOL_LIMITS,
type RelayRegion,
type RegionCorrectionRequest,
type RegionCorrectionResponse,
type IdleRegionalRehomeRequest,
} from '@orca-cloud/relay-contract'
import {
cellAdmissionState,
cellAdmissionStates,
ensureCellAdmission,
parseCellAdmissionState,
RelayCellAdmissionSelector,
setCellAdmissionBeforeBoundary,
stateFromEnabled,
synchronizeCellAdmissionBoundary,
type CellAdmissionMembership,
type CellAdmissionSelectorInspection,
type CellAdmissionState
} from './cell-admission-selector.js'
import {
RelayMigrationCellRegistrar,
type MigrationCellRegistration
} from './cell-admission-migration-registration.js'
import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
import type { RelayCellConfig } from './config.js'
import type {
RelayDatabase,
RelayLockOptions,
RelayTransactionOptions,
SqlRow
} from './database.js'
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
import {
combineRegionalRehomeSafety,
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT,
regionalRehomePoolPressure,
regionalRehomeSafetyFailure
} from './regional-rehome-safety.js'
import {
ABANDONED_REGISTERED_MIGRATION,
DURABLY_FENCED_MIGRATION_SOURCE,
REGISTERED_MIGRATION_ABANDON_MS
} from './registered-migration-abandonment.js'
type AssignmentIdentity = { userId: string; relayHostId: string }
type CellHeartbeat = {
cellId: string
cellUrl: string
cellIncarnation: string
startedAt: number
ready: boolean
observedRequests: number
region?: RelayRegion
totalConnections?: number
inFlightConnections?: number
reservedConnectionUnits?: number
enforcedConnectionUnits?: number
connectionInclusionWatermark?: number
connectionHardCap?: number
connectionUnobservedBound?: number
}
type CellRegionalRehomeStatus = {
cellId: string
cellIncarnation: string
regionalRehomeProtocol: number
safety: RegionalRehomeSafetySnapshot
}
type RelayAssignmentStoreOptions = {
regionalRehomeCohortPercent?: number
requireLiveCells?: boolean
heartbeatTtlMs?: number
recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
}
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
| 'database_error'
const CONTROL_RENEWAL_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved'
])
export type RelayAssignment = AssignmentIdentity & {
cellId: string
cellUrl: string
assignmentEpoch: number
leaseExpiresAt: number
region?: RelayRegion
}
export type RelayRegionCatalogEntry = {
region: RelayRegion
probeOrigins: string[]
}
export type RelayAssignmentMigration = AssignmentIdentity & {
sourceCellId: string
targetCellId: string
previousEpoch: number
assignmentEpoch: number
expiresAt: number
targetRegisteredAt?: number
}
export type RegionalRehomeAttempt = AssignmentIdentity & {
attemptId: string
preferredRegion: RelayRegion
sourceCellId: string
sourceCellUrl: string
sourceCellIncarnation: string
targetCellId: string
targetCellIncarnation: string
previousEpoch: number
assignmentEpoch: number
drainGraceMs: number
sendAttempts: number
}
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
export type RegionalRehomeFleetSafety = RegionalRehomeSafetySnapshot & {
requiredCells: number
missingCells: number
maxReconnects: number
}
export type RegionalRehomeControl = {
generation: number
enabled: boolean
observationStartedAt: number
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}
type RegisteredEvacuationSupersessionInput = {
assignmentEpoch: number
sourceCellId: string
currentTargetCellId: string
replacementTargetCellId: string
}
export type DeadSourceCompletionResult = {
changed: boolean
assignmentEpoch: number
sourceCellId: string
targetCellId: string
}
export type CellEvacuationStatus = {
inProgress: number
oldestExpiresAt: number | null
oldestRemainingMs: number | null
targetRegistered: number
registeredSourceActive: number
registeredCompletable: number
registeredTargetInactive: number
completed: number
blocked: number
expiredUnregistered: number
repairableExpiredUnregistered: number
abortableExpiredUnregistered: number
blockedExpiredUnregistered: number
blockedExpiredOnNewerTargetAssignment: number
}
export type CellEvacuationCapacity = {
sourceAssignments: number
requiredTargetUnits: number
availableTargetUnits: number
}
export type CellDeploymentStatus = {
cellId: string
cellUrl: string
region: RelayRegion
enabled: boolean
admissionState: CellAdmissionState
capacityRequests: number
reservedRequests: number
assignments: number
activityLeases: number
activityRequestUnits: number
restartBlockingActivityLeases: number
restartBlockingActivityRequestUnits: number
restartBlockingReservedRequests: number
outgoingMigrations: number
incomingMigrations: number
connectionCapacity: null | {
hardCap: number
controlRebindReserve: number
ordinaryConnectionLimit: number
unobservedBound: number
normalAdmissionPause: number
observedConnections: number
inFlightConnections: number
reservedConnectionUnits: number
enforcedConnectionUnits: number
pendingControlReservations: number
heartbeatFresh: boolean
}
runtime: null | {
cellUrl: string
cellIncarnation: string
startedAt: number
ready: boolean
observedRequests: number
lastHeartbeatAt: number
heartbeatFresh: boolean
regionalRehomeProtocol: number
}
}
export type CellFenceAttemptEvidence = {
attemptId: string
environment: 'staging' | 'production'
cellId: string
cellIncarnation: string
migName: string
instanceGroup: string
generationIdentity: string
fenceCommit: string
planSha256: string
planObjectName: string
planObjectGeneration?: string
varFileSha256: string
terraformStateLineage: string
terraformStateSerial: number
terraformStateObjectGeneration: string
terraformStateObjectSha256: string
requestReason: string
}
export type CellFenceApplyInvocation = {
invocationId: string
requestReason: string
startedAt: number
gceOperation?: string
}
export type CellFenceAttempt = CellFenceAttemptEvidence & {
applyInvocations?: CellFenceApplyInvocation[]
gceOperation?: string
createdAt: number
expiresAt: number
applyStartedAt?: number
completedAt?: number
abortedAt?: number
}
export type CellDrainAttemptState =
| 'prepared'
| 'send-may-have-started'
| 'application-receipt'
| 'proven-not-delivered'
export type CellDrainAttempt = {
attemptId: string
cellId: string
cellIncarnation: string
traceValue: string
plannedGraceMs: number
state: CellDrainAttemptState
preparedAt: number
sendMayHaveStartedAt?: number
sendPermitExpiresAt?: number
applicationReceiptAt?: number
backendSuccessStatus?: number
backendInstance?: string
receiptCellIncarnation?: string
retryAfter?: number
recoverForwardAttemptedAt?: number
provenNotDeliveredAt?: number
}
export type AssignmentActivityKind =
| 'control'
| 'splice'
| 'invite'
| 'install'
| 'confirmation'
| 'migration'
const ACTIVITY_COLUMN: Record<AssignmentActivityKind, string> = {
control: 'reserved_controls',
splice: 'reserved_splices',
invite: 'reserved_invites',
install: 'pending_installs',
confirmation: 'pending_confirmations',
migration: 'migration_leases'
}
const ACTIVITY_REQUEST_UNITS: Record<AssignmentActivityKind, number> = {
control: 1,
splice: 2,
invite: 1,
install: 1,
confirmation: 1,
migration: 1
}
const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000
// Why: one global FOR UPDATE over a 23-row table serialises every director and
// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled
// client for a full second, so the queue converts contention into pool
// exhaustion. The lock is held to COMMIT and the assignment path runs many
// statements after taking it, and no hold-time telemetry existed before this
// change, so 500ms is a first value to tune once cellInventoryHoldMsMax lands.
export const CELL_INVENTORY_LOCK_TIMEOUT_MS = 500
// The same inventory lock is taken by live requests and by background sweeps,
// and the right failure mode differs per caller.
export type CellInventoryLockMode =
// Bound the wait so a blocked request stops occupying a pooled client.
| 'request'
// Never queue: the caller handles database_lock_unavailable and moves on.
| 'nowait'
// A sweep can enter here, so keep the pool default. Failing sooner would turn
// ordinary contention into a 55P03 the retry wrapper reports as terminal, which
// spends the incident gate's bounded exhausted-retry budget (300 per 5 min).
| 'pool-default'
// Why: stranded detection (issue #225) needs a grant old enough that a real
// attach would have registered (the 90s activity lease covers dial +
// activation), yet recent enough to prove an active retry loop rather than
// ordinary dormancy — which stays governed by the 24h rule.
const STRANDED_MIN_GRANT_AGE_MS = 60_000
const STRANDED_RECENT_ACTIVITY_MS = 15 * 60_000
const REGION_PREFERENCE_RETENTION_MS = 30 * 24 * 60 * 60_000
const REGIONAL_REHOME_UNREGISTERED_REFRESH_MS = 5 * 60_000
const REGIONAL_REHOME_MAX_REFRESH_MS = 24 * 60 * 60_000
// Drain grace is enforced by session-scoped cell state that any control
// reconnect sheds, so receipted attempts can stall dual-homed past grace;
// redrains re-dispatch them with the elapsed (zero) grace until they detach.
export const REGIONAL_REHOME_REDRAIN_INTERVAL_MS = 60_000
export const REGIONAL_REHOME_REDRAIN_SEND_LIMIT = 20
export const REGIONAL_REHOME_QUARANTINE_FAILURES = 3
export const REGIONAL_REHOME_QUARANTINE_MS = 15 * 60_000
const REGIONAL_REHOME_QUARANTINE_EXCLUSION_LIMIT = 50
const REGIONAL_REHOME_QUARANTINE_MEMORY_LIMIT = 1_000
// Consecutive drain-dispatch failures that latch the durable control off. Any
// drain receipt and any enable reset it, so it reads "dispatch is broken now".
const REGIONAL_REHOME_FAILURE_BUDGET = 3
const REGIONAL_REHOME_OBSERVATION_MS = 24 * 60 * 60_000
const ASSIGNMENT_LOCK_RETRY_MAX_DELAY_MS = 50
type AssignmentInventoryScope = 'none' | 'general' | 'all'
type RetriedAssignmentInventoryScope = Exclude<AssignmentInventoryScope, 'none'>
class AssignmentInventoryLockUnavailable extends Error {
constructor(readonly inventoryScope: RetriedAssignmentInventoryScope) {
super('database_lock_unavailable')
}
}
class AssignmentInventoryScopeChanged extends Error {
constructor() {
super('assignment_inventory_scope_changed')
}
}
// Debt holds connection headroom for a control that may still arrive shortly
// after its director-side timeout. Nothing legitimately arrives minutes late
// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that
// never return otherwise starves connection headroom fleet-wide and turns
// every placement into relay_capacity_exhausted.
const LATE_ARRIVAL_DEBT_RETENTION_MS = 10 * 60 * 1_000
const CELL_FENCE_TTL_MS = 5 * 60 * 1_000
const CELL_FENCE_ATTEMPT_TTL_MS = 60 * 60 * 1_000
const CELL_DRAIN_SEND_PERMIT_MS = 30_000
const MAX_DRAIN_ACCOUNTING_REPAIR_ATTEMPTS = 3
const CELL_FENCE_ATTEMPT_SELECT = `
SELECT attempts.*, bindings.plan_object_name, bindings.plan_object_generation,
bindings.var_file_sha256, bindings.terraform_state_lineage,
bindings.terraform_state_serial, bindings.terraform_state_object_generation,
bindings.terraform_state_object_sha256, bindings.request_reason
FROM relay_cell_fence_attempts attempts
JOIN relay_cell_fence_plan_bindings bindings
ON bindings.attempt_id = attempts.attempt_id`
export const STRANDED_MIGRATION_ABANDON_MS = REGISTERED_MIGRATION_ABANDON_MS
const ABORTABLE_EXPIRED_MIGRATION = `(
(
migration.target_registered_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
WHERE pin.user_id = migration.user_id
AND pin.relay_host_id = migration.relay_host_id
AND pin.assignment_epoch = migration.assignment_epoch
)
)
OR ${ABANDONED_REGISTERED_MIGRATION}
)`
export class RelayAssignmentStore {
private readonly regionalRehomeCohortPercent: number
private readonly requireLiveCells: boolean
private readonly heartbeatTtlMs: number
// Poisoned attempts never complete or abort and stay the oldest rows, so
// unquarantined they eventually fill the sweeps' LIMIT page and starve
// every healthy candidate. Process-local on purpose: it resets on deploy
// and each director relearns within a few ticks.
private readonly regionalRehomeCandidateQuarantine = new Map<
string,
{ failures: number; until: number }
>()
private readonly recordControlRenewal?: RelayAssignmentStoreOptions['recordControlRenewal']
private readonly admissionSelector: RelayCellAdmissionSelector
private readonly migrationCellRegistrar: RelayMigrationCellRegistrar
private readonly activityQueue = new AssignmentIdentityQueue()
private assignmentTail: Promise<void> = Promise.resolve()
constructor(
private readonly database: RelayDatabase,
private readonly now: () => number = Date.now,
options: RelayAssignmentStoreOptions = {}
) {
this.regionalRehomeCohortPercent = options.regionalRehomeCohortPercent ?? 0
if (
!Number.isInteger(this.regionalRehomeCohortPercent) ||
this.regionalRehomeCohortPercent < 0 ||
this.regionalRehomeCohortPercent > 100
)
throw new Error('invalid_regional_rehome_cohort')
this.requireLiveCells = options.requireLiveCells ?? false
this.heartbeatTtlMs = options.heartbeatTtlMs ?? 45_000
this.recordControlRenewal = options.recordControlRenewal
this.admissionSelector = new RelayCellAdmissionSelector(database, now)
this.migrationCellRegistrar = new RelayMigrationCellRegistrar(database, now)
}
async reconcileCells(cells: RelayCellConfig[], disableMissing = true): Promise<void> {
await this.reconcileCellsWithOptions(cells, disableMissing)
}
async reconcileCellsAtStartup(cells: RelayCellConfig[]): Promise<void> {
await this.reconcileCellsWithOptions(cells, false, { reportRetries: false })
}
private async reconcileCellsWithOptions(
cells: RelayCellConfig[],
disableMissing: boolean,
transactionOptions?: RelayTransactionOptions
): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`)
// Capacity rows share one global lock order with assignment transactions.
for (const cell of [...cells].sort((left, right) => left.id.localeCompare(right.id))) {
// Operator-owned enabled state and tagged URLs must survive revision restarts.
await transaction.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
capacity_requests = excluded.capacity_requests,
last_heartbeat_at = excluded.last_heartbeat_at,
updated_at = excluded.updated_at`,
[
cell.id,
cell.url,
cell.initiallyEnabled === false ? 0 : 1,
cell.capacityRequests,
0,
0,
now,
now
]
)
await transaction.query(
`INSERT INTO relay_cell_regions (cell_id, region) VALUES (?, ?)
ON CONFLICT (cell_id) DO UPDATE SET region = excluded.region`,
[cell.id, cell.region ?? RELAY_DEFAULT_REGION]
)
await ensureCellAdmission(
transaction,
cell.id,
stateFromEnabled(cell.initiallyEnabled !== false),
now
)
if (
cell.connectionHardCap !== undefined &&
cell.connectionUnobservedBound !== undefined
) {
const currentLimit = (
await transaction.queryLocked(
`SELECT hard_cap, unobserved_bound FROM relay_cell_connection_limits
WHERE cell_id = ?`,
[cell.id]
)
)[0]
const limitChanged =
currentLimit !== undefined &&
(integer(currentLimit, 'hard_cap') !== cell.connectionHardCap ||
integer(currentLimit, 'unobserved_bound') !==
cell.connectionUnobservedBound)
await transaction.query(
`INSERT INTO relay_cell_connection_limits
(cell_id, hard_cap, unobserved_bound, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
hard_cap = excluded.hard_cap,
unobserved_bound = excluded.unobserved_bound,
updated_at = excluded.updated_at`,
[
cell.id,
cell.connectionHardCap,
cell.connectionUnobservedBound,
now
]
)
if (limitChanged) {
await transaction.query(
`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[cell.id]
)
await transaction.query(
`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`,
[cell.id]
)
}
}
}
const current = await transaction.query(
`SELECT cell_id, enabled FROM relay_cells ORDER BY cell_id ASC`
)
for (const row of current) {
await ensureCellAdmission(
transaction,
text(row, 'cell_id'),
stateFromEnabled(integer(row, 'enabled') === 1),
now
)
}
if (disableMissing) {
const ids = new Set(cells.map(({ id }) => id))
for (const row of current) {
const id = text(row, 'cell_id')
if (
!ids.has(id) &&
(await cellAdmissionState(transaction, id)) !== 'existing-only'
) {
await setCellAdmissionBeforeBoundary(transaction, id, 'existing-only', now)
}
}
}
await synchronizeCellAdmissionBoundary(transaction, now)
}, transactionOptions)
}
async assign(
identity: AssignmentIdentity,
preferredRegion?: RelayRegion,
placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION,
// evacuateDeadCells re-enters placement from a sweep; it must not take the
// bounded wait, whose 55P03 would surface as a terminal sweep failure.
lockMode: CellInventoryLockMode = 'request'
): Promise<RelayAssignment> {
const sticky = await this.assignStickyWithLockRetry(identity, lockMode, preferredRegion)
if (sticky) return sticky
// Only placement needs the global inventory critical section; queueing those
// attempts locally avoids turning true placement bursts into NOWAIT storms.
return await this.serializeAssignment(
async () =>
await this.assignWithLockRetry(identity, lockMode, preferredRegion, placementRegion)
)
}
private async assignStickyWithLockRetry(
identity: AssignmentIdentity,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion
): Promise<RelayAssignment | null> {
return await this.withAssignmentLockRetry(
async (inventoryFirst) =>
await this.assignStickyOnce(identity, inventoryFirst, lockMode, preferredRegion)
)
}
private async assignWithLockRetry(
identity: AssignmentIdentity,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion,
placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION
): Promise<RelayAssignment> {
const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS
let inventoryScope: AssignmentInventoryScope = 'none'
while (true) {
try {
return await this.assignOnce(
identity,
inventoryScope,
lockMode,
preferredRegion,
placementRegion
)
} catch (error) {
if (error instanceof AssignmentInventoryScopeChanged) {
inventoryScope = 'all'
continue
}
if (
!(error instanceof AssignmentInventoryLockUnavailable) ||
Date.now() >= deadline
) {
throw error
}
inventoryScope = error.inventoryScope
}
await waitForAssignmentLockRetry(deadline)
}
}
private async withAssignmentLockRetry<T>(
operation: (inventoryFirst: boolean) => Promise<T>
): Promise<T> {
const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS
let inventoryFirst = false
while (true) {
try {
return await operation(inventoryFirst)
} catch (error) {
if (!isDatabaseLockUnavailable(error) || Date.now() >= deadline) throw error
inventoryFirst = true
}
// The retry joins the cell queue without holding later legacy-cycle locks.
await waitForAssignmentLockRetry(deadline)
}
}
private async assignStickyOnce(
identity: AssignmentIdentity,
inventoryFirst: boolean,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion
): Promise<RelayAssignment | null> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
// Why: the retry exists to take a cell row before the assignment row, the
// order placement uses. It only ever needs the one cell this host is
// pinned to, so read the pin unlocked and lock that row alone; taking all
// 23 queued every sticky refresh in the fleet behind every other one.
const pinnedCellId = inventoryFirst
? await this.pinnedCellId(transaction, identity)
: undefined
const lockedCells =
pinnedCellId === undefined
? undefined
: await this.lockCellRows(transaction, [pinnedCellId], lockMode)
const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
if (!existing) return null
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
await this.recordRegionPreference(transaction, identity, preferredRegion, now)
if (mayNormallyReassign(activity(existing), now)) return null
// Why: a stranded host must fall through to placement — re-granting the
// pinned cell here is what refreshes its own activity and sustains the
// loop (issue #225).
if (await this.assignmentStrandedOnUnservedCell(transaction, identity, existing, now)) {
return null
}
const currentCellId = text(existing, 'cell_id')
// The pin moved between the unlocked read and the assignment lock, so the
// row held is the wrong one. Same recovery as losing the lock: retry.
if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
throw new Error('database_lock_unavailable')
}
const hadControl = holdsControlLease(
activityLeases,
currentCellId,
integer(existing, 'assignment_epoch')
)
const currentRow = lockedCells
? lockedCells.find((row) => text(row, 'cell_id') === currentCellId)
: hadControl
? (
await transaction.query(`SELECT * FROM relay_cells WHERE cell_id = ?`, [
currentCellId
])
)[0]
: (
await transaction.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id = ?`,
[currentCellId],
{ failIfUnavailable: true }
)
)[0]
if (!currentRow) throw new Error('assigned_cell_missing')
if (
this.requireLiveCells &&
!(await this.cellIsLive(transaction, currentCellId, now))
) {
return null
}
if (
!hadControl &&
!(await this.cellHasConnectionHeadroom(transaction, currentCellId))
) {
if (requestUnits(existing) === 0) return null
throw new Error('relay_connection_headroom_exhausted')
}
const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
if (hadControl) {
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
} else {
// Delta, not the value read from the snapshot: an absolute write here
// would clobber any concurrent movement of the same counter.
await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
await this.insertPendingControlLease(
transaction,
identity,
currentCellId,
integer(existing, 'assignment_epoch'),
now
)
}
return this.result(
identity,
existing,
cell(currentRow, await this.cellRegion(transaction, currentCellId)),
leaseExpiresAt
)
})
}
// Why: PR #194 pins assignments on existing-only cells so capacity pressure
// cannot scatter existing hosts — assuming those cells still serve them. A
// decommissioning cell (C3) broke that assumption: existing-only AND
// rejecting attaches, so pinned hosts loop forever while each grant
// refreshes their own last_activity_at, keeping dormancy-based
// reassignment permanently out of reach (issue #225). "Stranded" therefore
// requires proof the cell is not serving this host: a recent grant
// (last_activity_at inside the window) that had ample time to attach and
// still produced no live real activity. The evidence must come from the
// assignment row itself — reservation rows do not exist for cells without
// connection limits (C3), and expired pending leases are cleaned within a
// maintenance cycle, so neither reliably survives until the next grant.
private async assignmentStrandedOnUnservedCell(
transaction: RelayDatabase,
identity: AssignmentIdentity,
existing: SqlRow,
now: number
): Promise<boolean> {
const lastActivityAt = integer(existing, 'last_activity_at')
if (
now < lastActivityAt + STRANDED_MIN_GRANT_AGE_MS ||
now >= lastActivityAt + STRANDED_RECENT_ACTIVITY_MS
) {
return false
}
const admissionRow = (
await transaction.query(
`SELECT admission_state FROM relay_cell_admission WHERE cell_id = ?`,
[text(existing, 'cell_id')]
)
)[0]
// Unknown or missing admission fails safe: the pin stays.
if (admissionRow?.['admission_state'] !== 'existing-only') {
return false
}
const liveLeases = (
await transaction.query(
`SELECT COUNT(*) AS live FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND expires_at > ?
AND activity_id NOT LIKE 'control-pending:%'`,
[identity.userId, identity.relayHostId, now]
)
)[0]
return integer(liveLeases!, 'live') === 0
}
private async serializeAssignment<T>(operation: () => Promise<T>): Promise<T> {
const previous = this.assignmentTail
let release!: () => void
this.assignmentTail = new Promise((resolve) => (release = resolve))
await previous
try {
return await operation()
} finally {
release()
}
}
private async assignOnce(
identity: AssignmentIdentity,
inventoryScope: AssignmentInventoryScope,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion,
placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION
): Promise<RelayAssignment> {
const now = this.now()
let retryScope: RetriedAssignmentInventoryScope =
inventoryScope === 'all' ? 'all' : 'general'
return await this.database.transaction(async (transaction) => {
let lockedCells =
inventoryScope === 'all'
? await this.lockCellInventory(transaction, lockMode)
: inventoryScope === 'general'
? await this.lockGeneralCellInventory(transaction, lockMode)
: undefined
const existing = await this.assignmentRow(
transaction,
identity,
inventoryScope !== 'none'
)
retryScope = existing ? 'all' : 'general'
if (inventoryScope === 'general' && existing) {
throw new AssignmentInventoryScopeChanged()
}
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
await this.recordRegionPreference(transaction, identity, preferredRegion, now)
let forcedDeadReassignment = false
let connectionHeadroomReassignment = false
let strandedReassignment = false
if (existing && !mayNormallyReassign(activity(existing), now)) {
lockedCells ??= await this.lockCellInventory(transaction, 'nowait')
const admission = await cellAdmissionStates(transaction)
const currentRow = lockedCells.find(
(row) => text(row, 'cell_id') === text(existing, 'cell_id')
)
if (!currentRow) throw new Error('assigned_cell_missing')
const current = cell(
currentRow,
await this.cellRegion(transaction, text(existing, 'cell_id'))
)
// Why: an existing-only cell that rejects attaches (C3's decommission
// posture) must not re-pin the hosts it refuses; the dead-cell fence
// path below does not apply either — the cell is live, just unwilling.
strandedReassignment = await this.assignmentStrandedOnUnservedCell(
transaction,
identity,
existing,
now
)
if (
!strandedReassignment &&
(!this.requireLiveCells || (await this.cellIsLive(transaction, current.cellId, now)))
) {
const hadControl = holdsControlLease(
activityLeases,
current.cellId,
integer(existing, 'assignment_epoch')
)
const hasConnectionHeadroom =
hadControl ||
(await this.cellHasConnectionHeadroom(transaction, current.cellId))
if (hasConnectionHeadroom) {
const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
if (hadControl) {
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
} else {
await this.adjustCellReservation(transaction, current.cellId, 1)
await this.adjustActivityCount(
transaction,
identity,
'control',
1,
leaseExpiresAt,
now
)
await this.insertPendingControlLease(
transaction,
identity,
current.cellId,
integer(existing, 'assignment_epoch'),
now
)
}
return this.result(identity, existing, current, leaseExpiresAt)
}
if (requestUnits(existing) > 0) {
throw new Error('relay_connection_headroom_exhausted')
}
if (admission.get(current.cellId) !== 'general') {
throw new Error('relay_connection_headroom_exhausted')
}
connectionHeadroomReassignment = true
}
if (!strandedReassignment && !connectionHeadroomReassignment) {
if (
(await this.deadCellRequiresCommittedFence(
transaction,
identity,
current.cellId,
integer(existing, 'assignment_epoch')
)) &&
!(await this.cellHasCommittedFence(transaction, current.cellId, now))
) {
throw new Error('relay_capacity_exhausted')
}
forcedDeadReassignment = true
}
}
lockedCells ??= existing
? await this.lockCellInventory(transaction, 'nowait')
: await this.lockGeneralCellInventory(transaction, 'nowait')
const target = await this.leastLoadedCell(
transaction,
lockedCells,
placementRegion
)
if (!target) throw new Error('relay_capacity_exhausted')
const previousUnits = existing ? requestUnits(existing) : 0
if (existing) {
await this.adjustCellReservation(transaction, text(existing, 'cell_id'), -previousUnits)
if (forcedDeadReassignment || strandedReassignment) {
// A fenced/dead incarnation cannot own drainable work, and a
// stranded host's only leases are the unclaimed grant artifacts of
// its own loop. Removing them prevents late expiry from
// decrementing the replacement.
await transaction.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
}
}
await this.adjustCellReservation(transaction, target.cellId, 1)
const assignmentEpoch = existing ? integer(existing, 'assignment_epoch') + 1 : 1
const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
await transaction.query(
`INSERT INTO relay_assignments
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
pending_installs, pending_confirmations, migration_leases)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
cell_id = excluded.cell_id,
assignment_epoch = excluded.assignment_epoch,
lease_expires_at = excluded.lease_expires_at,
last_activity_at = excluded.last_activity_at,
reserved_controls = excluded.reserved_controls,
reserved_splices = excluded.reserved_splices,
reserved_invites = excluded.reserved_invites,
pending_installs = excluded.pending_installs,
pending_confirmations = excluded.pending_confirmations,
migration_leases = excluded.migration_leases`,
[
identity.userId,
identity.relayHostId,
target.cellId,
assignmentEpoch,
leaseExpiresAt,
now,
1,
0,
0,
0,
0,
0
]
)
if (existing) {
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
text(existing, 'cell_id'),
integer(existing, 'assignment_epoch'),
now
)
}
await this.insertPendingControlLease(
transaction,
identity,
target.cellId,
assignmentEpoch,
now
)
return { ...identity, ...target, assignmentEpoch, leaseExpiresAt }
}).catch((error: unknown) => {
if (isDatabaseLockUnavailable(error)) {
throw new AssignmentInventoryLockUnavailable(retryScope)
}
throw error
})
}
async resolve(identity: AssignmentIdentity): Promise<RelayAssignment | null> {
const rows = await this.database.query(
`SELECT assignment.*, cell.cell_url, region.region
FROM relay_assignments assignment
JOIN relay_cells cell ON cell.cell_id = assignment.cell_id
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
const row = rows[0]
if (
row &&
this.requireLiveCells &&
!(await this.cellIsLive(this.database, text(row, 'cell_id'), this.now()))
) {
return null
}
return row
? {
...identity,
cellId: text(row, 'cell_id'),
cellUrl: text(row, 'cell_url'),
region: optionalRelayRegion(row, 'region') ?? RELAY_DEFAULT_REGION,
assignmentEpoch: integer(row, 'assignment_epoch'),
leaseExpiresAt: integer(row, 'lease_expires_at')
}
: null
}
async setCellEnabled(cellId: string, enabled: boolean): Promise<void> {
await this.setCellAdmissionState(cellId, stateFromEnabled(enabled))
}
async setCellAdmissionState(cellId: string, state: CellAdmissionState): Promise<void> {
await this.database.transaction(
async (transaction) =>
await setCellAdmissionBeforeBoundary(transaction, cellId, state, this.now())
)
}
async applyCellAdmissionSelector(input: {
attemptId: string
expectedGeneration: number
expectedMembershipSha256?: string
membership: CellAdmissionMembership
}): Promise<{
changed: boolean
selector: {
generation: number
attemptId: string | null
membership: CellAdmissionMembership
}
}> {
return await this.admissionSelector.apply(input)
}
async inspectCellAdmissionSelector(
attemptId?: string
): Promise<CellAdmissionSelectorInspection> {
return await this.admissionSelector.inspect(attemptId)
}
async addMigrationCells(input: {
attemptId: string
expectedGeneration: number
cells: MigrationCellRegistration[]
}): Promise<{
changed: boolean
selector: {
generation: number
attemptId: string | null
membership: CellAdmissionMembership
}
}> {
return await this.migrationCellRegistrar.add(input)
}
async recordCellHeartbeat(input: CellHeartbeat): Promise<void> {
const now = this.now()
let inclusionWatermark: number | undefined
await this.database.transaction(async (transaction) => {
const configured = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [input.cellId])
)[0]
if (!configured) throw new Error('cell_not_found')
if (text(configured, 'cell_url') !== input.cellUrl) throw new Error('cell_origin_mismatch')
if (
(await this.cellRegion(transaction, input.cellId)) !==
(input.region ?? RELAY_DEFAULT_REGION)
) {
throw new Error('cell_region_mismatch')
}
const current = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (
current &&
((text(current, 'cell_incarnation') === input.cellIncarnation &&
input.startedAt !== integer(current, 'started_at')) ||
(text(current, 'cell_incarnation') !== input.cellIncarnation &&
input.startedAt <= integer(current, 'started_at')))
) {
throw new Error('stale_cell_incarnation')
}
const connectionLimit = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_connection_limits WHERE cell_id = ?`,
[input.cellId]
)
)[0]
if (connectionLimit) {
if (
input.totalConnections === undefined ||
input.inFlightConnections === undefined ||
input.reservedConnectionUnits === undefined ||
input.enforcedConnectionUnits === undefined ||
input.enforcedConnectionUnits !==
input.totalConnections +
input.inFlightConnections +
input.reservedConnectionUnits ||
input.connectionHardCap !== integer(connectionLimit, 'hard_cap') ||
input.connectionUnobservedBound !==
integer(connectionLimit, 'unobserved_bound')
) {
throw new Error('cell_connection_telemetry_mismatch')
}
} else if (
input.totalConnections !== undefined ||
input.inFlightConnections !== undefined ||
input.reservedConnectionUnits !== undefined ||
input.enforcedConnectionUnits !== undefined ||
input.connectionInclusionWatermark !== undefined ||
input.connectionHardCap !== undefined ||
input.connectionUnobservedBound !== undefined
) {
throw new Error('cell_connection_limit_not_configured')
}
await transaction.query(
`INSERT INTO relay_cell_runtime
(cell_id, cell_url, cell_incarnation, started_at, ready,
observed_requests, last_heartbeat_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_url = excluded.cell_url,
cell_incarnation = excluded.cell_incarnation,
started_at = excluded.started_at,
ready = excluded.ready,
observed_requests = excluded.observed_requests,
last_heartbeat_at = excluded.last_heartbeat_at,
updated_at = excluded.updated_at`,
[
input.cellId,
input.cellUrl,
input.cellIncarnation,
input.startedAt,
input.ready ? 1 : 0,
input.observedRequests,
now,
now
]
)
// Live directors read relay_cell_runtime; keep heartbeats off the placement capacity lock.
if (!this.requireLiveCells) {
await transaction.query(
`UPDATE relay_cells SET observed_requests = ?, last_heartbeat_at = ?, updated_at = ?
WHERE cell_id = ?`,
[input.observedRequests, now, now, input.cellId]
)
}
if (connectionLimit) {
const currentSnapshot = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[input.cellId]
)
)[0]
const previousWatermark =
currentSnapshot &&
text(currentSnapshot, 'cell_incarnation') === input.cellIncarnation
? integer(currentSnapshot, 'inclusion_watermark')
: -1
inclusionWatermark = input.connectionInclusionWatermark ?? previousWatermark + 1
if (
input.connectionInclusionWatermark !== undefined &&
inclusionWatermark <= previousWatermark
) {
throw new Error('stale_connection_snapshot')
}
await transaction.query(
`INSERT INTO relay_cell_connection_runtime
(cell_id, cell_incarnation, total_connections, in_flight_connections,
reserved_connection_units, enforced_connection_units, last_heartbeat_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
total_connections = excluded.total_connections,
in_flight_connections = excluded.in_flight_connections,
reserved_connection_units = excluded.reserved_connection_units,
enforced_connection_units = excluded.enforced_connection_units,
last_heartbeat_at = excluded.last_heartbeat_at,
updated_at = excluded.updated_at`,
[
input.cellId,
input.cellIncarnation,
input.totalConnections,
input.inFlightConnections,
input.reservedConnectionUnits,
input.enforcedConnectionUnits,
now,
now
]
)
await transaction.query(
`INSERT INTO relay_cell_connection_snapshots
(cell_id, cell_incarnation, inclusion_watermark, total_connections,
in_flight_connections, reserved_connection_units,
enforced_connection_units, snapshot_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
inclusion_watermark = excluded.inclusion_watermark,
total_connections = excluded.total_connections,
in_flight_connections = excluded.in_flight_connections,
reserved_connection_units = excluded.reserved_connection_units,
enforced_connection_units = excluded.enforced_connection_units,
snapshot_at = excluded.snapshot_at`,
[
input.cellId,
input.cellIncarnation,
inclusionWatermark,
input.totalConnections,
input.inFlightConnections,
input.reservedConnectionUnits,
input.enforcedConnectionUnits,
now
]
)
}
await transaction.query(`DELETE FROM relay_cell_fences WHERE cell_id = ?`, [input.cellId])
await transaction.query(`DELETE FROM relay_cell_committed_fences WHERE cell_id = ?`, [
input.cellId
])
await transaction.query(
`DELETE FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
[input.cellId]
)
})
if (inclusionWatermark !== undefined) {
await this.releaseHeartbeatConnectionReservationDebt(
input.cellId,
input.cellIncarnation,
inclusionWatermark,
now
)
}
}
async recordCellRegionalRehomeStatus(input: CellRegionalRehomeStatus): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
const runtime = (
await transaction.queryLocked(
`SELECT cell_incarnation FROM relay_cell_runtime WHERE cell_id = ?`,
[input.cellId]
)
)[0]
if (!runtime || text(runtime, 'cell_incarnation') !== input.cellIncarnation) {
throw new Error('stale_cell_incarnation')
}
await transaction.query(
`INSERT INTO relay_cell_capabilities
(cell_id, cell_incarnation, regional_rehome_protocol, last_heartbeat_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
regional_rehome_protocol = excluded.regional_rehome_protocol,
last_heartbeat_at = excluded.last_heartbeat_at`,
[input.cellId, input.cellIncarnation, input.regionalRehomeProtocol, now]
)
const safety = input.safety
await transaction.query(
`INSERT INTO relay_cell_rehome_safety
(cell_id, cell_incarnation, observed_at, sql_failures, reconnects,
control_activity_recovery_failures, database_pool_waiting,
database_pool_waiters_max, database_pool_wait_ms_max)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
observed_at = excluded.observed_at,
sql_failures = excluded.sql_failures,
reconnects = excluded.reconnects,
control_activity_recovery_failures =
excluded.control_activity_recovery_failures,
database_pool_waiting = excluded.database_pool_waiting,
database_pool_waiters_max = excluded.database_pool_waiters_max,
database_pool_wait_ms_max = excluded.database_pool_wait_ms_max`,
[
input.cellId,
input.cellIncarnation,
safety.observedAt,
safety.sqlFailures,
safety.reconnects,
safety.controlActivityRecoveryFailures,
safety.databasePoolWaiting,
safety.databasePoolWaitersMax,
safety.databasePoolWaitMsMax
]
)
})
}
private async releaseHeartbeatConnectionReservationDebt(
cellId: string,
cellIncarnation: string,
inclusionWatermark: number,
now: number
): Promise<void> {
await this.database.transaction(async (transaction) => {
const runtime = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`,
[cellId]
)
)[0]
const snapshot = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[cellId]
)
)[0]
if (
!runtime ||
!snapshot ||
text(runtime, 'cell_incarnation') !== cellIncarnation ||
text(snapshot, 'cell_incarnation') !== cellIncarnation ||
integer(snapshot, 'inclusion_watermark') !== inclusionWatermark
) {
return
}
await transaction.query(
`UPDATE relay_control_connection_reservations
SET state = 'released', released_at = ?, updated_at = ?
WHERE cell_id = ? AND state = 'claimed'
AND inclusion_watermark IS NOT NULL
AND inclusion_watermark <= ?`,
[now, now, cellId, inclusionWatermark]
)
await transaction.query(
`UPDATE relay_control_connection_reservations
SET state = 'released', released_at = ?, updated_at = ?
WHERE reservation_id IN (
SELECT duplicate.reservation_id
FROM relay_control_connection_reservations duplicate
WHERE duplicate.cell_id = ?
AND duplicate.state = 'late-arrival-debt'
AND duplicate.claim_activity_id IS NULL
AND duplicate.timeout_at <= ?
AND EXISTS (
SELECT 1
FROM relay_control_connection_reservations retained
WHERE retained.user_id = duplicate.user_id
AND retained.relay_host_id = duplicate.relay_host_id
AND retained.assignment_epoch = duplicate.assignment_epoch
AND retained.cell_id = duplicate.cell_id
AND retained.state = 'late-arrival-debt'
AND retained.claim_activity_id IS NULL
AND retained.timeout_at <= ?
AND (
retained.created_at < duplicate.created_at OR
(
retained.created_at = duplicate.created_at AND
retained.reservation_id < duplicate.reservation_id
)
)
)
)`,
[now, now, cellId, now, now]
)
await transaction.query(
`UPDATE relay_control_connection_reservations
SET state = 'released', released_at = ?, updated_at = ?
WHERE cell_id = ? AND state = 'late-arrival-debt'
AND claim_activity_id IS NULL AND timeout_at <= ?
AND EXISTS (
SELECT 1 FROM relay_assignments assignment
WHERE assignment.user_id =
relay_control_connection_reservations.user_id
AND assignment.relay_host_id =
relay_control_connection_reservations.relay_host_id
AND assignment.assignment_epoch >
relay_control_connection_reservations.assignment_epoch
)
AND EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id =
relay_control_connection_reservations.user_id
AND migration.relay_host_id =
relay_control_connection_reservations.relay_host_id
AND migration.assignment_epoch =
relay_control_connection_reservations.assignment_epoch
AND migration.target_cell_id =
relay_control_connection_reservations.cell_id
AND migration.aborted_at IS NOT NULL
)`,
[now, now, cellId, now]
)
})
}
async attestCellFence(cellId: string, cellIncarnation: string): Promise<number> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
cellId
])
)[0]
if (
!runtime ||
text(runtime, 'cell_incarnation') !== cellIncarnation ||
integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs
) {
throw new Error('cell_fence_runtime_not_stale')
}
const expiresAt = now + CELL_FENCE_TTL_MS
await transaction.query(
`INSERT INTO relay_cell_fences
(cell_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[cellId, cellIncarnation, now, expiresAt]
)
return expiresAt
})
}
async adoptLegacyCellFence(cellId: string, cellIncarnation: string): Promise<number> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
cellId
])
)[0]
if (
!runtime ||
text(runtime, 'cell_incarnation') !== cellIncarnation ||
integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs
) {
throw new Error('cell_fence_runtime_not_stale')
}
const attempt = (
await transaction.queryLocked(
`SELECT attempt_id FROM relay_cell_fence_attempts
WHERE cell_id = ? ORDER BY created_at DESC LIMIT 1`,
[cellId]
)
)[0]
if (attempt) throw new Error('legacy_cell_fence_attempt_exists')
const expiresAt = now + CELL_FENCE_TTL_MS
await transaction.query(
`INSERT INTO relay_cell_fences
(cell_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[cellId, cellIncarnation, now, expiresAt]
)
return expiresAt
})
}
async commitLegacyCellFenceAdoption(
cellId: string,
cellIncarnation: string
): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
cellId
])
)[0]
const fence = (
await transaction.queryLocked(`SELECT * FROM relay_cell_fences WHERE cell_id = ?`, [
cellId
])
)[0]
const attempt = (
await transaction.queryLocked(
`SELECT attempt_id FROM relay_cell_fence_attempts
WHERE cell_id = ? ORDER BY created_at DESC LIMIT 1`,
[cellId]
)
)[0]
if (attempt) throw new Error('legacy_cell_fence_attempt_exists')
if (
!runtime ||
!fence ||
text(runtime, 'cell_incarnation') !== cellIncarnation ||
text(fence, 'cell_incarnation') !== cellIncarnation ||
integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs ||
integer(fence, 'attested_at') < integer(runtime, 'last_heartbeat_at') ||
integer(fence, 'expires_at') <= now
) {
throw new Error('legacy_cell_fence_not_active')
}
await transaction.query(
`INSERT INTO relay_cell_legacy_fence_adoptions
(cell_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[cellId, cellIncarnation, now, integer(fence, 'expires_at')]
)
})
}
async prepareCellFenceAttempt(input: CellFenceAttemptEvidence): Promise<CellFenceAttempt> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
let createdAt = now
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (!runtime || text(runtime, 'cell_incarnation') !== input.cellIncarnation) {
throw new Error('cell_fence_runtime_mismatch')
}
const existing = (
await transaction.queryLocked(
`${CELL_FENCE_ATTEMPT_SELECT}
WHERE attempts.cell_id = ? ORDER BY attempts.created_at DESC LIMIT 1`,
[input.cellId]
)
)[0]
if (existing) {
const attempt = cellFenceAttempt(existing)
if (!attempt.completedAt && !attempt.abortedAt) {
assertCellFenceAttemptBase(existing, input)
return attempt
}
createdAt = Math.max(now, attempt.createdAt + 1)
} else {
const activeFence = (
await transaction.queryLocked(
`SELECT cell_id FROM relay_cell_fences
WHERE cell_id = ? AND cell_incarnation = ? AND expires_at > ?`,
[input.cellId, input.cellIncarnation, now]
)
)[0]
if (activeFence) throw new Error('cell_fence_already_attested')
}
const expiresAt = createdAt + CELL_FENCE_ATTEMPT_TTL_MS
await transaction.query(
`INSERT INTO relay_cell_fence_attempts
(attempt_id, environment, cell_id, cell_incarnation, mig_name, instance_group,
generation_identity, fence_commit, plan_sha256, gce_operation, created_at,
expires_at, apply_started_at, completed_at, aborted_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, NULL, NULL, NULL)`,
[
input.attemptId,
input.environment,
input.cellId,
input.cellIncarnation,
input.migName,
input.instanceGroup,
input.generationIdentity,
input.fenceCommit,
input.planSha256,
createdAt,
expiresAt
]
)
await transaction.query(
`INSERT INTO relay_cell_fence_plan_bindings
(attempt_id, plan_object_name, plan_object_generation, var_file_sha256,
terraform_state_lineage, terraform_state_serial,
terraform_state_object_generation, terraform_state_object_sha256,
request_reason)
VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?)`,
[
input.attemptId,
input.planObjectName,
input.varFileSha256,
input.terraformStateLineage,
input.terraformStateSerial,
input.terraformStateObjectGeneration,
input.terraformStateObjectSha256,
input.requestReason
]
)
const { planObjectGeneration: _ignored, ...prepared } = input
return { ...prepared, createdAt, expiresAt }
})
}
async bindCellFencePlanGeneration(
input: CellFenceAttemptEvidence,
planObjectGeneration: string
): Promise<CellFenceAttempt> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const row = await lockedCellFenceAttempt(transaction, input.attemptId)
assertCellFenceAttemptBase(row, input)
if (integer(row, 'expires_at') <= now) throw new Error('cell_fence_attempt_expired')
if (row.apply_started_at !== null) throw new Error('cell_fence_apply_already_started')
if (row.completed_at !== null || row.aborted_at !== null) {
throw new Error('cell_fence_attempt_terminal')
}
const existing = optionalText(row, 'plan_object_generation')
if (existing && existing !== planObjectGeneration) {
throw new Error('cell_fence_plan_generation_mismatch')
}
if (!existing) {
await transaction.query(
`UPDATE relay_cell_fence_plan_bindings
SET plan_object_generation = ? WHERE attempt_id = ?`,
[planObjectGeneration, input.attemptId]
)
row.plan_object_generation = planObjectGeneration
}
return cellFenceAttempt(row)
})
}
async startCellFenceApply(
input: CellFenceAttemptEvidence,
invocationId: string,
requestReason: string
): Promise<{ attempt: CellFenceAttempt; invocation: CellFenceApplyInvocation }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
if (requestReason !== `${input.requestReason}/${invocationId}`) {
throw new Error('cell_fence_invocation_mismatch')
}
const row = await lockedCellFenceAttempt(transaction, input.attemptId)
assertActiveCellFenceAttempt(row, input, now)
const existing = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_fence_apply_invocations WHERE invocation_id = ?`,
[invocationId]
)
)[0]
if (existing) {
if (
text(existing, 'attempt_id') !== input.attemptId ||
text(existing, 'request_reason') !== requestReason
) {
throw new Error('cell_fence_invocation_mismatch')
}
return {
attempt: cellFenceAttempt(row),
invocation: cellFenceApplyInvocation(existing)
}
}
if (row.apply_started_at === null) {
await transaction.query(
`UPDATE relay_cell_fence_attempts SET apply_started_at = ? WHERE attempt_id = ?`,
[now, input.attemptId]
)
row.apply_started_at = now
}
await transaction.query(
`INSERT INTO relay_cell_fence_apply_invocations
(invocation_id, attempt_id, request_reason, started_at, gce_operation)
VALUES (?, ?, ?, ?, NULL)`,
[invocationId, input.attemptId, requestReason, now]
)
return {
attempt: cellFenceAttempt(row),
invocation: { invocationId, requestReason, startedAt: now }
}
})
}
async recordCellFenceOperation(
input: CellFenceAttemptEvidence,
invocationId: string,
requestReason: string,
gceOperation: string
): Promise<{ attempt: CellFenceAttempt; invocation: CellFenceApplyInvocation }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
if (requestReason !== `${input.requestReason}/${invocationId}`) {
throw new Error('cell_fence_invocation_mismatch')
}
const row = await lockedCellFenceAttempt(transaction, input.attemptId)
assertActiveCellFenceAttempt(row, input, now)
if (row.apply_started_at === null) throw new Error('cell_fence_apply_not_started')
const invocation = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_fence_apply_invocations WHERE invocation_id = ?`,
[invocationId]
)
)[0]
if (
!invocation ||
text(invocation, 'attempt_id') !== input.attemptId ||
text(invocation, 'request_reason') !== requestReason
) {
throw new Error('cell_fence_invocation_mismatch')
}
const existing = invocation.gce_operation
if (existing !== null && existing !== gceOperation) {
throw new Error('cell_fence_operation_mismatch')
}
await transaction.query(
`UPDATE relay_cell_fence_apply_invocations
SET gce_operation = ? WHERE invocation_id = ?`,
[gceOperation, invocationId]
)
invocation.gce_operation = gceOperation
await transaction.query(
`UPDATE relay_cell_fence_attempts SET gce_operation = ? WHERE attempt_id = ?`,
[gceOperation, input.attemptId]
)
row.gce_operation = gceOperation
return {
attempt: cellFenceAttempt(row),
invocation: cellFenceApplyInvocation(invocation)
}
})
}
async cellFenceAttempt(cellId: string): Promise<CellFenceAttempt | null> {
const rows = await this.database.query(
`${CELL_FENCE_ATTEMPT_SELECT}
WHERE attempts.cell_id = ? ORDER BY attempts.created_at DESC LIMIT 1`,
[cellId]
)
if (rows.length === 0) return null
const attempt = cellFenceAttempt(rows[0]!)
const invocations = await this.database.query(
`SELECT * FROM relay_cell_fence_apply_invocations
WHERE attempt_id = ? ORDER BY started_at, invocation_id`,
[attempt.attemptId]
)
return {
...attempt,
applyInvocations: invocations.map(cellFenceApplyInvocation)
}
}
async abortCellFenceAttempt(input: CellFenceAttemptEvidence): Promise<CellFenceAttempt> {
return await this.database.transaction(async (transaction) => {
const row = await lockedCellFenceAttempt(transaction, input.attemptId)
assertCellFenceAttemptBase(row, input)
if (row.completed_at !== null) throw new Error('cell_fence_attempt_completed')
if (row.aborted_at !== null) throw new Error('cell_fence_attempt_aborted')
if (row.apply_started_at !== null || row.gce_operation !== null) {
throw new Error('cell_fence_apply_may_have_started')
}
const abortedAt = this.now()
await transaction.query(
`UPDATE relay_cell_fence_attempts SET aborted_at = ? WHERE attempt_id = ?`,
[abortedAt, input.attemptId]
)
row.aborted_at = abortedAt
return cellFenceAttempt(row)
})
}
async attestCellFenceAttempt(
input: CellFenceAttemptEvidence,
gceOperation: string
): Promise<{ expiresAt: number; attempt: CellFenceAttempt }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const attemptRow = await lockedCellFenceAttempt(transaction, input.attemptId)
assertCellFenceAttemptEvidence(attemptRow, input)
if (attemptRow.completed_at !== null) {
if (attemptRow.gce_operation !== gceOperation) {
throw new Error('cell_fence_operation_not_attested')
}
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (
!runtime ||
text(runtime, 'cell_incarnation') !== input.cellIncarnation ||
integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs
) {
throw new Error('cell_fence_runtime_not_stale')
}
const expiresAt = now + CELL_FENCE_TTL_MS
await transaction.query(
`INSERT INTO relay_cell_fences
(cell_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[input.cellId, input.cellIncarnation, now, expiresAt]
)
await this.recordCommittedCellFence(
transaction,
input.cellId,
input.cellIncarnation,
input.attemptId,
now,
expiresAt
)
return {
expiresAt,
attempt: cellFenceAttempt(attemptRow)
}
}
assertActiveCellFenceAttempt(attemptRow, input, now)
if (
attemptRow.apply_started_at === null ||
attemptRow.gce_operation !== gceOperation
) {
throw new Error('cell_fence_operation_not_attested')
}
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (!cell) throw new Error('cell_not_found')
if (integer(cell, 'enabled') !== 0) throw new Error('cell_fence_admission_enabled')
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
input.cellId
])
)[0]
if (
!runtime ||
text(runtime, 'cell_incarnation') !== input.cellIncarnation ||
integer(runtime, 'last_heartbeat_at') > now - this.heartbeatTtlMs
) {
throw new Error('cell_fence_runtime_not_stale')
}
const expiresAt = now + CELL_FENCE_TTL_MS
await transaction.query(
`INSERT INTO relay_cell_fences
(cell_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[input.cellId, input.cellIncarnation, now, expiresAt]
)
await this.recordCommittedCellFence(
transaction,
input.cellId,
input.cellIncarnation,
input.attemptId,
now,
expiresAt
)
await transaction.query(
`UPDATE relay_cell_fence_attempts SET completed_at = ? WHERE attempt_id = ?`,
[now, input.attemptId]
)
attemptRow.completed_at = now
return { expiresAt, attempt: cellFenceAttempt(attemptRow) }
})
}
async prepareCellDrainAttempt(input: {
attemptId: string
cellId: string
cellIncarnation: string
traceValue: string
plannedGraceMs: number
}): Promise<CellDrainAttempt & { shouldSend: false }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.assertDrainCellGeneration(
transaction,
input.cellId,
input.cellIncarnation
)
const existing = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_drain_attempt_states
WHERE cell_id = ? ORDER BY prepared_at DESC LIMIT 1`,
[input.cellId]
)
)[0]
if (existing) {
if (text(existing, 'attempt_id') === input.attemptId) {
if (
text(existing, 'cell_incarnation') !== input.cellIncarnation ||
text(existing, 'trace_value') !== input.traceValue ||
integer(existing, 'planned_grace_ms') !== input.plannedGraceMs
) {
throw new Error('drain_attempt_generation_mismatch')
}
return { ...cellDrainAttempt(existing), shouldSend: false }
}
if (
text(existing, 'state') !== 'proven-not-delivered' ||
text(existing, 'cell_incarnation') !== input.cellIncarnation
) {
throw new Error('drain_attempt_generation_mismatch')
}
}
await transaction.query(
`INSERT INTO relay_cell_drain_attempt_states
(attempt_id, cell_id, cell_incarnation, trace_value, planned_grace_ms,
state, prepared_at, send_may_have_started_at, send_permit_expires_at,
application_receipt_at, backend_success_status, backend_instance,
receipt_cell_incarnation, retry_after, recover_forward_attempted_at,
proven_not_delivered_at)
VALUES (?, ?, ?, ?, ?, 'prepared', ?, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL)`,
[
input.attemptId,
input.cellId,
input.cellIncarnation,
input.traceValue,
input.plannedGraceMs,
now
]
)
return {
attemptId: input.attemptId,
cellId: input.cellId,
cellIncarnation: input.cellIncarnation,
traceValue: input.traceValue,
plannedGraceMs: input.plannedGraceMs,
state: 'prepared',
preparedAt: now,
shouldSend: false
}
})
}
async beginCellDrainSend(input: {
attemptId: string
cellId: string
cellIncarnation: string
}): Promise<CellDrainAttempt & { shouldSend: boolean }> {
try {
return await this.beginCellDrainSendOnce(input)
} catch (error) {
if (
!(error instanceof Error) ||
error.message !== 'migration_activity_accounting_mismatch'
) {
throw error
}
const activityCells = await this.database.query(
`SELECT DISTINCT lease.cell_id
FROM relay_assignment_activity_leases lease
WHERE EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = lease.user_id
AND migration.relay_host_id = lease.relay_host_id
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY lease.cell_id`,
[input.cellId]
)
const cellIds = activityCells
.map((row) => text(row, 'cell_id'))
.filter((cellId) => cellId !== input.cellId)
for (const cellId of cellIds) {
await this.reconcileReservationAccounting(input.cellId, cellId)
}
await this.refreshDrainMigrationLeases(input)
return await this.beginCellDrainSendOnce(input)
}
}
private async refreshDrainMigrationLeases(input: {
cellId: string
cellIncarnation: string
}): Promise<void> {
await this.retireObsoleteDrainMigrations(input.cellId)
for (let repairAttempt = 0; ; repairAttempt++) {
try {
await this.refreshDrainMigrationLeasesOnce(input)
return
} catch (error) {
if (
!(error instanceof Error) ||
error.message !== 'migration_activity_accounting_mismatch' ||
repairAttempt >= MAX_DRAIN_ACCOUNTING_REPAIR_ATTEMPTS
) {
throw error
}
await this.reconcileDrainMigrationAccounting(input.cellId)
}
}
}
private async reconcileDrainMigrationAccounting(sourceCellId: string): Promise<void> {
const activityCells = await this.database.query(
`SELECT DISTINCT lease.cell_id
FROM relay_assignment_activity_leases lease
WHERE EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = lease.user_id
AND migration.relay_host_id = lease.relay_host_id
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY lease.cell_id`,
[sourceCellId]
)
for (const cellId of activityCells
.map((row) => text(row, 'cell_id'))
.filter((cellId) => cellId !== sourceCellId)) {
await this.reconcileReservationAccounting(sourceCellId, cellId)
}
}
private async retireObsoleteDrainMigrations(sourceCellId: string): Promise<void> {
const candidates = await this.database.query(
`SELECT migration.user_id, migration.relay_host_id,
migration.assignment_epoch
FROM relay_assignment_migrations migration
JOIN relay_assignments assignment
ON assignment.user_id = migration.user_id
AND assignment.relay_host_id = migration.relay_host_id
WHERE migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
AND assignment.assignment_epoch > migration.assignment_epoch
ORDER BY migration.user_id, migration.relay_host_id`,
[sourceCellId]
)
for (const candidate of candidates) {
await this.retireObsoleteDrainMigration(sourceCellId, candidate)
}
}
private async retireObsoleteDrainMigration(
sourceCellId: string,
candidate: SqlRow
): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
const identity = {
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
}
const assignment = await this.assignmentRow(transaction, identity)
const assignmentEpoch = integer(candidate, 'assignment_epoch')
const migrationRow = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND source_cell_id = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[identity.userId, identity.relayHostId, assignmentEpoch, sourceCellId]
)
)[0]
if (!migrationRow) return
if (!assignment || integer(assignment, 'assignment_epoch') <= assignmentEpoch) {
throw new Error('migration_assignment_mismatch')
}
const leases = await this.lockAssignmentActivities(transaction, identity)
const obsoleteActivityIds = new Set([
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
])
if (leases.some((lease) => obsoleteActivityIds.has(text(lease, 'activity_id')))) {
throw new Error('migration_activity_topology_mismatch')
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
text(migrationRow, 'target_cell_id'),
assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
})
}
private async refreshDrainMigrationLeasesOnce(input: {
cellId: string
cellIncarnation: string
}): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
await this.assertDrainCellGeneration(
transaction,
input.cellId,
input.cellIncarnation
)
const assignments = await transaction.queryLocked(
`SELECT assignment.*
FROM relay_assignments assignment
WHERE EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = assignment.user_id
AND migration.relay_host_id = assignment.relay_host_id
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY assignment.user_id, assignment.relay_host_id`,
[input.cellId]
)
const activityLeases = await transaction.queryLocked(
`SELECT lease.*
FROM relay_assignment_activity_leases lease
WHERE EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = lease.user_id
AND migration.relay_host_id = lease.relay_host_id
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`,
[input.cellId]
)
const migrations = await transaction.queryLocked(
`SELECT migration.*
FROM relay_assignment_migrations migration
WHERE migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id`,
[input.cellId]
)
const cells = await this.lockCellInventory(transaction, 'request')
const assignmentRows = createDrainMigrationRowLookup(assignments, text)
const leaseRows = createDrainMigrationRowLookup(activityLeases, text)
for (const migrationRow of migrations) {
const identity = {
userId: text(migrationRow, 'user_id'),
relayHostId: text(migrationRow, 'relay_host_id')
}
const assignment = assignmentRows.first(identity)
assertCurrentMigrationAssignment(assignment, migrationRow)
const leases = leaseRows.all(identity)
const migrationLeases = leases.filter(
(lease) => text(lease, 'activity_kind') === 'migration'
)
const assignmentEpoch = integer(migrationRow, 'assignment_epoch')
const sourceRequestUnits = integer(migrationRow, 'source_request_units')
const targetCellId = text(migrationRow, 'target_cell_id')
const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
if (migrationLeases.length > 0) {
assertAssignmentActivityAccounting(assignment, leases, migrationRow)
await transaction.query(
`UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ?
AND activity_id IN (?, ?)`,
[
expiresAt,
now,
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]
)
await transaction.query(
`UPDATE relay_assignments SET
lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[expiresAt, expiresAt, now, identity.userId, identity.relayHostId]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await this.refreshPendingControlReservation(
transaction,
identity,
targetCellId,
assignmentEpoch,
expiresAt,
now
)
continue
}
if (
optionalInteger(migrationRow, 'target_registered_at') === undefined ||
integer(migrationRow, 'expires_at') > now ||
sourceRequestUnits < 0 ||
integer(migrationRow, 'target_reserved_units') !== sourceRequestUnits + 1 ||
activityLeaseById(leases, migrationActivityId(assignmentEpoch)) ||
!cells.some((cell) => text(cell, 'cell_id') === targetCellId)
) {
throw new Error('migration_activity_lease_shape_mismatch')
}
await this.adjustCellReservation(transaction, targetCellId, sourceRequestUnits)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'migration', ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
migrationActivityId(assignmentEpoch),
targetCellId,
sourceRequestUnits,
expiresAt,
now
]
)
await transaction.query(
`UPDATE relay_assignments SET migration_leases = migration_leases + 1,
lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[expiresAt, expiresAt, now, identity.userId, identity.relayHostId]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await this.refreshPendingControlReservation(
transaction,
identity,
targetCellId,
assignmentEpoch,
expiresAt,
now
)
}
})
}
private async beginCellDrainSendOnce(input: {
attemptId: string
cellId: string
cellIncarnation: string
}): Promise<CellDrainAttempt & { shouldSend: boolean }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.assertDrainCellGeneration(
transaction,
input.cellId,
input.cellIncarnation
)
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_drain_attempt_states
WHERE attempt_id = ? AND cell_id = ?`,
[input.attemptId, input.cellId]
)
)[0]
if (!row || text(row, 'cell_incarnation') !== input.cellIncarnation) {
throw new Error('drain_attempt_not_found')
}
if (text(row, 'state') !== 'prepared') {
return { ...cellDrainAttempt(row), shouldSend: false }
}
const assignments = await transaction.queryLocked(
`SELECT assignment.*
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
AND migration.relay_host_id = assignment.relay_host_id
WHERE migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY assignment.user_id, assignment.relay_host_id`,
[input.cellId]
)
const activityLeases = await transaction.queryLocked(
`SELECT lease.*
FROM relay_assignment_activity_leases lease
WHERE EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = lease.user_id
AND migration.relay_host_id = lease.relay_host_id
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`,
[input.cellId]
)
const migrationIncarnations = await transaction.queryLocked(
`SELECT migration.*,
(
SELECT incarnation.source_cell_incarnation
FROM relay_assignment_migration_incarnations incarnation
WHERE incarnation.user_id = migration.user_id
AND incarnation.relay_host_id = migration.relay_host_id
AND incarnation.assignment_epoch = migration.assignment_epoch
) AS source_cell_incarnation,
(
SELECT incarnation.target_cell_incarnation
FROM relay_assignment_migration_incarnations incarnation
WHERE incarnation.user_id = migration.user_id
AND incarnation.relay_host_id = migration.relay_host_id
AND incarnation.assignment_epoch = migration.assignment_epoch
) AS target_cell_incarnation
FROM relay_assignment_migrations migration
WHERE migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id`,
[input.cellId]
)
if (
migrationIncarnations.some(
(migration) =>
optionalText(migration, 'source_cell_incarnation') !== input.cellIncarnation
)
) {
throw new Error('drain_migration_source_incarnation_mismatch')
}
const assignmentRows = createDrainMigrationRowLookup(assignments, text)
const leaseRows = createDrainMigrationRowLookup(activityLeases, text)
for (const migrationRow of migrationIncarnations) {
const identity = {
userId: text(migrationRow, 'user_id'),
relayHostId: text(migrationRow, 'relay_host_id')
}
const assignment = assignmentRows.first(identity)
assertCurrentMigrationAssignment(assignment, migrationRow)
assertAssignmentActivityAccounting(assignment, leaseRows.all(identity), migrationRow)
}
const sendPermitExpiresAt = now + CELL_DRAIN_SEND_PERMIT_MS
await transaction.query(
`UPDATE relay_cell_drain_attempt_states
SET state = 'send-may-have-started', send_may_have_started_at = ?,
send_permit_expires_at = ?
WHERE attempt_id = ?`,
[now, sendPermitExpiresAt, input.attemptId]
)
await transaction.query(
`INSERT INTO relay_post_drain_migration_pins
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
source_cell_id, source_cell_incarnation, target_cell_id,
target_cell_incarnation, source_request_units, target_reserved_units,
pinned_at)
SELECT migration.user_id, migration.relay_host_id,
migration.assignment_epoch, ?, migration.source_cell_id,
incarnation.source_cell_incarnation, migration.target_cell_id,
incarnation.target_cell_incarnation, migration.source_request_units,
migration.target_reserved_units, ?
FROM relay_assignment_migrations migration
JOIN relay_assignment_migration_incarnations incarnation
ON incarnation.user_id = migration.user_id
AND incarnation.relay_host_id = migration.relay_host_id
AND incarnation.assignment_epoch = migration.assignment_epoch
WHERE migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ON CONFLICT (user_id, relay_host_id, assignment_epoch) DO NOTHING`,
[input.attemptId, now, input.cellId]
)
row.state = 'send-may-have-started'
row.send_may_have_started_at = now
row.send_permit_expires_at = sendPermitExpiresAt
return { ...cellDrainAttempt(row), shouldSend: true }
})
}
async recordCellDrainApplicationReceipt(input: {
attemptId: string
cellId: string
cellIncarnation: string
traceValue: string
backendStatus: number
backendInstance?: string
}): Promise<CellDrainAttempt> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.assertDrainCellGeneration(
transaction,
input.cellId,
input.cellIncarnation
)
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_drain_attempt_states
WHERE attempt_id = ? AND cell_id = ?`,
[input.attemptId, input.cellId]
)
)[0]
if (
!row ||
text(row, 'cell_incarnation') !== input.cellIncarnation ||
text(row, 'trace_value') !== input.traceValue
) {
throw new Error('drain_attempt_not_found')
}
if (text(row, 'state') === 'application-receipt') return cellDrainAttempt(row)
if (
text(row, 'state') !== 'send-may-have-started' ||
input.backendStatus < 200 ||
input.backendStatus >= 300
) {
throw new Error('drain_application_receipt_invalid')
}
const retryAfter = now + integer(row, 'planned_grace_ms') + 30_000
await transaction.query(
`UPDATE relay_cell_drain_attempt_states
SET state = 'application-receipt', application_receipt_at = ?,
backend_success_status = ?, backend_instance = ?,
receipt_cell_incarnation = ?, retry_after = ?
WHERE attempt_id = ?`,
[
now,
input.backendStatus,
input.backendInstance,
input.cellIncarnation,
retryAfter,
input.attemptId
]
)
row.state = 'application-receipt'
row.application_receipt_at = now
row.backend_success_status = input.backendStatus
row.backend_instance = input.backendInstance ?? null
row.receipt_cell_incarnation = input.cellIncarnation
row.retry_after = retryAfter
return cellDrainAttempt(row)
})
}
async proveCellDrainNotDelivered(input: {
attemptId: string
cellId: string
cellIncarnation: string
}): Promise<CellDrainAttempt> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_drain_attempt_states
WHERE attempt_id = ? AND cell_id = ?`,
[input.attemptId, input.cellId]
)
)[0]
if (!row || text(row, 'cell_incarnation') !== input.cellIncarnation) {
throw new Error('drain_attempt_not_found')
}
if (text(row, 'state') === 'proven-not-delivered') return cellDrainAttempt(row)
if (text(row, 'state') !== 'send-may-have-started') {
throw new Error('drain_delivery_proof_invalid')
}
await transaction.query(
`UPDATE relay_cell_drain_attempt_states
SET state = 'proven-not-delivered', proven_not_delivered_at = ?
WHERE attempt_id = ?`,
[now, input.attemptId]
)
row.state = 'proven-not-delivered'
row.proven_not_delivered_at = now
return cellDrainAttempt(row)
})
}
async prepareCellDrainRecovery(input: {
attemptId?: string
cellId: string
cellIncarnation: string
}): Promise<{
shouldSend: boolean
retryAfter: number
preparedAttempt?: CellDrainAttempt
}> {
await this.refreshDrainMigrationLeases(input)
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.assertDrainCellGeneration(
transaction,
input.cellId,
input.cellIncarnation
)
const attempt = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_drain_attempt_states
WHERE cell_id = ?
ORDER BY prepared_at DESC LIMIT 1`,
[input.cellId]
)
)[0]
if (!attempt || (input.attemptId && text(attempt, 'attempt_id') !== input.attemptId)) {
throw new Error('drain_application_receipt_missing')
}
if (text(attempt, 'state') === 'prepared') {
if (text(attempt, 'cell_incarnation') !== input.cellIncarnation) {
throw new Error('drain_application_receipt_missing')
}
return {
shouldSend: false,
retryAfter: now,
preparedAttempt: cellDrainAttempt(attempt)
}
}
const applicationReceiptAt = optionalInteger(attempt, 'application_receipt_at')
const backendSuccessStatus = optionalInteger(attempt, 'backend_success_status')
const retryAfter = optionalInteger(attempt, 'retry_after')
if (
text(attempt, 'state') !== 'application-receipt' ||
optionalText(attempt, 'receipt_cell_incarnation') !==
text(attempt, 'cell_incarnation') ||
applicationReceiptAt === undefined ||
backendSuccessStatus === undefined ||
backendSuccessStatus < 200 ||
backendSuccessStatus >= 300 ||
retryAfter === undefined
) {
throw new Error('drain_application_receipt_missing')
}
if (now < retryAfter) throw new Error('drain_recovery_too_early')
const attemptId = text(attempt, 'attempt_id')
if (text(attempt, 'cell_incarnation') !== input.cellIncarnation) {
const priorRecovery = (
await transaction.queryLocked(
`SELECT attempted_at FROM relay_cell_drain_recovery_attempts
WHERE drain_attempt_id = ? AND cell_incarnation = ?`,
[attemptId, input.cellIncarnation]
)
)[0]
if (priorRecovery) return { shouldSend: false, retryAfter }
await transaction.query(
`INSERT INTO relay_cell_drain_recovery_attempts
(drain_attempt_id, cell_incarnation, attempted_at) VALUES (?, ?, ?)`,
[attemptId, input.cellIncarnation, now]
)
return { shouldSend: true, retryAfter }
}
if (optionalInteger(attempt, 'recover_forward_attempted_at') !== undefined) {
return { shouldSend: false, retryAfter }
}
await transaction.query(
`UPDATE relay_cell_drain_attempt_states SET recover_forward_attempted_at = ?
WHERE attempt_id = ? AND recover_forward_attempted_at IS NULL`,
[now, attemptId]
)
return { shouldSend: true, retryAfter }
})
}
async evacuateDeadCells(limit = 100): Promise<number> {
if (!this.requireLiveCells) return 0
const cutoff = this.now() - this.heartbeatTtlMs
const rows = await this.database.query(
`SELECT assignment.user_id, assignment.relay_host_id, assignment.cell_id
FROM relay_assignments assignment
JOIN relay_cells cell ON cell.cell_id = assignment.cell_id
LEFT JOIN relay_cell_committed_fences committed
ON committed.cell_id = assignment.cell_id
LEFT JOIN relay_cell_fence_attempts attempt
ON attempt.attempt_id = committed.attempt_id
LEFT JOIN relay_cell_fences fence ON fence.cell_id = assignment.cell_id
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
WHERE (runtime.cell_id IS NULL OR runtime.ready != ? OR runtime.last_heartbeat_at <= ?)
AND (
(
cell.enabled = 1
AND NOT EXISTS (
SELECT 1 FROM relay_cell_connection_limits limits
WHERE limits.cell_id = assignment.cell_id
)
AND NOT EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
JOIN relay_assignment_migrations migration
ON migration.user_id = pin.user_id
AND migration.relay_host_id = pin.relay_host_id
AND migration.assignment_epoch = pin.assignment_epoch
WHERE pin.user_id = assignment.user_id
AND pin.relay_host_id = assignment.relay_host_id
AND pin.assignment_epoch = assignment.assignment_epoch
AND pin.target_cell_id = assignment.cell_id
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
)
OR (
cell.enabled = 0
AND (
EXISTS (
SELECT 1 FROM relay_cell_connection_limits limits
WHERE limits.cell_id = assignment.cell_id
)
OR EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
JOIN relay_assignment_migrations migration
ON migration.user_id = pin.user_id
AND migration.relay_host_id = pin.relay_host_id
AND migration.assignment_epoch = pin.assignment_epoch
WHERE pin.user_id = assignment.user_id
AND pin.relay_host_id = assignment.relay_host_id
AND pin.assignment_epoch = assignment.assignment_epoch
AND pin.target_cell_id = assignment.cell_id
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
)
AND attempt.completed_at IS NOT NULL
AND attempt.aborted_at IS NULL
AND committed.cell_incarnation = runtime.cell_incarnation
AND fence.cell_incarnation = committed.cell_incarnation
AND committed.attested_at >= runtime.last_heartbeat_at
AND committed.expires_at > ?
AND fence.expires_at > ?
)
)
ORDER BY assignment.user_id, assignment.relay_host_id LIMIT ?`,
[1, cutoff, this.now(), this.now(), limit]
)
let moved = 0
for (const row of rows) {
try {
const assignment = await this.assign(
{ userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') },
undefined,
undefined,
'pool-default'
)
if (assignment.cellId !== text(row, 'cell_id')) moved++
} catch (error) {
if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error
}
}
return moved
}
async configureCell(
cell: RelayCellConfig,
admission: boolean | CellAdmissionState
): Promise<void> {
const now = this.now()
const state = typeof admission === 'boolean' ? stateFromEnabled(admission) : admission
await this.database.transaction(async (transaction) => {
const cells = await transaction.queryLocked(
`SELECT * FROM relay_cells ORDER BY cell_id ASC`
)
const current = cells.find((row) => text(row, 'cell_id') === cell.id)
if (current && integer(current, 'reserved_requests') > cell.capacityRequests) {
throw new Error('cell_capacity_below_reserved')
}
// Deploy automation owns tagged URLs; a restarted revision must not overwrite them.
await transaction.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
cell_url = excluded.cell_url,
enabled = excluded.enabled,
capacity_requests = excluded.capacity_requests,
updated_at = excluded.updated_at`,
[
cell.id,
cell.url,
state === 'existing-only' ? 0 : 1,
cell.capacityRequests,
0,
0,
now,
now
]
)
await transaction.query(
`INSERT INTO relay_cell_regions (cell_id, region) VALUES (?, ?)
ON CONFLICT (cell_id) DO UPDATE SET region = excluded.region`,
[cell.id, cell.region ?? RELAY_DEFAULT_REGION]
)
await ensureCellAdmission(transaction, cell.id, state, now)
await setCellAdmissionBeforeBoundary(transaction, cell.id, state, now)
if (
cell.connectionHardCap !== undefined &&
cell.connectionUnobservedBound !== undefined
) {
const currentLimit = (
await transaction.queryLocked(
`SELECT hard_cap, unobserved_bound FROM relay_cell_connection_limits
WHERE cell_id = ?`,
[cell.id]
)
)[0]
const limitChanged =
currentLimit !== undefined &&
(integer(currentLimit, 'hard_cap') !== cell.connectionHardCap ||
integer(currentLimit, 'unobserved_bound') !==
cell.connectionUnobservedBound)
await transaction.query(
`INSERT INTO relay_cell_connection_limits
(cell_id, hard_cap, unobserved_bound, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
hard_cap = excluded.hard_cap,
unobserved_bound = excluded.unobserved_bound,
updated_at = excluded.updated_at`,
[cell.id, cell.connectionHardCap, cell.connectionUnobservedBound, now]
)
if (limitChanged) {
await transaction.query(
`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[cell.id]
)
await transaction.query(
`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`,
[cell.id]
)
}
}
})
}
async startActiveCellEvacuations(
sourceCellId: string,
targetCellId: string,
limit: number
): Promise<number> {
const rows = await this.database.query(
`SELECT user_id, relay_host_id FROM relay_assignments
WHERE cell_id = ? AND
(reserved_controls > 0 OR reserved_splices > 0 OR reserved_invites > 0 OR
pending_installs > 0 OR pending_confirmations > 0 OR migration_leases > 0)
ORDER BY user_id, relay_host_id LIMIT ?`,
[sourceCellId, limit]
)
let started = 0
for (const row of rows) {
const migration = await this.startEvacuation(
{ userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') },
targetCellId,
sourceCellId
)
if (migration) started++
}
return started
}
async cellEvacuationCapacity(
sourceCellId: string,
targetCellId: string
): Promise<CellEvacuationCapacity> {
const cells = await this.database.query(
`SELECT * FROM relay_cells WHERE cell_id IN (?, ?) ORDER BY cell_id`,
[sourceCellId, targetCellId]
)
if (!cells.some((row) => text(row, 'cell_id') === sourceCellId)) {
throw new Error('source_cell_not_found')
}
const target = cells.find((row) => text(row, 'cell_id') === targetCellId)
if (!target) throw new Error('target_cell_not_found')
if (!(await this.cellIsLive(this.database, targetCellId, this.now()))) {
throw new Error('target_cell_unavailable')
}
const summary = (
await this.database.query(
`SELECT COUNT(*) AS assignments,
COALESCE(SUM((SELECT COALESCE(SUM(lease.request_units), 0)
FROM relay_assignment_activity_leases lease
WHERE lease.user_id = assignment.user_id
AND lease.relay_host_id = assignment.relay_host_id)), 0) AS source_units
FROM relay_assignments assignment
WHERE assignment.cell_id = ? AND
(assignment.reserved_controls > 0 OR assignment.reserved_splices > 0 OR
assignment.reserved_invites > 0 OR assignment.pending_installs > 0 OR
assignment.pending_confirmations > 0 OR assignment.migration_leases > 0)`,
[sourceCellId]
)
)[0]!
const sourceAssignments = integer(summary, 'assignments')
return {
sourceAssignments,
// Each migration reserves its future target control plus all source-owned units.
requiredTargetUnits: integer(summary, 'source_units') + sourceAssignments,
availableTargetUnits:
integer(target, 'capacity_requests') - integer(target, 'reserved_requests')
}
}
async cellEvacuationStatus(
sourceCellId: string,
targetCellId: string,
completeReady: boolean
): Promise<CellEvacuationStatus> {
let completed = 0
let blocked = 0
const sourceFenced = await this.cellHasActiveFence(sourceCellId)
if (completeReady) {
const candidates = await this.activeCellMigrations(sourceCellId, targetCellId)
for (const [index, row] of candidates.entries()) {
try {
const identity = {
userId: text(row, 'user_id'),
relayHostId: text(row, 'relay_host_id')
}
const assignmentEpoch = integer(row, 'assignment_epoch')
if (sourceFenced) {
await this.completeEvacuationFromDeadSource(identity, {
assignmentEpoch,
sourceCellId,
targetCellId
})
} else {
await this.completeEvacuation(identity, assignmentEpoch)
}
completed++
} catch (error) {
if (error instanceof Error && error.message === 'migration_cell_inventory_busy') {
blocked += candidates.length - index
break
}
if (isIncompleteMigration(error)) blocked++
else if (!(error instanceof Error && error.message === 'migration_not_found')) throw error
}
}
}
const active = await this.activeCellMigrations(sourceCellId, targetCellId)
const oldestExpiresAt =
active.length === 0 ? null : Math.min(...active.map((row) => integer(row, 'expires_at')))
if (completeReady && active.length === 0) {
await this.reconcileReservationAccounting(sourceCellId, targetCellId)
}
const registered = active.filter(
(row) => optionalInteger(row, 'target_registered_at') !== undefined
)
const registeredSourceActive = registered.filter(
(row) => integer(row, 'source_activity_units') > 0
).length
const registeredCompletable = registered.filter(
(row) =>
integer(row, 'source_activity_units') === 0 &&
integer(row, 'target_control_current') === 1
).length
const registeredTargetInactive = registered.filter(
(row) =>
integer(row, 'source_activity_units') === 0 &&
integer(row, 'target_control_current') === 0
).length
const expiredUnregistered = active.filter(
(row) =>
optionalInteger(row, 'target_registered_at') === undefined &&
integer(row, 'expires_at') <= this.now()
)
const repairableExpiredUnregistered = expiredUnregistered.filter(
(row) => migrationHasExactActiveTarget(row, targetCellId)
).length
const abortableExpiredUnregistered = expiredUnregistered.filter(
(row) => integer(row, 'target_control_active') === 0
).length
const blockedExpiredUnregistered =
expiredUnregistered.length -
repairableExpiredUnregistered -
abortableExpiredUnregistered
return {
inProgress: active.length,
oldestExpiresAt,
oldestRemainingMs: oldestExpiresAt === null ? null : oldestExpiresAt - this.now(),
targetRegistered: registered.length,
registeredSourceActive,
registeredCompletable,
registeredTargetInactive,
completed,
blocked,
expiredUnregistered: expiredUnregistered.length,
repairableExpiredUnregistered,
abortableExpiredUnregistered,
blockedExpiredUnregistered,
blockedExpiredOnNewerTargetAssignment: expiredUnregistered.filter(
(row) =>
integer(row, 'target_control_active') === 1 &&
optionalText(row, 'current_cell_id') === targetCellId &&
(optionalInteger(row, 'current_assignment_epoch') ?? 0) >
integer(row, 'assignment_epoch')
).length
}
}
async completeReadyEvacuations(limit = 100): Promise<number> {
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
throw new Error('invalid_evacuation_completion_limit')
}
const now = this.now()
const runtimeSafety = this.requireLiveCells
? `AND EXISTS (
SELECT 1 FROM relay_cells source_cell
WHERE source_cell.cell_id = migration.source_cell_id
AND source_cell.enabled = 0
)
AND EXISTS (
SELECT 1 FROM relay_cells target_cell
WHERE target_cell.cell_id = migration.target_cell_id
AND target_cell.enabled = 1
)
AND EXISTS (
SELECT 1 FROM relay_cell_runtime source_runtime
WHERE source_runtime.cell_id = migration.source_cell_id
AND source_runtime.ready = 1
AND source_runtime.observed_requests = 0
AND source_runtime.last_heartbeat_at > ?
)
AND EXISTS (
SELECT 1 FROM relay_cell_runtime target_runtime
WHERE target_runtime.cell_id = migration.target_cell_id
AND target_runtime.ready = 1
AND target_runtime.last_heartbeat_at > ?
)`
: ''
const targetIncarnation = this.requireLiveCells
? `AND target_control.updated_at >= (
SELECT target_runtime.started_at FROM relay_cell_runtime target_runtime
WHERE target_runtime.cell_id = migration.target_cell_id
)`
: ''
// Selection avoids polling offline desktops; completeEvacuation rechecks
// current target activity and zero source ownership under row locks.
const candidates = await this.database.query(
`SELECT migration.user_id, migration.relay_host_id, migration.source_cell_id,
migration.target_cell_id, migration.assignment_epoch
FROM relay_assignment_migrations migration
WHERE migration.target_registered_at IS NOT NULL
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
${runtimeSafety}
AND NOT EXISTS (
SELECT 1 FROM relay_assignment_activity_leases source_lease
WHERE source_lease.user_id = migration.user_id
AND source_lease.relay_host_id = migration.relay_host_id
AND source_lease.cell_id = migration.source_cell_id
)
AND EXISTS (
SELECT 1 FROM relay_assignment_activity_leases target_control
WHERE target_control.user_id = migration.user_id
AND target_control.relay_host_id = migration.relay_host_id
AND target_control.cell_id = migration.target_cell_id
AND target_control.activity_kind = 'control'
AND target_control.activity_id NOT LIKE 'control-pending:%'
AND target_control.expires_at > ?
${targetIncarnation}
)
ORDER BY migration.user_id, migration.relay_host_id
LIMIT ?`,
[
...(this.requireLiveCells
? [now - this.heartbeatTtlMs, now - this.heartbeatTtlMs]
: []),
now,
limit
]
)
const pairs = new Map<string, { sourceCellId: string; targetCellId: string }>()
let completed = 0
for (const row of candidates) {
const sourceCellId = text(row, 'source_cell_id')
const targetCellId = text(row, 'target_cell_id')
pairs.set(JSON.stringify([sourceCellId, targetCellId]), { sourceCellId, targetCellId })
try {
await this.completeEvacuation(
{ userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') },
integer(row, 'assignment_epoch')
)
completed++
} catch (error) {
if (!isIncompleteMigration(error) && !isMissingMigration(error)) throw error
}
}
for (const { sourceCellId, targetCellId } of pairs.values()) {
if ((await this.activeCellMigrations(sourceCellId, targetCellId)).length === 0) {
await this.reconcileReservationAccounting(sourceCellId, targetCellId)
}
}
return completed
}
async cellDeploymentStatus(cellId: string): Promise<CellDeploymentStatus> {
const cellRow = (
await this.database.query(
`WITH activity AS (
SELECT COUNT(*) AS activity_lease_count,
COALESCE(SUM(request_units), 0) AS activity_request_units,
COALESCE(SUM(CASE WHEN activity_kind = 'control'
AND activity_id LIKE 'control-pending:%'
AND request_units = 1
THEN 1 ELSE 0 END), 0) AS pending_control_units
FROM relay_assignment_activity_leases WHERE cell_id = ?
)
SELECT cell.*, runtime.cell_url AS runtime_cell_url,
admission.admission_state,
runtime.cell_incarnation AS runtime_cell_incarnation,
runtime.started_at AS runtime_started_at, runtime.ready AS runtime_ready,
runtime.observed_requests AS runtime_observed_requests,
runtime.last_heartbeat_at AS runtime_last_heartbeat_at,
COALESCE(capabilities.regional_rehome_protocol, 0)
AS runtime_regional_rehome_protocol,
activity.activity_lease_count, activity.activity_request_units,
activity.activity_lease_count - activity.pending_control_units
AS restart_blocking_activity_leases,
activity.activity_request_units - activity.pending_control_units
AS restart_blocking_activity_request_units,
cell.reserved_requests - activity.pending_control_units
AS restart_blocking_reserved_requests
FROM relay_cells cell
CROSS JOIN activity
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
LEFT JOIN relay_cell_capabilities capabilities
ON capabilities.cell_id = runtime.cell_id
AND capabilities.cell_incarnation = runtime.cell_incarnation
WHERE cell.cell_id = ?`,
[cellId, cellId]
)
)[0]
if (!cellRow) throw new Error('cell_not_found')
const assignmentRow = (
await this.database.query(
`SELECT COUNT(*) AS assignment_count FROM relay_assignments WHERE cell_id = ?`,
[cellId]
)
)[0]!
const migrationRow = (
await this.database.query(
`SELECT
COALESCE(SUM(CASE WHEN source_cell_id = ? THEN 1 ELSE 0 END), 0)
AS outgoing_migrations,
COALESCE(SUM(CASE WHEN target_cell_id = ? THEN 1 ELSE 0 END), 0)
AS incoming_migrations
FROM relay_assignment_migrations
WHERE completed_at IS NULL AND aborted_at IS NULL
AND (source_cell_id = ? OR target_cell_id = ?)`,
[cellId, cellId, cellId, cellId]
)
)[0]!
const connectionRow = (
await this.database.query(
`SELECT limits.hard_cap, limits.unobserved_bound,
connection_runtime.total_connections, connection_runtime.in_flight_connections,
connection_runtime.reserved_connection_units,
connection_runtime.enforced_connection_units,
connection_runtime.last_heartbeat_at,
current_runtime.cell_incarnation AS current_incarnation,
connection_runtime.cell_incarnation AS connection_incarnation,
(SELECT COUNT(*) FROM relay_control_connection_reservations reservation
WHERE reservation.cell_id = limits.cell_id
AND reservation.state IN
('reserved', 'late-arrival-debt', 'claimed')) AS outstanding_reservations
FROM relay_cell_connection_limits limits
LEFT JOIN relay_cell_connection_runtime connection_runtime
ON connection_runtime.cell_id = limits.cell_id
LEFT JOIN relay_cell_runtime current_runtime
ON current_runtime.cell_id = limits.cell_id
WHERE limits.cell_id = ?`,
[cellId]
)
)[0]
const runtimeHeartbeat = optionalInteger(cellRow, 'runtime_last_heartbeat_at')
const connectionHeartbeat = connectionRow
? optionalInteger(connectionRow, 'last_heartbeat_at')
: undefined
return {
cellId,
cellUrl: text(cellRow, 'cell_url'),
region: await this.cellRegion(this.database, cellId),
enabled: integer(cellRow, 'enabled') === 1,
admissionState: parseCellAdmissionState(text(cellRow, 'admission_state')),
capacityRequests: integer(cellRow, 'capacity_requests'),
reservedRequests: integer(cellRow, 'reserved_requests'),
assignments: integer(assignmentRow, 'assignment_count'),
activityLeases: integer(cellRow, 'activity_lease_count'),
activityRequestUnits: integer(cellRow, 'activity_request_units'),
restartBlockingActivityLeases: integer(
cellRow,
'restart_blocking_activity_leases'
),
restartBlockingActivityRequestUnits: integer(
cellRow,
'restart_blocking_activity_request_units'
),
restartBlockingReservedRequests: integer(
cellRow,
'restart_blocking_reserved_requests'
),
outgoingMigrations: integer(migrationRow, 'outgoing_migrations'),
incomingMigrations: integer(migrationRow, 'incoming_migrations'),
connectionCapacity: connectionRow
? {
hardCap: integer(connectionRow, 'hard_cap'),
controlRebindReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls,
ordinaryConnectionLimit:
integer(connectionRow, 'hard_cap') -
RELAY_ADMISSION_BUDGETS.reservedHostControls,
unobservedBound: integer(connectionRow, 'unobserved_bound'),
normalAdmissionPause:
integer(connectionRow, 'hard_cap') -
RELAY_ADMISSION_BUDGETS.reservedHostControls -
integer(connectionRow, 'unobserved_bound'),
observedConnections:
optionalInteger(connectionRow, 'total_connections') ?? 0,
inFlightConnections:
optionalInteger(connectionRow, 'in_flight_connections') ?? 0,
reservedConnectionUnits:
optionalInteger(connectionRow, 'reserved_connection_units') ?? 0,
enforcedConnectionUnits:
optionalInteger(connectionRow, 'enforced_connection_units') ?? 0,
pendingControlReservations: integer(
connectionRow,
'outstanding_reservations'
),
heartbeatFresh:
connectionHeartbeat !== undefined &&
connectionHeartbeat > this.now() - this.heartbeatTtlMs &&
optionalText(connectionRow, 'current_incarnation') ===
optionalText(connectionRow, 'connection_incarnation')
}
: null,
runtime:
runtimeHeartbeat === undefined
? null
: {
cellUrl: text(cellRow, 'runtime_cell_url'),
cellIncarnation: text(cellRow, 'runtime_cell_incarnation'),
startedAt: integer(cellRow, 'runtime_started_at'),
ready: integer(cellRow, 'runtime_ready') === 1,
observedRequests: integer(cellRow, 'runtime_observed_requests'),
lastHeartbeatAt: runtimeHeartbeat,
heartbeatFresh: runtimeHeartbeat > this.now() - this.heartbeatTtlMs,
regionalRehomeProtocol: integer(
cellRow,
'runtime_regional_rehome_protocol'
)
}
}
}
async verifyCellAssignment(input: AssignmentIdentity & {
cellId: string
assignmentEpoch: number
}): Promise<boolean> {
const rows = await this.database.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[input.userId, input.relayHostId]
)
return Boolean(
rows[0] &&
text(rows[0], 'cell_id') === input.cellId &&
integer(rows[0], 'assignment_epoch') === input.assignmentEpoch
)
}
async changeActivity(
identity: AssignmentIdentity,
kind: AssignmentActivityKind,
delta: 1 | -1
): Promise<void> {
await this.activityQueue.run(identity, async () => {
const column = ACTIVITY_COLUMN[kind]
const now = this.now()
await this.database.transaction(async (transaction) => {
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (!row) return
const before = integer(row, column)
const after = Math.max(0, before + delta)
await transaction.query(
`UPDATE relay_assignments SET ${column} = ?, lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[after, now + ASSIGNMENT_LIMITS.activityLeaseMs, now, identity.userId, identity.relayHostId]
)
const requestDelta = ACTIVITY_REQUEST_UNITS[kind] * (after - before)
if (requestDelta !== 0) {
await this.adjustCellReservationAtomically(transaction, text(row, 'cell_id'), requestDelta)
}
})
})
}
async acquireActivity(
identity: AssignmentIdentity,
input: {
activityId: string
kind: AssignmentActivityKind
cellId: string
expiresAt?: number
}
): Promise<void> {
validateActivityId(input.activityId)
await this.activityQueue.run(identity, async () => {
const now = this.now()
await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
if (!assignment) throw new Error('assignment_not_found')
const assignmentCellId = text(assignment, 'cell_id')
if (input.cellId !== assignmentCellId) {
// Origin controls may renew work only while the exact forward
// migration is active; completion must fence late source activity.
const migration = (
await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
AND source_cell_id = ? AND target_cell_id = ?
AND assignment_epoch = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[
identity.userId,
identity.relayHostId,
input.cellId,
assignmentCellId,
integer(assignment, 'assignment_epoch')
]
)
)[0]
if (!migration) throw new Error('activity_cell_not_authoritative')
}
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
const existing = activityLeaseById(activityLeases, input.activityId)
const expiresAt = input.expiresAt ?? now + ASSIGNMENT_LIMITS.activityLeaseMs
if (!Number.isSafeInteger(expiresAt) || expiresAt <= now) {
throw new Error('invalid_activity_expiry')
}
if (existing && text(existing, 'activity_kind') === input.kind && text(existing, 'cell_id') === input.cellId) {
await transaction.query(
`UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, input.activityId]
)
await this.touchAssignment(transaction, identity, expiresAt, now)
return
}
const units = ACTIVITY_REQUEST_UNITS[input.kind]
if (existing) {
// Why: a client-chosen activity id can move between cells, so lock the
// one or two rows this path touches in cell_id order, the same order
// placement takes the inventory in, and no cycle can form.
await this.lockCellRows(transaction, [text(existing, 'cell_id'), input.cellId])
await this.removeActivityLease(transaction, identity, existing, now)
await this.adjustCellReservationAtomically(transaction, input.cellId, units)
}
await this.adjustActivityCount(transaction, identity, input.kind, 1, expiresAt, now)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
input.activityId,
input.kind,
input.cellId,
units,
expiresAt,
now
]
)
// Keep the contended cell row locked for only the final write and commit.
if (!existing) {
await this.adjustCellReservationAtomically(transaction, input.cellId, units)
}
})
})
}
async exchangeRegionCorrection(
identity: AssignmentIdentity,
request: RegionCorrectionRequest,
assignmentEpoch: number
): Promise<RegionCorrectionResponse> {
return exchangeRegionCorrection(this.database, identity, request, assignmentEpoch, this.now())
}
async regionCorrectionOutcomes() {
return readRegionCorrectionOutcomes(this.database, this.now())
}
async previewRegionCorrection(): Promise<Record<string, number>> {
return previewRegionCorrection(this.database, this.now())
}
private idleRegionalCandidateOffset = 0
async selectIdleRegionalRehomeCandidates(
processSafety?: RegionalRehomeSafetySnapshot
): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const now = this.now()
if (!processSafety || this.regionalRehomeCohortPercent === 0) return []
const control = (await this.database.query(
"SELECT enabled, not_before FROM relay_region_rehome_control WHERE control_id = 'global'"
))[0]
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) return []
const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now)
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return []
const candidates = await selectIdleRegionalRehomes({
database: this.database, now, heartbeatTtlMs: this.heartbeatTtlMs,
cohortPercent: this.regionalRehomeCohortPercent, offset: this.idleRegionalCandidateOffset,
connectionHeadroom: await this.connectionHeadroomByCell(this.database),
cellIsClean: regionalRehomeCellSafetyIsClean
})
this.idleRegionalCandidateOffset = candidates.length < IDLE_REHOME_PAGE_SIZE
? 0 : this.idleRegionalCandidateOffset + candidates.length
return candidates
}
async commitIdleRegionalRehome(
request: IdleRegionalRehomeRequest,
processSafety?: RegionalRehomeSafetySnapshot,
cohortPercent = this.regionalRehomeCohortPercent
): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> {
const prior = await this.reconcileIdleRegionalRehome(request)
if (prior !== 'not-committed') return { outcome: prior }
if (!processSafety || !Number.isInteger(cohortPercent) || cohortPercent <= 0 || cohortPercent > 100) {
return { outcome: 'deferred' }
}
let safetyDisable: Record<string, string | number> | null = null
const result = await this.database.transaction(async (transaction): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> => {
safetyDisable = null
const now = this.now()
const control = (await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`
))[0]
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) {
return { outcome: 'deferred' }
}
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
VALUES ('global', 0, 0, 0, ?) ON CONFLICT (worker_id) DO NOTHING`, [now]
)
const worker = (await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'`
))[0]!
if (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now) {
return { outcome: 'deferred' }
}
const open = (await transaction.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE completed_at IS NULL AND aborted_at IS NULL`
))[0]
if (Number(open?.count ?? 0) >= REGIONAL_REHOME_CONCURRENT_LIMIT) return { outcome: 'deferred' }
const attempt = await this.startRegionalRehomeCandidate(transaction, {
identity: request,
sourceCellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
preferenceCutoff: now - Number(control.preference_max_age_ms),
cooldownCutoff: now - Number(control.host_cooldown_ms),
drainGraceMs: 0,
processSafety,
worker,
now,
skips: [],
idleRequest: request,
cohortPercent,
onSafetyDisabled: (event) => { safetyDisable = event }
})
if (!attempt) return { outcome: 'deferred' }
await this.markRegionalRehomeDispatchClaimed(
transaction, request.attemptId, now, Math.ceil(60_000 / Number(control.rate_per_minute))
)
await transaction.query(
`UPDATE relay_region_rehome_attempts SET drain_receipt_at = ?, drain_outcome = 'accepted'
WHERE attempt_id = ?`, [now, request.attemptId]
)
return { outcome: 'committed' }
})
if (safetyDisable) console.warn(JSON.stringify(safetyDisable))
return result
}
async reconcileIdleRegionalRehome(request: IdleRegionalRehomeRequest): Promise<'committed' | 'not-committed' | 'stale'> {
return this.database.transaction(async (transaction) => {
// Absence is definitive only after the same assignment lock as commit/activation.
const assignment = await this.assignmentRow(transaction, request)
const attempt = (await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
))[0]
if (attempt) {
return attempt.user_id === request.userId &&
attempt.relay_host_id === request.relayHostId &&
attempt.source_cell_id === request.sourceCellId &&
attempt.source_cell_incarnation === request.sourceCellIncarnation &&
Number(attempt.previous_epoch) === request.sourceAssignmentEpoch &&
Number(attempt.source_generation) === request.sourceGeneration &&
attempt.target_cell_id === request.targetCellId &&
attempt.aborted_at == null
? 'committed' : 'stale'
}
if (!assignment || assignment.cell_id !== request.sourceCellId ||
Number(assignment.assignment_epoch) !== request.sourceAssignmentEpoch) return 'stale'
const control = (await transaction.query(
`SELECT capability.generation, capability.cell_incarnation
FROM relay_control_capabilities capability
JOIN relay_assignment_activity_leases lease
ON lease.user_id = capability.user_id AND lease.relay_host_id = capability.relay_host_id
AND lease.activity_id = capability.activity_id
WHERE capability.user_id = ? AND capability.relay_host_id = ?
AND capability.cell_id = ? AND capability.assignment_epoch = ?
AND lease.activity_kind = 'control' AND lease.expires_at > ?
ORDER BY capability.generation DESC LIMIT 1`,
[request.userId, request.relayHostId, request.sourceCellId, request.sourceAssignmentEpoch, this.now()]
))[0]
return control && Number(control.generation) === request.sourceGeneration &&
control.cell_incarnation === request.sourceCellIncarnation ? 'not-committed' : 'stale'
})
}
async previewRegionalRehomeEligibility(
processSafety?: RegionalRehomeSafetySnapshot
): Promise<RegionCorrectionPreview> {
const now = this.now()
const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now)
return previewRegionalRehomeEligibility({
database: this.database,
now,
heartbeatTtlMs: this.heartbeatTtlMs,
cohortPercent: this.regionalRehomeCohortPercent,
globalSafetyFailure: processSafety
? regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)
: 'process-safety-unavailable',
connectionHeadroom: await this.connectionHeadroomByCell(this.database),
cellIsClean: regionalRehomeCellSafetyIsClean
})
}
async renewControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number }
): Promise<void> {
validateActivityId(input.activityId)
const now = this.now()
const maximumExpiresAt =
now +
ASSIGNMENT_LIMITS.activityLeaseMs +
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
if (
!Number.isSafeInteger(input.expiresAt) ||
input.expiresAt <= now ||
input.expiresAt > maximumExpiresAt
) {
throw new Error('invalid_activity_expiry')
}
const startedAt = performance.now()
let outcome: ControlRenewalOutcome = 'database_error'
try {
outcome =
this.database.dialect === 'postgres'
? await this.renewPostgresControlActivity(identity, input, now)
: await this.renewTransactionalControlActivity(identity, input, now)
if (outcome !== 'renewed') throw new Error(outcome)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) {
outcome = message as ControlRenewalOutcome
}
throw error
} finally {
this.recordControlRenewal?.(performance.now() - startedAt, outcome)
}
}
private async renewPostgresControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number },
now: number
): Promise<ControlRenewalOutcome> {
const row = (
await this.database.query(
`WITH assignment_state AS MATERIALIZED (
SELECT cell_id, assignment_epoch
FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?
FOR UPDATE
), migration_state AS MATERIALIZED (
SELECT migration.assignment_epoch
FROM relay_assignment_migrations migration
JOIN assignment_state assignment
ON migration.target_cell_id = assignment.cell_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT 1 AS authorized
FROM assignment_state assignment
WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state)
), lease_state AS MATERIALIZED (
SELECT lease.activity_kind, lease.cell_id
FROM relay_assignment_activity_leases lease
CROSS JOIN authorization_state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, ?),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
AND state.activity_kind = 'control' AND state.cell_id = ?
RETURNING 1
), renewed_assignment AS (
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
AND EXISTS (SELECT 1 FROM renewed_lease)
RETURNING 1
)
SELECT CASE
WHEN NOT EXISTS (SELECT 1 FROM assignment_state)
THEN 'assignment_not_found'
WHEN NOT EXISTS (SELECT 1 FROM authorization_state)
THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (SELECT 1 FROM lease_state)
THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state
WHERE activity_kind <> 'control' OR cell_id <> ?
) THEN 'control_activity_moved'
WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome`,
[
identity.userId,
identity.relayHostId,
identity.userId,
identity.relayHostId,
input.cellId,
input.cellId,
identity.userId,
identity.relayHostId,
input.activityId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.activityId,
input.cellId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.cellId
]
)
)[0]
if (!row) throw new Error('missing_control_renewal_outcome')
const outcome = text(row, 'outcome') as ControlRenewalOutcome
if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome')
return outcome
}
private async renewTransactionalControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number },
now: number
): Promise<ControlRenewalOutcome> {
// Bypass the process queue so a network-stalled activity call cannot suppress renewal.
await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
if (!assignment) throw new Error('assignment_not_found')
const assignmentCellId = text(assignment, 'cell_id')
if (input.cellId !== assignmentCellId) {
const migration = (
await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
AND source_cell_id = ? AND target_cell_id = ?
AND assignment_epoch = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[
identity.userId,
identity.relayHostId,
input.cellId,
assignmentCellId,
integer(assignment, 'assignment_epoch')
]
)
)[0]
if (!migration) throw new Error('activity_cell_not_authoritative')
}
const lease = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[identity.userId, identity.relayHostId, input.activityId]
)
)[0]
if (!lease) throw new Error('control_activity_not_found')
if (
text(lease, 'activity_kind') !== 'control' ||
text(lease, 'cell_id') !== input.cellId
) {
throw new Error('control_activity_moved')
}
await transaction.query(
`UPDATE relay_assignment_activity_leases SET
expires_at = CASE WHEN expires_at > ? THEN expires_at ELSE ? END,
updated_at = CASE WHEN updated_at > ? THEN updated_at ELSE ? END
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[
input.expiresAt,
input.expiresAt,
now,
now,
identity.userId,
identity.relayHostId,
input.activityId
]
)
await transaction.query(
`UPDATE relay_assignments SET
lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = CASE WHEN last_activity_at > ? THEN last_activity_at ELSE ? END
WHERE user_id = ? AND relay_host_id = ?`,
[input.expiresAt, input.expiresAt, now, now, identity.userId, identity.relayHostId]
)
})
return 'renewed'
}
async releaseActivity(identity: AssignmentIdentity, activityId: string): Promise<boolean> {
validateActivityId(activityId)
return await this.activityQueue.run(identity, async () => {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.assignmentRow(transaction, identity)
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
const existing = activityLeaseById(activityLeases, activityId)
if (!existing) return false
await transaction.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[identity.userId, identity.relayHostId, activityId]
)
await this.adjustActivityCount(
transaction,
identity,
activityKind(existing),
-1,
now,
now
)
// Release paths can safely defer the cell-row lock until their final write.
await this.adjustCellReservationAtomically(
transaction,
text(existing, 'cell_id'),
-integer(existing, 'request_units')
)
return true
})
})
}
async activateControl(
identity: AssignmentIdentity,
input: {
cellId: string
assignmentEpoch: number
generation: number
idleRegionalRehome?: boolean
cellIncarnation?: string
connectionInclusionWatermark?: number
}
): Promise<string> {
const activityId = `control:${input.cellId}:${input.generation}`
validateActivityId(activityId)
return await this.activityQueue.run(identity, async () => {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
if (
!assignment ||
text(assignment, 'cell_id') !== input.cellId ||
integer(assignment, 'assignment_epoch') !== input.assignmentEpoch
) {
throw new Error('wrong_assignment')
}
const expiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
const existing = activityLeaseById(activityLeases, activityId)
const pendingId = pendingControlActivityId(input.assignmentEpoch)
const pending = activityLeaseById(activityLeases, pendingId)
const retainedActivityId =
existing || (pending && text(pending, 'cell_id') === input.cellId)
? existing
? activityId
: pendingId
: activityId
await this.removeSupersededSameCellControls(
transaction,
identity,
activityLeases,
input.cellId,
retainedActivityId,
now
)
if (existing) {
await transaction.query(
`UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, activityId]
)
await this.touchAssignment(transaction, identity, expiresAt, now)
} else {
if (pending && text(pending, 'cell_id') === input.cellId) {
await transaction.query(
`UPDATE relay_assignment_activity_leases
SET activity_id = ?, expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[activityId, expiresAt, now, identity.userId, identity.relayHostId, pendingId]
)
await this.touchAssignment(transaction, identity, expiresAt, now)
} else {
await this.adjustCellReservationAtomically(transaction, input.cellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
activityId,
'control',
input.cellId,
1,
expiresAt,
now
]
)
}
}
await this.claimControlConnectionReservation(
transaction,
identity,
input.cellId,
input.assignmentEpoch,
activityId,
input.connectionInclusionWatermark,
now
)
await transaction.query(
`DELETE FROM relay_control_capabilities WHERE user_id = ? AND relay_host_id = ?
AND NOT EXISTS (SELECT 1 FROM relay_assignment_activity_leases lease
WHERE lease.user_id = relay_control_capabilities.user_id
AND lease.relay_host_id = relay_control_capabilities.relay_host_id
AND lease.activity_id = relay_control_capabilities.activity_id)`,
[identity.userId, identity.relayHostId]
)
await transaction.query(
`INSERT INTO relay_control_capabilities
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing, idle_regional_rehome)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (user_id, relay_host_id, activity_id) DO UPDATE SET
cell_incarnation = excluded.cell_incarnation, assignment_epoch = excluded.assignment_epoch,
generation = excluded.generation, finish_existing = excluded.finish_existing, idle_regional_rehome = excluded.idle_regional_rehome`,
[
identity.userId,
identity.relayHostId,
activityId,
input.cellId,
input.cellIncarnation ?? '',
input.assignmentEpoch,
input.generation,
0,
input.idleRegionalRehome && input.cellIncarnation ? 1 : 0
]
)
return activityId
})
})
}
async startEvacuation(
identity: AssignmentIdentity,
targetCellId: string
): Promise<RelayAssignmentMigration>
async startEvacuation(
identity: AssignmentIdentity,
targetCellId: string,
expectedSourceCellId: string
): Promise<RelayAssignmentMigration | null>
async startEvacuation(
identity: AssignmentIdentity,
targetCellId: string,
expectedSourceCellId?: string
): Promise<RelayAssignmentMigration | null> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
if (!assignment) throw new Error('assignment_not_found')
const sourceCellId = text(assignment, 'cell_id')
// Bulk selection is intentionally staleable; fence it before considering
// an existing migration that already moved the assignment to its target.
if (expectedSourceCellId && sourceCellId !== expectedSourceCellId) return null
const existing = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND completed_at IS NULL
AND aborted_at IS NULL AND expires_at > ?
ORDER BY assignment_epoch DESC LIMIT 1`,
[identity.userId, identity.relayHostId, now]
)
)[0]
if (existing) {
if (text(existing, 'target_cell_id') !== targetCellId) {
throw new Error('migration_in_progress')
}
return migration(identity, existing)
}
if (sourceCellId === targetCellId) throw new Error('target_matches_source')
await this.lockAssignmentActivities(transaction, identity)
const cells = await this.lockCellInventory(transaction, 'request')
const target = cells.find((row) => text(row, 'cell_id') === targetCellId)
if (!target || integer(target, 'enabled') !== 1) throw new Error('target_cell_unavailable')
if (!(await this.cellIsLive(transaction, targetCellId, now))) {
throw new Error('target_cell_unavailable')
}
let sourceCellIncarnation: string | undefined
let targetCellIncarnation: string | undefined
if (this.requireLiveCells) {
const runtimes = await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`,
[sourceCellId, targetCellId]
)
const sourceRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === sourceCellId
)
const targetRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === targetCellId
)
if (
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs
) {
throw new Error('source_cell_unavailable')
}
if (
!targetRuntime ||
integer(targetRuntime, 'ready') !== 1 ||
integer(targetRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs
) {
throw new Error('target_cell_unavailable')
}
sourceCellIncarnation = text(sourceRuntime, 'cell_incarnation')
targetCellIncarnation = text(targetRuntime, 'cell_incarnation')
}
await this.assertCellConnectionHeadroom(transaction, targetCellId)
const sourceUnitsRow = (
await transaction.query(
`SELECT COALESCE(SUM(request_units), 0) AS units
FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND cell_id = ?`,
[identity.userId, identity.relayHostId, sourceCellId]
)
)[0]
const sourceRequestUnits = integer(sourceUnitsRow!, 'units')
const targetReservedUnits = sourceRequestUnits + 1
await this.adjustCellReservation(transaction, targetCellId, targetReservedUnits)
const previousEpoch = integer(assignment, 'assignment_epoch')
const assignmentEpoch = previousEpoch + 1
const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
reserved_controls = reserved_controls + 1,
migration_leases = migration_leases + 1,
lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
targetCellId,
assignmentEpoch,
expiresAt,
now,
identity.userId,
identity.relayHostId
]
)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
'control',
targetCellId,
1,
expiresAt,
now,
identity.userId,
identity.relayHostId,
migrationActivityId(assignmentEpoch),
'migration',
targetCellId,
sourceRequestUnits,
expiresAt,
now
]
)
await this.insertControlConnectionReservation(
transaction,
identity,
targetCellId,
assignmentEpoch,
expiresAt,
now
)
await transaction.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id,
previous_epoch, assignment_epoch, source_request_units,
target_reserved_units, expires_at, target_registered_at,
completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`,
[
identity.userId,
identity.relayHostId,
sourceCellId,
targetCellId,
previousEpoch,
assignmentEpoch,
sourceRequestUnits,
targetReservedUnits,
expiresAt,
now,
now
]
)
if (sourceCellIncarnation && targetCellIncarnation) {
await transaction.query(
`INSERT INTO relay_assignment_migration_incarnations
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
target_cell_incarnation)
VALUES (?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
sourceCellIncarnation,
targetCellIncarnation
]
)
}
return {
...identity,
sourceCellId,
targetCellId,
previousEpoch,
assignmentEpoch,
expiresAt
}
})
}
async markMigrationTargetRegistered(
identity: AssignmentIdentity,
input: { cellId: string; assignmentEpoch: number }
): Promise<boolean> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const rows = await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND target_cell_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[
identity.userId,
identity.relayHostId,
input.assignmentEpoch,
input.cellId
]
)
if (!rows[0]) return false
await transaction.query(
`UPDATE relay_assignment_migrations
SET target_registered_at = COALESCE(target_registered_at, ?), updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, input.assignmentEpoch]
)
return true
})
}
async completeEvacuationFromDeadSource(
identity: AssignmentIdentity,
input: {
assignmentEpoch: number
sourceCellId: string
targetCellId: string
}
): Promise<DeadSourceCompletionResult> {
try {
return await this.withAssignmentLockRetry(
async (inventoryFirst) =>
await this.completeEvacuationFromDeadSourceOnce(identity, input, inventoryFirst)
)
} catch (error) {
if (isDatabaseLockUnavailable(error)) {
throw new Error('migration_cell_inventory_busy')
}
throw error
}
}
private async completeEvacuationFromDeadSourceOnce(
identity: AssignmentIdentity,
input: {
assignmentEpoch: number
sourceCellId: string
targetCellId: string
},
inventoryFirst: boolean
): Promise<DeadSourceCompletionResult> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
let lockedCells: SqlRow[] | undefined
if (inventoryFirst) {
try {
lockedCells = await this.lockCellInventory(transaction, 'request')
} catch (error) {
if (isDatabaseLockTimeout(error)) {
throw new Error('database_lock_unavailable')
}
throw error
}
}
const assignment = await this.assignmentRow(transaction, identity, inventoryFirst)
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
if (!row) throw new Error('migration_not_found')
assertMigrationPair(row, input.sourceCellId, input.targetCellId)
if (optionalInteger(row, 'completed_at') !== undefined) {
return deadSourceCompletionResult(row, false)
}
if (optionalInteger(row, 'aborted_at') !== undefined) {
throw new Error('migration_already_aborted')
}
if (optionalInteger(row, 'target_registered_at') === undefined) {
throw new Error('migration_target_not_registered')
}
assertCurrentMigrationAssignment(assignment, row)
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
assertAssignmentActivityAccounting(assignment, activityLeases, row)
if (
activityLeases.some(
(lease) =>
![input.sourceCellId, input.targetCellId].includes(text(lease, 'cell_id'))
)
) {
throw new Error('migration_activity_topology_mismatch')
}
if (activityUnitsForCell(activityLeases, input.sourceCellId) > 0) {
throw new Error('migration_source_still_active')
}
const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait'))
const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId)
const target = cells.find((cell) => text(cell, 'cell_id') === input.targetCellId)
if (!source || integer(source, 'enabled') !== 0) {
throw new Error('migration_source_admission_changed')
}
if (!target || integer(target, 'enabled') !== 1) {
throw new Error('migration_target_admission_changed')
}
await assertCellReservationAccounting(transaction, cells, [
input.sourceCellId,
input.targetCellId
])
const runtimes = await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`,
[input.sourceCellId, input.targetCellId]
)
const sourceRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === input.sourceCellId
)
const targetRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === input.targetCellId
)
const freshAfter = now - this.heartbeatTtlMs
await this.requireCellFence(transaction, input.sourceCellId, sourceRuntime, now)
if (!sourceRuntime || integer(sourceRuntime, 'last_heartbeat_at') > freshAfter) {
throw new Error('migration_source_runtime_not_dead')
}
if (
!targetRuntime ||
integer(targetRuntime, 'ready') !== 1 ||
integer(targetRuntime, 'last_heartbeat_at') <= freshAfter
) {
throw new Error('migration_target_runtime_not_ready')
}
const targetIsActive = activityLeases.some(
(lease) =>
text(lease, 'cell_id') === input.targetCellId &&
text(lease, 'activity_kind') === 'control' &&
!text(lease, 'activity_id').startsWith('control-pending:') &&
integer(lease, 'expires_at') > now &&
integer(lease, 'updated_at') >= integer(targetRuntime, 'started_at')
)
if (!targetIsActive) {
const inactiveTargetControls = activityLeases.filter(
(lease) =>
text(lease, 'cell_id') === input.targetCellId &&
text(lease, 'activity_kind') === 'control'
)
for (const lease of inactiveTargetControls) {
await this.removeActivityLease(transaction, identity, lease, now)
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
input.targetCellId,
input.assignmentEpoch,
now
)
}
const migrationLease = activityLeaseById(
activityLeases,
migrationActivityId(input.assignmentEpoch)
)
if (migrationLease) {
await this.removeActivityLease(transaction, identity, migrationLease, now)
}
await transaction.query(
`UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, input.assignmentEpoch]
)
return deadSourceCompletionResult(row, true)
})
}
async supersedeRegisteredEvacuation(
identity: AssignmentIdentity,
input: RegisteredEvacuationSupersessionInput,
preservedActivityCellIds: readonly string[] = []
): Promise<RelayAssignmentMigration> {
if (input.assignmentEpoch >= Number.MAX_SAFE_INTEGER) {
throw new Error('assignment_epoch_exhausted')
}
return await this.withAssignmentLockRetry(
async (inventoryFirst) =>
await this.supersedeRegisteredEvacuationOnce(
identity,
input,
inventoryFirst,
preservedActivityCellIds
)
)
}
private async supersedeRegisteredEvacuationOnce(
identity: AssignmentIdentity,
input: RegisteredEvacuationSupersessionInput,
inventoryFirst: boolean,
preservedActivityCellIds: readonly string[]
): Promise<RelayAssignmentMigration> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const lockedCells = inventoryFirst
? await this.lockCellInventory(transaction, 'request')
: undefined
const assignment = await this.assignmentRow(transaction, identity, inventoryFirst)
const existing = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
if (!existing) throw new Error('migration_not_found')
assertMigrationPair(existing, input.sourceCellId, input.currentTargetCellId)
if (optionalInteger(existing, 'completed_at') !== undefined) {
throw new Error('migration_already_completed')
}
if (optionalInteger(existing, 'aborted_at') !== undefined) {
const successor = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND previous_epoch = ?
AND source_cell_id = ? AND target_cell_id = ?
ORDER BY assignment_epoch DESC LIMIT 1`,
[
identity.userId,
identity.relayHostId,
input.assignmentEpoch,
input.sourceCellId,
input.replacementTargetCellId
]
)
)[0]
if (!successor) throw new Error('migration_already_superseded')
return migration(identity, successor)
}
if (optionalInteger(existing, 'target_registered_at') === undefined) {
throw new Error('migration_target_not_registered')
}
const existingIncarnation = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migration_incarnations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
const existingPin = (
await transaction.queryLocked(
`SELECT * FROM relay_post_drain_migration_pins
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
if (existingPin && !existingIncarnation) {
throw new Error('drain_migration_source_incarnation_mismatch')
}
assertCurrentMigrationAssignment(assignment, existing)
if (input.currentTargetCellId === input.replacementTargetCellId) {
throw new Error('target_matches_source')
}
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
assertAssignmentActivityAccounting(assignment, activityLeases, existing)
const additionalActivityCellIds = new Set(
activityLeases
.map((lease) => text(lease, 'cell_id'))
.filter(
(cellId) =>
![input.sourceCellId, input.currentTargetCellId].includes(cellId)
)
)
if (
preservedActivityCellIds.includes(input.replacementTargetCellId) ||
!matchesExactCellSet(additionalActivityCellIds, preservedActivityCellIds)
) {
throw new Error('migration_activity_topology_mismatch')
}
const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait'))
const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId)
const currentTarget = cells.find(
(cell) => text(cell, 'cell_id') === input.currentTargetCellId
)
const replacement = cells.find(
(cell) => text(cell, 'cell_id') === input.replacementTargetCellId
)
if (!source || integer(source, 'enabled') !== 0) {
throw new Error('migration_source_admission_changed')
}
if (!currentTarget || integer(currentTarget, 'enabled') !== 0) {
throw new Error('migration_target_still_enabled')
}
if (!replacement || integer(replacement, 'enabled') !== 1) {
throw new Error('replacement_target_unavailable')
}
await assertCellReservationAccounting(transaction, cells, [
input.sourceCellId,
input.currentTargetCellId,
input.replacementTargetCellId,
...preservedActivityCellIds
])
const runtimeCellIds = [
input.currentTargetCellId,
input.replacementTargetCellId,
...preservedActivityCellIds
]
const runtimes = await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime
WHERE cell_id IN (${runtimeCellIds.map(() => '?').join(', ')})
ORDER BY cell_id`,
runtimeCellIds
)
const currentRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === input.currentTargetCellId
)
const replacementRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === input.replacementTargetCellId
)
const freshAfter = now - this.heartbeatTtlMs
await this.requireCellFence(
transaction,
input.currentTargetCellId,
currentRuntime,
now
)
if (
currentRuntime &&
integer(currentRuntime, 'ready') === 1 &&
integer(currentRuntime, 'last_heartbeat_at') > freshAfter
) {
throw new Error('migration_target_still_available')
}
if (
!replacementRuntime ||
integer(replacementRuntime, 'ready') !== 1 ||
integer(replacementRuntime, 'last_heartbeat_at') <= freshAfter
) {
throw new Error('replacement_target_unavailable')
}
if (
preservedActivityCellIds.some((cellId) => {
const runtime = runtimes.find((row) => text(row, 'cell_id') === cellId)
return (
!runtime ||
integer(runtime, 'ready') !== 1 ||
integer(runtime, 'last_heartbeat_at') <= freshAfter
)
})
) {
throw new Error('migration_preserved_activity_cell_unavailable')
}
await this.assertCellConnectionHeadroom(
transaction,
input.replacementTargetCellId
)
for (const lease of activityLeases.filter(
(candidate) => text(candidate, 'cell_id') === input.currentTargetCellId
)) {
await this.removeActivityLease(transaction, identity, lease, now)
}
const sourceRequestUnits = activityUnitsForCell(activityLeases, input.sourceCellId)
const targetReservedUnits = sourceRequestUnits + 1
await this.adjustCellReservation(
transaction,
input.replacementTargetCellId,
targetReservedUnits
)
const assignmentEpoch = input.assignmentEpoch + 1
const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
reserved_controls = reserved_controls + 1,
migration_leases = migration_leases + 1,
lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
input.replacementTargetCellId,
assignmentEpoch,
expiresAt,
now,
identity.userId,
identity.relayHostId
]
)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
'control',
input.replacementTargetCellId,
1,
expiresAt,
now,
identity.userId,
identity.relayHostId,
migrationActivityId(assignmentEpoch),
'migration',
input.replacementTargetCellId,
sourceRequestUnits,
expiresAt,
now
]
)
await this.insertControlConnectionReservation(
transaction,
identity,
input.replacementTargetCellId,
assignmentEpoch,
expiresAt,
now
)
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
input.currentTargetCellId,
input.assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, input.assignmentEpoch]
)
await transaction.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id,
previous_epoch, assignment_epoch, source_request_units,
target_reserved_units, expires_at, target_registered_at,
completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`,
[
identity.userId,
identity.relayHostId,
input.sourceCellId,
input.replacementTargetCellId,
input.assignmentEpoch,
assignmentEpoch,
sourceRequestUnits,
targetReservedUnits,
expiresAt,
now,
now
]
)
if (existingIncarnation) {
await transaction.query(
`INSERT INTO relay_assignment_migration_incarnations
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
target_cell_incarnation)
VALUES (?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
text(existingIncarnation, 'source_cell_incarnation'),
text(replacementRuntime, 'cell_incarnation')
]
)
}
if (existingPin) {
await transaction.query(
`INSERT INTO relay_post_drain_migration_pins
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
source_cell_id, source_cell_incarnation, target_cell_id,
target_cell_incarnation, source_request_units, target_reserved_units,
pinned_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
text(existingPin, 'drain_attempt_id'),
input.sourceCellId,
text(existingPin, 'source_cell_incarnation'),
input.replacementTargetCellId,
text(replacementRuntime, 'cell_incarnation'),
sourceRequestUnits,
targetReservedUnits,
now
]
)
}
return {
...identity,
sourceCellId: input.sourceCellId,
targetCellId: input.replacementTargetCellId,
previousEpoch: input.assignmentEpoch,
assignmentEpoch,
expiresAt
}
})
}
async supersedeRegisteredCellEvacuations(
sourceCellId: string,
currentTargetCellId: string,
replacementTargetCellId: string,
limit: number
): Promise<number> {
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
throw new Error('invalid_evacuation_limit')
}
const rows = await this.database.query(
`SELECT user_id, relay_host_id, assignment_epoch
FROM relay_assignment_migrations
WHERE source_cell_id = ? AND target_cell_id = ?
AND target_registered_at IS NOT NULL
AND completed_at IS NULL AND aborted_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM relay_assignment_migrations earlier
WHERE earlier.user_id = relay_assignment_migrations.user_id
AND earlier.relay_host_id = relay_assignment_migrations.relay_host_id
AND earlier.source_cell_id = relay_assignment_migrations.source_cell_id
AND earlier.target_cell_id = relay_assignment_migrations.target_cell_id
AND earlier.target_registered_at IS NOT NULL
AND earlier.completed_at IS NULL AND earlier.aborted_at IS NULL
AND earlier.assignment_epoch < relay_assignment_migrations.assignment_epoch
)
ORDER BY user_id, relay_host_id, assignment_epoch
LIMIT ?`,
[sourceCellId, currentTargetCellId, limit]
)
let superseded = 0
let reconciledAccounting = false
for (const row of rows) {
const identity = {
userId: text(row, 'user_id'),
relayHostId: text(row, 'relay_host_id')
}
const input = {
assignmentEpoch: integer(row, 'assignment_epoch'),
sourceCellId,
currentTargetCellId,
replacementTargetCellId
}
const prepared = await this.prepareRegisteredCellSupersession(identity, input)
if (prepared.retired) {
input.assignmentEpoch = integer(row, 'assignment_epoch')
await this.supersedeRegisteredEvacuation(identity, input)
superseded++
continue
}
input.assignmentEpoch = prepared.assignmentEpoch
try {
await this.supersedeRegisteredEvacuation(identity, input)
} catch (error) {
if (
error instanceof Error &&
error.message === 'migration_activity_topology_mismatch'
) {
const preservedActivityCellIds = await this.additionalActivityCellIds(
identity,
sourceCellId,
currentTargetCellId
)
if (
preservedActivityCellIds.length === 0 ||
preservedActivityCellIds.includes(replacementTargetCellId)
) {
throw error
}
await this.reconcileReservationAccounting(sourceCellId, currentTargetCellId)
await this.reconcileReservationAccounting(sourceCellId, replacementTargetCellId)
for (const cellId of preservedActivityCellIds) {
await this.reconcileReservationAccounting(sourceCellId, cellId)
}
await this.supersedeRegisteredEvacuation(
identity,
input,
preservedActivityCellIds
)
superseded++
continue
}
if (
reconciledAccounting ||
!(error instanceof Error) ||
error.message !== 'migration_cell_reservation_accounting_mismatch'
) {
throw error
}
await this.reconcileReservationAccounting(sourceCellId, currentTargetCellId)
await this.reconcileReservationAccounting(sourceCellId, replacementTargetCellId)
reconciledAccounting = true
await this.supersedeRegisteredEvacuation(identity, input)
}
superseded++
}
return superseded
}
private async prepareRegisteredCellSupersession(
identity: AssignmentIdentity,
input: RegisteredEvacuationSupersessionInput
): Promise<{ assignmentEpoch: number; retired: boolean }> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
const staleMigration = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
if (!assignment || !staleMigration) throw new Error('migration_assignment_mismatch')
assertMigrationPair(
staleMigration,
input.sourceCellId,
input.currentTargetCellId
)
if (
optionalInteger(staleMigration, 'completed_at') !== undefined ||
optionalInteger(staleMigration, 'aborted_at') !== undefined
) {
return { assignmentEpoch: integer(assignment, 'assignment_epoch'), retired: true }
}
if (optionalInteger(staleMigration, 'target_registered_at') === undefined) {
throw new Error('migration_not_registered_for_supersession')
}
const assignmentEpoch = integer(assignment, 'assignment_epoch')
if (assignmentEpoch < input.assignmentEpoch) {
throw new Error('migration_assignment_mismatch')
}
const assignmentCellId = text(assignment, 'cell_id')
if (
![input.currentTargetCellId, input.replacementTargetCellId].includes(
assignmentCellId
)
) {
throw new Error('migration_assignment_mismatch')
}
const leases = await this.lockAssignmentActivities(transaction, identity)
const staleIncarnation = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migration_incarnations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
const stalePin = (
await transaction.queryLocked(
`SELECT * FROM relay_post_drain_migration_pins
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, input.assignmentEpoch]
)
)[0]
assertMigrationRecoveryMetadata(staleMigration, staleIncarnation, stalePin)
const failedRuntime = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`,
[input.currentTargetCellId]
)
)[0]
await this.requireCellFence(
transaction,
input.currentTargetCellId,
failedRuntime,
now
)
if (
assignmentEpoch > input.assignmentEpoch &&
assignmentCellId === input.replacementTargetCellId
) {
const obsoleteActivityIds = new Set([
pendingControlActivityId(input.assignmentEpoch),
migrationActivityId(input.assignmentEpoch)
])
const obsoleteLeases = leases.filter((lease) =>
obsoleteActivityIds.has(text(lease, 'activity_id'))
)
assertAssignmentActivityCounts(
assignment,
leases,
leases.filter((lease) => activityKind(lease) === 'migration').length
)
for (const lease of obsoleteLeases) {
const kind = activityKind(lease)
const expectedUnits =
kind === 'migration'
? integer(staleMigration, 'source_request_units')
: ACTIVITY_REQUEST_UNITS.control
if (
text(lease, 'cell_id') !== input.currentTargetCellId ||
!['control', 'migration'].includes(kind) ||
integer(lease, 'request_units') !== expectedUnits
) {
throw new Error('migration_activity_topology_mismatch')
}
}
if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'request')
for (const lease of obsoleteLeases) {
await this.removeActivityLease(transaction, identity, lease, now)
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
input.currentTargetCellId,
input.assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, input.assignmentEpoch]
)
return { assignmentEpoch, retired: true }
}
let migrationRow = staleMigration
if (assignmentEpoch > input.assignmentEpoch) {
const existingCurrent = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
if (existingCurrent) {
assertMigrationPair(
existingCurrent,
input.sourceCellId,
input.currentTargetCellId
)
if (
optionalInteger(existingCurrent, 'completed_at') !== undefined ||
optionalInteger(existingCurrent, 'aborted_at') !== undefined ||
optionalInteger(existingCurrent, 'target_registered_at') === undefined
) {
throw new Error('migration_activity_topology_mismatch')
}
assertCurrentMigrationAssignment(assignment, existingCurrent)
const currentIncarnation = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migration_incarnations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
const currentPin = (
await transaction.queryLocked(
`SELECT * FROM relay_post_drain_migration_pins
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
assertMigrationRecoveryMetadata(
existingCurrent,
currentIncarnation,
currentPin
)
migrationRow = existingCurrent
} else {
if (leases.length !== 0) {
throw new Error('migration_activity_topology_mismatch')
}
assertAssignmentActivityCounts(assignment, leases, 0)
const currentIncarnations = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migration_incarnations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
const currentPins = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_post_drain_migration_pins
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
if (currentIncarnations.length > 0 || currentPins.length > 0) {
throw new Error('migration_activity_topology_mismatch')
}
await transaction.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id,
previous_epoch, assignment_epoch, source_request_units,
target_reserved_units, expires_at, target_registered_at,
completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 0, 1, ?, ?, NULL, NULL, ?, ?)`,
[
identity.userId,
identity.relayHostId,
input.sourceCellId,
input.currentTargetCellId,
input.assignmentEpoch,
assignmentEpoch,
now - 1,
now,
now,
now
]
)
migrationRow = {
...staleMigration,
assignment_epoch: assignmentEpoch,
previous_epoch: input.assignmentEpoch,
source_request_units: 0,
target_reserved_units: 1,
expires_at: now - 1,
target_registered_at: now,
completed_at: null,
aborted_at: null
}
if (staleIncarnation) {
await transaction.query(
`INSERT INTO relay_assignment_migration_incarnations
(user_id, relay_host_id, assignment_epoch,
source_cell_incarnation, target_cell_incarnation)
VALUES (?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
text(staleIncarnation, 'source_cell_incarnation'),
text(staleIncarnation, 'target_cell_incarnation')
]
)
}
if (stalePin) {
await transaction.query(
`INSERT INTO relay_post_drain_migration_pins
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
source_cell_id, source_cell_incarnation, target_cell_id,
target_cell_incarnation, source_request_units,
target_reserved_units, pinned_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 1, ?)`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
text(stalePin, 'drain_attempt_id'),
input.sourceCellId,
text(stalePin, 'source_cell_incarnation'),
input.currentTargetCellId,
text(stalePin, 'target_cell_incarnation'),
now
]
)
}
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
input.currentTargetCellId,
input.assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, input.assignmentEpoch]
)
}
const migrationLeases = leases.filter(
(lease) => activityKind(lease) === 'migration'
)
if (migrationLeases.length > 0) {
assertAssignmentActivityAccounting(assignment, leases, migrationRow)
return { assignmentEpoch, retired: false }
}
assertAssignmentActivityCounts(assignment, leases, 0)
const sourceRequestUnits = integer(migrationRow, 'source_request_units')
if (
integer(migrationRow, 'expires_at') > now ||
sourceRequestUnits < 0 ||
integer(migrationRow, 'target_reserved_units') !== sourceRequestUnits + 1
) {
throw new Error('migration_activity_lease_shape_mismatch')
}
await this.lockCellInventory(transaction, 'request')
await this.adjustCellReservation(
transaction,
input.currentTargetCellId,
sourceRequestUnits
)
const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'migration', ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
migrationActivityId(assignmentEpoch),
input.currentTargetCellId,
sourceRequestUnits,
expiresAt,
now
]
)
await transaction.query(
`UPDATE relay_assignments SET migration_leases = migration_leases + 1,
lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[expiresAt, expiresAt, now, identity.userId, identity.relayHostId]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
return { assignmentEpoch, retired: false }
})
}
private async additionalActivityCellIds(
identity: AssignmentIdentity,
sourceCellId: string,
currentTargetCellId: string
): Promise<string[]> {
const rows = await this.database.query(
`SELECT DISTINCT cell_id FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ?
AND cell_id NOT IN (?, ?)
ORDER BY cell_id`,
[identity.userId, identity.relayHostId, sourceCellId, currentTargetCellId]
)
return rows.map((row) => text(row, 'cell_id'))
}
async completeEvacuation(
identity: AssignmentIdentity,
assignmentEpoch: number
): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
const row = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
if (!row) throw new Error('migration_not_found')
if (optionalInteger(row, 'target_registered_at') === undefined) {
throw new Error('migration_target_not_registered')
}
const sourceCellId = text(row, 'source_cell_id')
const targetCellId = text(row, 'target_cell_id')
if (
!assignment ||
text(assignment, 'cell_id') !== targetCellId ||
integer(assignment, 'assignment_epoch') !== assignmentEpoch
) {
throw new Error('migration_assignment_mismatch')
}
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
const sourceUnits = activityUnitsForCell(activityLeases, sourceCellId)
if (sourceUnits > 0) throw new Error('migration_source_still_active')
let cellsLocked = false
let targetStartedAt = 0
if (this.requireLiveCells) {
let cells: SqlRow[]
try {
cells = await this.lockCellInventory(transaction, 'nowait')
} catch (error) {
if (isDatabaseLockUnavailable(error)) {
// Mixed-version workers may still hold a cell-first lock; defer
// instead of waiting long enough to form their legacy lock cycle.
throw new Error('migration_cell_inventory_busy')
}
throw error
}
cellsLocked = true
const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId)
const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId)
if (!source || integer(source, 'enabled') !== 0) {
throw new Error('migration_source_admission_changed')
}
if (!target || integer(target, 'enabled') !== 1) {
throw new Error('migration_target_admission_changed')
}
const runtimes = await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id IN (?, ?) ORDER BY cell_id`,
[sourceCellId, targetCellId]
)
const sourceRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === sourceCellId
)
const targetRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === targetCellId
)
const freshAfter = now - this.heartbeatTtlMs
if (
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'observed_requests') !== 0 ||
integer(sourceRuntime, 'last_heartbeat_at') <= freshAfter
) {
throw new Error('migration_source_runtime_not_quiescent')
}
if (
!targetRuntime ||
integer(targetRuntime, 'ready') !== 1 ||
integer(targetRuntime, 'last_heartbeat_at') <= freshAfter
) {
throw new Error('migration_target_runtime_not_ready')
}
targetStartedAt = integer(targetRuntime, 'started_at')
}
const targetIsActive = activityLeases.some(
(lease) =>
text(lease, 'cell_id') === targetCellId &&
text(lease, 'activity_kind') === 'control' &&
!text(lease, 'activity_id').startsWith('control-pending:') &&
integer(lease, 'expires_at') > now &&
integer(lease, 'updated_at') >= targetStartedAt
)
if (!targetIsActive) throw new Error('migration_target_not_active')
const lease = activityLeaseById(activityLeases, migrationActivityId(assignmentEpoch))
if (lease && !cellsLocked) await this.lockCellInventory(transaction, 'pool-default')
if (lease) await this.removeActivityLease(transaction, identity, lease, now)
await transaction.query(
`UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
})
}
async rebalanceDormant(
identity: AssignmentIdentity,
targetCellId: string
): Promise<RelayAssignment> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
if (!assignment) throw new Error('assignment_not_found')
if (!mayNormallyReassign(activity(assignment), now)) throw new Error('assignment_active')
const sourceCellId = text(assignment, 'cell_id')
if (sourceCellId === targetCellId) throw new Error('target_matches_source')
await this.lockAssignmentActivities(transaction, identity)
const cells = await this.lockCellInventory(transaction, 'request')
const admission = await cellAdmissionStates(transaction)
const targetRow = cells.find(
(row) =>
text(row, 'cell_id') === targetCellId &&
admission.get(targetCellId) === 'general'
)
if (!targetRow) throw new Error('target_cell_unavailable')
if (!(await this.cellIsLive(transaction, targetCellId, now))) {
throw new Error('target_cell_unavailable')
}
await this.assertCellConnectionHeadroom(transaction, targetCellId)
const target = cell(targetRow, await this.cellRegion(transaction, targetCellId))
await this.adjustCellReservation(transaction, targetCellId, 1)
const assignmentEpoch = integer(assignment, 'assignment_epoch') + 1
const leaseExpiresAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
reserved_controls = 1, lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
targetCellId,
assignmentEpoch,
leaseExpiresAt,
now,
identity.userId,
identity.relayHostId
]
)
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
sourceCellId,
assignmentEpoch - 1,
now
)
await this.insertPendingControlLease(
transaction,
identity,
targetCellId,
assignmentEpoch,
now
)
return {
...identity,
...target,
assignmentEpoch,
leaseExpiresAt
}
})
}
async inspectRegionalRehomeControl(): Promise<RegionalRehomeControl> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.initializeRegionalRehomeControl(transaction, now)
const row = (
await transaction.query(
`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`
)
)[0]!
return regionalRehomeControl(row)
})
}
async applyRegionalRehomeControl(input: {
expectedGeneration: number
enabled: boolean
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}): Promise<RegionalRehomeControl> {
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
throw new Error('invalid_regional_rehome_generation')
}
if (!Number.isSafeInteger(input.notBefore) || input.notBefore < 0) {
throw new Error('invalid_regional_rehome_not_before')
}
if (!Number.isSafeInteger(input.ratePerMinute) || input.ratePerMinute < 1 || input.ratePerMinute > 120) {
throw new Error('invalid_regional_rehome_rate')
}
if (
!Number.isSafeInteger(input.preferenceMaxAgeMs) ||
input.preferenceMaxAgeMs < 60_000 ||
input.preferenceMaxAgeMs > 30 * 24 * 60 * 60_000
) {
throw new Error('invalid_regional_rehome_preference_age')
}
if (
!Number.isSafeInteger(input.hostCooldownMs) ||
input.hostCooldownMs < 60_000 ||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
) {
throw new Error('invalid_regional_rehome_host_cooldown')
}
if (
!Number.isSafeInteger(input.drainGraceMs) ||
input.drainGraceMs < 60_000 ||
input.drainGraceMs > 60 * 60_000
) {
throw new Error('invalid_regional_rehome_drain_grace')
}
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.initializeRegionalRehomeControl(transaction, now)
const current = (
await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`
)
)[0]!
if (integer(current, 'generation') !== input.expectedGeneration) {
throw new Error('regional_rehome_generation_mismatch')
}
if (
input.enabled &&
input.notBefore <
integer(current, 'observation_started_at') + REGIONAL_REHOME_OBSERVATION_MS
) {
throw new Error('regional_rehome_observation_window_incomplete')
}
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = ?, not_before = ?,
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
drain_grace_ms = ?, updated_at = ?
WHERE control_id = 'global'`,
[
input.enabled ? 1 : 0,
input.notBefore,
input.ratePerMinute,
input.preferenceMaxAgeMs,
input.hostCooldownMs,
input.drainGraceMs,
now
]
)
if (input.enabled) {
// A budget spent under a previous enable is not evidence about this one.
// Without this an old counter latches the fresh enable straight back off
// on its first transient failure.
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
VALUES ('global', 0, 0, 0, ?)
ON CONFLICT (worker_id) DO UPDATE
SET paused_until = 0, consecutive_failures = 0,
updated_at = excluded.updated_at`,
[now]
)
}
const updated = (
await transaction.query(
`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`
)
)[0]!
return regionalRehomeControl(updated)
})
}
async disableRegionalRehomeControl(): Promise<boolean> {
const now = this.now()
const result = await this.database.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1`,
[now]
)
return integer(result[0]!, 'changes') === 1
}
private async initializeRegionalRehomeControl(
database: RelayDatabase,
now: number
): Promise<void> {
await database.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
ON CONFLICT (control_id) DO NOTHING`,
[
now,
24 * 60 * 60_000,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
60 * 60_000,
now
]
)
}
async regionalRehomeFleetSafety(): Promise<RegionalRehomeFleetSafety> {
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
}
// The rehome fleet is every general cell that can be drained: those are the
// sources and, because a host must be movable back out again, the only legal
// targets. The region join stays so a cell with no region row is excluded.
private async readRegionalRehomeFleetSafety(
database: RelayDatabase,
now: number
): Promise<RegionalRehomeFleetSafety> {
const rows = await database.query(
`SELECT runtime.ready, runtime.last_heartbeat_at,
safety.cell_id AS safety_cell_id, safety.observed_at,
safety.sql_failures, safety.reconnects,
safety.control_activity_recovery_failures,
safety.database_pool_waiting, safety.database_pool_waiters_max,
safety.database_pool_wait_ms_max
FROM relay_cells cell
JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
LEFT JOIN relay_cell_capabilities capability ON capability.cell_id = cell.cell_id
LEFT JOIN relay_cell_rehome_safety safety
ON safety.cell_id = runtime.cell_id
AND safety.cell_incarnation = runtime.cell_incarnation
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
AND capability.regional_rehome_protocol >= 1`
)
const valid = rows.filter(
(row) =>
integer(row, 'ready') === 1 &&
integer(row, 'last_heartbeat_at') > now - this.heartbeatTtlMs &&
optionalText(row, 'safety_cell_id') !== undefined &&
integer(row, 'observed_at') > now - 60_000
)
const missingCells = rows.length === 0 ? 1 : rows.length - valid.length
return {
requiredCells: rows.length,
missingCells,
observedAt:
missingCells > 0
? 0
: Math.min(...valid.map((row) => integer(row, 'observed_at'))),
sqlFailures: valid.reduce((total, row) => total + integer(row, 'sql_failures'), 0),
reconnects: valid.reduce((total, row) => total + integer(row, 'reconnects'), 0),
maxReconnects: Math.max(0, ...valid.map((row) => integer(row, 'reconnects'))),
controlActivityRecoveryFailures: valid.reduce(
(total, row) => total + integer(row, 'control_activity_recovery_failures'),
0
),
databasePoolWaiting: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_waiting'))
),
databasePoolWaitersMax: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_waiters_max'))
),
databasePoolWaitMsMax: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_wait_ms_max'))
)
}
}
private async startRegionalRehomeCandidate(
transaction: RelayDatabase,
input: {
identity: AssignmentIdentity
sourceCellId: string
assignmentEpoch: number
preferenceCutoff: number
cooldownCutoff: number
drainGraceMs: number
processSafety: RegionalRehomeSafetySnapshot
worker: SqlRow
now: number
skips: RegionalRehomeCandidateSkip[]
idleRequest: IdleRegionalRehomeRequest
cohortPercent: number
onSafetyDisabled: (event: Record<string, string | number> | null) => void
}
): Promise<Omit<RegionalRehomeAttempt, 'sendAttempts'> | null> {
const assignment = await this.assignmentRow(transaction, input.identity)
if (
!assignment ||
text(assignment, 'cell_id') !== input.sourceCellId ||
integer(assignment, 'assignment_epoch') !== input.assignmentEpoch
) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const preference = (
await transaction.queryLocked(
`SELECT * FROM relay_region_decisions
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
)[0]
if (
!preference ||
integer(preference, 'observed_at') < input.preferenceCutoff ||
Number(preference.expires_at) <= input.now ||
preference.outcome !== 'conclusive' ||
Number(preference.policy_version) !== 1 ||
Number(preference.assignment_epoch) !== input.assignmentEpoch ||
!preference.preferred_region ||
Number(preference.cohort_bucket) >= input.cohortPercent
) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const preferredRegion = relayRegion(preference, 'preferred_region')
const activeMigration = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[input.identity.userId, input.identity.relayHostId]
)
if (activeMigration.length > 0) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
// Re-read under the claim: an attempt committed between the scan and here
// would otherwise start a second move for the same host.
const recentAttempt = await transaction.query(
`SELECT 1 FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
LIMIT 1`,
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
)
if (recentAttempt.length > 0) {
input.skips.push({ reason: 'host_cooldown' })
return null
}
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
assertAssignmentActivityCounts(assignment, activityLeases, 0)
const cells = await this.lockCellInventory(transaction, 'nowait')
const admission = await cellAdmissionStates(transaction)
const regions = new Map(
(await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
text(row, 'cell_id'),
relayRegion(row, 'region')
])
)
const runtimes = await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime ORDER BY cell_id`
)
const capabilities = await transaction.queryLocked(
`SELECT * FROM relay_cell_capabilities ORDER BY cell_id`
)
const safetyRows = await transaction.queryLocked(
`SELECT * FROM relay_cell_rehome_safety ORDER BY cell_id`
)
const source = cells.find((row) => text(row, 'cell_id') === input.sourceCellId)
const sourceRuntime = runtimes.find(
(row) => text(row, 'cell_id') === input.sourceCellId
)
const sourceCapability = capabilities.find(
(row) => text(row, 'cell_id') === input.sourceCellId
)
const sourceSafety = safetyRows.find(
(row) => text(row, 'cell_id') === input.sourceCellId
)
const fleetSafety = regionalRehomeFleetSafetyFromInventory({
cells,
admission,
regions,
runtimes,
capabilities,
safetyRows,
now: input.now,
heartbeatTtlMs: this.heartbeatTtlMs
})
const safetyFailure = regionalRehomeFleetSafetyFailure(
input.processSafety,
fleetSafety,
input.now
)
if (safetyFailure) {
input.onSafetyDisabled(await this.pauseRegionalRehomeForSafety(
transaction,
input.worker,
input.now,
safetyFailure,
fleetSafety
))
return null
}
// The preference read under lock can now agree with the cell the host is
// already on: nothing to move, in either direction.
if (regions.get(input.sourceCellId) === preferredRegion) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
if (
!source ||
integer(source, 'enabled') !== 1 ||
admission.get(input.sourceCellId) !== 'general' ||
regions.get(input.sourceCellId) === undefined ||
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
!sourceCapability ||
text(sourceCapability, 'cell_incarnation') !== text(sourceRuntime, 'cell_incarnation') ||
integer(sourceCapability, 'regional_rehome_protocol') < 3 ||
sourceRuntime.cell_incarnation !== input.idleRequest.sourceCellIncarnation
) {
input.skips.push({ reason: 'source_ineligible', cellId: input.sourceCellId })
return null
}
if (!regionalRehomeCellSafetyIsClean(sourceSafety, sourceRuntime, input.now)) {
input.skips.push(cellUncleanSkip('source_unclean', input.sourceCellId, sourceSafety))
return null
}
const hostCapability = (
await transaction.query(
`SELECT capability.* FROM relay_control_capabilities capability
JOIN relay_assignment_activity_leases lease
ON lease.user_id = capability.user_id AND lease.relay_host_id = capability.relay_host_id
AND lease.activity_id = capability.activity_id
WHERE capability.user_id = ? AND capability.relay_host_id = ?
AND capability.cell_id = ? AND capability.assignment_epoch = ?
AND capability.cell_incarnation = ? AND capability.idle_regional_rehome = 1
AND lease.expires_at > ? AND lease.activity_kind = 'control'
ORDER BY capability.generation DESC LIMIT 1`,
[
input.identity.userId,
input.identity.relayHostId,
input.sourceCellId,
input.assignmentEpoch,
sourceRuntime.cell_incarnation,
input.now
]
)
)[0]
if (!hostCapability || preference.incumbent_region !== regions.get(input.sourceCellId) ||
Number(hostCapability.generation) !== input.idleRequest.sourceGeneration) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const sourceControlActive = activityLeases.some(
(lease) =>
text(lease, 'cell_id') === input.sourceCellId &&
text(lease, 'activity_kind') === 'control' &&
!text(lease, 'activity_id').startsWith('control-pending:') &&
integer(lease, 'expires_at') > input.now &&
integer(lease, 'updated_at') >= integer(sourceRuntime, 'started_at')
)
if (!sourceControlActive) {
input.skips.push({ reason: 'source_control_inactive', cellId: input.sourceCellId })
return null
}
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
// A target must be drainable too, or the host lands somewhere it can never
// be rehomed out of again -- the trap this bidirectional move exists to undo.
const eligibleTargets = cells.filter((row) => {
const cellId = text(row, 'cell_id')
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
const capability = capabilities.find(
(candidate) => text(candidate, 'cell_id') === cellId
)
return (
cellId !== input.sourceCellId &&
integer(row, 'enabled') === 1 &&
admission.get(cellId) === 'general' &&
regions.get(cellId) === preferredRegion &&
runtime !== undefined &&
integer(runtime, 'ready') === 1 &&
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
capability !== undefined &&
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
integer(capability, 'regional_rehome_protocol') >= 3 &&
cellId === input.idleRequest.targetCellId
)
})
const targetIsClean = (row: SqlRow): boolean => {
const cellId = text(row, 'cell_id')
return regionalRehomeCellSafetyIsClean(
safetyRows.find((safety) => text(safety, 'cell_id') === cellId),
runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)!,
input.now
)
}
const targetCandidates = eligibleTargets.filter(
(row) => targetIsClean(row) && connectionHeadroom.get(text(row, 'cell_id')) !== false
)
if (targetCandidates.length === 0) {
const unclean = eligibleTargets.filter((row) => !targetIsClean(row))
for (const row of unclean) {
const cellId = text(row, 'cell_id')
input.skips.push(
cellUncleanSkip(
'target_unclean',
cellId,
safetyRows.find((safety) => text(safety, 'cell_id') === cellId)
)
)
}
if (unclean.length === 0) {
input.skips.push({
reason: eligibleTargets.length === 0 ? 'no_eligible_target' : 'no_target_headroom'
})
}
return null
}
targetCandidates.sort((left, right) => {
const leftRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === text(left, 'cell_id')
)!
const rightRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === text(right, 'cell_id')
)!
const leftLoad =
(integer(left, 'reserved_requests') + integer(leftRuntime, 'observed_requests')) /
integer(left, 'capacity_requests')
const rightLoad =
(integer(right, 'reserved_requests') + integer(rightRuntime, 'observed_requests')) /
integer(right, 'capacity_requests')
return leftLoad - rightLoad ||
text(left, 'cell_id').localeCompare(text(right, 'cell_id'))
})
const sourceRequestUnits = activityUnitsForCell(activityLeases, input.sourceCellId)
const targetReservedUnits = sourceRequestUnits + 1
const target = targetCandidates.find(
(row) =>
integer(row, 'reserved_requests') + targetReservedUnits <=
integer(row, 'capacity_requests')
)
if (!target) {
input.skips.push({ reason: 'no_target_headroom' })
return null
}
const targetCellId = text(target, 'cell_id')
const targetRuntime = runtimes.find(
(runtime) => text(runtime, 'cell_id') === targetCellId
)!
await this.adjustCellReservation(transaction, targetCellId, targetReservedUnits)
const previousEpoch = integer(assignment, 'assignment_epoch')
const assignmentEpoch = previousEpoch + 1
const expiresAt = input.now + ASSIGNMENT_LIMITS.migrationLeaseMs
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
reserved_controls = reserved_controls + 1,
migration_leases = migration_leases + 1,
lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
targetCellId,
assignmentEpoch,
expiresAt,
input.now,
input.identity.userId,
input.identity.relayHostId
]
)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'control', ?, 1, ?, ?),
(?, ?, ?, 'migration', ?, ?, ?, ?)`,
[
input.identity.userId,
input.identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
targetCellId,
expiresAt,
input.now,
input.identity.userId,
input.identity.relayHostId,
migrationActivityId(assignmentEpoch),
targetCellId,
sourceRequestUnits,
expiresAt,
input.now
]
)
await this.insertControlConnectionReservation(
transaction,
input.identity,
targetCellId,
assignmentEpoch,
expiresAt,
input.now
)
await transaction.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id,
previous_epoch, assignment_epoch, source_request_units,
target_reserved_units, expires_at, target_registered_at,
completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`,
[
input.identity.userId,
input.identity.relayHostId,
input.sourceCellId,
targetCellId,
previousEpoch,
assignmentEpoch,
sourceRequestUnits,
targetReservedUnits,
expiresAt,
input.now,
input.now
]
)
await transaction.query(
`INSERT INTO relay_assignment_migration_incarnations
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
target_cell_incarnation)
VALUES (?, ?, ?, ?, ?)`,
[
input.identity.userId,
input.identity.relayHostId,
assignmentEpoch,
text(sourceRuntime, 'cell_incarnation'),
text(targetRuntime, 'cell_incarnation')
]
)
const attemptId = input.idleRequest.attemptId
await transaction.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region,
source_cell_id, source_cell_incarnation, target_cell_id,
target_cell_incarnation, previous_epoch, assignment_epoch,
drain_grace_ms, send_attempts, last_send_attempt_at,
drain_receipt_at, drain_outcome, completed_at, aborted_at,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
NULL, NULL, NULL, NULL, ?, ?)`,
[
attemptId,
input.identity.userId,
input.identity.relayHostId,
preferredRegion,
input.sourceCellId,
text(sourceRuntime, 'cell_incarnation'),
targetCellId,
text(targetRuntime, 'cell_incarnation'),
previousEpoch,
assignmentEpoch,
input.drainGraceMs,
input.now,
input.now
]
)
await transaction.query(
`UPDATE relay_region_rehome_attempts SET source_generation = ? WHERE attempt_id = ?`,
[input.idleRequest.sourceGeneration, attemptId]
)
return {
...input.identity,
attemptId,
preferredRegion,
sourceCellId: input.sourceCellId,
sourceCellUrl: text(source, 'cell_url'),
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
targetCellId,
targetCellIncarnation: text(targetRuntime, 'cell_incarnation'),
previousEpoch,
assignmentEpoch,
drainGraceMs: input.drainGraceMs
}
}
private async pauseRegionalRehomeForSafety(
transaction: RelayDatabase,
worker: SqlRow,
now: number,
reason: string,
fleetSafety: RegionalRehomeFleetSafety
): Promise<Record<string, string | number> | null> {
const disabled = await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1
RETURNING generation`,
[now]
)
// The durable disable is otherwise invisible: nothing else records why
// claims stopped and inspection only shows enabled=false. Logged after
// the transaction commits so a rollback cannot fabricate the record.
let event: Record<string, string | number> | null = null
if (disabled.length > 0) {
event = {
event: 'orca_relay_regional_rehome_safety_disabled',
reason,
controlGeneration: integer(disabled[0]!, 'generation'),
now,
requiredCells: fleetSafety.requiredCells,
missingCells: fleetSafety.missingCells,
observedAt: fleetSafety.observedAt,
sqlFailures: fleetSafety.sqlFailures,
reconnects: fleetSafety.reconnects,
maxReconnects: fleetSafety.maxReconnects,
controlActivityRecoveryFailures: fleetSafety.controlActivityRecoveryFailures,
databasePoolWaiting: fleetSafety.databasePoolWaiting,
databasePoolWaitersMax: fleetSafety.databasePoolWaitersMax,
databasePoolWaitMsMax: fleetSafety.databasePoolWaitMsMax
}
}
await this.incrementRegionalRehomeWorkerFailure(transaction, worker, now)
return event
}
private async markRegionalRehomeDispatchClaimed(
transaction: RelayDatabase,
attemptId: string,
now: number,
intervalMs: number
): Promise<void> {
await transaction.query(
`UPDATE relay_region_rehome_attempts
SET send_attempts = send_attempts + 1, last_send_attempt_at = ?, updated_at = ?
WHERE attempt_id = ?`,
[now, now, attemptId]
)
await transaction.query(
`UPDATE relay_region_rehome_worker_state
SET next_dispatch_at = ?, updated_at = ? WHERE worker_id = 'global'`,
[now + intervalMs, now]
)
}
// Returns the durable disable this failure caused, for the caller to log once
// its transaction commits; null when the budget survives or was already spent.
private async incrementRegionalRehomeWorkerFailure(
transaction: RelayDatabase,
worker: SqlRow,
now: number
): Promise<Record<string, string | number> | null> {
const failures = integer(worker, 'consecutive_failures') + 1
const spent = failures >= REGIONAL_REHOME_FAILURE_BUDGET
await transaction.query(
`UPDATE relay_region_rehome_worker_state
SET consecutive_failures = ?, paused_until = ?, updated_at = ?
WHERE worker_id = 'global'`,
[failures, spent ? now + 5 * 60_000 : 0, now]
)
if (!spent) return null
const disabled = await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1
RETURNING generation`,
[now]
)
// The disable is otherwise invisible: inspection only shows enabled=false and
// nothing records that the failure budget, not an operator, turned it off.
if (disabled.length === 0) return null
return {
event: 'orca_relay_regional_rehome_failure_budget_disabled',
controlGeneration: integer(disabled[0]!, 'generation'),
consecutiveFailures: failures,
now
}
}
async completeReadyRegionalRehomes(limit = 10): Promise<number> {
const now = this.now()
const quarantined = this.quarantinedRegionalRehomeAttemptIds(now)
const exclusion = quarantined.length
? ` AND attempt.attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})`
: ''
const candidates = await this.database.query(
`SELECT attempt.attempt_id, attempt.user_id, attempt.relay_host_id,
attempt.assignment_epoch
FROM relay_region_rehome_attempts attempt
JOIN relay_assignment_migrations migration
ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id
AND migration.assignment_epoch = attempt.assignment_epoch
WHERE attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
AND migration.target_registered_at IS NOT NULL
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM relay_assignment_activity_leases source_lease
WHERE source_lease.user_id = attempt.user_id
AND source_lease.relay_host_id = attempt.relay_host_id
AND source_lease.cell_id = attempt.source_cell_id
)${exclusion}
ORDER BY attempt.created_at, attempt.attempt_id
LIMIT ?`,
[...quarantined, limit]
)
let completed = 0
let inventoryBusy = 0
for (const candidate of candidates) {
// One poisoned row must not stall every later candidate: an invariant
// throw here blocked fleet completions head-of-line in production.
const attemptId = text(candidate, 'attempt_id')
try {
const changed = await this.completeRegionalRehomeCandidate(
{
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
},
integer(candidate, 'assignment_epoch'),
now
)
if (changed) completed++
this.regionalRehomeCandidateQuarantine.delete(attemptId)
} catch (error) {
if (isDatabaseLockUnavailable(error)) {
inventoryBusy++
continue
}
this.recordRegionalRehomeCandidateFailure('complete', attemptId, now, error)
}
}
warnSweepCellInventoryBusy('complete-ready-regional-rehomes', inventoryBusy)
return completed
}
// Repeated invariant failures quarantine the attempt out of the sweeps'
// LIMIT pages: poisoned rows are permanent and always the oldest, so
// without exclusion they eventually starve every healthy candidate.
private recordRegionalRehomeCandidateFailure(
operation: 'complete' | 'abort' | 'refresh',
attemptId: string,
now: number,
error: unknown
): void {
const entry = this.regionalRehomeCandidateQuarantine.get(attemptId) ?? {
failures: 0,
until: 0
}
entry.failures++
if (entry.failures >= REGIONAL_REHOME_QUARANTINE_FAILURES) {
entry.until = now + REGIONAL_REHOME_QUARANTINE_MS
}
this.regionalRehomeCandidateQuarantine.delete(attemptId)
this.regionalRehomeCandidateQuarantine.set(attemptId, entry)
if (this.regionalRehomeCandidateQuarantine.size > REGIONAL_REHOME_QUARANTINE_MEMORY_LIMIT) {
// Evict a non-quarantined entry first: mid-quarantine rows are excluded
// from the candidate pages and losing one returns it to the page with a
// reset counter.
let evict = this.regionalRehomeCandidateQuarantine.keys().next().value!
for (const [key, candidate] of this.regionalRehomeCandidateQuarantine) {
if (candidate.until <= now) {
evict = key
break
}
}
this.regionalRehomeCandidateQuarantine.delete(evict)
}
warnRegionalRehomeCandidateFailure(operation, attemptId, error)
}
private quarantinedRegionalRehomeAttemptIds(now: number): string[] {
const excluded: string[] = []
for (const [attemptId, entry] of this.regionalRehomeCandidateQuarantine) {
if (entry.until > now) excluded.push(attemptId)
if (excluded.length >= REGIONAL_REHOME_QUARANTINE_EXCLUSION_LIMIT) break
}
return excluded
}
async refreshRegionalRehomeLeases(limit = 100): Promise<number> {
const now = this.now()
const quarantined = this.quarantinedRegionalRehomeAttemptIds(now)
const exclusion = quarantined.length
? ` AND attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})`
: ''
const candidates = await this.database.query(
`SELECT attempt_id, user_id, relay_host_id, assignment_epoch
FROM relay_region_rehome_attempts
WHERE completed_at IS NULL AND aborted_at IS NULL${exclusion}
ORDER BY updated_at, attempt_id LIMIT ?`,
[...quarantined, limit]
)
let refreshed = 0
for (const candidate of candidates) {
const identity = {
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
}
const assignmentEpoch = integer(candidate, 'assignment_epoch')
const attemptId = text(candidate, 'attempt_id')
try {
const changed = await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
const attempt = (
await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
const migration = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
if (attempt && attempt.completed_at == null && attempt.aborted_at == null) {
await transaction.query(
`UPDATE relay_region_rehome_attempts SET updated_at = ? WHERE attempt_id = ?`,
[now, attempt.attempt_id]
)
}
if (
!assignment ||
!attempt ||
!migration ||
optionalInteger(attempt, 'completed_at') !== undefined ||
optionalInteger(attempt, 'aborted_at') !== undefined ||
optionalInteger(migration, 'completed_at') !== undefined ||
optionalInteger(migration, 'aborted_at') !== undefined
) {
return false
}
const attemptAgeMs = now - integer(attempt, 'created_at')
if (attemptAgeMs >= REGIONAL_REHOME_MAX_REFRESH_MS) {
return false
}
if (
optionalInteger(migration, 'target_registered_at') === undefined &&
attemptAgeMs >= REGIONAL_REHOME_UNREGISTERED_REFRESH_MS
) {
await transaction.query(
`UPDATE relay_assignment_activity_leases
SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END,
updated_at = ?
WHERE user_id = ? AND relay_host_id = ?
AND activity_id IN (?, ?)`,
[
now,
now,
now,
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]
)
await transaction.query(
`UPDATE relay_assignment_migrations
SET expires_at = CASE WHEN expires_at < ? THEN expires_at ELSE ? END,
updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
return false
}
const leases = await this.lockAssignmentActivities(transaction, identity)
const protectedIds = new Set([
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
])
const protectedLeases = leases.filter((lease) =>
protectedIds.has(text(lease, 'activity_id'))
)
if (protectedLeases.length === 0) return false
const expiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
await transaction.query(
`UPDATE relay_assignment_activity_leases
SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ?
AND activity_id IN (?, ?)`,
[
expiresAt,
now,
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[expiresAt, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await transaction.query(
`UPDATE relay_assignments SET lease_expires_at =
CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[expiresAt, expiresAt, now, identity.userId, identity.relayHostId]
)
return true
})
if (changed) refreshed++
this.regionalRehomeCandidateQuarantine.delete(attemptId)
} catch (error) {
if (!isDatabaseLockUnavailable(error))
this.recordRegionalRehomeCandidateFailure('refresh', attemptId, now, error)
}
}
return refreshed
}
private async completeRegionalRehomeCandidate(
identity: AssignmentIdentity,
assignmentEpoch: number,
now: number
): Promise<boolean> {
return await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
const attempt = (
await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
const migration = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
if (
!attempt ||
!migration ||
optionalInteger(attempt, 'completed_at') !== undefined ||
optionalInteger(attempt, 'aborted_at') !== undefined ||
optionalInteger(migration, 'target_registered_at') === undefined ||
optionalInteger(migration, 'completed_at') !== undefined ||
optionalInteger(migration, 'aborted_at') !== undefined
) {
return false
}
const sourceCellId = text(attempt, 'source_cell_id')
const targetCellId = text(attempt, 'target_cell_id')
if (
!assignment ||
text(assignment, 'cell_id') !== targetCellId ||
integer(assignment, 'assignment_epoch') !== assignmentEpoch ||
text(migration, 'source_cell_id') !== sourceCellId ||
text(migration, 'target_cell_id') !== targetCellId
) {
throw new Error('regional_rehome_assignment_mismatch')
}
const leases = await this.lockAssignmentActivities(transaction, identity)
if (activityUnitsForCell(leases, sourceCellId) !== 0) return false
await this.repairAssignmentActivityCounts(
transaction,
identity,
text(attempt, 'attempt_id'),
assignment,
leases,
migration
)
const cells = await this.lockCellInventory(transaction, 'nowait')
const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId)
const admission = await cellAdmissionStates(transaction)
if (
!target ||
integer(target, 'enabled') !== 1 ||
!['general', 'migration-only'].includes(admission.get(targetCellId) ?? '')
) {
return false
}
const targetRuntime = (
await transaction.queryLocked(
`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`,
[targetCellId]
)
)[0]
if (
!targetRuntime ||
integer(targetRuntime, 'ready') !== 1 ||
integer(targetRuntime, 'last_heartbeat_at') <= now - this.heartbeatTtlMs
) {
return false
}
const targetActive = leases.some(
(lease) =>
text(lease, 'cell_id') === targetCellId &&
text(lease, 'activity_kind') === 'control' &&
!text(lease, 'activity_id').startsWith('control-pending:') &&
integer(lease, 'expires_at') > now &&
integer(lease, 'updated_at') >= integer(targetRuntime, 'started_at')
)
if (!targetActive) return false
const migrationLease = activityLeaseById(
leases,
migrationActivityId(assignmentEpoch)
)
if (migrationLease) {
await this.removeActivityLease(transaction, identity, migrationLease, now)
}
await transaction.query(
`UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await transaction.query(
`UPDATE relay_region_rehome_attempts SET completed_at = ?, updated_at = ?
WHERE attempt_id = ?`,
[now, now, text(attempt, 'attempt_id')]
)
return true
})
}
// Why: counter skew left by a pre-fix sticky grant is reconstructible from the
// locked lease rows; lease shape and migration topology are not, so only a
// count mismatch is repaired and the re-assert still throws on anything else.
// reconcileReservationAccounting cannot stand in: it opens its own
// transaction, while this has to run inside the sweep's on rows it holds.
private async repairAssignmentActivityCounts(
transaction: RelayDatabase,
identity: AssignmentIdentity,
attemptId: string,
assignment: SqlRow,
leases: SqlRow[],
migrationRow: SqlRow
): Promise<void> {
try {
assertAssignmentActivityAccounting(assignment, leases, migrationRow)
return
} catch (error) {
if (
!(error instanceof Error) ||
error.message !== 'migration_activity_accounting_mismatch'
) {
throw error
}
}
const counts = activityCounts(leases)
await transaction.query(
`UPDATE relay_assignments SET reserved_controls = ?, reserved_splices = ?,
reserved_invites = ?, pending_installs = ?, pending_confirmations = ?,
migration_leases = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
counts.control,
counts.splice,
counts.invite,
counts.install,
counts.confirmation,
counts.migration,
identity.userId,
identity.relayHostId
]
)
const repaired = await this.assignmentRow(transaction, identity)
if (!repaired) throw new Error('regional_rehome_assignment_mismatch')
assertAssignmentActivityAccounting(repaired, leases, migrationRow)
noteRegionalRehomeActivityCountsRepaired(attemptId)
}
async reapRegionalRehomeAttempts(): Promise<number> {
const now = this.now()
const completed = await this.database.query(
`UPDATE relay_region_rehome_attempts
SET completed_at = COALESCE(completed_at, ?), updated_at = ?
WHERE completed_at IS NULL AND aborted_at IS NULL
AND EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = relay_region_rehome_attempts.user_id
AND migration.relay_host_id = relay_region_rehome_attempts.relay_host_id
AND migration.assignment_epoch = relay_region_rehome_attempts.assignment_epoch
AND migration.completed_at IS NOT NULL
)`,
[now, now]
)
const aborted = await this.database.query(
`UPDATE relay_region_rehome_attempts
SET aborted_at = COALESCE(aborted_at, ?), updated_at = ?
WHERE completed_at IS NULL AND aborted_at IS NULL
AND EXISTS (
SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = relay_region_rehome_attempts.user_id
AND migration.relay_host_id = relay_region_rehome_attempts.relay_host_id
AND migration.assignment_epoch = relay_region_rehome_attempts.assignment_epoch
AND migration.aborted_at IS NOT NULL
)`,
[now, now]
)
if (integer(aborted[0]!, 'changes') > 0) {
await this.disableRegionalRehomeControl()
}
return integer(completed[0]!, 'changes') + integer(aborted[0]!, 'changes')
}
async abortExpiredRegionalRehomes(limit = 100): Promise<number> {
const now = this.now()
const quarantined = this.quarantinedRegionalRehomeAttemptIds(now)
const exclusion = quarantined.length
? ` AND attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})`
: ''
const candidates = await this.database.query(
`SELECT attempt_id, user_id, relay_host_id, assignment_epoch
FROM relay_region_rehome_attempts
WHERE completed_at IS NULL AND aborted_at IS NULL
AND created_at <= ?${exclusion}
ORDER BY created_at, attempt_id LIMIT ?`,
[now - REGIONAL_REHOME_MAX_REFRESH_MS, ...quarantined, limit]
)
let aborted = 0
let inventoryBusy = 0
for (const candidate of candidates) {
const identity = {
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
}
const assignmentEpoch = integer(candidate, 'assignment_epoch')
const attemptId = text(candidate, 'attempt_id')
// Isolated like the completion sweep: one poisoned row must not stall
// every later candidate.
let changed = false
try {
changed = await this.database.transaction(async (transaction) => {
const assignment = await this.assignmentRow(transaction, identity)
const attempt = (
await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
const migration = (
await transaction.queryLocked(
`SELECT * FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
if (
!assignment ||
!attempt ||
!migration ||
optionalInteger(attempt, 'completed_at') !== undefined ||
optionalInteger(attempt, 'aborted_at') !== undefined ||
integer(attempt, 'created_at') > now - REGIONAL_REHOME_MAX_REFRESH_MS ||
optionalInteger(migration, 'completed_at') !== undefined ||
optionalInteger(migration, 'aborted_at') !== undefined
) {
return false
}
const sourceCellId = text(attempt, 'source_cell_id')
const targetCellId = text(attempt, 'target_cell_id')
if (
text(assignment, 'cell_id') !== targetCellId ||
integer(assignment, 'assignment_epoch') !== assignmentEpoch
) {
throw new Error('regional_rehome_assignment_mismatch')
}
const leases = await this.lockAssignmentActivities(transaction, identity)
if (activityUnitsForCell(leases, sourceCellId) > 0) return false
const targetActive = leases.some(
(lease) =>
text(lease, 'cell_id') === targetCellId &&
text(lease, 'activity_kind') === 'control' &&
!text(lease, 'activity_id').startsWith('control-pending:') &&
integer(lease, 'expires_at') > now
)
if (targetActive) return false
const cells = await this.lockCellInventory(transaction, 'nowait')
const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId)
const admission = await cellAdmissionStates(transaction)
if (
!source ||
integer(source, 'enabled') !== 1 ||
admission.get(sourceCellId) !== 'general' ||
!(await this.cellIsLive(transaction, sourceCellId, now))
) {
return false
}
for (const activityId of [
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]) {
const lease = activityLeaseById(leases, activityId)
if (lease) await this.removeActivityLease(transaction, identity, lease, now)
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
targetCellId,
assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
sourceCellId,
assignmentEpoch + 1,
now + ASSIGNMENT_LIMITS.activityLeaseMs,
now,
identity.userId,
identity.relayHostId
]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await transaction.query(
`UPDATE relay_region_rehome_attempts SET aborted_at = ?, updated_at = ?
WHERE attempt_id = ?`,
[now, now, text(attempt, 'attempt_id')]
)
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1`,
[now]
)
return true
})
this.regionalRehomeCandidateQuarantine.delete(attemptId)
} catch (error) {
if (isDatabaseLockUnavailable(error)) inventoryBusy++
else this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error)
}
if (changed) aborted++
}
warnSweepCellInventoryBusy('abort-expired-regional-rehomes', inventoryBusy)
return aborted
}
async abortExpiredEvacuations(): Promise<number> {
const now = this.now()
const abandonedBefore = now - STRANDED_MIGRATION_ABANDON_MS
const candidates = await this.database.query(
`SELECT migration.user_id, migration.relay_host_id, migration.assignment_epoch
FROM relay_assignment_migrations migration
WHERE migration.expires_at <= ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
AND ${ABORTABLE_EXPIRED_MIGRATION}
ORDER BY user_id, relay_host_id, assignment_epoch`,
[now, now, abandonedBefore, abandonedBefore]
)
let aborted = 0
let inventoryBusy = 0
for (const candidate of candidates) {
const didAbort = await this.database
.transaction(async (transaction) => {
const identity = {
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
}
// Migration cleanup follows the same assignment-first order as evacuation.
const assignment = await this.assignmentRow(transaction, identity)
const assignmentEpoch = integer(candidate, 'assignment_epoch')
const regionalAttempt = (
await transaction.queryLocked(
`SELECT attempt_id FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND completed_at IS NULL AND aborted_at IS NULL`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
)[0]
const row = (
await transaction.queryLocked(
`SELECT migration.* FROM relay_assignment_migrations migration
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.assignment_epoch = ?
AND migration.expires_at <= ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
AND ${ABORTABLE_EXPIRED_MIGRATION}`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
now,
now,
abandonedBefore,
abandonedBefore
]
)
)[0]
if (!row) return false
const targetCellId = text(row, 'target_cell_id')
const activityLeases = await this.lockAssignmentActivities(transaction, identity)
if (!assignment) throw new Error('migration_assignment_missing')
const currentAssignmentEpoch = integer(assignment, 'assignment_epoch')
const assignmentEpochMatches =
text(assignment, 'cell_id') === targetCellId &&
currentAssignmentEpoch === assignmentEpoch
const pendingTargetControl = activityLeaseById(
activityLeases,
pendingControlActivityId(assignmentEpoch)
)
const targetGrantIsFresh =
assignmentEpochMatches &&
pendingTargetControl !== undefined &&
text(pendingTargetControl, 'cell_id') === targetCellId &&
text(pendingTargetControl, 'activity_kind') === 'control' &&
integer(pendingTargetControl, 'expires_at') > now
const targetIsActive = activityLeases.some(
(lease) =>
text(lease, 'cell_id') === targetCellId &&
text(lease, 'activity_kind') === 'control' &&
text(lease, 'activity_id') !== pendingControlActivityId(assignmentEpoch)
)
if (targetGrantIsFresh) return false
if (targetIsActive && assignmentEpochMatches) {
// A committed target control is stronger evidence than a failed follow-up
// write; repair the marker instead of rolling a live desktop backward.
await transaction.query(
`UPDATE relay_assignment_migrations
SET target_registered_at = COALESCE(target_registered_at, ?), updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
return false
}
if (!assignmentEpochMatches) {
if (currentAssignmentEpoch <= assignmentEpoch) {
throw new Error('migration_assignment_mismatch')
}
// A newer assignment is authoritative regardless of where it landed.
// Retire only this obsolete migration; never rewrite the newer epoch.
const obsoleteLeases = [
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]
.map((activityId) => activityLeaseById(activityLeases, activityId))
.filter((lease): lease is SqlRow => lease !== undefined)
if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait')
for (const lease of obsoleteLeases) {
await this.removeActivityLease(transaction, identity, lease, now)
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
targetCellId,
assignmentEpoch,
now
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
return true
}
const cells = await this.lockCellInventory(transaction, 'nowait')
const sourceCellId = text(row, 'source_cell_id')
const admissionRows = await transaction.query(
`SELECT cell_id, admission_state, updated_at FROM relay_cell_admission
WHERE cell_id IN (?, ?)`,
[sourceCellId, targetCellId]
)
const sourceCell = cells.find((cell) => text(cell, 'cell_id') === sourceCellId)
const targetCell = cells.find((cell) => text(cell, 'cell_id') === targetCellId)
const sourceAdmission = admissionRows.find(
(admission) => text(admission, 'cell_id') === sourceCellId
)
const targetAdmission = admissionRows.find(
(admission) => text(admission, 'cell_id') === targetCellId
)
const registered = optionalInteger(row, 'target_registered_at') !== undefined
const sourceIsDurablyFenced =
registered &&
(
await transaction.query(
`SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.assignment_epoch = ?
AND ${DURABLY_FENCED_MIGRATION_SOURCE}`,
[identity.userId, identity.relayHostId, assignmentEpoch]
)
).length === 1
const retireOnTarget =
registered &&
activityUnitsForCell(activityLeases, sourceCellId) === 0 &&
sourceCell !== undefined &&
integer(sourceCell, 'enabled') === 0 &&
sourceAdmission !== undefined &&
text(sourceAdmission, 'admission_state') === 'existing-only' &&
(integer(sourceAdmission, 'updated_at') <= abandonedBefore || sourceIsDurablyFenced) &&
targetCell !== undefined &&
integer(targetCell, 'enabled') === 1 &&
targetAdmission !== undefined &&
['migration-only', 'general'].includes(text(targetAdmission, 'admission_state'))
const rollbackReason =
!registered ||
(targetCell !== undefined &&
integer(targetCell, 'enabled') === 0 &&
targetAdmission !== undefined &&
text(targetAdmission, 'admission_state') === 'existing-only' &&
integer(targetAdmission, 'updated_at') <= abandonedBefore)
const regionalRollbackSourceAvailable =
!regionalAttempt ||
(sourceCell !== undefined &&
integer(sourceCell, 'enabled') === 1 &&
sourceAdmission !== undefined &&
text(sourceAdmission, 'admission_state') === 'general' &&
(await this.cellIsLive(transaction, sourceCellId, now)))
const rollbackToSource = rollbackReason && regionalRollbackSourceAvailable
if (!retireOnTarget && !rollbackToSource) return false
for (const activityId of [
pendingControlActivityId(assignmentEpoch),
migrationActivityId(assignmentEpoch)
]) {
const lease = activityLeaseById(activityLeases, activityId)
if (lease) await this.removeActivityLease(transaction, identity, lease, now)
}
await this.releaseSupersededControlConnectionReservations(
transaction,
identity,
targetCellId,
assignmentEpoch,
now
)
if (retireOnTarget) {
await transaction.query(
`UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
return true
}
await transaction.query(
`UPDATE relay_assignments SET cell_id = ?, assignment_epoch = ?,
lease_expires_at = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
sourceCellId,
assignmentEpoch + 1,
now + ASSIGNMENT_LIMITS.activityLeaseMs,
now,
identity.userId,
identity.relayHostId
]
)
await transaction.query(
`UPDATE relay_assignment_migrations SET aborted_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
if (regionalAttempt) {
await transaction.query(
`UPDATE relay_region_rehome_attempts SET aborted_at = ?, updated_at = ? WHERE attempt_id = ?`,
[now, now, regionalAttempt.attempt_id]
)
}
return true
})
.catch((error: unknown): boolean => {
// Expiry is durable; another director settling this row is not a failure.
// Invariant failures remain fatal so operators see corrupt migration state.
if (!isDatabaseLockUnavailable(error)) throw error
inventoryBusy++
return false
})
if (didAbort) aborted++
}
warnSweepCellInventoryBusy('abort-expired-evacuations', inventoryBusy)
return aborted
}
async releaseExpiredActivityLeases(): Promise<number> {
const now = this.now()
await this.database.query(
`UPDATE relay_control_connection_reservations
SET state = 'late-arrival-debt', updated_at = ?
WHERE state = 'reserved' AND timeout_at <= ?`,
[now, now]
)
await this.database.query(
`UPDATE relay_control_connection_reservations
SET state = 'released', released_at = ?, updated_at = ?
WHERE state = 'late-arrival-debt'
AND claim_activity_id IS NULL
AND timeout_at <= ?`,
[now, now, now - LATE_ARRIVAL_DEBT_RETENTION_MS]
)
const candidates = await this.database.query(
`SELECT user_id, relay_host_id, activity_id
FROM relay_assignment_activity_leases lease
WHERE expires_at <= ?
AND NOT EXISTS (
SELECT 1 FROM relay_region_rehome_attempts rehome
WHERE rehome.user_id = lease.user_id
AND rehome.relay_host_id = lease.relay_host_id
AND rehome.completed_at IS NULL AND rehome.aborted_at IS NULL
AND lease.activity_id IN (
'control-pending:' || CAST(rehome.assignment_epoch AS TEXT),
'migration:' || CAST(rehome.assignment_epoch AS TEXT)
)
)
AND NOT EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
JOIN relay_assignment_migrations migration
ON migration.user_id = pin.user_id
AND migration.relay_host_id = pin.relay_host_id
AND migration.assignment_epoch = pin.assignment_epoch
WHERE pin.user_id = lease.user_id
AND pin.relay_host_id = lease.relay_host_id
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
AND lease.activity_id IN (
'control-pending:' || CAST(pin.assignment_epoch AS TEXT),
'migration:' || CAST(pin.assignment_epoch AS TEXT)
)
)`,
[now]
)
let released = 0
for (const candidate of candidates) {
let didRelease: boolean
try {
didRelease = await this.database.transaction(async (transaction) => {
const identity = {
userId: text(candidate, 'user_id'),
relayHostId: text(candidate, 'relay_host_id')
}
// Why: re-check under the canonical assignment-first lock so a concurrent
// renewal wins without cleanup reaping its refreshed lease.
// Several directors sweep the same expired rows. Skip a row another
// director is settling instead of waiting and retrying the transaction.
await this.assignmentRow(transaction, identity, true)
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
const lease = activityLeaseById(activityLeases, text(candidate, 'activity_id'))
if (!lease || integer(lease, 'expires_at') > now) return false
await this.lockCellInventory(transaction, 'nowait')
await this.removeActivityLease(transaction, identity, lease, now)
return true
})
} catch (error) {
// Released cell images can still hold their legacy cell-first lock;
// expiry is durable, so a later maintenance sweep can safely retry it.
if (isDatabaseLockUnavailable(error)) continue
throw error
}
if (didRelease) released++
}
return released
}
async releaseExpiredActivity(): Promise<number> {
const now = this.now()
try {
return await this.database.transaction(async (transaction) => {
const expired = await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE lease_expires_at <= ? AND
(reserved_controls > 0 OR reserved_splices > 0 OR reserved_invites > 0 OR
pending_installs > 0 OR pending_confirmations > 0 OR migration_leases > 0)
AND NOT EXISTS (
SELECT 1 FROM relay_region_rehome_attempts rehome
WHERE rehome.user_id = relay_assignments.user_id
AND rehome.relay_host_id = relay_assignments.relay_host_id
AND rehome.completed_at IS NULL AND rehome.aborted_at IS NULL
)
AND NOT EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
JOIN relay_assignment_migrations migration
ON migration.user_id = pin.user_id
AND migration.relay_host_id = pin.relay_host_id
AND migration.assignment_epoch = pin.assignment_epoch
WHERE pin.user_id = relay_assignments.user_id
AND pin.relay_host_id = relay_assignments.relay_host_id
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
)
ORDER BY user_id, relay_host_id`,
[now],
{ failIfUnavailable: true }
)
if (expired.length > 0) await this.lockCellInventory(transaction, 'nowait')
for (const row of expired) {
await this.adjustCellReservation(transaction, text(row, 'cell_id'), -requestUnits(row))
await transaction.query(
`UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0,
reserved_invites = 0, pending_installs = 0, pending_confirmations = 0,
migration_leases = 0
WHERE user_id = ? AND relay_host_id = ?`,
[text(row, 'user_id'), text(row, 'relay_host_id')]
)
}
return expired.length
})
} catch (error) {
// Aggregate expiry is reconstructible from durable leases; never wait
// long enough to form a mixed-version cell/assignment lock cycle.
if (isDatabaseLockUnavailable(error)) return 0
throw error
}
}
async releaseExpiredRegionPreferences(): Promise<number> {
const result = await this.database.query(
`DELETE FROM relay_assignment_region_preferences WHERE observed_at < ?`,
[this.now() - REGION_PREFERENCE_RETENTION_MS]
)
return integer(result[0]!, 'changes')
}
private async reconcileReservationAccounting(
sourceCellId: string,
targetCellId: string
): Promise<void> {
const now = this.now()
await this.database.transaction(async (transaction) => {
// Reconciliation takes the same assignment→activity→cell order as live
// mutations so correcting drift never races a credential or socket lease.
const assignments = await transaction.queryLocked(
`SELECT assignment.* FROM relay_assignments assignment
WHERE assignment.cell_id IN (?, ?) OR EXISTS (
SELECT 1 FROM relay_assignment_activity_leases scoped_lease
WHERE scoped_lease.user_id = assignment.user_id
AND scoped_lease.relay_host_id = assignment.relay_host_id
AND scoped_lease.cell_id IN (?, ?)
)
ORDER BY assignment.user_id, assignment.relay_host_id`,
[sourceCellId, targetCellId, sourceCellId, targetCellId]
)
const leases = await transaction.queryLocked(
`SELECT lease.* FROM relay_assignment_activity_leases lease
WHERE lease.cell_id IN (?, ?) OR EXISTS (
SELECT 1 FROM relay_assignments assignment
WHERE assignment.user_id = lease.user_id
AND assignment.relay_host_id = lease.relay_host_id
AND (
assignment.cell_id IN (?, ?) OR EXISTS (
SELECT 1 FROM relay_assignment_activity_leases scoped_lease
WHERE scoped_lease.user_id = lease.user_id
AND scoped_lease.relay_host_id = lease.relay_host_id
AND scoped_lease.cell_id IN (?, ?)
)
)
)
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id`,
[
sourceCellId,
targetCellId,
sourceCellId,
targetCellId,
sourceCellId,
targetCellId
]
)
// Only the two cells this repairs need holding. The id set below is an
// existence check against a table that only reconcileCells writes, so it
// reads unlocked instead of dragging the other 21 rows into the section.
const cellIds = new Set(
(await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
text(row, 'cell_id')
)
)
const cells = await this.lockCellRows(
transaction,
[sourceCellId, targetCellId],
'pool-default'
)
const assignmentKeys = new Set(
assignments.map((row) =>
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
)
)
const assignmentCounts = new Map<
string,
{ counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
>()
const cellUnits = new Map<string, number>()
for (const lease of leases) {
const key = assignmentKey(text(lease, 'user_id'), text(lease, 'relay_host_id'))
if (!assignmentKeys.has(key)) throw new Error('activity_lease_assignment_missing')
const cellId = text(lease, 'cell_id')
if (!cellIds.has(cellId)) throw new Error('activity_lease_cell_missing')
const current =
assignmentCounts.get(key) ?? {
counts: emptyActivityCounts(),
leaseExpiresAt: 0
}
const kind = activityKind(lease)
current.counts[kind]++
current.leaseExpiresAt = Math.max(current.leaseExpiresAt, integer(lease, 'expires_at'))
assignmentCounts.set(key, current)
cellUnits.set(cellId, (cellUnits.get(cellId) ?? 0) + integer(lease, 'request_units'))
}
for (const row of assignments) {
const current = assignmentCounts.get(
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
)
const counts = current?.counts ?? emptyActivityCounts()
const differs = (Object.keys(ACTIVITY_COLUMN) as AssignmentActivityKind[]).some(
(kind) => integer(row, ACTIVITY_COLUMN[kind]) !== counts[kind]
)
const expiryDiffers =
current !== undefined && integer(row, 'lease_expires_at') !== current.leaseExpiresAt
if (!differs && !expiryDiffers) continue
await transaction.query(
`UPDATE relay_assignments SET reserved_controls = ?, reserved_splices = ?,
reserved_invites = ?, pending_installs = ?, pending_confirmations = ?,
migration_leases = ?, lease_expires_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
counts.control,
counts.splice,
counts.invite,
counts.install,
counts.confirmation,
counts.migration,
current?.leaseExpiresAt ?? integer(row, 'lease_expires_at'),
text(row, 'user_id'),
text(row, 'relay_host_id')
]
)
}
for (const row of cells) {
const cellId = text(row, 'cell_id')
const expected = cellUnits.get(cellId) ?? 0
if (expected > integer(row, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
if (integer(row, 'reserved_requests') === expected) continue
await transaction.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[expected, now, cellId]
)
}
})
}
private async lockCellInventory(
database: RelayDatabase,
mode: CellInventoryLockMode
): Promise<SqlRow[]> {
// Every capacity-changing assignment takes the tiny cell inventory in one
// order; dynamically locking only the selected target allowed cross-cell cycles.
const rows = await database.queryLocked(
`SELECT * FROM relay_cells ORDER BY cell_id ASC`,
[],
cellInventoryLockOptions(mode)
)
return rows
}
// Per-connection paths touch one or two cells. Locking exactly those rows,
// in the same ascending order the inventory lock uses (ORDER BY fixes the
// row-lock order), keeps them off the fleet-wide lock without a cycle.
// The wait policy follows the caller for the same reason the inventory lock's
// does: a sweep must not fail terminally on ordinary contention. Hold time is
// deliberately not sampled here — the metric tracks the fleet-wide lock these
// rows replace, and mixing in short single-row holds would flatter it.
private async lockCellRows(
database: RelayDatabase,
cellIds: string[],
mode: CellInventoryLockMode = 'request'
): Promise<SqlRow[]> {
const distinct = [...new Set(cellIds)]
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
return await database.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
distinct,
wait
)
}
// Unlocked on purpose: this only names the row to lock next, and the caller
// re-checks the pin once the assignment row is held.
private async pinnedCellId(
database: RelayDatabase,
identity: AssignmentIdentity
): Promise<string | undefined> {
const row = (
await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
return row ? text(row, 'cell_id') : undefined
}
private async lockGeneralCellInventory(
database: RelayDatabase,
mode: CellInventoryLockMode
): Promise<SqlRow[]> {
const rows = await database.queryLocked(
`SELECT * FROM relay_cells
WHERE cell_id IN (
SELECT cell_id FROM relay_cell_admission WHERE admission_state = 'general'
)
ORDER BY cell_id ASC`,
[],
cellInventoryLockOptions(mode)
)
return rows
}
private async leastLoadedCell(
database: RelayDatabase,
// Required: the one caller has already locked the inventory it selects from,
// and an optional parameter left a second fleet-wide lock reachable here.
rows: SqlRow[],
preferredRegion: RelayRegion
): Promise<CellRow | null> {
const regions = new Map(
(await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
text(row, 'cell_id'),
relayRegion(row, 'region')
])
)
const admission = await cellAdmissionStates(database)
const runtimeLoad = this.requireLiveCells
? new Map(
(
await database.query(
`SELECT cell_id, observed_requests FROM relay_cell_runtime
WHERE ready = ? AND last_heartbeat_at > ?`,
[1, this.now() - this.heartbeatTtlMs]
)
).map((row) => [text(row, 'cell_id'), integer(row, 'observed_requests')])
)
: null
const connectionHeadroom = await this.connectionHeadroomByCell(database)
const candidates = rows.filter((row) => {
const cellId = text(row, 'cell_id')
return (
admission.get(cellId) === 'general' &&
integer(row, 'reserved_requests') < integer(row, 'capacity_requests') &&
(!runtimeLoad || runtimeLoad.has(cellId)) &&
connectionHeadroom.get(cellId) !== false
)
})
candidates.sort((left, right) => {
const leftLoad =
integer(left, 'reserved_requests') +
(runtimeLoad?.get(text(left, 'cell_id')) ?? integer(left, 'observed_requests'))
const rightLoad =
integer(right, 'reserved_requests') +
(runtimeLoad?.get(text(right, 'cell_id')) ?? integer(right, 'observed_requests'))
const loadDifference =
leftLoad / integer(left, 'capacity_requests') -
rightLoad / integer(right, 'capacity_requests')
return loadDifference || text(left, 'cell_id').localeCompare(text(right, 'cell_id'))
})
const preferred = candidates.filter(
(candidate) =>
(regions.get(text(candidate, 'cell_id')) ?? RELAY_DEFAULT_REGION) === preferredRegion
)
const selected = preferred[0] ?? candidates[0]
return selected
? cell(selected, regions.get(text(selected, 'cell_id')) ?? RELAY_DEFAULT_REGION)
: null
}
async regionCatalog(): Promise<RelayRegionCatalogEntry[]> {
const rows = await this.database.query(
`SELECT cell.cell_id, cell.cell_url, region.region
FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
WHERE cell.enabled = 1
AND admission.admission_state = 'general'
AND runtime.ready = ? AND runtime.last_heartbeat_at > ?
ORDER BY cell.cell_id ASC`,
[1, this.now() - this.heartbeatTtlMs]
)
const origins = new Map<RelayRegion, string[]>()
for (const row of rows) {
const region = optionalRelayRegion(row, 'region') ?? RELAY_DEFAULT_REGION
const current = origins.get(region) ?? []
if (current.length < 2) current.push(text(row, 'cell_url'))
origins.set(region, current)
}
return RELAY_REGIONS.flatMap((region) => {
const probeOrigins = origins.get(region)
return probeOrigins?.length ? [{ region, probeOrigins }] : []
})
}
private async recordRegionPreference(
database: RelayDatabase,
identity: AssignmentIdentity,
preferredRegion: RelayRegion | undefined,
observedAt: number
): Promise<void> {
if (!preferredRegion) return
await database.query(
`INSERT INTO relay_assignment_region_preferences
(user_id, relay_host_id, preferred_region, observed_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
preferred_region = CASE
WHEN excluded.observed_at >= relay_assignment_region_preferences.observed_at
THEN excluded.preferred_region
ELSE relay_assignment_region_preferences.preferred_region
END,
observed_at = CASE
WHEN excluded.observed_at >= relay_assignment_region_preferences.observed_at
THEN excluded.observed_at
ELSE relay_assignment_region_preferences.observed_at
END`,
[identity.userId, identity.relayHostId, preferredRegion, observedAt]
)
}
private async cellRegion(database: RelayDatabase, cellId: string): Promise<RelayRegion> {
const row = (
await database.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [cellId])
)[0]
return row ? relayRegion(row, 'region') : RELAY_DEFAULT_REGION
}
private async connectionHeadroomByCell(
database: RelayDatabase
): Promise<Map<string, boolean>> {
const now = this.now()
const rows = await database.query(ASSIGNMENT_CONNECTION_HEADROOM_QUERY)
return new Map(
rows.map((row) => {
const heartbeat = optionalInteger(row, 'last_heartbeat_at')
const hasFreshTelemetry =
heartbeat !== undefined &&
heartbeat > now - this.heartbeatTtlMs &&
optionalText(row, 'connection_incarnation') ===
optionalText(row, 'current_incarnation')
const hasHeadroom =
hasFreshTelemetry &&
integer(row, 'enforced_connection_units') +
integer(row, 'outstanding_reservations') +
integer(row, 'unobserved_bound') <
integer(row, 'hard_cap') -
RELAY_ADMISSION_BUDGETS.reservedHostControls
return [text(row, 'cell_id'), hasHeadroom] as const
})
)
}
private async assertCellConnectionHeadroom(
database: RelayDatabase,
cellId: string
): Promise<void> {
if (!(await this.cellHasConnectionHeadroom(database, cellId))) {
throw new Error('relay_connection_headroom_exhausted')
}
}
private async cellHasConnectionHeadroom(
database: RelayDatabase,
cellId: string
): Promise<boolean> {
return (await this.connectionHeadroomByCell(database)).get(cellId) !== false
}
private async cellIsLive(
database: RelayDatabase,
cellId: string,
now: number
): Promise<boolean> {
if (!this.requireLiveCells) return true
const rows = await database.query(
`SELECT cell.cell_id FROM relay_cells cell
JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
WHERE cell.cell_id = ? AND runtime.ready = ? AND runtime.last_heartbeat_at > ?`,
[cellId, 1, now - this.heartbeatTtlMs]
)
return rows.length === 1
}
private async cellHasActiveFence(cellId: string): Promise<boolean> {
const rows = await this.database.query(
`SELECT fence.cell_id FROM relay_cell_fences fence
JOIN relay_cells cell ON cell.cell_id = fence.cell_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = fence.cell_id
WHERE fence.cell_id = ? AND cell.enabled = 0
AND fence.cell_incarnation = runtime.cell_incarnation
AND fence.attested_at >= runtime.last_heartbeat_at
AND fence.expires_at > ?`,
[cellId, this.now()]
)
return rows.length === 1
}
private async recordCommittedCellFence(
transaction: RelayDatabase,
cellId: string,
cellIncarnation: string,
attemptId: string,
attestedAt: number,
expiresAt: number
): Promise<void> {
await transaction.query(
`INSERT INTO relay_cell_committed_fences
(cell_id, attempt_id, cell_incarnation, attested_at, expires_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (cell_id) DO UPDATE SET
attempt_id = excluded.attempt_id,
cell_incarnation = excluded.cell_incarnation,
attested_at = excluded.attested_at,
expires_at = excluded.expires_at`,
[cellId, attemptId, cellIncarnation, attestedAt, expiresAt]
)
}
private async cellHasCommittedFence(
database: RelayDatabase,
cellId: string,
now: number
): Promise<boolean> {
const rows = await database.query(
`SELECT committed.cell_id
FROM relay_cell_committed_fences committed
JOIN relay_cell_fence_attempts attempt
ON attempt.attempt_id = committed.attempt_id
JOIN relay_cell_fences fence ON fence.cell_id = committed.cell_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = committed.cell_id
JOIN relay_cells cell ON cell.cell_id = committed.cell_id
WHERE committed.cell_id = ?
AND attempt.completed_at IS NOT NULL
AND attempt.aborted_at IS NULL
AND cell.enabled = 0
AND committed.cell_incarnation = runtime.cell_incarnation
AND fence.cell_incarnation = committed.cell_incarnation
AND committed.attested_at >= runtime.last_heartbeat_at
AND committed.expires_at > ?
AND fence.expires_at > ?`,
[cellId, now, now]
)
return rows.length === 1
}
private async deadCellRequiresCommittedFence(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number
): Promise<boolean> {
const row = (
await database.query(
`SELECT
CASE WHEN EXISTS (
SELECT 1 FROM relay_cell_connection_limits limits
WHERE limits.cell_id = ?
) THEN 1 ELSE 0 END AS capped,
CASE WHEN EXISTS (
SELECT 1 FROM relay_post_drain_migration_pins pin
JOIN relay_assignment_migrations migration
ON migration.user_id = pin.user_id
AND migration.relay_host_id = pin.relay_host_id
AND migration.assignment_epoch = pin.assignment_epoch
WHERE pin.user_id = ? AND pin.relay_host_id = ?
AND pin.assignment_epoch = ?
AND pin.target_cell_id = ?
AND migration.completed_at IS NULL
AND migration.aborted_at IS NULL
) THEN 1 ELSE 0 END AS pinned`,
[cellId, identity.userId, identity.relayHostId, assignmentEpoch, cellId]
)
)[0]
if (!row) return false
return integer(row, 'capped') === 1 || integer(row, 'pinned') === 1
}
private async assertDrainCellGeneration(
transaction: RelayDatabase,
cellId: string,
cellIncarnation: string
): Promise<void> {
const cell = (
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId])
)[0]
const runtime = (
await transaction.queryLocked(`SELECT * FROM relay_cell_runtime WHERE cell_id = ?`, [
cellId
])
)[0]
if (!cell || integer(cell, 'enabled') !== 0) {
throw new Error('drain_attempt_admission_enabled')
}
if (!runtime || text(runtime, 'cell_incarnation') !== cellIncarnation) {
throw new Error('drain_attempt_generation_mismatch')
}
}
private async requireCellFence(
transaction: RelayDatabase,
cellId: string,
runtime: SqlRow | undefined,
now: number
): Promise<void> {
const fence = (
await transaction.queryLocked(`SELECT * FROM relay_cell_fences WHERE cell_id = ?`, [
cellId
])
)[0]
if (
!fence ||
!runtime ||
text(fence, 'cell_incarnation') !== text(runtime, 'cell_incarnation') ||
integer(fence, 'attested_at') < integer(runtime, 'last_heartbeat_at') ||
integer(fence, 'expires_at') <= now
) {
throw new Error('cell_fence_attestation_missing')
}
}
private async activeCellMigrations(sourceCellId: string, targetCellId: string): Promise<SqlRow[]> {
const targetRuntimeSafety = this.requireLiveCells
? `AND EXISTS (
SELECT 1 FROM relay_cell_runtime target_runtime
WHERE target_runtime.cell_id = migration.target_cell_id
AND lease.updated_at >= target_runtime.started_at
)`
: ''
return await this.database.query(
`SELECT migration.*, assignment.cell_id AS current_cell_id,
assignment.assignment_epoch AS current_assignment_epoch,
COALESCE((
SELECT SUM(source_lease.request_units)
FROM relay_assignment_activity_leases source_lease
WHERE source_lease.user_id = migration.user_id
AND source_lease.relay_host_id = migration.relay_host_id
AND source_lease.cell_id = migration.source_cell_id
), 0) AS source_activity_units,
CASE WHEN EXISTS (
SELECT 1 FROM relay_assignment_activity_leases lease
WHERE lease.user_id = migration.user_id
AND lease.relay_host_id = migration.relay_host_id
AND lease.cell_id = migration.target_cell_id
AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%'
) THEN 1 ELSE 0 END AS target_control_active,
CASE WHEN EXISTS (
SELECT 1 FROM relay_assignment_activity_leases lease
WHERE lease.user_id = migration.user_id
AND lease.relay_host_id = migration.relay_host_id
AND lease.cell_id = migration.target_cell_id
AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%'
AND lease.expires_at > ?
${targetRuntimeSafety}
) THEN 1 ELSE 0 END AS target_control_current
FROM relay_assignment_migrations migration
LEFT JOIN relay_assignments assignment
ON assignment.user_id = migration.user_id
AND assignment.relay_host_id = migration.relay_host_id
WHERE migration.source_cell_id = ? AND migration.target_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id`,
[this.now(), sourceCellId, targetCellId]
)
}
private async insertPendingControlLease(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number,
now: number
): Promise<void> {
const timeoutAt = now + ASSIGNMENT_LIMITS.activityLeaseMs
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (user_id, relay_host_id, activity_id) DO UPDATE SET
expires_at = excluded.expires_at, updated_at = excluded.updated_at`,
[
identity.userId,
identity.relayHostId,
pendingControlActivityId(assignmentEpoch),
'control',
cellId,
1,
timeoutAt,
now
]
)
await this.insertControlConnectionReservation(
database,
identity,
cellId,
assignmentEpoch,
timeoutAt,
now
)
}
private async insertControlConnectionReservation(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number,
timeoutAt: number,
now: number
): Promise<void> {
const existing = await database.queryLocked(
`SELECT reservation_id
FROM relay_control_connection_reservations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt')
AND claim_activity_id IS NULL
ORDER BY created_at ASC, reservation_id ASC
LIMIT 1`,
[identity.userId, identity.relayHostId, assignmentEpoch, cellId]
)
if (existing.length > 0) return
const reservationId = randomUUID()
await database.query(
`INSERT INTO relay_control_connection_reservations
(reservation_id, idempotency_key, user_id, relay_host_id,
assignment_epoch, cell_id, state, inclusion_watermark,
claim_activity_id, created_at, timeout_at, claimed_at, released_at,
updated_at)
SELECT ?, ?, ?, ?, ?, ?, 'reserved', NULL, NULL, ?, ?, NULL, NULL, ?
FROM relay_cell_connection_limits WHERE cell_id = ?`,
[
reservationId,
reservationId,
identity.userId,
identity.relayHostId,
assignmentEpoch,
cellId,
now,
timeoutAt,
now,
cellId
]
)
}
private async refreshPendingControlReservation(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number,
timeoutAt: number,
now: number
): Promise<void> {
await database.query(
`UPDATE relay_control_connection_reservations
SET state = 'reserved', timeout_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt')
AND claim_activity_id IS NULL`,
[
timeoutAt,
now,
identity.userId,
identity.relayHostId,
assignmentEpoch,
cellId
]
)
}
private async claimControlConnectionReservation(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number,
activityId: string,
inclusionWatermark: number | undefined,
now: number
): Promise<void> {
await database.query(
`UPDATE relay_control_connection_reservations
SET claim_activity_id = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND cell_id = ? AND state = 'claimed'
AND claim_activity_id IS NOT NULL AND claim_activity_id <> ?`,
[
activityId,
now,
identity.userId,
identity.relayHostId,
assignmentEpoch,
cellId,
activityId
]
)
// A released row's claim_activity_id is history, not a live claim: a control that
// reconnects under the same generation would otherwise leave its fresh reservation
// unclaimable, decaying through debt while the connection is already enforced.
const alreadyClaimed = await database.queryLocked(
`SELECT reservation_id FROM relay_control_connection_reservations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND cell_id = ? AND claim_activity_id = ? AND state <> 'released'
ORDER BY created_at ASC, reservation_id ASC`,
[
identity.userId,
identity.relayHostId,
assignmentEpoch,
cellId,
activityId
]
)
if (alreadyClaimed.length > 0) return
const reservation = (
await database.queryLocked(
`SELECT * FROM relay_control_connection_reservations
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
AND cell_id = ? AND state IN ('reserved', 'late-arrival-debt')
AND claim_activity_id IS NULL
ORDER BY created_at ASC, reservation_id ASC`,
[identity.userId, identity.relayHostId, assignmentEpoch, cellId]
)
)[0]
if (!reservation) return
await database.query(
`UPDATE relay_control_connection_reservations
SET state = 'claimed', inclusion_watermark = ?, claim_activity_id = ?,
claimed_at = ?, updated_at = ?
WHERE reservation_id = ?`,
[
inclusionWatermark,
activityId,
now,
now,
text(reservation, 'reservation_id')
]
)
}
private async releaseSupersededControlConnectionReservations(
database: RelayDatabase,
identity: AssignmentIdentity,
cellId: string,
assignmentEpoch: number,
now: number
): Promise<void> {
await database.query(
`UPDATE relay_control_connection_reservations
SET state = 'released', released_at = ?, updated_at = ?
WHERE user_id = ? AND relay_host_id = ? AND cell_id = ?
AND assignment_epoch = ? AND state <> 'released'`,
[
now,
now,
identity.userId,
identity.relayHostId,
cellId,
assignmentEpoch
]
)
}
private async assignmentRow(
database: RelayDatabase,
identity: AssignmentIdentity,
failIfUnavailable = false
): Promise<SqlRow | undefined> {
return (
await database.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId],
{ failIfUnavailable }
)
)[0]
}
private async lockAssignmentActivities(
database: RelayDatabase,
identity: AssignmentIdentity,
failIfUnavailable = false
): Promise<SqlRow[]> {
// Lease rows always precede the globally ordered capacity rows.
return await database.queryLocked(
`SELECT * FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`,
[identity.userId, identity.relayHostId],
{ failIfUnavailable }
)
}
private async removeActivityLease(
database: RelayDatabase,
identity: AssignmentIdentity,
lease: SqlRow,
now: number
): Promise<void> {
const kind = activityKind(lease)
await this.adjustCellReservation(
database,
text(lease, 'cell_id'),
-integer(lease, 'request_units')
)
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[identity.userId, identity.relayHostId, text(lease, 'activity_id')]
)
await this.adjustActivityCount(database, identity, kind, -1, now, now)
}
private async removeSupersededSameCellControls(
database: RelayDatabase,
identity: AssignmentIdentity,
leases: SqlRow[],
cellId: string,
retainedActivityId: string,
now: number
): Promise<void> {
const superseded = leases.filter(
(lease) =>
activityKind(lease) === 'control' &&
text(lease, 'cell_id') === cellId &&
text(lease, 'activity_id') !== retainedActivityId
)
if (superseded.length === 0) return
if (
superseded.some(
(lease) => integer(lease, 'request_units') !== ACTIVITY_REQUEST_UNITS.control
)
) {
throw new Error('activity_lease_shape_mismatch')
}
// Why: this recomputes one cell's reservation from its leases, so only that
// row needs to be held; the 23-row inventory lock here serialised every
// desktop control rebind in the fleet behind every other one.
const cellRow = (await this.lockCellRows(database, [cellId]))[0]
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'
AND cell_id = ? AND activity_id <> ?`,
[identity.userId, identity.relayHostId, cellId, retainedActivityId]
)
const remainingControls =
leases.filter((lease) => activityKind(lease) === 'control').length - superseded.length
await database.query(
`UPDATE relay_assignments SET reserved_controls = ?, last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[remainingControls, now, identity.userId, identity.relayHostId]
)
const cellUnitsRow = (
await database.query(
`SELECT COALESCE(SUM(request_units), 0) AS request_units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[cellId]
)
)[0]!
const cellUnits = integer(cellUnitsRow, 'request_units')
if (!cellRow) throw new Error('assigned_cell_missing')
if (cellUnits > integer(cellRow, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
await database.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[cellUnits, now, cellId]
)
}
private async adjustActivityCount(
database: RelayDatabase,
identity: AssignmentIdentity,
kind: AssignmentActivityKind,
delta: 1 | -1,
leaseExpiresAt: number,
now: number
): Promise<void> {
const column = ACTIVITY_COLUMN[kind]
await database.query(
`UPDATE relay_assignments SET ${column} =
CASE WHEN ${column} + ? < 0 THEN 0 ELSE ${column} + ? END,
lease_expires_at =
CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[delta, delta, leaseExpiresAt, leaseExpiresAt, now, identity.userId, identity.relayHostId]
)
}
private async touchAssignment(
database: RelayDatabase,
identity: AssignmentIdentity,
leaseExpiresAt: number,
now: number
): Promise<void> {
await database.query(
`UPDATE relay_assignments SET lease_expires_at =
CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END,
last_activity_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[leaseExpiresAt, leaseExpiresAt, now, identity.userId, identity.relayHostId]
)
}
private async adjustCellReservation(
database: RelayDatabase,
cellId: string,
delta: number
): Promise<void> {
const row = (await database.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellId]))[0]
if (!row) throw new Error('assigned_cell_missing')
const next = integer(row, 'reserved_requests') + delta
if (next > integer(row, 'capacity_requests')) throw new Error('relay_capacity_exhausted')
await database.query(
`UPDATE relay_cells SET reserved_requests =
CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END,
updated_at = ? WHERE cell_id = ?`,
[delta, delta, this.now(), cellId]
)
}
private async adjustCellReservationAtomically(
database: RelayDatabase,
cellId: string,
delta: number
): Promise<void> {
const rows = await database.query(
`UPDATE relay_cells SET reserved_requests =
CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END,
updated_at = ?
WHERE cell_id = ?
AND (? <= 0 OR reserved_requests + ? <= capacity_requests)
RETURNING cell_id`,
[delta, delta, this.now(), cellId, delta, delta]
)
if (rows.length > 0) return
const cell = (
await database.query(`SELECT cell_id FROM relay_cells WHERE cell_id = ?`, [cellId])
)[0]
if (!cell) throw new Error('assigned_cell_missing')
throw new Error('relay_capacity_exhausted')
}
private result(
identity: AssignmentIdentity,
row: SqlRow,
cellRow: CellRow,
leaseExpiresAt: number
): RelayAssignment {
return {
...identity,
...cellRow,
assignmentEpoch: integer(row, 'assignment_epoch'),
leaseExpiresAt
}
}
}
type CellRow = { cellId: string; cellUrl: string; region: RelayRegion }
function cell(row: SqlRow, region: RelayRegion): CellRow {
return { cellId: text(row, 'cell_id'), cellUrl: text(row, 'cell_url'), region }
}
function relayRegion(row: SqlRow, field: string): RelayRegion {
const value = text(row, field)
if (!RELAY_REGIONS.includes(value as RelayRegion)) throw new Error(`invalid region field ${field}`)
return value as RelayRegion
}
function optionalRelayRegion(row: SqlRow, field: string): RelayRegion | undefined {
const value = optionalText(row, field)
if (value === undefined) return undefined
if (!RELAY_REGIONS.includes(value as RelayRegion)) throw new Error(`invalid region field ${field}`)
return value as RelayRegion
}
function activityLeaseById(rows: SqlRow[], activityId: string): SqlRow | undefined {
return rows.find((row) => text(row, 'activity_id') === activityId)
}
// Why: mid-rehome a host legitimately holds the source control plus the
// target's pending control, so only the lease rows — never the counter a grant
// would overwrite — can say whether this cell is already reserved for it.
function holdsControlLease(
rows: SqlRow[],
cellId: string,
assignmentEpoch: number
): boolean {
const pendingId = pendingControlActivityId(assignmentEpoch)
return rows.some((row) => {
if (activityKind(row) !== 'control' || text(row, 'cell_id') !== cellId) return false
const activityId = text(row, 'activity_id')
return activityId === pendingId || !activityId.startsWith('control-pending:')
})
}
function activityCounts(rows: SqlRow[]): Record<AssignmentActivityKind, number> {
const counts = emptyActivityCounts()
for (const row of rows) counts[activityKind(row)]++
return counts
}
function activityUnitsForCell(rows: SqlRow[], cellId: string): number {
return rows.reduce(
(total, row) =>
text(row, 'cell_id') === cellId ? total + integer(row, 'request_units') : total,
0
)
}
function activity(row: SqlRow) {
return {
relayHostId: text(row, 'relay_host_id'),
cellId: text(row, 'cell_id'),
assignmentEpoch: integer(row, 'assignment_epoch'),
leaseExpiresAt: integer(row, 'lease_expires_at'),
lastActivityAt: integer(row, 'last_activity_at'),
reservedControls: integer(row, 'reserved_controls'),
reservedSplices: integer(row, 'reserved_splices'),
reservedInvites: integer(row, 'reserved_invites'),
pendingInstalls: integer(row, 'pending_installs'),
pendingConfirmations: integer(row, 'pending_confirmations'),
migrationLeases: integer(row, 'migration_leases')
}
}
function requestUnits(row: SqlRow): number {
return (
integer(row, 'reserved_controls') +
2 * integer(row, 'reserved_splices') +
integer(row, 'reserved_invites') +
integer(row, 'pending_installs') +
integer(row, 'pending_confirmations') +
integer(row, 'migration_leases')
)
}
function integer(row: SqlRow, field: string): number {
const value = Number(row[field])
if (!Number.isSafeInteger(value)) throw new Error(`invalid_${field}`)
return value
}
function text(row: SqlRow, field: string): string {
const value = row[field]
if (typeof value !== 'string') throw new Error(`invalid_${field}`)
return value
}
function cellFenceAttempt(row: SqlRow): CellFenceAttempt {
const environment = text(row, 'environment')
if (!['staging', 'production'].includes(environment)) {
throw new Error('invalid_environment')
}
return {
attemptId: text(row, 'attempt_id'),
environment: environment as CellFenceAttempt['environment'],
cellId: text(row, 'cell_id'),
cellIncarnation: text(row, 'cell_incarnation'),
migName: text(row, 'mig_name'),
instanceGroup: text(row, 'instance_group'),
generationIdentity: text(row, 'generation_identity'),
fenceCommit: text(row, 'fence_commit'),
planSha256: text(row, 'plan_sha256'),
planObjectName: text(row, 'plan_object_name'),
planObjectGeneration: optionalText(row, 'plan_object_generation'),
varFileSha256: text(row, 'var_file_sha256'),
terraformStateLineage: text(row, 'terraform_state_lineage'),
terraformStateSerial: integer(row, 'terraform_state_serial'),
terraformStateObjectGeneration: text(
row,
'terraform_state_object_generation'
),
terraformStateObjectSha256: text(row, 'terraform_state_object_sha256'),
requestReason: text(row, 'request_reason'),
gceOperation: optionalText(row, 'gce_operation'),
createdAt: integer(row, 'created_at'),
expiresAt: integer(row, 'expires_at'),
applyStartedAt: optionalInteger(row, 'apply_started_at'),
completedAt: optionalInteger(row, 'completed_at'),
abortedAt: optionalInteger(row, 'aborted_at')
}
}
function cellFenceApplyInvocation(row: SqlRow): CellFenceApplyInvocation {
return {
invocationId: text(row, 'invocation_id'),
requestReason: text(row, 'request_reason'),
startedAt: integer(row, 'started_at'),
gceOperation: optionalText(row, 'gce_operation')
}
}
function assertCellFenceAttemptBase(
row: SqlRow,
input: CellFenceAttemptEvidence
): void {
const fields: [keyof CellFenceAttemptEvidence, string][] = [
['attemptId', 'attempt_id'],
['environment', 'environment'],
['cellId', 'cell_id'],
['cellIncarnation', 'cell_incarnation'],
['migName', 'mig_name'],
['instanceGroup', 'instance_group'],
['generationIdentity', 'generation_identity'],
['fenceCommit', 'fence_commit'],
['planSha256', 'plan_sha256'],
['planObjectName', 'plan_object_name'],
['varFileSha256', 'var_file_sha256'],
['terraformStateLineage', 'terraform_state_lineage'],
['terraformStateObjectGeneration', 'terraform_state_object_generation'],
['terraformStateObjectSha256', 'terraform_state_object_sha256'],
['requestReason', 'request_reason']
]
if (
fields.some(([inputField, rowField]) => input[inputField] !== text(row, rowField)) ||
input.terraformStateSerial !== integer(row, 'terraform_state_serial')
) {
throw new Error('cell_fence_attempt_evidence_mismatch')
}
}
function assertCellFenceAttemptEvidence(
row: SqlRow,
input: CellFenceAttemptEvidence
): void {
assertCellFenceAttemptBase(row, input)
if (
!input.planObjectGeneration ||
input.planObjectGeneration !== optionalText(row, 'plan_object_generation')
) {
throw new Error('cell_fence_attempt_evidence_mismatch')
}
}
function assertActiveCellFenceAttempt(
row: SqlRow,
input: CellFenceAttemptEvidence,
now: number
): void {
assertCellFenceAttemptEvidence(row, input)
if (
integer(row, 'expires_at') <= now &&
optionalInteger(row, 'apply_started_at') === undefined
) {
throw new Error('cell_fence_attempt_expired')
}
if (row.completed_at !== null) throw new Error('cell_fence_attempt_completed')
if (row.aborted_at !== null) throw new Error('cell_fence_attempt_aborted')
}
async function lockedCellFenceAttempt(
transaction: RelayDatabase,
attemptId: string
): Promise<SqlRow> {
const row = (
await transaction.queryLocked(
`${CELL_FENCE_ATTEMPT_SELECT} WHERE attempts.attempt_id = ?`,
[attemptId]
)
)[0]
if (!row) throw new Error('cell_fence_attempt_not_found')
return row
}
function pendingControlActivityId(assignmentEpoch: number): string {
return `control-pending:${assignmentEpoch}`
}
function validateActivityId(activityId: string): void {
if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id')
}
function activityKind(row: SqlRow): AssignmentActivityKind {
const value = text(row, 'activity_kind')
if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind')
return value as AssignmentActivityKind
}
function migrationActivityId(assignmentEpoch: number): string {
return `migration:${assignmentEpoch}`
}
function isIncompleteMigration(error: unknown): boolean {
return (
error instanceof Error &&
[
'migration_target_not_registered',
'migration_source_still_active',
'migration_target_not_active',
'migration_assignment_mismatch',
'migration_source_admission_changed',
'migration_target_admission_changed',
'migration_source_runtime_not_quiescent',
'migration_source_runtime_not_dead',
'migration_target_runtime_not_ready',
'migration_cell_inventory_busy',
'migration_activity_accounting_mismatch',
'migration_activity_topology_mismatch',
'migration_activity_lease_shape_mismatch',
'migration_cell_reservation_accounting_mismatch',
'cell_fence_attestation_missing'
].includes(error.message)
)
}
function isMissingMigration(error: unknown): boolean {
return error instanceof Error && error.message === 'migration_not_found'
}
function isDatabaseLockUnavailable(error: unknown): boolean {
return error instanceof Error && error.message === 'database_lock_unavailable'
}
export function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions {
if (mode === 'nowait') return { failIfUnavailable: true, measureHoldMs: true }
if (mode === 'pool-default') return { measureHoldMs: true }
return { lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS, measureHoldMs: true }
}
// Background sweeps take the cell inventory NOWAIT so they never queue ahead of
// assignment traffic. A skipped candidate is re-derived from durable state on
// the next tick, so it is ordinary contention, not a sweep failure: one summary
// line per tick, never an error and never a quarantine.
function warnSweepCellInventoryBusy(sweep: string, skipped: number): void {
if (skipped === 0) return
console.warn(
JSON.stringify({ event: 'orca_relay_sweep_cell_inventory_busy', sweep, skipped })
)
}
function isDatabaseLockTimeout(error: unknown): boolean {
return String((error as { code?: unknown }).code) === '55P03'
}
async function waitForAssignmentLockRetry(deadline: number): Promise<void> {
const remainingMs = Math.max(0, deadline - Date.now())
const maxDelayMs = Math.min(ASSIGNMENT_LOCK_RETRY_MAX_DELAY_MS, remainingMs)
const delayMs = Math.floor(Math.random() * (maxDelayMs + 1))
await new Promise((resolve) => setTimeout(resolve, delayMs))
}
function migration(identity: AssignmentIdentity, row: SqlRow): RelayAssignmentMigration {
const targetRegisteredAt = optionalInteger(row, 'target_registered_at')
return {
...identity,
sourceCellId: text(row, 'source_cell_id'),
targetCellId: text(row, 'target_cell_id'),
previousEpoch: integer(row, 'previous_epoch'),
assignmentEpoch: integer(row, 'assignment_epoch'),
expiresAt: integer(row, 'expires_at'),
...(targetRegisteredAt === undefined ? {} : { targetRegisteredAt })
}
}
// Attempt ids are server-minted UUIDs and this codebase's invariant messages
// are snake_case slugs; anything else could carry secrets and logs redacted.
function warnRegionalRehomeCandidateFailure(
operation: 'complete' | 'abort' | 'refresh',
attemptId: string,
error: unknown
): void {
const message = error instanceof Error ? error.message : ''
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_candidate_failed',
operation,
attemptId,
reason: /^[a-z0-9_]{1,64}$/.test(message) ? message : 'redacted'
})
)
}
// The attempt id is the only field: counts would say which host holds what.
function noteRegionalRehomeActivityCountsRepaired(attemptId: string): void {
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_activity_counts_repaired',
attemptId
})
)
}
function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
return {
generation: integer(row, 'generation'),
enabled: integer(row, 'enabled') === 1,
observationStartedAt: integer(row, 'observation_started_at'),
notBefore: integer(row, 'not_before'),
ratePerMinute: integer(row, 'rate_per_minute'),
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
hostCooldownMs: integer(row, 'host_cooldown_ms'),
drainGraceMs: integer(row, 'drain_grace_ms')
}
}
function regionalRehomeFleetSafetyFromInventory(input: {
cells: SqlRow[]
admission: ReadonlyMap<string, CellAdmissionState>
regions: ReadonlyMap<string, RelayRegion>
runtimes: SqlRow[]
capabilities: SqlRow[]
safetyRows: SqlRow[]
now: number
heartbeatTtlMs: number
}): RegionalRehomeFleetSafety {
const capabilities = new Map(
input.capabilities.map((row) => [text(row, 'cell_id'), row])
)
const runtimes = new Map(input.runtimes.map((row) => [text(row, 'cell_id'), row]))
const safetyRows = new Map(
input.safetyRows.map((row) => [text(row, 'cell_id'), row])
)
const required = input.cells.filter((row) => {
const cellId = text(row, 'cell_id')
const capability = capabilities.get(cellId)
return (
integer(row, 'enabled') === 1 &&
input.admission.get(cellId) === 'general' &&
input.regions.get(cellId) !== undefined &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const valid = required.flatMap((row) => {
const cellId = text(row, 'cell_id')
const runtime = runtimes.get(cellId)
const safety = safetyRows.get(cellId)
if (
!runtime ||
integer(runtime, 'ready') !== 1 ||
integer(runtime, 'last_heartbeat_at') <= input.now - input.heartbeatTtlMs ||
!safety ||
text(safety, 'cell_incarnation') !== text(runtime, 'cell_incarnation') ||
integer(safety, 'observed_at') <= input.now - 60_000
) {
return []
}
return [safety]
})
const missingCells = required.length === 0 ? 1 : required.length - valid.length
return {
requiredCells: required.length,
missingCells,
observedAt:
missingCells > 0
? 0
: Math.min(...valid.map((row) => integer(row, 'observed_at'))),
sqlFailures: valid.reduce((total, row) => total + integer(row, 'sql_failures'), 0),
reconnects: valid.reduce((total, row) => total + integer(row, 'reconnects'), 0),
maxReconnects: Math.max(0, ...valid.map((row) => integer(row, 'reconnects'))),
controlActivityRecoveryFailures: valid.reduce(
(total, row) => total + integer(row, 'control_activity_recovery_failures'),
0
),
databasePoolWaiting: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_waiting'))
),
databasePoolWaitersMax: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_waiters_max'))
),
databasePoolWaitMsMax: Math.max(
0,
...valid.map((row) => integer(row, 'database_pool_wait_ms_max'))
)
}
}
function regionalRehomeFleetSafetyFailure(
processSafety: RegionalRehomeSafetySnapshot,
fleetSafety: RegionalRehomeFleetSafety,
now: number
): string | null {
if (fleetSafety.maxReconnects > REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT) {
return 'elevated_reconnects'
}
return regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
now,
fleetSafety.requiredCells
)
}
type RegionalRehomeCandidateSkip = {
reason:
| 'candidate_stale'
| 'host_cooldown'
| 'source_ineligible'
| 'source_unclean'
| 'source_control_inactive'
| 'target_unclean'
| 'no_eligible_target'
| 'no_target_headroom'
cellId?: string
sqlFailures?: number
reconnects?: number
}
function cellUncleanSkip(
reason: 'source_unclean' | 'target_unclean',
cellId: string,
safety: SqlRow | undefined
): RegionalRehomeCandidateSkip {
return {
reason,
cellId,
sqlFailures: safety === undefined ? undefined : integer(safety, 'sql_failures'),
reconnects: safety === undefined ? undefined : integer(safety, 'reconnects')
}
}
// Candidate skips are otherwise invisible: they neither latch the control off
// nor produce attempts, so an operator cannot tell "skipping" from "idle".
// Cell ids and counters only — never free-form error text.
function regionalRehomeCellSafetyIsClean(
safety: SqlRow | undefined,
runtime: SqlRow,
now: number
): boolean {
return (
safety !== undefined &&
text(safety, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
integer(safety, 'observed_at') > now - 60_000 &&
integer(safety, 'sql_failures') <= REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT &&
integer(safety, 'reconnects') <= REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT &&
integer(safety, 'control_activity_recovery_failures') === 0 &&
!regionalRehomePoolPressure({
databasePoolWaitersMax: integer(safety, 'database_pool_waiters_max'),
databasePoolWaitMsMax: integer(safety, 'database_pool_wait_ms_max')
})
)
}
function assertMigrationPair(row: SqlRow, sourceCellId: string, targetCellId: string): void {
if (
text(row, 'source_cell_id') !== sourceCellId ||
text(row, 'target_cell_id') !== targetCellId
) {
throw new Error('migration_pair_mismatch')
}
}
function matchesExactCellSet(
actual: ReadonlySet<string>,
expected: readonly string[]
): boolean {
const expectedSet = new Set(expected)
return (
expectedSet.size === expected.length &&
actual.size === expectedSet.size &&
[...actual].every((cellId) => expectedSet.has(cellId))
)
}
function assertCurrentMigrationAssignment(
assignment: SqlRow | undefined,
migrationRow: SqlRow
): asserts assignment is SqlRow {
if (
!assignment ||
text(assignment, 'cell_id') !== text(migrationRow, 'target_cell_id') ||
integer(assignment, 'assignment_epoch') !== integer(migrationRow, 'assignment_epoch')
) {
throw new Error('migration_assignment_mismatch')
}
}
function assertMigrationRecoveryMetadata(
migrationRow: SqlRow,
incarnation: SqlRow | undefined,
pin: SqlRow | undefined
): void {
if (pin && !incarnation) {
throw new Error('drain_migration_source_incarnation_mismatch')
}
if (
pin &&
incarnation &&
(text(pin, 'source_cell_id') !== text(migrationRow, 'source_cell_id') ||
text(pin, 'target_cell_id') !== text(migrationRow, 'target_cell_id') ||
text(pin, 'source_cell_incarnation') !==
text(incarnation, 'source_cell_incarnation') ||
text(pin, 'target_cell_incarnation') !==
text(incarnation, 'target_cell_incarnation') ||
integer(pin, 'source_request_units') !==
integer(migrationRow, 'source_request_units') ||
integer(pin, 'target_reserved_units') !==
integer(migrationRow, 'target_reserved_units'))
) {
throw new Error('migration_activity_topology_mismatch')
}
}
function assertAssignmentActivityAccounting(
assignment: SqlRow,
leases: SqlRow[],
migrationRow: SqlRow
): void {
if (
integer(migrationRow, 'target_reserved_units') !==
integer(migrationRow, 'source_request_units') + 1
) {
throw new Error('migration_activity_lease_shape_mismatch')
}
const counts = emptyActivityCounts()
for (const lease of leases) {
const kind = activityKind(lease)
const requestUnits = integer(lease, 'request_units')
if (kind === 'migration') {
if (
text(lease, 'activity_id') !==
migrationActivityId(integer(migrationRow, 'assignment_epoch')) ||
text(lease, 'cell_id') !== text(migrationRow, 'target_cell_id') ||
requestUnits !== integer(migrationRow, 'source_request_units')
) {
throw new Error('migration_activity_lease_shape_mismatch')
}
} else if (requestUnits !== ACTIVITY_REQUEST_UNITS[kind]) {
throw new Error('migration_activity_lease_shape_mismatch')
}
counts[kind]++
}
assertAssignmentActivityCounts(assignment, leases, 1)
}
function assertAssignmentActivityCounts(
assignment: SqlRow,
leases: SqlRow[],
expectedMigrations: number
): void {
const counts = activityCounts(leases)
if (
(Object.keys(ACTIVITY_COLUMN) as AssignmentActivityKind[]).some(
(kind) => integer(assignment, ACTIVITY_COLUMN[kind]) !== counts[kind]
) ||
counts.migration !== expectedMigrations
) {
throw new Error('migration_activity_accounting_mismatch')
}
}
async function assertCellReservationAccounting(
transaction: RelayDatabase,
cells: SqlRow[],
cellIds: string[]
): Promise<void> {
const uniqueCellIds = [...new Set(cellIds)]
const rows = await transaction.query(
`SELECT cell_id, COALESCE(SUM(request_units), 0) AS request_units
FROM relay_assignment_activity_leases
WHERE cell_id IN (${uniqueCellIds.map(() => '?').join(', ')})
GROUP BY cell_id`,
uniqueCellIds
)
const unitsByCell = new Map(
rows.map((row) => [text(row, 'cell_id'), integer(row, 'request_units')])
)
for (const cellId of uniqueCellIds) {
const cell = cells.find((row) => text(row, 'cell_id') === cellId)
if (!cell || integer(cell, 'reserved_requests') !== (unitsByCell.get(cellId) ?? 0)) {
throw new Error('migration_cell_reservation_accounting_mismatch')
}
}
}
function deadSourceCompletionResult(
row: SqlRow,
changed: boolean
): DeadSourceCompletionResult {
return {
changed,
assignmentEpoch: integer(row, 'assignment_epoch'),
sourceCellId: text(row, 'source_cell_id'),
targetCellId: text(row, 'target_cell_id')
}
}
function cellDrainAttempt(row: SqlRow): CellDrainAttempt {
const state = text(row, 'state')
if (
![
'prepared',
'send-may-have-started',
'application-receipt',
'proven-not-delivered'
].includes(state)
) {
throw new Error('invalid_drain_attempt_state')
}
return {
attemptId: text(row, 'attempt_id'),
cellId: text(row, 'cell_id'),
cellIncarnation: text(row, 'cell_incarnation'),
traceValue: text(row, 'trace_value'),
plannedGraceMs: integer(row, 'planned_grace_ms'),
state: state as CellDrainAttemptState,
preparedAt: integer(row, 'prepared_at'),
sendMayHaveStartedAt: optionalInteger(row, 'send_may_have_started_at'),
sendPermitExpiresAt: optionalInteger(row, 'send_permit_expires_at'),
applicationReceiptAt: optionalInteger(row, 'application_receipt_at'),
backendSuccessStatus: optionalInteger(row, 'backend_success_status'),
backendInstance: optionalText(row, 'backend_instance'),
receiptCellIncarnation: optionalText(row, 'receipt_cell_incarnation'),
retryAfter: optionalInteger(row, 'retry_after'),
recoverForwardAttemptedAt: optionalInteger(row, 'recover_forward_attempted_at'),
provenNotDeliveredAt: optionalInteger(row, 'proven_not_delivered_at')
}
}
function optionalInteger(row: SqlRow, field: string): number | undefined {
if (row[field] === null || row[field] === undefined) return undefined
return integer(row, field)
}
function optionalText(row: SqlRow, field: string): string | undefined {
if (row[field] === null || row[field] === undefined) return undefined
return text(row, field)
}
function migrationHasExactActiveTarget(row: SqlRow, targetCellId: string): boolean {
return (
integer(row, 'target_control_active') === 1 &&
optionalText(row, 'current_cell_id') === targetCellId &&
optionalInteger(row, 'current_assignment_epoch') === integer(row, 'assignment_epoch')
)
}
function assignmentKey(userId: string, relayHostId: string): string {
return `${userId}\u0000${relayHostId}`
}
function emptyActivityCounts(): Record<AssignmentActivityKind, number> {
return {
control: 0,
splice: 0,
invite: 0,
install: 0,
confirmation: 0,
migration: 0
}
}