mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
feat(relay): correct regional placement only when the source is idle (#20105)
* feat(relay): correct regional placement only at an idle source * test(relay): lock source activity capacity semantics
This commit is contained in:
@@ -13,6 +13,11 @@ on:
|
||||
default: preserve
|
||||
type: choice
|
||||
options: [preserve, enable, disable]
|
||||
region-correction-cohort-percent:
|
||||
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
|
||||
required: true
|
||||
default: preserve
|
||||
type: string
|
||||
prune-incompatible-revisions:
|
||||
description: Retain only the newly verified serving and rollback revisions
|
||||
required: true
|
||||
@@ -62,6 +67,7 @@ jobs:
|
||||
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
|
||||
IMAGE_DIGEST: ${{ inputs.image-digest }}
|
||||
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
|
||||
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
|
||||
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
|
||||
# Floor the served revision must keep, matching relay_min_instances in
|
||||
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
|
||||
@@ -106,8 +112,11 @@ jobs:
|
||||
echo "image-digest must be an immutable lowercase sha256 digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
|
||||
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
|
||||
fi
|
||||
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
|
||||
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
|
||||
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
|
||||
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
@@ -218,7 +227,8 @@ jobs:
|
||||
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
|
||||
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
|
||||
--release-id "${RELEASE_ID}" \
|
||||
--regional-placement-secret-version "${target_version}"
|
||||
--regional-placement-secret-version "${target_version}" \
|
||||
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
|
||||
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
|
||||
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
|
||||
'/v1/admin/cell-status',
|
||||
'/v1/admin/evacuation-status',
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'/v1/admin/regional-rehome-preview',
|
||||
'/v1/admin/runtime-status'
|
||||
] as const
|
||||
|
||||
|
||||
+137
-51
@@ -1,5 +1,9 @@
|
||||
import {
|
||||
AssignmentRequestSchema,
|
||||
IdleRegionalRehomeRequestSchema,
|
||||
type IdleRegionalRehomeRequest,
|
||||
type IdleRegionalRehomeOutcome,
|
||||
type RegionCorrectionResponse,
|
||||
isRelayCellConnectionHardCap,
|
||||
RELAY_ADMISSION_BUDGETS,
|
||||
RELAY_DEFAULT_REGION,
|
||||
@@ -39,7 +43,7 @@ import {
|
||||
type AssignmentAdmissionRejection
|
||||
} from './public-assignment-admission.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayRuntimeCounts } from './relay-observability.js'
|
||||
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
|
||||
import {
|
||||
isRegionalRehomeTrustProbe,
|
||||
probeRegionalRehomeTrust
|
||||
@@ -68,21 +72,28 @@ export function createRelayApp(
|
||||
store: RelayCredentialStore
|
||||
assignments: RelayAssignmentStore
|
||||
drain: (graceMs: number) => void
|
||||
idleRehome?: (input: IdleRegionalRehomeRequest & {
|
||||
cohortPercent: number
|
||||
directorSafety: RegionalRehomeSafetySnapshot
|
||||
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
|
||||
drainHost?: (input: {
|
||||
attemptId: string
|
||||
userId: string
|
||||
relayHostId: string
|
||||
sourceAssignmentEpoch: number
|
||||
sourceCellIncarnation: string
|
||||
graceMs: number
|
||||
}) => 'accepted' | 'already-accepted' | 'host-not-connected'
|
||||
}) =>
|
||||
| 'accepted'
|
||||
| 'already-accepted'
|
||||
| 'host-not-connected'
|
||||
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
|
||||
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
|
||||
regionalRehomeFetch?: typeof fetch
|
||||
regionalRehomeTrustProbeHostExists?: (input: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
}) => boolean
|
||||
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
|
||||
cellIncarnation?: string
|
||||
isDraining?: () => boolean
|
||||
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
|
||||
runtimeCounts?: () => RelayRuntimeCounts
|
||||
ready: () => Promise<boolean>
|
||||
recordAssignmentAdmission?: (
|
||||
@@ -226,7 +237,8 @@ export function createRelayApp(
|
||||
return context.json({ error: 'host_identity_mismatch' }, 403)
|
||||
}
|
||||
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
|
||||
const requestedRegion = body.data.preferredRegion
|
||||
const requestedRegion =
|
||||
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
|
||||
const targetRegion =
|
||||
config.regionalPlacementEnabled !== false && requestedRegion
|
||||
? requestedRegion
|
||||
@@ -295,10 +307,30 @@ export function createRelayApp(
|
||||
}
|
||||
}
|
||||
let assignment: RelayAssignment
|
||||
let regionCorrection: RegionCorrectionResponse | undefined
|
||||
try {
|
||||
assignment = requestedRegion
|
||||
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
|
||||
: await operations.assignments.assign(identity)
|
||||
if (body.data.regionCorrection?.action === 'report') {
|
||||
const current = await operations.assignments.resolve(identity)
|
||||
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
|
||||
assignment = current
|
||||
} else {
|
||||
assignment = requestedRegion
|
||||
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
|
||||
: await operations.assignments.assign(identity)
|
||||
}
|
||||
if (body.data.regionCorrection) {
|
||||
try {
|
||||
regionCorrection = await operations.assignments.exchangeRegionCorrection(
|
||||
identity,
|
||||
body.data.regionCorrection,
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
} catch (error) {
|
||||
if (body.data.regionCorrection.action === 'report') throw error
|
||||
// Optional measurement setup must not discard an otherwise valid placement.
|
||||
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
|
||||
logAssignmentRejection({
|
||||
@@ -353,13 +385,16 @@ export function createRelayApp(
|
||||
v: 1,
|
||||
cellUrl: assignment.cellUrl,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
lease
|
||||
lease,
|
||||
...(regionCorrection ? { regionCorrection } : {})
|
||||
})
|
||||
})
|
||||
app.post('/v1/resolve', async (context) => {
|
||||
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
|
||||
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
|
||||
if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
|
||||
if (
|
||||
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
|
||||
) {
|
||||
return context.json({ error: 'request_too_large' }, 413)
|
||||
}
|
||||
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
|
||||
@@ -433,6 +468,34 @@ export function createRelayApp(
|
||||
operations.drain(body.data.graceMs)
|
||||
return context.json({ ok: true })
|
||||
})
|
||||
app.post('/v1/admin/host-idle-rehome', async (context) => {
|
||||
if (config.role !== 'cell' || !operations.idleRehome) {
|
||||
return context.json({ error: 'cell_only' }, 404)
|
||||
}
|
||||
const bearer = readBearer(context.req.header('authorization'))
|
||||
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
|
||||
return context.json({ error: 'invalid_token' }, 401)
|
||||
}
|
||||
if (requestTooLarge(context.req.header('content-length'))) {
|
||||
return context.json({ error: 'request_too_large' }, 413)
|
||||
}
|
||||
const body = IdleRegionalRehomeCommandSchema.safeParse(
|
||||
await context.req.json().catch(() => null)
|
||||
)
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
if (
|
||||
body.data.sourceCellId !== config.cellId ||
|
||||
!operations.cellIncarnation ||
|
||||
body.data.sourceCellIncarnation !== operations.cellIncarnation
|
||||
) {
|
||||
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
|
||||
}
|
||||
try {
|
||||
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/host-drain', async (context) => {
|
||||
if (config.role !== 'cell' || !operations.drainHost) {
|
||||
return context.json({ error: 'cell_only' }, 404)
|
||||
@@ -474,7 +537,7 @@ export function createRelayApp(
|
||||
}
|
||||
sharedRuntimeIdentityRejected = true
|
||||
}
|
||||
const outcome = operations.drainHost(body.data)
|
||||
const outcome = await operations.drainHost(body.data)
|
||||
return context.json({
|
||||
v: 1,
|
||||
outcome,
|
||||
@@ -502,8 +565,7 @@ export function createRelayApp(
|
||||
region: config.region ?? RELAY_DEFAULT_REGION,
|
||||
imageDigest: config.imageDigest ?? null,
|
||||
draining: operations.isDraining?.() ?? false,
|
||||
regionalRehomeProtocol:
|
||||
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
|
||||
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
|
||||
connectionCapacity:
|
||||
config.connectionHardCap === undefined
|
||||
? null
|
||||
@@ -559,6 +621,18 @@ export function createRelayApp(
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
}
|
||||
})
|
||||
app.get('/v1/admin/regional-rehome-preview', async (context) => {
|
||||
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
|
||||
const bearer = readBearer(context.req.header('authorization'))
|
||||
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
|
||||
return context.json({ error: 'invalid_token' }, 401)
|
||||
}
|
||||
const preview = await operations.assignments.previewRegionalRehomeEligibility(
|
||||
operations.regionalRehomeSafetySnapshot?.()
|
||||
)
|
||||
const outcomes = await operations.assignments.regionCorrectionOutcomes()
|
||||
return context.json({ v: 1, preview, outcomes })
|
||||
})
|
||||
app.post('/v1/admin/regional-rehome-control', async (context) => {
|
||||
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
|
||||
const bearer = readBearer(context.req.header('authorization'))
|
||||
@@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z
|
||||
})
|
||||
.strict()
|
||||
|
||||
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
|
||||
cohortPercent: z.number().int().min(0).max(100),
|
||||
directorSafety: RegionalRehomeSafetySchema
|
||||
})
|
||||
|
||||
const CellHeartbeatSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
@@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z
|
||||
v: z.literal(1),
|
||||
cellId: z.string().min(1).max(128),
|
||||
cellIncarnation: z.string().uuid(),
|
||||
regionalRehomeProtocol: z.number().int().min(0).max(1),
|
||||
regionalRehomeProtocol: z.number().int().min(0).max(3),
|
||||
safety: RegionalRehomeSafetySchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
|
||||
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
|
||||
z.object({
|
||||
v: z.literal(1),
|
||||
action: z.literal('apply'),
|
||||
expectedGeneration: z.number().int().nonnegative(),
|
||||
enabled: z.boolean(),
|
||||
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
|
||||
ratePerMinute: z.number().int().min(1).max(120),
|
||||
preferenceMaxAgeMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
hostCooldownMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
|
||||
confirmation: z.enum([
|
||||
'ENABLE_REGIONAL_REHOMING',
|
||||
'DISABLE_REGIONAL_REHOMING'
|
||||
])
|
||||
}).strict()
|
||||
]).superRefine((value, context) => {
|
||||
if (value.action !== 'apply') return
|
||||
const expected = value.enabled
|
||||
? 'ENABLE_REGIONAL_REHOMING'
|
||||
: 'DISABLE_REGIONAL_REHOMING'
|
||||
if (value.confirmation !== expected) {
|
||||
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
|
||||
}
|
||||
})
|
||||
const RegionalRehomeControlSchema = z
|
||||
.discriminatedUnion('action', [
|
||||
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
|
||||
z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
action: z.literal('apply'),
|
||||
expectedGeneration: z.number().int().nonnegative(),
|
||||
enabled: z.boolean(),
|
||||
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
|
||||
ratePerMinute: z.number().int().min(1).max(120),
|
||||
preferenceMaxAgeMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
hostCooldownMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
drainGraceMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(60 * 60_000),
|
||||
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
|
||||
})
|
||||
.strict()
|
||||
])
|
||||
.superRefine((value, context) => {
|
||||
if (value.action !== 'apply') return
|
||||
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
|
||||
if (value.confirmation !== expected) {
|
||||
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
|
||||
}
|
||||
})
|
||||
|
||||
const RegionalRehomeTrustProbeSchema = z
|
||||
.object({
|
||||
@@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z
|
||||
sourceCellId: z.string().min(1).max(128),
|
||||
sourceCellIncarnation: z.string().uuid(),
|
||||
sourceAssignmentEpoch: z.number().int().positive(),
|
||||
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
|
||||
graceMs: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.max(60 * 60 * 1000)
|
||||
})
|
||||
.strict()
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -92,7 +92,7 @@ describe('cell heartbeat client', () => {
|
||||
client.stop()
|
||||
|
||||
expect(JSON.parse(String(requests[1]!.body))).toMatchObject({
|
||||
regionalRehomeProtocol: 1,
|
||||
regionalRehomeProtocol: 3,
|
||||
safety: {
|
||||
observedAt: 120,
|
||||
sqlFailures: 0,
|
||||
@@ -149,28 +149,34 @@ describe('cell heartbeat client', () => {
|
||||
|
||||
it('does not start outside an explicitly configured cell role', () => {
|
||||
expect(
|
||||
startCellHeartbeat({ ...CONFIG, role: 'director' }, {
|
||||
ready: async () => true,
|
||||
observedRequests: () => 0,
|
||||
connectionCounts: () => ({
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0
|
||||
})
|
||||
})
|
||||
startCellHeartbeat(
|
||||
{ ...CONFIG, role: 'director' },
|
||||
{
|
||||
ready: async () => true,
|
||||
observedRequests: () => 0,
|
||||
connectionCounts: () => ({
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0
|
||||
})
|
||||
}
|
||||
)
|
||||
).toBeNull()
|
||||
expect(
|
||||
startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, {
|
||||
ready: async () => true,
|
||||
observedRequests: () => 0,
|
||||
connectionCounts: () => ({
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0
|
||||
})
|
||||
})
|
||||
startCellHeartbeat(
|
||||
{ ...CONFIG, directorUrl: undefined },
|
||||
{
|
||||
ready: async () => true,
|
||||
observedRequests: () => 0,
|
||||
connectionCounts: () => ({
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0
|
||||
})
|
||||
}
|
||||
)
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -70,8 +70,7 @@ export function startCellHeartbeat(
|
||||
inFlightConnections: connectionCounts!.inFlightConnections,
|
||||
reservedConnectionUnits: connectionCounts!.reservedConnectionUnits,
|
||||
enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits,
|
||||
connectionInclusionWatermark:
|
||||
connectionCounts!.inclusionWatermark,
|
||||
connectionInclusionWatermark: connectionCounts!.inclusionWatermark,
|
||||
connectionHardCap: config.connectionHardCap,
|
||||
connectionUnobservedBound: config.connectionUnobservedBound
|
||||
})
|
||||
@@ -94,7 +93,7 @@ export function startCellHeartbeat(
|
||||
cellId: config.cellId,
|
||||
cellIncarnation,
|
||||
regionalRehomeProtocol:
|
||||
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
|
||||
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
|
||||
safety: options.regionalRehomeSafety()
|
||||
}),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
@@ -106,7 +105,10 @@ export function startCellHeartbeat(
|
||||
}
|
||||
} catch (error) {
|
||||
// A heartbeat must fail closed without ever logging its bearer token.
|
||||
console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '')
|
||||
console.warn(
|
||||
'[orca-relay] cell heartbeat failed',
|
||||
error instanceof Error ? error.message : ''
|
||||
)
|
||||
} finally {
|
||||
inFlight = false
|
||||
}
|
||||
|
||||
@@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [
|
||||
{ method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
|
||||
{ method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
|
||||
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
|
||||
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
|
||||
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }
|
||||
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
|
||||
// repairs exactly two cells' counters and now holds only those rows, and the
|
||||
// second selects from the inventory its single caller has already locked.
|
||||
@@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map<string, Set<string>> {
|
||||
bounds.forEach((method, index) => {
|
||||
const end = bounds[index + 1]?.start ?? lines.length
|
||||
const names = callees.get(method.name) ?? new Set<string>()
|
||||
for (const call of lines.slice(method.start, end).join('\n').matchAll(
|
||||
/this\.([A-Za-z_][\w]*)\s*\(/g
|
||||
)) {
|
||||
for (const call of lines
|
||||
.slice(method.start, end)
|
||||
.join('\n')
|
||||
.matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) {
|
||||
names.add(call[1]!)
|
||||
}
|
||||
callees.set(method.name, names)
|
||||
@@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
|
||||
|
||||
describe('cell inventory lock call-site census', () => {
|
||||
it('classifies every call site exactly as recorded', () => {
|
||||
expect(readCallSites()).toEqual(
|
||||
CENSUS.map(({ method, mode }) => ({ method, mode }))
|
||||
)
|
||||
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
|
||||
})
|
||||
|
||||
// Why: the census only sees lockCellInventory calls, so a hand-written
|
||||
|
||||
@@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
|
||||
}
|
||||
|
||||
describe('GCE relay capacity configuration', () => {
|
||||
it('defaults optional region correction off and bounds the cohort', () => {
|
||||
const env = cellEnvironment(4_000)
|
||||
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0)
|
||||
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5'
|
||||
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5)
|
||||
for (const invalid of ['-1', '101', '1.5', 'not-a-number']) {
|
||||
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid
|
||||
expect(() => loadRelayConfig(env)).toThrow()
|
||||
}
|
||||
})
|
||||
|
||||
it('requires distinct dedicated admin identities and accepts omitted values', () => {
|
||||
const env = cellEnvironment(4_000)
|
||||
expect(loadRelayConfig(env)).toMatchObject({
|
||||
|
||||
@@ -75,11 +75,15 @@ const EnvSchema = z.object({
|
||||
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(),
|
||||
ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(),
|
||||
ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(),
|
||||
ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(),
|
||||
ORCA_RELAY_IMAGE_DIGEST: z
|
||||
.string()
|
||||
.regex(/^sha256:[a-f0-9]{64}$/)
|
||||
.optional(),
|
||||
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
|
||||
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
|
||||
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
|
||||
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
|
||||
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
|
||||
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2),
|
||||
ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1),
|
||||
ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64),
|
||||
@@ -185,6 +189,7 @@ export type RelayConfig = {
|
||||
databasePoolMax: number
|
||||
publicAssignmentsEnabled: boolean
|
||||
regionalPlacementEnabled?: boolean
|
||||
regionCorrectionCohortPercent?: number
|
||||
publicAssignmentConcurrency: number
|
||||
publicAssignmentQueueMax: number
|
||||
publicAssignmentWaitMs: number
|
||||
@@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
|
||||
databasePoolMax,
|
||||
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
|
||||
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
|
||||
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
|
||||
publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY,
|
||||
publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX,
|
||||
publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS,
|
||||
|
||||
@@ -18,6 +18,34 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('relay database', () => {
|
||||
it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => {
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-'))
|
||||
temporaryDirectories.push(dataDir)
|
||||
const legacy = await openRelayDatabase({ dataDir })
|
||||
await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome')
|
||||
await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation')
|
||||
await legacy.query(
|
||||
`INSERT INTO relay_control_capabilities
|
||||
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing)
|
||||
VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)`
|
||||
)
|
||||
await legacy.close()
|
||||
const upgraded = await openRelayDatabase({ dataDir })
|
||||
try {
|
||||
expect(
|
||||
await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities')
|
||||
).toEqual([{ idle_regional_rehome: 0 }])
|
||||
const columns = await upgraded.query(
|
||||
"SELECT * FROM pragma_table_info('relay_region_rehome_attempts')"
|
||||
)
|
||||
expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({
|
||||
dflt_value: '0'
|
||||
})
|
||||
} finally {
|
||||
await upgraded.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('creates every durable relay state table', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const rows = await database.query(
|
||||
@@ -54,6 +82,7 @@ describe('relay database', () => {
|
||||
'relay_confirm_results',
|
||||
'relay_confirmable_splices',
|
||||
'relay_connection_bases',
|
||||
'relay_control_capabilities',
|
||||
'relay_control_connection_reservations',
|
||||
'relay_devices',
|
||||
'relay_direct_authorizations',
|
||||
@@ -62,6 +91,7 @@ describe('relay database', () => {
|
||||
'relay_migration_leases',
|
||||
'relay_post_drain_migration_pins',
|
||||
'relay_rate_windows',
|
||||
'relay_region_decisions',
|
||||
'relay_region_rehome_attempts',
|
||||
'relay_region_rehome_control',
|
||||
'relay_region_rehome_worker_state'
|
||||
@@ -140,9 +170,7 @@ describe('relay database', () => {
|
||||
|
||||
const second = await openRelayDatabase({ dataDir })
|
||||
expect(
|
||||
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
|
||||
'legacy-cell'
|
||||
])
|
||||
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell'])
|
||||
).toEqual([{ region: 'us-central1' }])
|
||||
await second.close()
|
||||
})
|
||||
@@ -165,9 +193,7 @@ describe('relay database', () => {
|
||||
'relay_region_rehome_attempts'
|
||||
])
|
||||
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
|
||||
expect(
|
||||
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
|
||||
).toBe(true)
|
||||
expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
|
||||
@@ -197,6 +197,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
|
||||
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
|
||||
ON relay_assignment_region_preferences(observed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_region_decisions (
|
||||
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
|
||||
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
|
||||
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
|
||||
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
|
||||
cohort_bucket BIGINT NOT NULL DEFAULT 0,
|
||||
last_considered_at BIGINT NOT NULL DEFAULT 0,
|
||||
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
|
||||
PRIMARY KEY (user_id, relay_host_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
|
||||
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
|
||||
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
|
||||
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
|
||||
finish_existing BIGINT NOT NULL,
|
||||
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (user_id, relay_host_id, activity_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
|
||||
worker_id TEXT PRIMARY KEY,
|
||||
next_dispatch_at BIGINT NOT NULL,
|
||||
@@ -228,6 +246,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||||
CHECK (preferred_region IN (${REGION_LIST})),
|
||||
source_cell_id TEXT NOT NULL,
|
||||
source_cell_incarnation TEXT NOT NULL,
|
||||
source_generation BIGINT NOT NULL DEFAULT 0,
|
||||
target_cell_id TEXT NOT NULL,
|
||||
target_cell_incarnation TEXT NOT NULL,
|
||||
previous_epoch BIGINT NOT NULL,
|
||||
@@ -600,6 +619,8 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
|
||||
// auto-named; the replacement is named, so both statements are no-ops on a
|
||||
// database the current schema created and neither can drop the other.
|
||||
export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||||
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
@@ -607,7 +628,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||||
CHECK (preferred_region IN (${REGION_LIST}))`,
|
||||
`ALTER TABLE relay_region_rehome_control
|
||||
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
|
||||
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
|
||||
]
|
||||
|
||||
function postgresSql(sql: string): string {
|
||||
@@ -1013,6 +1036,15 @@ async function applySchema(database: RelayDatabase): Promise<void> {
|
||||
for (const statement of SCHEMA.split(';')) {
|
||||
if (statement.trim()) await database.query(statement)
|
||||
}
|
||||
for (const [table, column] of [
|
||||
['relay_control_capabilities', 'idle_regional_rehome'],
|
||||
['relay_region_rehome_attempts', 'source_generation']
|
||||
]) {
|
||||
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
|
||||
if (!columns.some((existing) => existing.name === column)) {
|
||||
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
|
||||
|
||||
@@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('does not admit new source work after a drain crosses activity acquisition', async () => {
|
||||
const h = harness()
|
||||
const control = await activeHost(h)
|
||||
const slow = deferred<void>()
|
||||
h.acquireActivity.mockReturnValueOnce(slow.promise)
|
||||
const client = new FakeSocket()
|
||||
const capacity = { bind: vi.fn(), release: vi.fn() }
|
||||
const accepting = h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential',
|
||||
capacity
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
h.registry.drainHost({
|
||||
attemptId: 'attempt',
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
sourceAssignmentEpoch: 1,
|
||||
graceMs: 60_000
|
||||
})
|
||||
slow.resolve()
|
||||
await accepting
|
||||
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
|
||||
expect(capacity.bind).not.toHaveBeenCalled()
|
||||
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
|
||||
expect(h.releaseActivity).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not splice an attachment whose generation retired during basis persistence', async () => {
|
||||
const h = harness()
|
||||
await activeHost(h)
|
||||
const client = new FakeSocket()
|
||||
await h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
const pending = [...session.pendingConns.values()][0]!
|
||||
const slow = deferred<void>()
|
||||
h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise)
|
||||
const host = new FakeSocket()
|
||||
const attaching = h.registry.acceptHostData(
|
||||
host as unknown as WebSocket,
|
||||
pending.connId,
|
||||
pending.connTicket,
|
||||
1
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
slow.resolve()
|
||||
expect(await attaching).toBe(false)
|
||||
expect(session.activeSplices.size).toBe(0)
|
||||
expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId)
|
||||
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true'))
|
||||
expect(host.close).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops after a slow activity acquire when the phone already hung up', async () => {
|
||||
const h = harness()
|
||||
const control = await activeHost(h)
|
||||
@@ -390,6 +450,7 @@ describe('successful client accept timing', () => {
|
||||
relayHostIdDigest: string
|
||||
}
|
||||
expect(event.credentialKind).toBe('resume')
|
||||
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
|
||||
// Joins the line back to the emitting process, like the runtime metrics event.
|
||||
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
|
||||
expect(Object.keys(event.stageMs).sort()).toEqual([
|
||||
@@ -460,6 +521,9 @@ describe('control round-trip sampling', () => {
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
rttMsMedian: 40,
|
||||
assignmentEpoch: 1,
|
||||
controlGeneration: 1,
|
||||
drainMode: 'none',
|
||||
sampleCount: 4
|
||||
})
|
||||
expect(rttLines()[0]).not.toContain(identity.relayHostId)
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
CONTROL_CONTINUITY_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
|
||||
RELAY_PROTOCOL_LIMITS
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -140,7 +141,10 @@ function createRegistry(
|
||||
store as RelayCredentialStore,
|
||||
assignments,
|
||||
new ProcessQueuedByteBudget(),
|
||||
observer
|
||||
observer,
|
||||
Date.now,
|
||||
Math.random,
|
||||
'incarnation-1'
|
||||
)
|
||||
// Mirrors the production signature exactly so a future positional shift fails to compile.
|
||||
const bound = (
|
||||
@@ -166,7 +170,14 @@ function createRegistry(
|
||||
assignmentEpoch,
|
||||
appVersion = '1.4.173'
|
||||
) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion)
|
||||
return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer }
|
||||
return {
|
||||
registry,
|
||||
activate,
|
||||
acquireActivity,
|
||||
renewControlActivity,
|
||||
releaseActivity,
|
||||
observer
|
||||
}
|
||||
}
|
||||
|
||||
describe('host session cleanup races', () => {
|
||||
@@ -398,26 +409,20 @@ describe('host session cleanup races', () => {
|
||||
attemptId: '22222222-2222-4222-8222-222222222222'
|
||||
})
|
||||
).toThrow('regional_rehome_attempt_conflict')
|
||||
expect(() =>
|
||||
registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })
|
||||
).toThrow('regional_rehome_assignment_epoch_mismatch')
|
||||
expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow(
|
||||
'regional_rehome_assignment_epoch_mismatch'
|
||||
)
|
||||
|
||||
const rebound = new FakeSocket()
|
||||
await activate(
|
||||
rebound as unknown as WebSocket,
|
||||
identity,
|
||||
registry.get(request),
|
||||
1,
|
||||
true,
|
||||
7
|
||||
)
|
||||
await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7)
|
||||
expect(registry.get(request)?.state).toBe('drain-only')
|
||||
expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(registry.get(request)).toBeNull()
|
||||
expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }))
|
||||
.not.toBeNull()
|
||||
expect(
|
||||
registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })
|
||||
).not.toBeNull()
|
||||
expect(secondSocket.close).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -513,14 +518,7 @@ describe('host session cleanup races', () => {
|
||||
expect(original).not.toBeNull()
|
||||
|
||||
const rebindSocket = new FakeSocket()
|
||||
const rebinding = activate(
|
||||
rebindSocket as unknown as WebSocket,
|
||||
identity,
|
||||
original,
|
||||
1,
|
||||
true,
|
||||
1
|
||||
)
|
||||
const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1)
|
||||
rebindSocket.close()
|
||||
blocked.resolve('control:production-gce-c3:1')
|
||||
await rebinding
|
||||
@@ -659,14 +657,7 @@ describe('host session cleanup races', () => {
|
||||
originalSocket.close()
|
||||
|
||||
const replacementSocket = new FakeSocket()
|
||||
await activate(
|
||||
replacementSocket as unknown as WebSocket,
|
||||
identity,
|
||||
original,
|
||||
2,
|
||||
false,
|
||||
1
|
||||
)
|
||||
await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1)
|
||||
const replacement = registry.get({
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId
|
||||
@@ -696,14 +687,7 @@ describe('host session cleanup races', () => {
|
||||
})
|
||||
expect(original).not.toBeNull()
|
||||
|
||||
await activate(
|
||||
new FakeSocket() as unknown as WebSocket,
|
||||
identity,
|
||||
original,
|
||||
2,
|
||||
false,
|
||||
1
|
||||
)
|
||||
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
|
||||
vi.advanceTimersByTime(15_000)
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledOnce()
|
||||
@@ -718,6 +702,53 @@ describe('host session cleanup races', () => {
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
|
||||
it('ignores a denial belonging to the socket before a same-generation rebind', async () => {
|
||||
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
|
||||
const oldSocket = new FakeSocket()
|
||||
await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
let reject!: (error: Error) => void
|
||||
h.renewControlActivity.mockReturnValueOnce(
|
||||
new Promise<void>((_, fail) => {
|
||||
reject = fail
|
||||
})
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
const replacement = new FakeSocket()
|
||||
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
|
||||
reject(new Error('activity_cell_not_authoritative'))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(replacement.close).not.toHaveBeenCalled()
|
||||
expect(session.socket).toBe(replacement)
|
||||
expect(session.generation).toBe(1)
|
||||
})
|
||||
|
||||
it('ignores missing-activity recovery denial after an authority transition', async () => {
|
||||
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
|
||||
const socket = new FakeSocket()
|
||||
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found'))
|
||||
let reject!: (error: Error) => void
|
||||
h.acquireActivity.mockReturnValueOnce(
|
||||
new Promise<void>((_, fail) => {
|
||||
reject = fail
|
||||
})
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
h.registry.drainHost({
|
||||
attemptId: 'attempt',
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
sourceAssignmentEpoch: 1,
|
||||
graceMs: 60_000
|
||||
})
|
||||
reject(new Error('activity_cell_not_authoritative'))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
expect(session.state).toBe('drain-only')
|
||||
})
|
||||
|
||||
it('keeps 15s pings while halving steady-state control renewals', async () => {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
@@ -732,9 +763,7 @@ describe('host session cleanup races', () => {
|
||||
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
|
||||
}
|
||||
|
||||
const pings = socket.send.mock.calls.filter((call) =>
|
||||
String(call[0]).includes('"ping"')
|
||||
)
|
||||
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
|
||||
expect(pings).toHaveLength(4)
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt)
|
||||
@@ -1118,3 +1147,277 @@ describe('host hello ack pending connections', () => {
|
||||
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
})
|
||||
})
|
||||
|
||||
describe('source-owned idle cutover', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
const request = {
|
||||
attemptId: 'idle-1',
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
sourceAssignmentEpoch: 1,
|
||||
sourceGeneration: 1,
|
||||
sourceCellIncarnation: 'incarnation-1',
|
||||
targetCellId: 'target'
|
||||
}
|
||||
async function source(store: Partial<RelayCredentialStore> = {}) {
|
||||
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store)
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
undefined,
|
||||
new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME])
|
||||
)
|
||||
socket.removeAllListeners('message')
|
||||
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
return { ...h, socket, session: h.registry.get(request)! }
|
||||
}
|
||||
it('keeps either established client busy until both actually leave', async () => {
|
||||
const h = await source()
|
||||
h.session.activeConnIds.add('phone')
|
||||
h.session.activeConnIds.add('ipad')
|
||||
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
|
||||
h.session.activeConnIds.delete('ipad')
|
||||
expect(
|
||||
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'busy' })
|
||||
expect(commit).not.toHaveBeenCalled()
|
||||
h.session.activeConnIds.delete('phone')
|
||||
expect(
|
||||
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'committed' })
|
||||
expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
expect(h.releaseActivity).toHaveBeenCalled()
|
||||
})
|
||||
it.each([
|
||||
{ userId: 'other-user' },
|
||||
{ sourceAssignmentEpoch: 2 },
|
||||
{ sourceGeneration: 2 },
|
||||
{ sourceCellIncarnation: 'other-incarnation' },
|
||||
{ targetCellId: 'other-target' }
|
||||
])('rejects a reused operation ID with changed authority %j', async (change) => {
|
||||
const h = await source()
|
||||
const result = deferred<{ outcome: 'deferred' }>()
|
||||
const commit = vi.fn().mockReturnValue(result.promise)
|
||||
const reconcile = vi.fn().mockResolvedValue('not-committed')
|
||||
const moving = h.registry.idleRehome(request, commit, reconcile)
|
||||
const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile)
|
||||
result.resolve({ outcome: 'deferred' })
|
||||
expect(await conflicting).toEqual({ outcome: 'stale' })
|
||||
expect(await moving).toEqual({ outcome: 'deferred' })
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(h.socket.close).not.toHaveBeenCalled()
|
||||
})
|
||||
it('accounts for accepts before credential identity resolves', async () => {
|
||||
const lookup = deferred<null>()
|
||||
const h = await source({
|
||||
resolveResume: vi.fn().mockReturnValue(lookup.promise),
|
||||
resolveInviteForMove: vi.fn().mockResolvedValue(null)
|
||||
})
|
||||
const client = new FakeSocket()
|
||||
const accept = h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
expect(
|
||||
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'busy' })
|
||||
lookup.resolve(null)
|
||||
await accept
|
||||
expect(h.socket.close).not.toHaveBeenCalled()
|
||||
})
|
||||
it('rejects new accepts and replacements synchronously while a commit awaits', async () => {
|
||||
const h = await source()
|
||||
const result = deferred<{ outcome: 'deferred' }>()
|
||||
const commit = vi.fn().mockReturnValue(result.promise)
|
||||
const moving = h.registry.idleRehome(
|
||||
request,
|
||||
commit,
|
||||
vi.fn().mockResolvedValue('not-committed')
|
||||
)
|
||||
const duplicate = h.registry.idleRehome(
|
||||
request,
|
||||
commit,
|
||||
vi.fn().mockResolvedValue('not-committed')
|
||||
)
|
||||
const client = new FakeSocket()
|
||||
const release = vi.fn()
|
||||
await h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential',
|
||||
{ release } as never
|
||||
)
|
||||
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
|
||||
expect(release).toHaveBeenCalledOnce()
|
||||
const replacement = new FakeSocket()
|
||||
await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1)
|
||||
expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
|
||||
result.resolve({ outcome: 'deferred' })
|
||||
await moving
|
||||
await duplicate
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(h.socket.close).not.toHaveBeenCalled()
|
||||
expect(
|
||||
await h.registry.idleRehome(
|
||||
{ ...request, attemptId: 'next' },
|
||||
vi.fn().mockResolvedValue({ outcome: 'committed' }),
|
||||
vi.fn().mockResolvedValue('not-committed')
|
||||
)
|
||||
).toEqual({ outcome: 'committed' })
|
||||
})
|
||||
it.each(['ambiguous', 'deferred'])(
|
||||
'keeps %s outcomes fenced until locked reconciliation succeeds',
|
||||
async (claim) => {
|
||||
const h = await source()
|
||||
const reconcile = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('database unavailable'))
|
||||
.mockRejectedValueOnce(new Error('database unavailable'))
|
||||
.mockResolvedValue('not-committed')
|
||||
const moving = h.registry.idleRehome(
|
||||
request,
|
||||
claim === 'ambiguous'
|
||||
? vi.fn().mockRejectedValue(new Error('lost commit reply'))
|
||||
: vi.fn().mockResolvedValue({ outcome: 'deferred' }),
|
||||
reconcile
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(50)
|
||||
expect(
|
||||
await h.registry.idleRehome(
|
||||
{ ...request, attemptId: 'other' },
|
||||
vi.fn(),
|
||||
vi.fn().mockResolvedValue('not-committed')
|
||||
)
|
||||
).toEqual({ outcome: 'busy' })
|
||||
expect(h.socket.close).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
expect(await moving).toEqual({ outcome: 'deferred' })
|
||||
expect(reconcile).toHaveBeenCalledTimes(3)
|
||||
expect(h.socket.close).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
it('owns accepted control mutations before the handler first awaits', async () => {
|
||||
const mutation = deferred<RelayTokenClaims | null>()
|
||||
const h = await source()
|
||||
;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi
|
||||
.fn()
|
||||
.mockReturnValue(mutation.promise)
|
||||
h.socket.emit(
|
||||
'message',
|
||||
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })),
|
||||
false
|
||||
)
|
||||
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
|
||||
expect(
|
||||
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'busy' })
|
||||
mutation.resolve(identity)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(
|
||||
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'deferred' })
|
||||
})
|
||||
it('owns queued replacement activation before its first persistence await', async () => {
|
||||
const h = await source()
|
||||
const activation = deferred<string>()
|
||||
const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } })
|
||||
.assignments
|
||||
assignments.activateControl = vi.fn().mockReturnValue(activation.promise)
|
||||
const replacement = new FakeSocket()
|
||||
const activating = h.activate(
|
||||
replacement as unknown as WebSocket,
|
||||
identity,
|
||||
h.session,
|
||||
2,
|
||||
false,
|
||||
1
|
||||
)
|
||||
expect(
|
||||
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
|
||||
).toEqual({ outcome: 'busy' })
|
||||
activation.resolve('control:2')
|
||||
await activating
|
||||
})
|
||||
it('retires changed authority even when the claim definitively deferred', async () => {
|
||||
const h = await source()
|
||||
expect(
|
||||
await h.registry.idleRehome(
|
||||
request,
|
||||
vi.fn().mockResolvedValue({ outcome: 'deferred' }),
|
||||
vi.fn().mockResolvedValue('stale')
|
||||
)
|
||||
).toEqual({ outcome: 'stale' })
|
||||
expect(h.session.state).toBe('closed')
|
||||
expect(h.releaseActivity).toHaveBeenCalled()
|
||||
})
|
||||
it('holds attach ownership through basis failure reservation cleanup', async () => {
|
||||
const basis = deferred<void>()
|
||||
const cleanup = deferred<void>()
|
||||
const h = await source({
|
||||
recordConnectionBasis: vi.fn().mockImplementation(async () => {
|
||||
await basis.promise
|
||||
throw new Error('basis failed')
|
||||
}),
|
||||
failReservation: vi.fn().mockReturnValue(cleanup.promise)
|
||||
})
|
||||
const client = new FakeSocket()
|
||||
h.session.pendingConns.set('conn', {
|
||||
connId: 'conn',
|
||||
connTicket: 'ticket',
|
||||
client: client as unknown as WebSocket,
|
||||
reservation: {
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
credentialKind: 'invite',
|
||||
leaseExpiresAt: Date.now() + 1000
|
||||
},
|
||||
attachTimer: setTimeout(() => {}, 1000),
|
||||
credentialActivityId: null
|
||||
} as never)
|
||||
const attached = h.registry.acceptHostData(
|
||||
new FakeSocket() as unknown as WebSocket,
|
||||
'conn',
|
||||
'ticket',
|
||||
1
|
||||
)
|
||||
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
|
||||
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
|
||||
basis.resolve()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(h.session.activeConnIds.size).toBe(0)
|
||||
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
|
||||
expect(commit).not.toHaveBeenCalled()
|
||||
cleanup.resolve()
|
||||
await attached
|
||||
})
|
||||
it('returns the durable operation outcome after source retirement', async () => {
|
||||
const h = await source()
|
||||
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
|
||||
await h.registry.idleRehome(request, commit, vi.fn())
|
||||
expect(
|
||||
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed'))
|
||||
).toEqual({ outcome: 'committed' })
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
})
|
||||
it('does not reopen a source overtaken by emergency drain', async () => {
|
||||
const h = await source()
|
||||
const result = deferred<{ outcome: 'deferred' }>()
|
||||
const moving = h.registry.idleRehome(
|
||||
request,
|
||||
() => result.promise,
|
||||
vi.fn().mockResolvedValue('not-committed')
|
||||
)
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
result.resolve({ outcome: 'deferred' })
|
||||
await moving
|
||||
expect(h.session.state).toBe('closed')
|
||||
expect(h.registry.get(request)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
HostHelloSchema,
|
||||
InviteCreateSchema,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
type RelayHostCloseReason,
|
||||
@@ -25,10 +26,7 @@ import type WebSocket from 'ws'
|
||||
import type { RawData } from 'ws'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import {
|
||||
RelayCredentialStore,
|
||||
type CredentialReservation
|
||||
} from './credential-store.js'
|
||||
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
@@ -78,7 +76,7 @@ export type HostSession = {
|
||||
identity: RelayTokenClaims
|
||||
readonly relayHostId: string
|
||||
readonly generation: number
|
||||
readonly assignmentEpoch: number
|
||||
assignmentEpoch: number
|
||||
readonly controlActivityId: string | null
|
||||
readonly controlResumeSecret: string
|
||||
// Why: reconnect churn is only actionable once it can be pinned to a client build.
|
||||
@@ -94,6 +92,7 @@ export type HostSession = {
|
||||
pendingPingAt: number | null
|
||||
controlRttSamplesMs: number[]
|
||||
controlRttLoggedAt: number | null
|
||||
authorityRevision: number
|
||||
activityRenewalDueAt: number
|
||||
activityRenewalAttempt: number
|
||||
activityRenewalCompletedAttempt: number
|
||||
@@ -108,10 +107,7 @@ export type HostSession = {
|
||||
regionalDrainExpiresAt: number | null
|
||||
}
|
||||
|
||||
export type RegionalHostDrainOutcome =
|
||||
| 'accepted'
|
||||
| 'already-accepted'
|
||||
| 'host-not-connected'
|
||||
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
|
||||
|
||||
type PendingConnection = {
|
||||
connId: string
|
||||
@@ -184,6 +180,113 @@ export class HostSessionRegistry {
|
||||
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
|
||||
private draining = false
|
||||
|
||||
private readonly idleWork = new Map<string, number>()
|
||||
private readonly idleAttempts = new Map<
|
||||
string,
|
||||
{
|
||||
attemptId: string
|
||||
authorityKey: string
|
||||
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
|
||||
}
|
||||
>()
|
||||
|
||||
async idleRehome(
|
||||
input: {
|
||||
attemptId: string
|
||||
userId: string
|
||||
relayHostId: string
|
||||
sourceAssignmentEpoch: number
|
||||
sourceGeneration: number
|
||||
sourceCellIncarnation: string
|
||||
targetCellId: string
|
||||
},
|
||||
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
|
||||
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
|
||||
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
|
||||
const authorityKey = JSON.stringify([
|
||||
input.userId,
|
||||
input.sourceAssignmentEpoch,
|
||||
input.sourceGeneration,
|
||||
input.sourceCellIncarnation,
|
||||
input.targetCellId
|
||||
])
|
||||
const prior = this.idleAttempts.get(input.relayHostId)
|
||||
if (prior) {
|
||||
if (prior.attemptId !== input.attemptId) return { outcome: 'busy' }
|
||||
return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' }
|
||||
}
|
||||
const session = this.get(input)
|
||||
if (
|
||||
this.draining ||
|
||||
!session ||
|
||||
session.state !== 'active' ||
|
||||
session.generation !== input.sourceGeneration ||
|
||||
session.assignmentEpoch !== input.sourceAssignmentEpoch ||
|
||||
this.cellIncarnation !== input.sourceCellIncarnation
|
||||
) {
|
||||
const durable = await reconcile()
|
||||
return { outcome: durable === 'committed' ? 'committed' : 'stale' }
|
||||
}
|
||||
if (
|
||||
!session.socket ||
|
||||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
|
||||
)
|
||||
return { outcome: 'deferred' }
|
||||
if (
|
||||
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
|
||||
session.activeConnIds.size !== 0 ||
|
||||
session.activeSplices.size !== 0 ||
|
||||
session.pendingConns.size !== 0
|
||||
)
|
||||
return { outcome: 'busy' }
|
||||
const revision = session.authorityRevision
|
||||
const promise = Promise.resolve().then(async () => {
|
||||
let outcome: 'committed' | 'deferred' | 'stale'
|
||||
try {
|
||||
outcome = (await commit()).outcome
|
||||
if (outcome === 'deferred') {
|
||||
const durable = await reconcile()
|
||||
outcome = durable === 'not-committed' ? 'deferred' : durable
|
||||
}
|
||||
} catch {
|
||||
let delay = 100
|
||||
for (;;) {
|
||||
try {
|
||||
const durable = await reconcile()
|
||||
outcome = durable === 'not-committed' ? 'deferred' : durable
|
||||
break
|
||||
} catch {
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(resolve, delay)
|
||||
timer.unref?.()
|
||||
})
|
||||
delay = Math.min(delay * 2, 5000)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.get(input) === session) {
|
||||
if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) {
|
||||
this.closeDrainedSession(session)
|
||||
}
|
||||
}
|
||||
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
|
||||
this.idleAttempts.delete(input.relayHostId)
|
||||
return { outcome }
|
||||
})
|
||||
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
|
||||
return promise
|
||||
}
|
||||
|
||||
private beginIdleWork(hostId: string): (() => void) | null {
|
||||
if (this.idleAttempts.has(hostId)) return null
|
||||
this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1)
|
||||
return () => {
|
||||
const remaining = (this.idleWork.get(hostId) ?? 1) - 1
|
||||
if (remaining === 0) this.idleWork.delete(hostId)
|
||||
else this.idleWork.set(hostId, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly config: RelayConfig,
|
||||
private readonly verifyRelayToken: VerifyRelayToken,
|
||||
@@ -192,7 +295,8 @@ export class HostSessionRegistry {
|
||||
private readonly queuedByteBudget: ProcessQueuedByteBudget,
|
||||
private readonly observer: RelayRuntimeObserver,
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly random: () => number = Math.random
|
||||
private readonly random: () => number = Math.random,
|
||||
private readonly cellIncarnation?: string
|
||||
) {}
|
||||
|
||||
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
|
||||
@@ -206,6 +310,25 @@ export class HostSessionRegistry {
|
||||
hostId: string,
|
||||
credential: string,
|
||||
capacityReservation?: PendingHostDataReservation
|
||||
): Promise<void> {
|
||||
const release = this.beginIdleWork(hostId)
|
||||
if (!release) {
|
||||
capacityReservation?.release()
|
||||
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation)
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
private async acceptClientUnfenced(
|
||||
socket: WebSocket,
|
||||
hostId: string,
|
||||
credential: string,
|
||||
capacityReservation?: PendingHostDataReservation
|
||||
): Promise<void> {
|
||||
if (this.draining) {
|
||||
capacityReservation?.release()
|
||||
@@ -295,6 +418,7 @@ export class HostSessionRegistry {
|
||||
this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED)
|
||||
return
|
||||
}
|
||||
const admittingSocket = session.socket
|
||||
const connId = randomUUID()
|
||||
const connTicket = randomBytes(32).toString('base64url')
|
||||
const identity = { userId: reservation.userId, relayHostId: hostId }
|
||||
@@ -324,6 +448,20 @@ export class HostSessionRegistry {
|
||||
) {
|
||||
return
|
||||
}
|
||||
// Admission may have crossed a drain or control replacement while persisting activity.
|
||||
if (
|
||||
this.draining ||
|
||||
this.sessions.get(sessionKey) !== session ||
|
||||
session.state !== 'active' ||
|
||||
session.socket !== admittingSocket ||
|
||||
admittingSocket.readyState !== admittingSocket.OPEN
|
||||
) {
|
||||
capacityReservation?.release()
|
||||
this.failReservationBestEffort(reservation)
|
||||
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
|
||||
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
|
||||
return
|
||||
}
|
||||
markStage('activity')
|
||||
const attachTimer = setTimeout(() => {
|
||||
session.pendingConns.delete(connId)
|
||||
@@ -372,6 +510,27 @@ export class HostSessionRegistry {
|
||||
connId: string,
|
||||
connTicket: string,
|
||||
generation: number
|
||||
): Promise<boolean> {
|
||||
const owner = [...this.sessions.values()].find((candidate) =>
|
||||
candidate.pendingConns.has(connId)
|
||||
)
|
||||
const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {}
|
||||
if (!release) {
|
||||
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
|
||||
return false
|
||||
}
|
||||
try {
|
||||
return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation)
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
private async acceptHostDataUnfenced(
|
||||
socket: WebSocket,
|
||||
connId: string,
|
||||
connTicket: string,
|
||||
generation: number
|
||||
): Promise<boolean> {
|
||||
const session = [...this.sessions.values()].find((candidate) =>
|
||||
candidate.pendingConns.has(connId)
|
||||
@@ -427,6 +586,27 @@ export class HostSessionRegistry {
|
||||
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed')
|
||||
return false
|
||||
}
|
||||
// Already admitted attachments may finish a regional drain, but never a retired generation.
|
||||
if (
|
||||
this.draining ||
|
||||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
|
||||
this.get(identity)?.state === 'closed' ||
|
||||
!session.activeConnIds.has(connId) ||
|
||||
socket.readyState !== socket.OPEN ||
|
||||
pending.client.readyState !== pending.client.OPEN
|
||||
) {
|
||||
session.activeConnIds.delete(connId)
|
||||
pending.capacityReservation?.release()
|
||||
this.deactivateBasisBestEffort(connId)
|
||||
this.failReservationBestEffort(pending.reservation)
|
||||
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
|
||||
if (pending.credentialActivityId) {
|
||||
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
|
||||
}
|
||||
this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING)
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment')
|
||||
return false
|
||||
}
|
||||
const close = wireSplice({
|
||||
client: pending.client,
|
||||
host: socket,
|
||||
@@ -505,6 +685,7 @@ export class HostSessionRegistry {
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_client_accept_completed',
|
||||
...this.logIdentity(),
|
||||
...this.sessionPlacementLogFields(session),
|
||||
credentialKind: pending.reservation.credentialKind,
|
||||
stageMs,
|
||||
totalMs,
|
||||
@@ -513,6 +694,14 @@ export class HostSessionRegistry {
|
||||
)
|
||||
}
|
||||
|
||||
private sessionPlacementLogFields(session: HostSession) {
|
||||
return {
|
||||
assignmentEpoch: session.assignmentEpoch,
|
||||
controlGeneration: session.generation,
|
||||
drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none'
|
||||
}
|
||||
}
|
||||
|
||||
// Matches the runtime metrics event so a log line and a metric point can be
|
||||
// joined back to the process that emitted them.
|
||||
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
|
||||
@@ -549,6 +738,7 @@ export class HostSessionRegistry {
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
...this.logIdentity(),
|
||||
...this.sessionPlacementLogFields(session),
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
|
||||
rttMsMedian: percentile(samples, 0.5),
|
||||
sampleCount: samples.length
|
||||
@@ -562,6 +752,10 @@ export class HostSessionRegistry {
|
||||
connectionInclusionWatermark?: number,
|
||||
hostCapabilities?: ReadonlySet<string>
|
||||
): void {
|
||||
if (this.idleAttempts.has(identity.relayHostId)) {
|
||||
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
|
||||
return
|
||||
}
|
||||
// Keyed by socket, not session: a rebind swaps the session's socket, and the
|
||||
// successor's own advertisement is the only one that describes its decoder.
|
||||
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
|
||||
@@ -602,8 +796,7 @@ export class HostSessionRegistry {
|
||||
socket: WebSocket | null,
|
||||
context: string
|
||||
): void {
|
||||
void Promise.resolve()
|
||||
.then(task)
|
||||
void (async () => task())()
|
||||
.catch((error: unknown) => {
|
||||
const message = (error instanceof Error ? error.message : 'unknown')
|
||||
// Untruncated, unlike peer-supplied close reasons: this is the
|
||||
@@ -653,6 +846,7 @@ export class HostSessionRegistry {
|
||||
this.draining = true
|
||||
for (const session of this.sessions.values()) {
|
||||
if (session.state === 'closed') continue
|
||||
session.authorityRevision += 1
|
||||
session.state = 'drain-only'
|
||||
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
|
||||
setTimeout(() => this.closeDrainedSession(session), graceMs)
|
||||
@@ -665,7 +859,8 @@ export class HostSessionRegistry {
|
||||
relayHostId: string
|
||||
sourceAssignmentEpoch: number
|
||||
graceMs: number
|
||||
}): RegionalHostDrainOutcome {
|
||||
sourceCellIncarnation?: string
|
||||
}): RegionalHostDrainOutcome | Promise<RegionalHostDrainOutcome> {
|
||||
const session = this.get(input)
|
||||
if (!session || session.state === 'closed') return 'host-not-connected'
|
||||
if (session.assignmentEpoch !== input.sourceAssignmentEpoch) {
|
||||
@@ -678,13 +873,11 @@ export class HostSessionRegistry {
|
||||
this.reassertRegionalDrain(session)
|
||||
return 'already-accepted'
|
||||
}
|
||||
session.authorityRevision += 1
|
||||
session.regionalDrainAttemptId = input.attemptId
|
||||
session.regionalDrainExpiresAt = this.now() + input.graceMs
|
||||
this.reassertRegionalDrain(session)
|
||||
session.regionalDrainTimer = setTimeout(
|
||||
() => this.closeDrainedSession(session),
|
||||
input.graceMs
|
||||
)
|
||||
session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs)
|
||||
return 'accepted'
|
||||
}
|
||||
|
||||
@@ -740,9 +933,9 @@ export class HostSessionRegistry {
|
||||
const existing = this.sessions.get(key)
|
||||
const rebind = Boolean(
|
||||
existing &&
|
||||
hello.data.controlResumeSecret &&
|
||||
hello.data.controlResumeSecret === existing.controlResumeSecret &&
|
||||
(existing.state === 'orphaned' || existing.state === 'active')
|
||||
hello.data.controlResumeSecret &&
|
||||
hello.data.controlResumeSecret === existing.controlResumeSecret &&
|
||||
(existing.state === 'orphaned' || existing.state === 'active')
|
||||
)
|
||||
const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1
|
||||
const ephemeral = nacl.box.keyPair()
|
||||
@@ -784,7 +977,9 @@ export class HostSessionRegistry {
|
||||
}, 10_000)
|
||||
socket.once('message', (raw, isBinary) => {
|
||||
clearTimeout(proofTimer)
|
||||
const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
|
||||
const ack = isBinary
|
||||
? null
|
||||
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
|
||||
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
|
||||
if (
|
||||
!ack?.success ||
|
||||
@@ -826,6 +1021,11 @@ export class HostSessionRegistry {
|
||||
appVersion: string,
|
||||
connectionInclusionWatermark?: number
|
||||
): Promise<void> {
|
||||
const release = this.beginIdleWork(identity.relayHostId)
|
||||
if (!release) {
|
||||
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
|
||||
return Promise.resolve()
|
||||
}
|
||||
const key = this.key(identity.sub, identity.relayHostId)
|
||||
const previous = this.activationQueues.get(key) ?? Promise.resolve()
|
||||
// The timeout only fails this waiting socket; the queue entry still chains
|
||||
@@ -836,26 +1036,29 @@ export class HostSessionRegistry {
|
||||
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
|
||||
}, ACTIVATION_QUEUE_WAIT_MS)
|
||||
queueWaitTimer.unref?.()
|
||||
const activation = previous.catch(() => undefined).then(async () => {
|
||||
clearTimeout(queueWaitTimer)
|
||||
if (queueWaitExpired) return
|
||||
if ((this.sessions.get(key) ?? null) !== existing) {
|
||||
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
|
||||
return
|
||||
}
|
||||
await this.activateCurrent(
|
||||
socket,
|
||||
identity,
|
||||
existing,
|
||||
generation,
|
||||
rebind,
|
||||
assignmentEpoch,
|
||||
appVersion,
|
||||
connectionInclusionWatermark
|
||||
)
|
||||
})
|
||||
const activation = previous
|
||||
.catch(() => undefined)
|
||||
.then(async () => {
|
||||
clearTimeout(queueWaitTimer)
|
||||
if (queueWaitExpired) return
|
||||
if ((this.sessions.get(key) ?? null) !== existing) {
|
||||
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
|
||||
return
|
||||
}
|
||||
await this.activateCurrent(
|
||||
socket,
|
||||
identity,
|
||||
existing,
|
||||
generation,
|
||||
rebind,
|
||||
assignmentEpoch,
|
||||
appVersion,
|
||||
connectionInclusionWatermark
|
||||
)
|
||||
})
|
||||
this.activationQueues.set(key, activation)
|
||||
const cleanup = (): void => {
|
||||
release()
|
||||
if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key)
|
||||
}
|
||||
void activation.then(cleanup, cleanup)
|
||||
@@ -881,7 +1084,11 @@ export class HostSessionRegistry {
|
||||
cellId: this.config.cellId,
|
||||
assignmentEpoch,
|
||||
generation,
|
||||
connectionInclusionWatermark
|
||||
connectionInclusionWatermark,
|
||||
idleRegionalRehome:
|
||||
this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ??
|
||||
false,
|
||||
cellIncarnation: this.cellIncarnation
|
||||
}
|
||||
)
|
||||
await this.assignments.markMigrationTargetRegistered(
|
||||
@@ -918,14 +1125,15 @@ export class HostSessionRegistry {
|
||||
const previousSocket = existing.socket
|
||||
if (existing.orphanTimer) clearTimeout(existing.orphanTimer)
|
||||
existing.orphanTimer = null
|
||||
existing.authorityRevision += 1
|
||||
existing.assignmentEpoch = assignmentEpoch
|
||||
existing.socket = socket
|
||||
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
|
||||
existing.appVersion = appVersion
|
||||
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
|
||||
existing.lastPongAt = this.now()
|
||||
existing.pendingPingAt = null
|
||||
existing.activityRenewalDueAt =
|
||||
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
this.wireActiveControl(existing)
|
||||
this.sendHelloAck(existing)
|
||||
if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing)
|
||||
@@ -980,6 +1188,7 @@ export class HostSessionRegistry {
|
||||
pendingPingAt: null,
|
||||
controlRttSamplesMs: [],
|
||||
controlRttLoggedAt: null,
|
||||
authorityRevision: 0,
|
||||
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
|
||||
activityRenewalAttempt: 0,
|
||||
activityRenewalCompletedAttempt: 0,
|
||||
@@ -1028,7 +1237,9 @@ export class HostSessionRegistry {
|
||||
` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` +
|
||||
` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` +
|
||||
` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` +
|
||||
(socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`)
|
||||
(socketError === null
|
||||
? ''
|
||||
: ` error=${JSON.stringify(printableCloseReason(socketError))}`)
|
||||
)
|
||||
})
|
||||
socket.on('message', (raw, isBinary) => {
|
||||
@@ -1077,6 +1288,19 @@ export class HostSessionRegistry {
|
||||
}
|
||||
|
||||
private async acceptRefresh(session: HostSession, raw: RawData): Promise<void> {
|
||||
const release = this.beginIdleWork(session.relayHostId)
|
||||
if (!release) {
|
||||
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.acceptRefreshUnfenced(session, raw)
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise<void> {
|
||||
const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh'))
|
||||
if (!parsed.success) return
|
||||
const refreshed = await this.verifyRelayToken(parsed.data.relayJwt)
|
||||
@@ -1107,6 +1331,16 @@ export class HostSessionRegistry {
|
||||
if (controlActivityId && now >= session.activityRenewalDueAt) {
|
||||
const attempt = ++session.activityRenewalAttempt
|
||||
const startedAt = now
|
||||
const socket = session.socket
|
||||
const authorityRevision = session.authorityRevision
|
||||
const current = (): boolean =>
|
||||
this.sessions.get(key) === session &&
|
||||
session.state !== 'closed' &&
|
||||
session.socket === socket &&
|
||||
socket.readyState === socket.OPEN &&
|
||||
session.controlActivityId === controlActivityId &&
|
||||
session.authorityRevision === authorityRevision &&
|
||||
attempt > session.activityRenewalCompletedAttempt
|
||||
void this.assignments
|
||||
.renewControlActivity(
|
||||
{ userId: session.identity.sub, relayHostId: session.relayHostId },
|
||||
@@ -1117,11 +1351,16 @@ export class HostSessionRegistry {
|
||||
}
|
||||
)
|
||||
.then(() => {
|
||||
if (attempt <= session.activityRenewalCompletedAttempt) return
|
||||
if (!current()) return
|
||||
session.activityRenewalCompletedAttempt = attempt
|
||||
session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS
|
||||
})
|
||||
.catch(async (error: unknown) => {
|
||||
if (!current()) return
|
||||
if (error instanceof Error && error.message === 'assignment_not_found') {
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing')
|
||||
return
|
||||
}
|
||||
if (error instanceof Error && error.message === 'activity_cell_not_authoritative') {
|
||||
// Completion fences a late drain-only heartbeat after all source work is gone.
|
||||
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed')
|
||||
@@ -1144,8 +1383,22 @@ export class HostSessionRegistry {
|
||||
cellId: this.config.cellId
|
||||
}
|
||||
)
|
||||
if (!current()) {
|
||||
// A replaced activity must not remain leased after its owner disappears.
|
||||
if (
|
||||
!this.sessions.get(key) ||
|
||||
this.sessions.get(key)?.controlActivityId !== controlActivityId
|
||||
) {
|
||||
this.releaseActivityBestEffort(
|
||||
{ userId: session.identity.sub, relayHostId: session.relayHostId },
|
||||
controlActivityId
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
this.observer.recordControlActivityRecovery?.(true)
|
||||
} catch (acquireError: unknown) {
|
||||
if (!current()) return
|
||||
this.observer.recordControlActivityRecovery?.(false)
|
||||
if (
|
||||
acquireError instanceof Error &&
|
||||
@@ -1218,6 +1471,21 @@ export class HostSessionRegistry {
|
||||
|
||||
private closeDrainedSession(session: HostSession): void {
|
||||
if (session.state === 'closed') return
|
||||
const forcedConnections = session.activeConnIds.size + session.pendingConns.size
|
||||
if (forcedConnections > 0) {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_host_drain_forced_close',
|
||||
...this.logIdentity(),
|
||||
...this.sessionPlacementLogFields(session),
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
|
||||
reason: this.draining ? 'emergency' : 'regional-deadline',
|
||||
forcedConnections,
|
||||
splices: session.activeSplices.size,
|
||||
pending: session.pendingConns.size
|
||||
})
|
||||
)
|
||||
}
|
||||
if (session.heartbeatTimer) clearInterval(session.heartbeatTimer)
|
||||
if (session.orphanTimer) clearTimeout(session.orphanTimer)
|
||||
if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer)
|
||||
@@ -1250,11 +1518,7 @@ export class HostSessionRegistry {
|
||||
session.pendingConns.clear()
|
||||
session.state = 'closed'
|
||||
if (session.socket) {
|
||||
closeRelayWebSocket(
|
||||
session.socket,
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
'resolve configured director'
|
||||
)
|
||||
closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director')
|
||||
}
|
||||
const key = this.key(session.identity.sub, session.relayHostId)
|
||||
if (this.sessions.get(key) === session) this.sessions.delete(key)
|
||||
@@ -1278,6 +1542,23 @@ export class HostSessionRegistry {
|
||||
session: HostSession,
|
||||
type: unknown,
|
||||
raw: RawData
|
||||
): Promise<void> {
|
||||
const release = this.beginIdleWork(session.relayHostId)
|
||||
if (!release) {
|
||||
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.acceptControlCommandUnfenced(session, type, raw)
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
private async acceptControlCommandUnfenced(
|
||||
session: HostSession,
|
||||
type: unknown,
|
||||
raw: RawData
|
||||
): Promise<void> {
|
||||
if (typeof type !== 'string' || !session.socket) return
|
||||
try {
|
||||
@@ -1315,10 +1596,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
if (type === 'device-credential-install') {
|
||||
const request = DeviceCredentialInstallSchema.parse(payload(raw, type))
|
||||
if (
|
||||
session.state !== 'active' &&
|
||||
request.authorization.mode === 'authenticated-direct'
|
||||
) {
|
||||
if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') {
|
||||
throw new Error('authorization_expired')
|
||||
}
|
||||
const installActivityId = `install:${request.reqId}`
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
|
||||
|
||||
const request = {
|
||||
v: 1 as const,
|
||||
attemptId: '33333333-3333-4333-8333-333333333333',
|
||||
userId: 'idle-reconciliation-test',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'source',
|
||||
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
sourceAssignmentEpoch: 1,
|
||||
sourceGeneration: 7,
|
||||
targetCellId: 'target'
|
||||
}
|
||||
const databases: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
for (const database of databases.splice(0)) await database.close()
|
||||
})
|
||||
|
||||
async function setup() {
|
||||
const database = await openIdleRehomeTestDatabase()
|
||||
databases.push(database)
|
||||
const store = new RelayAssignmentStore(database, () => 100_000_000)
|
||||
await store.reconcileCells([
|
||||
{ id: 'source', url: 'https://source.example.test', capacityRequests: 100 },
|
||||
{ id: 'target', url: 'https://target.example.test', capacityRequests: 100 }
|
||||
])
|
||||
await store.assign(request)
|
||||
await store.activateControl(request, {
|
||||
cellId: request.sourceCellId,
|
||||
assignmentEpoch: request.sourceAssignmentEpoch,
|
||||
generation: request.sourceGeneration,
|
||||
cellIncarnation: request.sourceCellIncarnation
|
||||
})
|
||||
return { database, store }
|
||||
}
|
||||
|
||||
describe('idle cutover durable reconciliation', () => {
|
||||
it('only permits reopening when the exact source still owns the assignment', async () => {
|
||||
const { store } = await setup()
|
||||
expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed')
|
||||
await store.activateControl(request, {
|
||||
cellId: request.sourceCellId,
|
||||
assignmentEpoch: request.sourceAssignmentEpoch,
|
||||
generation: request.sourceGeneration + 1,
|
||||
cellIncarnation: request.sourceCellIncarnation
|
||||
})
|
||||
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
|
||||
})
|
||||
|
||||
it('does not reopen an obsolete source after an assignment change', async () => {
|
||||
const { store } = await setup()
|
||||
await store.startEvacuation(request, request.targetCellId)
|
||||
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
|
||||
})
|
||||
|
||||
it('propagates unavailable durable state instead of declaring rollback', async () => {
|
||||
const { database, store } = await setup()
|
||||
vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable'))
|
||||
await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable')
|
||||
})
|
||||
|
||||
it('waits for an outstanding assignment transaction before deciding authority', async () => {
|
||||
const { database, store } = await setup()
|
||||
let release!: () => void
|
||||
let entered!: () => void
|
||||
const locked = new Promise<void>((resolve) => {
|
||||
entered = resolve
|
||||
})
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve
|
||||
})
|
||||
const commit = database.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?',
|
||||
[request.userId, request.relayHostId]
|
||||
)
|
||||
entered()
|
||||
await gate
|
||||
await transaction.query(
|
||||
'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?',
|
||||
[request.userId, request.relayHostId]
|
||||
)
|
||||
})
|
||||
await locked
|
||||
let settled = false
|
||||
const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => {
|
||||
settled = true
|
||||
})
|
||||
try {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
expect(settled).toBe(false)
|
||||
} finally {
|
||||
release()
|
||||
await commit
|
||||
await reconciliation
|
||||
}
|
||||
expect(await reconciliation).toBe('stale')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,117 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract'
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
|
||||
export const IDLE_REHOME_PAGE_SIZE = 100
|
||||
|
||||
export async function selectIdleRegionalRehomes(input: {
|
||||
database: RelayDatabase
|
||||
now: number
|
||||
heartbeatTtlMs: number
|
||||
cohortPercent: number
|
||||
offset: number
|
||||
connectionHeadroom: Map<string, boolean>
|
||||
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
|
||||
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
|
||||
const [runtimes, safetyRows] = await Promise.all([
|
||||
input.database.query('SELECT * FROM relay_cell_runtime'),
|
||||
input.database.query('SELECT * FROM relay_cell_rehome_safety')
|
||||
])
|
||||
const cleanCells = runtimes
|
||||
.filter((runtime) =>
|
||||
input.cellIsClean(
|
||||
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
|
||||
runtime,
|
||||
input.now
|
||||
)
|
||||
)
|
||||
.map((runtime) => String(runtime.cell_id))
|
||||
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
|
||||
if (!cleanCells.length || !targetCells.length) return []
|
||||
const rows = await input.database.query(
|
||||
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
|
||||
a.assignment_epoch, host.generation, r.cell_incarnation,
|
||||
s.cell_url, target.cell_id AS target_cell_id
|
||||
FROM relay_region_rehome_control policy
|
||||
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
|
||||
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
|
||||
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
|
||||
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
|
||||
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
|
||||
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
|
||||
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
|
||||
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
|
||||
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
|
||||
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
|
||||
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
|
||||
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
|
||||
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
|
||||
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
|
||||
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
|
||||
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
|
||||
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
|
||||
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
|
||||
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
|
||||
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
|
||||
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
|
||||
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
|
||||
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
|
||||
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
|
||||
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
|
||||
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
|
||||
-- Reserve the moving host's source activity plus its assignment on the target.
|
||||
AND target.reserved_requests + 1 + (
|
||||
SELECT COALESCE(SUM(activity.request_units), 0)
|
||||
FROM relay_assignment_activity_leases activity
|
||||
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
|
||||
AND activity.cell_id = a.cell_id
|
||||
) <= target.capacity_requests
|
||||
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
|
||||
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
|
||||
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
|
||||
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
|
||||
AND attempt.created_at > ? - policy.host_cooldown_ms)
|
||||
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
|
||||
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
|
||||
target.cell_id
|
||||
LIMIT ? OFFSET ?`,
|
||||
[
|
||||
input.now,
|
||||
input.now,
|
||||
input.now,
|
||||
input.cohortPercent,
|
||||
input.now,
|
||||
input.now - input.heartbeatTtlMs,
|
||||
input.now - input.heartbeatTtlMs,
|
||||
...cleanCells,
|
||||
...targetCells,
|
||||
input.now,
|
||||
IDLE_REHOME_PAGE_SIZE,
|
||||
input.offset
|
||||
]
|
||||
)
|
||||
return rows.map((row) => {
|
||||
const request = {
|
||||
v: 1 as const,
|
||||
userId: String(row.user_id),
|
||||
relayHostId: String(row.relay_host_id),
|
||||
sourceCellId: String(row.source_cell_id),
|
||||
sourceCellIncarnation: String(row.cell_incarnation),
|
||||
sourceAssignmentEpoch: Number(row.assignment_epoch),
|
||||
sourceGeneration: Number(row.generation),
|
||||
targetCellId: String(row.target_cell_id)
|
||||
}
|
||||
// UUIDv5 keeps retries on every director bound to the same source authority and target.
|
||||
const digest = createHash('sha1')
|
||||
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
|
||||
.update(JSON.stringify(request))
|
||||
.digest()
|
||||
digest[6] = (digest[6]! & 0x0f) | 0x50
|
||||
digest[8] = (digest[8]! & 0x3f) | 0x80
|
||||
const hex = digest.subarray(0, 16).toString('hex')
|
||||
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
|
||||
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayDatabase, RelayLockOptions } from './database.js'
|
||||
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
|
||||
|
||||
const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' }
|
||||
const incarnations = [
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'22222222-2222-4222-8222-222222222222'
|
||||
]
|
||||
const cells = [
|
||||
{
|
||||
id: 'source',
|
||||
url: 'https://source.example.test',
|
||||
region: 'us-central1' as const,
|
||||
capacityRequests: 100
|
||||
},
|
||||
{
|
||||
id: 'target',
|
||||
url: 'https://target.example.test',
|
||||
region: 'asia-east2' as const,
|
||||
capacityRequests: 100
|
||||
}
|
||||
]
|
||||
const databases: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
for (const database of databases.splice(0)) await database.close()
|
||||
})
|
||||
|
||||
async function setup() {
|
||||
const database = await openIdleRehomeTestDatabase()
|
||||
databases.push(database)
|
||||
let now = 100_000_000
|
||||
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
|
||||
await store.inspectRegionalRehomeControl()
|
||||
now += 86_400_000
|
||||
await store.applyRegionalRehomeControl({
|
||||
expectedGeneration: 0,
|
||||
enabled: true,
|
||||
notBefore: now,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
await store.reconcileCells(cells)
|
||||
const safety = {
|
||||
observedAt: now,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
for (const [index, cell] of cells.entries()) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
region: cell.region,
|
||||
cellIncarnation: incarnations[index]!,
|
||||
startedAt: now - 1_000,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
await store.recordCellRegionalRehomeStatus({
|
||||
cellId: cell.id,
|
||||
cellIncarnation: incarnations[index]!,
|
||||
regionalRehomeProtocol: 3,
|
||||
safety
|
||||
})
|
||||
}
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
await store.activateControl(identity, {
|
||||
cellId: cells[0]!.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 7,
|
||||
cellIncarnation: incarnations[0],
|
||||
idleRegionalRehome: true
|
||||
})
|
||||
const issued = await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: issued.window!.generation,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 180, 'asia-east2': 40 }
|
||||
},
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
const request = {
|
||||
v: 1 as const,
|
||||
...identity,
|
||||
attemptId: '33333333-3333-4333-8333-333333333333',
|
||||
sourceCellId: cells[0]!.id,
|
||||
sourceCellIncarnation: incarnations[0]!,
|
||||
sourceAssignmentEpoch: assignment.assignmentEpoch,
|
||||
sourceGeneration: 7,
|
||||
targetCellId: cells[1]!.id
|
||||
}
|
||||
return { store, database, safety, request }
|
||||
}
|
||||
|
||||
describe('constrained idle regional assignment transaction', () => {
|
||||
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
|
||||
const { store, database, safety, request } = await setup()
|
||||
// Model three source activity units and seven units already reserved at the target.
|
||||
await database.query(
|
||||
'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?',
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'")
|
||||
await database.query(
|
||||
"UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'",
|
||||
[capacity]
|
||||
)
|
||||
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
|
||||
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
|
||||
outcome: capacity === 11 ? 'committed' : 'deferred'
|
||||
})
|
||||
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
|
||||
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
|
||||
expect(await store.resolve(identity)).toMatchObject({
|
||||
cellId: capacity === 11 ? 'target' : 'source',
|
||||
assignmentEpoch: capacity === 11 ? 2 : 1
|
||||
})
|
||||
})
|
||||
|
||||
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
for (const table of [
|
||||
'relay_assignments',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_control_capabilities',
|
||||
'relay_region_decisions'
|
||||
]) {
|
||||
const template = (
|
||||
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
])
|
||||
)[0]!
|
||||
const columns = Object.keys(template)
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const values = columns.map((column) =>
|
||||
column === 'user_id' || column === 'relay_host_id' ? '?' : column
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
`idle-store-test-${String(index).padStart(3, '0')}`,
|
||||
`pagehost${String(index).padStart(8, '0')}`,
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
]
|
||||
)
|
||||
}
|
||||
}
|
||||
const first = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
const next = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(first).toHaveLength(100)
|
||||
expect(next).toHaveLength(1)
|
||||
expect(next[0]!.relayHostId).toBe('pagehost00000099')
|
||||
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
|
||||
regionalRehomeCohortPercent: 100
|
||||
})
|
||||
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
|
||||
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
|
||||
})
|
||||
|
||||
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
|
||||
'rechecks generation when replacement wins after the initial authority lookup',
|
||||
async () => {
|
||||
const { store, safety, request, database } = await setup()
|
||||
const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control')
|
||||
const commit = store.commitIdleRegionalRehome(request, safety)
|
||||
await held.entered
|
||||
try {
|
||||
await store.activateControl(identity, {
|
||||
cellId: 'source',
|
||||
assignmentEpoch: 1,
|
||||
generation: 8,
|
||||
cellIncarnation: incarnations[0],
|
||||
idleRegionalRehome: true
|
||||
})
|
||||
} finally {
|
||||
held.release()
|
||||
}
|
||||
expect(await commit).toEqual({ outcome: 'deferred' })
|
||||
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
}
|
||||
)
|
||||
|
||||
it('rejects source replacement when the cutover already holds assignment authority', async () => {
|
||||
const { store, safety, request, database } = await setup()
|
||||
const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id')
|
||||
const commit = store.commitIdleRegionalRehome(request, safety)
|
||||
await held.entered
|
||||
const replacement = store.activateControl(identity, {
|
||||
cellId: 'source',
|
||||
assignmentEpoch: 1,
|
||||
generation: 8,
|
||||
cellIncarnation: incarnations[0],
|
||||
idleRegionalRehome: true
|
||||
})
|
||||
const rejected = expect(replacement).rejects.toThrow('wrong_assignment')
|
||||
held.release()
|
||||
expect(await commit).toEqual({ outcome: 'committed' })
|
||||
await rejected
|
||||
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
|
||||
})
|
||||
|
||||
it('finds the committed attempt after its database reply is lost', async () => {
|
||||
const { store, safety, request, database } = await setup()
|
||||
const transaction = database.transaction.bind(database)
|
||||
const intercepted = vi
|
||||
.spyOn(database, 'transaction')
|
||||
.mockImplementation(async (operation, options) => {
|
||||
let changed = false
|
||||
const result = await transaction(
|
||||
async (tx) =>
|
||||
operation(
|
||||
new Proxy(tx, {
|
||||
get(target, key) {
|
||||
if (key === 'query')
|
||||
return async (sql: string, params?: unknown[]) => {
|
||||
if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true
|
||||
return target.query(sql, params)
|
||||
}
|
||||
const value = Reflect.get(target, key)
|
||||
return typeof value === 'function' ? value.bind(target) : value
|
||||
}
|
||||
})
|
||||
),
|
||||
options
|
||||
)
|
||||
if (changed) throw new Error('simulated_commit_reply_lost')
|
||||
return result
|
||||
})
|
||||
await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow(
|
||||
'simulated_commit_reply_lost'
|
||||
)
|
||||
intercepted.mockRestore()
|
||||
expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed')
|
||||
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('commits the requested move once and records its outcome without source retention', async () => {
|
||||
const { store, database, safety, request } = await setup()
|
||||
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
|
||||
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
|
||||
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
|
||||
const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts')
|
||||
expect(attempts).toHaveLength(1)
|
||||
expect(attempts[0]!.attempt_id).toBe(request.attemptId)
|
||||
expect(Number(attempts[0]!.source_generation)).toBe(7)
|
||||
})
|
||||
|
||||
it('rejects a replaced control and never substitutes a different target', async () => {
|
||||
const { store, safety, request } = await setup()
|
||||
expect(
|
||||
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
|
||||
).toEqual({ outcome: 'deferred' })
|
||||
await store.activateControl(identity, {
|
||||
cellId: 'source',
|
||||
assignmentEpoch: 1,
|
||||
generation: 8,
|
||||
cellIncarnation: incarnations[0],
|
||||
idleRegionalRehome: true
|
||||
})
|
||||
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' })
|
||||
expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 })
|
||||
})
|
||||
|
||||
it('does not commit without process safety or cohort authorization', async () => {
|
||||
const { store, safety, request, database } = await setup()
|
||||
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
|
||||
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
|
||||
outcome: 'deferred'
|
||||
})
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
|
||||
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
|
||||
const { store, safety, database } = await setup()
|
||||
const before = await database.query('SELECT * FROM relay_assignments')
|
||||
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(candidates).toHaveLength(1)
|
||||
expect(candidates[0]).toMatchObject({
|
||||
sourceGeneration: 7,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target'
|
||||
})
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
|
||||
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
function holdStatement(database: RelayDatabase, fragment: string) {
|
||||
let entered!: () => void
|
||||
let release!: () => void
|
||||
const arrival = new Promise<void>((resolve) => {
|
||||
entered = resolve
|
||||
})
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve
|
||||
})
|
||||
let held = false
|
||||
const transaction = database.transaction.bind(database)
|
||||
vi.spyOn(database, 'transaction').mockImplementation((operation, options) =>
|
||||
transaction(async (tx) => {
|
||||
return operation(
|
||||
new Proxy(tx, {
|
||||
get(target, key) {
|
||||
if (key === 'query' || key === 'queryLocked')
|
||||
return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => {
|
||||
if (!held && sql.includes(fragment)) {
|
||||
held = true
|
||||
entered()
|
||||
await gate
|
||||
}
|
||||
return key === 'queryLocked'
|
||||
? target.queryLocked(sql, params, lockOptions)
|
||||
: target.query(sql, params)
|
||||
}
|
||||
const value = Reflect.get(target, key)
|
||||
return typeof value === 'function' ? value.bind(target) : value
|
||||
}
|
||||
})
|
||||
)
|
||||
}, options)
|
||||
)
|
||||
return { entered: arrival, release }
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import pg from 'pg'
|
||||
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
export async function openIdleRehomeTestDatabase(): Promise<RelayDatabase> {
|
||||
const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL
|
||||
if (!configured) return openInMemoryRelayDatabase()
|
||||
const url = new URL(configured)
|
||||
if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
|
||||
throw new Error('idle_rehome_tests_require_local_postgres_55440')
|
||||
}
|
||||
const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}`
|
||||
const admin = new pg.Client({ connectionString: configured })
|
||||
await admin.connect()
|
||||
try {
|
||||
await admin.query(`CREATE SCHEMA ${schema}`)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
|
||||
const close = database.close.bind(database)
|
||||
database.close = async () => {
|
||||
try {
|
||||
await close()
|
||||
} finally {
|
||||
try {
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
|
||||
} finally {
|
||||
await admin.end()
|
||||
}
|
||||
}
|
||||
}
|
||||
return database
|
||||
} catch (error) {
|
||||
try {
|
||||
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
} finally {
|
||||
await admin.end()
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
|
||||
|
||||
const candidate = {
|
||||
v: 1,
|
||||
attemptId: '11111111-1111-4111-8111-111111111111',
|
||||
userId: 'private-user',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'source',
|
||||
sourceCellUrl: 'https://source.example.test',
|
||||
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
|
||||
sourceAssignmentEpoch: 7,
|
||||
sourceGeneration: 3,
|
||||
targetCellId: 'target'
|
||||
}
|
||||
const config = {
|
||||
role: 'director',
|
||||
regionCorrectionCohortPercent: 100,
|
||||
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
|
||||
rehomeDirectorServiceAccount: 'director@example.test'
|
||||
} as RelayConfig
|
||||
|
||||
function setup(fetch: typeof globalThis.fetch) {
|
||||
const selectIdleRegionalRehomeCandidates = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValue([candidate])
|
||||
const claimRegionalRehome = vi.fn()
|
||||
const recordRegionalRehomeDispatchFailure = vi.fn()
|
||||
const worker = startRegionalRehomeWorker(
|
||||
config,
|
||||
{
|
||||
selectIdleRegionalRehomeCandidates,
|
||||
claimRegionalRehome,
|
||||
recordRegionalRehomeDispatchFailure
|
||||
} as unknown as RelayAssignmentStore,
|
||||
{
|
||||
safetySnapshot: () => ({ observedAt: 100 }) as never,
|
||||
intervalMs: 60_000,
|
||||
identityToken: async () => 'private-token',
|
||||
fetch
|
||||
}
|
||||
)!
|
||||
return {
|
||||
worker,
|
||||
selectIdleRegionalRehomeCandidates,
|
||||
claimRegionalRehome,
|
||||
recordRegionalRehomeDispatchFailure
|
||||
}
|
||||
}
|
||||
|
||||
describe('idle regional worker dispatch', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
it('sends an idle request without claiming an assignment first', async () => {
|
||||
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
|
||||
Response.json({ v: 1, outcome: 'committed' })
|
||||
)
|
||||
const c = setup(fetch)
|
||||
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
|
||||
await c.worker.run()
|
||||
c.worker.stop()
|
||||
expect(c.claimRegionalRehome).not.toHaveBeenCalled()
|
||||
expect(fetch).toHaveBeenCalledOnce()
|
||||
const [url, init] = fetch.mock.calls[0]!
|
||||
expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome')
|
||||
const { sourceCellUrl: _, ...request } = candidate
|
||||
expect(JSON.parse(String(init?.body))).toEqual({
|
||||
...request,
|
||||
cohortPercent: 100,
|
||||
directorSafety: { observedAt: 100 }
|
||||
})
|
||||
})
|
||||
it('progresses past busy hosts without charging a dispatch failure', async () => {
|
||||
const fetch = vi
|
||||
.fn<typeof globalThis.fetch>()
|
||||
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' }))
|
||||
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' }))
|
||||
const c = setup(fetch)
|
||||
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
|
||||
c.selectIdleRegionalRehomeCandidates.mockResolvedValue([
|
||||
candidate,
|
||||
{
|
||||
...candidate,
|
||||
relayHostId: 'ponmlkjihgfedcba',
|
||||
attemptId: '33333333-3333-4333-8333-333333333333'
|
||||
}
|
||||
])
|
||||
await c.worker.run()
|
||||
c.worker.stop()
|
||||
expect(fetch).toHaveBeenCalledTimes(2)
|
||||
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
|
||||
})
|
||||
it('does not charge a lost response as a claimed migration failure', async () => {
|
||||
const fetch = vi.fn<typeof globalThis.fetch>(async () => {
|
||||
throw new Error('response lost')
|
||||
})
|
||||
const c = setup(fetch)
|
||||
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
|
||||
await c.worker.run()
|
||||
c.worker.stop()
|
||||
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
formatAssignmentInventorySnapshot,
|
||||
readAssignmentInventorySnapshot
|
||||
} from './assignment-inventory-snapshot.js'
|
||||
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { loadRelayConfig } from './config.js'
|
||||
import { startCellHeartbeat } from './cell-heartbeat-client.js'
|
||||
@@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
|
||||
void runRelayBackgroundOperation(async () => {
|
||||
const inventory = await readRegisteredMigrationInventory(database, Date.now())
|
||||
for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line)
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_region_correction_outcomes',
|
||||
observedAt: Date.now(),
|
||||
outcomes: await readRegionCorrectionOutcomes(database, Date.now())
|
||||
})
|
||||
)
|
||||
}, '[orca-relay] migration inventory failed')
|
||||
}, 5 * 60_000)
|
||||
: null
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { RelayDatabase } from './database.js'
|
||||
|
||||
export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) {
|
||||
const rows = await database.query(
|
||||
`SELECT attempt.source_cell_id, attempt.target_cell_id,
|
||||
CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted'
|
||||
WHEN attempt.completed_at IS NOT NULL THEN 'completed'
|
||||
WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state,
|
||||
COUNT(*) AS count,
|
||||
COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
|
||||
THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms,
|
||||
COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
|
||||
THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units
|
||||
FROM relay_region_rehome_attempts attempt
|
||||
JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id
|
||||
AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch
|
||||
GROUP BY attempt.source_cell_id, attempt.target_cell_id, state
|
||||
ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`,
|
||||
[now]
|
||||
)
|
||||
return rows.map((row) => ({
|
||||
sourceCellId: String(row.source_cell_id),
|
||||
targetCellId: String(row.target_cell_id),
|
||||
state: String(row.state),
|
||||
count: Number(row.count),
|
||||
oldestOpenMs: Number(row.oldest_open_ms),
|
||||
targetReservedUnits: Number(row.target_reserved_units)
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
|
||||
import {
|
||||
REGIONAL_REHOME_CONCURRENT_LIMIT,
|
||||
REGION_DECISION_TTL_MS
|
||||
} from './region-correction-state.js'
|
||||
|
||||
export type RegionCorrectionPreview = {
|
||||
observedAt: number
|
||||
newClaimsEnabled: boolean
|
||||
cohortPercent: number
|
||||
openMigrations: number
|
||||
availableMigrationSlots: number
|
||||
globalSafetyFailure: string | null
|
||||
counts: Record<string, number>
|
||||
}
|
||||
|
||||
export async function previewRegionalRehomeEligibility(input: {
|
||||
database: RelayDatabase
|
||||
now: number
|
||||
heartbeatTtlMs: number
|
||||
cohortPercent: number
|
||||
globalSafetyFailure: string | null
|
||||
connectionHeadroom: ReadonlyMap<string, boolean>
|
||||
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
|
||||
}): Promise<RegionCorrectionPreview> {
|
||||
const { database, now } = input
|
||||
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
|
||||
await Promise.all([
|
||||
database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at,
|
||||
decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version,
|
||||
decision.observed_at, decision.cohort_bucket,
|
||||
(SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt
|
||||
WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at,
|
||||
(SELECT COUNT(*) FROM relay_assignment_migrations migration
|
||||
WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations,
|
||||
(SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease
|
||||
WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id
|
||||
AND lease.cell_id = assignment.cell_id) AS source_units,
|
||||
(SELECT COUNT(*) FROM relay_control_capabilities host_capability
|
||||
JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id
|
||||
AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id
|
||||
JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id
|
||||
AND runtime.cell_incarnation = host_capability.cell_incarnation
|
||||
WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id
|
||||
AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch
|
||||
AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control'
|
||||
AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ?
|
||||
AND lease.updated_at >= runtime.started_at) AS capable_controls
|
||||
FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision
|
||||
ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`,
|
||||
[now]
|
||||
),
|
||||
database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell
|
||||
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
|
||||
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`),
|
||||
database.query(`SELECT * FROM relay_cell_runtime`),
|
||||
database.query(`SELECT * FROM relay_cell_capabilities`),
|
||||
database.query(`SELECT * FROM relay_cell_rehome_safety`),
|
||||
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
|
||||
database.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
|
||||
)
|
||||
])
|
||||
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
|
||||
const runtimes = byCell(runtimeRows)
|
||||
const capabilities = byCell(capabilityRows)
|
||||
const safety = byCell(safetyRows)
|
||||
const inventory = byCell(cells)
|
||||
const control = controls[0]
|
||||
const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
|
||||
const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS)
|
||||
const openMigrations = Number(migrations[0]?.count ?? 0)
|
||||
const counts: Record<string, number> = {}
|
||||
const count = (reason: string) => {
|
||||
counts[reason] = (counts[reason] ?? 0) + 1
|
||||
}
|
||||
const available = (cell: SqlRow): boolean => {
|
||||
const id = String(cell.cell_id)
|
||||
const runtime = runtimes.get(id)
|
||||
const capability = capabilities.get(id)
|
||||
return (
|
||||
Number(cell.enabled) === 1 &&
|
||||
cell.admission_state === 'general' &&
|
||||
cell.region != null &&
|
||||
runtime !== undefined &&
|
||||
Number(runtime.ready) === 1 &&
|
||||
Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs &&
|
||||
capability !== undefined &&
|
||||
capability.cell_incarnation === runtime.cell_incarnation &&
|
||||
Number(capability.regional_rehome_protocol) >= 3
|
||||
)
|
||||
}
|
||||
for (const host of hosts) {
|
||||
let reason: string | null = null
|
||||
const source = inventory.get(String(host.cell_id))
|
||||
if (host.generation == null) reason = 'no-verified-decision'
|
||||
else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge)
|
||||
reason = 'expired'
|
||||
else if (
|
||||
Number(host.decision_epoch) !== Number(host.assignment_epoch) ||
|
||||
host.incumbent_region !== source?.region
|
||||
)
|
||||
reason = 'basis-changed'
|
||||
else if (
|
||||
host.outcome !== 'conclusive' ||
|
||||
Number(host.policy_version) !== 1 ||
|
||||
host.preferred_region == null
|
||||
)
|
||||
reason = 'inconclusive-or-insufficient-improvement'
|
||||
else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort'
|
||||
else if (Number(host.open_migrations) > 0) reason = 'migration-open'
|
||||
else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown)
|
||||
reason = 'host-cooldown'
|
||||
else if (!source || !available(source)) reason = 'source-ineligible'
|
||||
else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive'
|
||||
else if (
|
||||
!input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now)
|
||||
)
|
||||
reason = 'source-unclean'
|
||||
if (reason) {
|
||||
count(reason)
|
||||
continue
|
||||
}
|
||||
const targets = cells.filter(
|
||||
(cell) =>
|
||||
cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell)
|
||||
)
|
||||
const clean = targets.filter((cell) =>
|
||||
input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now)
|
||||
)
|
||||
const capacity = clean.filter(
|
||||
(cell) =>
|
||||
input.connectionHeadroom.get(String(cell.cell_id)) !== false &&
|
||||
Number(cell.reserved_requests) + Number(host.source_units) + 1 <=
|
||||
Number(cell.capacity_requests)
|
||||
)
|
||||
if (targets.length === 0) count('no-eligible-target')
|
||||
else if (clean.length === 0) count('target-unclean')
|
||||
else if (capacity.length === 0) count('no-target-headroom')
|
||||
else if (input.globalSafetyFailure) count('global-safety-blocked')
|
||||
else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap')
|
||||
else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`)
|
||||
}
|
||||
return {
|
||||
observedAt: now,
|
||||
newClaimsEnabled: Number(control?.enabled ?? 0) === 1,
|
||||
cohortPercent: input.cohortPercent,
|
||||
openMigrations,
|
||||
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
|
||||
globalSafetyFailure: input.globalSafetyFailure,
|
||||
counts
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' }
|
||||
const paths: string[] = []
|
||||
const databases = new Set<RelayDatabase>()
|
||||
afterEach(async () => {
|
||||
for (const database of databases) await database.close()
|
||||
databases.clear()
|
||||
for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function setup() {
|
||||
const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-'))
|
||||
paths.push(dataDir)
|
||||
let now = 1_000_000_000
|
||||
const open = async () => {
|
||||
const database = await openRelayDatabase({ dataDir })
|
||||
databases.add(database)
|
||||
return { database, store: new RelayAssignmentStore(database, () => now) }
|
||||
}
|
||||
const first = await open()
|
||||
const cell = {
|
||||
id: 'restart-us',
|
||||
url: 'https://restart-us.example.test',
|
||||
region: 'us-central1' as const,
|
||||
capacityRequests: 100
|
||||
}
|
||||
await first.store.reconcileCells([cell])
|
||||
await first.store.setCellEnabled(cell.id, true)
|
||||
await first.store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
region: cell.region,
|
||||
startedAt: now - 1_000,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
const assignment = await first.store.assign(identity)
|
||||
const issue = (store: RelayAssignmentStore) =>
|
||||
store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
|
||||
const window = (await issue(first.store)).window!
|
||||
const report = {
|
||||
v: 1 as const,
|
||||
action: 'report' as const,
|
||||
generation: window.generation,
|
||||
assignmentEpoch: window.assignmentEpoch,
|
||||
policyVersion: 1 as const,
|
||||
outcome: 'conclusive' as const,
|
||||
measurements: { 'us-central1': 200, 'asia-east2': 40 }
|
||||
}
|
||||
const restart = async () => {
|
||||
await first.database.close()
|
||||
databases.delete(first.database)
|
||||
return open()
|
||||
}
|
||||
return {
|
||||
...first, window, report, issue, restart,
|
||||
setNow: (value: number) => { now = value }
|
||||
}
|
||||
}
|
||||
|
||||
describe('persisted region decisions across director restart', () => {
|
||||
it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => {
|
||||
const context = await setup()
|
||||
const epoch = context.window.assignmentEpoch
|
||||
await context.store.exchangeRegionCorrection(identity, {
|
||||
v: 1, action: 'report', generation: context.window.generation,
|
||||
assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter'
|
||||
}, epoch)
|
||||
const restarted = await context.restart()
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
|
||||
.toMatchObject({ reportStatus: 'duplicate' })
|
||||
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
|
||||
expect(row.outcome).toBe('inconclusive')
|
||||
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
})
|
||||
|
||||
it('uses server expiry after a restart regardless of an old client report', async () => {
|
||||
const context = await setup()
|
||||
context.setNow(context.window.expiresAt)
|
||||
const restarted = await context.restart()
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
|
||||
.toMatchObject({ reportStatus: 'expired' })
|
||||
expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 })
|
||||
})
|
||||
|
||||
it('does not interpret a persisted future-policy window using the old policy after rollback', async () => {
|
||||
const context = await setup()
|
||||
await context.database.query('UPDATE relay_region_decisions SET policy_version = 2')
|
||||
const restarted = await context.restart()
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
|
||||
expect(row.outcome).toBe('pending')
|
||||
expect(row.preferred_region).toBeNull()
|
||||
expect(row.report_json).toBeNull()
|
||||
})
|
||||
|
||||
it('keeps generation ordering when the server clock moves backwards across restart', async () => {
|
||||
const context = await setup()
|
||||
context.setNow(1_000_000_000 - 60_000)
|
||||
const restarted = await context.restart()
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,158 @@
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import { createHash } from 'node:crypto'
|
||||
import type {
|
||||
RegionCorrectionRequest,
|
||||
RegionCorrectionResponse,
|
||||
RelayRegion
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
|
||||
type Identity = { userId: string; relayHostId: string }
|
||||
export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000
|
||||
export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8
|
||||
|
||||
export async function exchangeRegionCorrection(
|
||||
database: RelayDatabase,
|
||||
identity: Identity,
|
||||
request: RegionCorrectionRequest,
|
||||
assignmentEpoch: number,
|
||||
now: number
|
||||
): Promise<RegionCorrectionResponse> {
|
||||
const result: RegionCorrectionResponse = await database.transaction(async (transaction) => {
|
||||
const assignment = (
|
||||
await transaction.queryLocked(
|
||||
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
const region =
|
||||
assignment &&
|
||||
(
|
||||
await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
|
||||
assignment.cell_id
|
||||
])
|
||||
)[0]
|
||||
if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) {
|
||||
return { v: 1, reportStatus: 'basis-changed' }
|
||||
}
|
||||
const prior = (
|
||||
await transaction.queryLocked(
|
||||
`SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
if (request.action === 'issue-window') {
|
||||
const generation = Number(prior?.generation ?? 0) + 1
|
||||
if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted')
|
||||
const expiresAt = now + REGION_DECISION_TTL_MS
|
||||
const cohortBucket =
|
||||
createHash('sha256')
|
||||
.update(JSON.stringify([identity.userId, identity.relayHostId]))
|
||||
.digest()
|
||||
.readUInt32BE(0) % 100
|
||||
await transaction.query(
|
||||
`INSERT INTO relay_region_decisions
|
||||
(user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region,
|
||||
policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?)
|
||||
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
|
||||
generation = excluded.generation, expires_at = excluded.expires_at,
|
||||
assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region,
|
||||
policy_version = 1, outcome = 'pending', preferred_region = NULL,
|
||||
observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
generation,
|
||||
expiresAt,
|
||||
assignmentEpoch,
|
||||
region.region,
|
||||
now,
|
||||
cohortBucket
|
||||
]
|
||||
)
|
||||
return {
|
||||
v: 1,
|
||||
window: {
|
||||
generation,
|
||||
expiresAt,
|
||||
assignmentEpoch,
|
||||
incumbentRegion: region.region as RelayRegion,
|
||||
policyVersion: 1
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!prior || Number(prior.generation) !== request.generation)
|
||||
return { v: 1, reportStatus: 'stale' }
|
||||
if (Number(prior.policy_version) !== request.policyVersion)
|
||||
return { v: 1, reportStatus: 'stale' }
|
||||
if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' }
|
||||
if (
|
||||
request.assignmentEpoch !== assignmentEpoch ||
|
||||
Number(prior.assignment_epoch) !== assignmentEpoch ||
|
||||
prior.incumbent_region !== region.region
|
||||
) {
|
||||
return { v: 1, reportStatus: 'basis-changed' }
|
||||
}
|
||||
// The first report wins, including an inconclusive tombstone.
|
||||
if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' }
|
||||
let preferredRegion: RelayRegion | null = null
|
||||
if (request.outcome === 'conclusive') {
|
||||
const incumbent = request.measurements[region.region as RelayRegion]
|
||||
const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1'
|
||||
const targetRtt = request.measurements[target]
|
||||
if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target
|
||||
}
|
||||
await transaction.query(
|
||||
`UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ?
|
||||
WHERE user_id = ? AND relay_host_id = ? AND generation = ?`,
|
||||
[
|
||||
request.outcome,
|
||||
preferredRegion,
|
||||
JSON.stringify(request),
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
request.generation
|
||||
]
|
||||
)
|
||||
return { v: 1, reportStatus: 'accepted' }
|
||||
})
|
||||
if (request.action === 'report' && result.reportStatus === 'accepted') {
|
||||
const digest = relayHostLogDigest(identity.relayHostId)
|
||||
// Stable sampling includes unchanged hosts for before/after comparisons.
|
||||
if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_region_comparison',
|
||||
relayHostIdDigest: digest,
|
||||
assignmentEpoch,
|
||||
generation: request.generation,
|
||||
policyVersion: request.policyVersion,
|
||||
outcome: request.outcome,
|
||||
...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {})
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export async function previewRegionCorrection(
|
||||
database: RelayDatabase,
|
||||
now: number
|
||||
): Promise<Record<string, number>> {
|
||||
const rows = await database.query(
|
||||
`SELECT CASE WHEN decision.expires_at <= ? THEN 'expired'
|
||||
WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed'
|
||||
WHEN decision.outcome = 'pending' THEN 'pending'
|
||||
WHEN decision.preferred_region IS NULL THEN 'ineligible'
|
||||
ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason,
|
||||
COUNT(*) AS count
|
||||
FROM relay_region_decisions decision
|
||||
JOIN relay_assignments assignment ON assignment.user_id = decision.user_id
|
||||
AND assignment.relay_host_id = decision.relay_host_id
|
||||
GROUP BY reason`,
|
||||
[now]
|
||||
)
|
||||
return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)]))
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' }
|
||||
const cells = [
|
||||
{
|
||||
id: 'decision-us',
|
||||
url: 'https://decision-us.example.test',
|
||||
region: 'us-central1' as const,
|
||||
capacityRequests: 100
|
||||
},
|
||||
{
|
||||
id: 'decision-asia',
|
||||
url: 'https://decision-asia.example.test',
|
||||
region: 'asia-east2' as const,
|
||||
capacityRequests: 100
|
||||
}
|
||||
]
|
||||
const incarnations = [
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'22222222-2222-4222-8222-222222222222'
|
||||
]
|
||||
const opened: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
for (const database of opened.splice(0)) {
|
||||
if (database.dialect === 'postgres') await cleanupPostgres(database)
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
async function cleanupPostgres(database: RelayDatabase) {
|
||||
for (const table of [
|
||||
'relay_control_connection_reservations',
|
||||
'relay_region_decisions',
|
||||
'relay_control_capabilities',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignment_migration_incarnations',
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_region_rehome_attempts',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
|
||||
}
|
||||
for (const table of [
|
||||
'relay_cell_rehome_safety',
|
||||
'relay_cell_capabilities',
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await database.query(
|
||||
`DELETE FROM ${table} WHERE cell_id IN (?, ?)`,
|
||||
cells.map((cell) => cell.id)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function setup() {
|
||||
const database =
|
||||
process.env.ORCA_REGION_CORRECTION_POSTGRES === '1'
|
||||
? await openRelayDatabase({
|
||||
databaseUrl: requiredPostgresUrl(),
|
||||
dataDir: '/tmp/orca-region-correction-unused'
|
||||
})
|
||||
: await openInMemoryRelayDatabase()
|
||||
opened.push(database)
|
||||
if (database.dialect === 'postgres') await cleanupPostgres(database)
|
||||
let clock = 100_000_000
|
||||
const store = new RelayAssignmentStore(database, () => clock, {
|
||||
regionalRehomeCohortPercent: 100
|
||||
})
|
||||
await store.reconcileCells(cells)
|
||||
for (const cell of cells) await store.setCellEnabled(cell.id, true)
|
||||
for (const [index, cell] of cells.entries()) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
region: cell.region,
|
||||
cellIncarnation: incarnations[index]!,
|
||||
startedAt: clock - 1_000,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
}
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
const activityId = await store.activateControl(identity, {
|
||||
cellId: cells[0]!.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 7,
|
||||
cellIncarnation: incarnations[0],
|
||||
idleRegionalRehome: true
|
||||
})
|
||||
return {
|
||||
database,
|
||||
store,
|
||||
assignment,
|
||||
activityId,
|
||||
now: () => clock,
|
||||
advance: (ms: number) => {
|
||||
clock += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function requiredPostgresUrl(): string {
|
||||
const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
if (!url || new URL(url).port !== '55440')
|
||||
throw new Error('PostgreSQL tests require configured port 55440')
|
||||
return url
|
||||
}
|
||||
|
||||
async function window(context: Awaited<ReturnType<typeof setup>>) {
|
||||
const result = await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
context.assignment.assignmentEpoch
|
||||
)
|
||||
return result.window!
|
||||
}
|
||||
|
||||
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
|
||||
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
|
||||
const attemptId = '33333333-3333-4333-8333-333333333333'
|
||||
await context.database.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
|
||||
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
|
||||
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
|
||||
[
|
||||
attemptId,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
cells[0]!.id,
|
||||
incarnations[0],
|
||||
cells[1]!.id,
|
||||
incarnations[1],
|
||||
migration.previousEpoch,
|
||||
migration.assignmentEpoch,
|
||||
context.now(),
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
return { migration }
|
||||
}
|
||||
|
||||
describe('ordered region decisions and migration outcomes', () => {
|
||||
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
|
||||
const context = await setup()
|
||||
const { migration } = await regionalMigration(context)
|
||||
context.advance(1_000)
|
||||
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
|
||||
const outcomes = await context.store.regionCorrectionOutcomes()
|
||||
expect(outcomes).toEqual([
|
||||
expect.objectContaining({
|
||||
sourceCellId: cells[0]!.id,
|
||||
targetCellId: cells[1]!.id,
|
||||
state: 'registering',
|
||||
count: 1,
|
||||
oldestOpenMs: 1_000
|
||||
})
|
||||
])
|
||||
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
|
||||
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
|
||||
before
|
||||
)
|
||||
await context.store.activateControl(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.markMigrationTargetRegistered(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch
|
||||
})
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({ state: 'registered' })
|
||||
])
|
||||
await context.store.releaseActivity(identity, context.activityId)
|
||||
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({
|
||||
state: 'completed',
|
||||
targetReservedUnits: 0,
|
||||
oldestOpenMs: 0
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => {
|
||||
const context = await setup()
|
||||
const first = await window(context)
|
||||
const second = await window(context)
|
||||
expect(second.generation).toBe(first.generation + 1)
|
||||
const report = {
|
||||
v: 1 as const,
|
||||
action: 'report' as const,
|
||||
assignmentEpoch: first.assignmentEpoch,
|
||||
policyVersion: 1 as const,
|
||||
outcome: 'conclusive' as const,
|
||||
measurements: { 'us-central1': 200, 'asia-east2': 40 }
|
||||
}
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ ...report, generation: first.generation },
|
||||
first.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'stale' })
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' },
|
||||
second.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'accepted' })
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ ...report, generation: second.generation },
|
||||
second.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'duplicate' })
|
||||
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
|
||||
})
|
||||
|
||||
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
|
||||
const context = await setup()
|
||||
const query = context.database.query.bind(context.database)
|
||||
const transaction = context.database.transaction.bind(context.database)
|
||||
const queryLocked = context.database.queryLocked.bind(context.database)
|
||||
context.database.query = async (sql, params) => {
|
||||
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
|
||||
return query(sql, params)
|
||||
}
|
||||
context.database.transaction = async () => {
|
||||
throw new Error('preview_must_not_open_mutating_transaction')
|
||||
}
|
||||
context.database.queryLocked = async () => {
|
||||
throw new Error('preview_must_not_lock')
|
||||
}
|
||||
try {
|
||||
const preview = await context.store.previewRegionalRehomeEligibility()
|
||||
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
|
||||
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.userId)
|
||||
} finally {
|
||||
context.database.query = query
|
||||
context.database.transaction = transaction
|
||||
context.database.queryLocked = queryLocked
|
||||
}
|
||||
})
|
||||
|
||||
it('allocates distinct ordered generations for concurrent window issuers', async () => {
|
||||
const context = await setup()
|
||||
const replies = await Promise.all([window(context), window(context), window(context)])
|
||||
expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3])
|
||||
const older = replies.find((reply) => reply.generation === 2)!
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: older.generation,
|
||||
assignmentEpoch: older.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'inconclusive',
|
||||
reason: 'delayed'
|
||||
},
|
||||
older.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'stale' })
|
||||
})
|
||||
|
||||
it('compares with assigned region, preserves hints, and never extends a window on report', async () => {
|
||||
const context = await setup()
|
||||
await context.store.assign(identity, 'asia-east2')
|
||||
const issued = await window(context)
|
||||
expect(issued.incumbentRegion).toBe('us-central1')
|
||||
context.advance(50)
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: issued.generation,
|
||||
assignmentEpoch: issued.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 110, 'asia-east2': 90 }
|
||||
},
|
||||
issued.assignmentEpoch
|
||||
)
|
||||
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
|
||||
const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]!
|
||||
expect(Number(row.expires_at)).toBe(issued.expiresAt)
|
||||
const hint = (
|
||||
await context.database.query(
|
||||
`SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
)[0]
|
||||
expect(hint?.preferred_region).toBe('asia-east2')
|
||||
context.advance(24 * 60 * 60_000)
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: issued.generation,
|
||||
assignmentEpoch: issued.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'inconclusive',
|
||||
reason: 'late'
|
||||
},
|
||||
issued.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'expired' })
|
||||
})
|
||||
|
||||
it('rejects stale assignment basis and requires both thresholds', async () => {
|
||||
const context = await setup()
|
||||
const issued = await window(context)
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: issued.generation,
|
||||
assignmentEpoch: issued.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 150, 'asia-east2': 100 }
|
||||
},
|
||||
issued.assignmentEpoch
|
||||
)
|
||||
expect(await context.store.previewRegionCorrection()).toEqual({
|
||||
'us-central1-to-asia-east2': 1
|
||||
})
|
||||
await context.store.startEvacuation(identity, cells[1]!.id)
|
||||
expect(
|
||||
await context.store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
issued.assignmentEpoch
|
||||
)
|
||||
).toMatchObject({ reportStatus: 'basis-changed' })
|
||||
})
|
||||
})
|
||||
@@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({
|
||||
createAdminTokenVerifier: () => async (token: string, route?: string) =>
|
||||
token === 'deploy-token' ||
|
||||
(token === 'monitor-token' &&
|
||||
(!route || route === '/v1/admin/regional-rehome-control')),
|
||||
(!route ||
|
||||
route === '/v1/admin/regional-rehome-control' ||
|
||||
route === '/v1/admin/regional-rehome-preview')),
|
||||
createReadOnlyAdminTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) =>
|
||||
token === 'deploy-token',
|
||||
@@ -41,7 +43,83 @@ const request = {
|
||||
graceMs: 60_000
|
||||
}
|
||||
|
||||
describe('idle regional cutover endpoint', () => {
|
||||
it('authenticates and fences the source before invoking a cutover', async () => {
|
||||
const idleRehome = vi.fn(async () => ({ outcome: 'busy' }))
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {} as never,
|
||||
drain: vi.fn(),
|
||||
idleRehome,
|
||||
cellIncarnation,
|
||||
ready: vi.fn(async () => true)
|
||||
} as Parameters<typeof createRelayApp>[1])
|
||||
const input = {
|
||||
v: 1,
|
||||
attemptId: request.attemptId,
|
||||
userId: request.userId,
|
||||
relayHostId: request.relayHostId,
|
||||
sourceCellId: request.sourceCellId,
|
||||
sourceCellIncarnation: cellIncarnation,
|
||||
sourceAssignmentEpoch: 7,
|
||||
sourceGeneration: 1,
|
||||
targetCellId: 'target-cell',
|
||||
cohortPercent: 100,
|
||||
directorSafety: {
|
||||
observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0
|
||||
}
|
||||
}
|
||||
const path = '/v1/admin/host-idle-rehome'
|
||||
expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401)
|
||||
expect(
|
||||
(
|
||||
await postPath(app, path, 'rehome-token', {
|
||||
...input,
|
||||
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
|
||||
})
|
||||
).status
|
||||
).toBe(409)
|
||||
expect(idleRehome).not.toHaveBeenCalled()
|
||||
const response = await postPath(app, path, 'rehome-token', input)
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toEqual({ v: 1, outcome: 'busy' })
|
||||
expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input)
|
||||
})
|
||||
})
|
||||
|
||||
describe('regional host drain endpoint', () => {
|
||||
it('exposes aggregate preview to monitors without a mutation path', async () => {
|
||||
const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } }
|
||||
const safety = { observedAt: 100 }
|
||||
const previewRegionalRehomeEligibility = vi.fn(async () => preview)
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
previewRegionalRehomeEligibility,
|
||||
regionCorrectionOutcomes: async () => []
|
||||
} as never,
|
||||
regionalRehomeSafetySnapshot: () => safety as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const path = '/v1/admin/regional-rehome-preview'
|
||||
expect((await app.request(path)).status).toBe(401)
|
||||
expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled()
|
||||
const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } })
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] })
|
||||
expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety)
|
||||
expect(
|
||||
(
|
||||
await app.request(path, {
|
||||
method: 'POST',
|
||||
headers: { authorization: 'Bearer deploy-token' }
|
||||
})
|
||||
).status
|
||||
).toBe(404)
|
||||
})
|
||||
|
||||
it('accepts only the dedicated identity and exact cell generation', async () => {
|
||||
const drainHost = vi.fn(() => 'accepted' as const)
|
||||
const app = createRelayApp(config(), {
|
||||
@@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => {
|
||||
|
||||
expect((await post(app, 'deploy-token', request)).status).toBe(401)
|
||||
expect(
|
||||
(await post(app, 'rehome-token', {
|
||||
...request,
|
||||
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
|
||||
})).status
|
||||
(
|
||||
await post(app, 'rehome-token', {
|
||||
...request,
|
||||
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
|
||||
})
|
||||
).status
|
||||
).toBe(409)
|
||||
expect(drainHost).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('waits for an asynchronous drain operation before acknowledging', async () => {
|
||||
let grant!: (value: 'accepted') => void
|
||||
let entered!: () => void
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve
|
||||
})
|
||||
const drainHost = vi.fn(() => {
|
||||
entered()
|
||||
return new Promise<'accepted'>((resolve) => {
|
||||
grant = resolve
|
||||
})
|
||||
})
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {} as never,
|
||||
drain: vi.fn(),
|
||||
drainHost,
|
||||
cellIncarnation,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const pending = post(app, 'rehome-token', request)
|
||||
let acknowledged = false
|
||||
void pending.then(() => {
|
||||
acknowledged = true
|
||||
})
|
||||
await started
|
||||
await Promise.resolve()
|
||||
expect(acknowledged).toBe(false)
|
||||
grant('accepted')
|
||||
const response = await pending
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' })
|
||||
})
|
||||
|
||||
it('rejects a failed asynchronous drain instead of acknowledging it', async () => {
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {} as never,
|
||||
drain: vi.fn(),
|
||||
drainHost: async () => {
|
||||
throw new Error('activity_cell_not_authoritative')
|
||||
},
|
||||
cellIncarnation,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const response = await post(app, 'rehome-token', request)
|
||||
expect(response.status).toBe(409)
|
||||
expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' })
|
||||
})
|
||||
|
||||
it('rejects malformed identities before touching the session registry', async () => {
|
||||
const drainHost = vi.fn(() => 'accepted' as const)
|
||||
const app = createRelayApp(config(), {
|
||||
@@ -224,7 +354,7 @@ describe('regional rehome director controls', () => {
|
||||
v: 1,
|
||||
cellId: 'production-gce-c7',
|
||||
cellIncarnation,
|
||||
regionalRehomeProtocol: 1,
|
||||
regionalRehomeProtocol: 2,
|
||||
safety: {
|
||||
observedAt: 100,
|
||||
sqlFailures: 0,
|
||||
@@ -235,12 +365,7 @@ describe('regional rehome director controls', () => {
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
}
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/cell-rehome-status',
|
||||
'runtime-token',
|
||||
body
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
|
||||
@@ -266,18 +391,13 @@ describe('regional rehome director controls', () => {
|
||||
v: 1,
|
||||
cellId: 'production-gce-c7',
|
||||
cellIncarnation,
|
||||
regionalRehomeProtocol: 1,
|
||||
regionalRehomeProtocol: 2,
|
||||
safety: {
|
||||
...observability.regionalRehomeRuntimeSafety(),
|
||||
...emptyPostgresPoolPressureCounts()
|
||||
}
|
||||
}
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/cell-rehome-status',
|
||||
'runtime-token',
|
||||
body
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
|
||||
@@ -301,12 +421,14 @@ describe('regional rehome director controls', () => {
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
{ v: 1, action: 'inspect' }
|
||||
)).status).toBe(200)
|
||||
expect(
|
||||
(
|
||||
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})
|
||||
).status
|
||||
).toBe(200)
|
||||
const apply = {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
@@ -319,39 +441,35 @@ describe('regional rehome director controls', () => {
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
}
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
apply
|
||||
)).status).toBe(200)
|
||||
expect(
|
||||
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status
|
||||
).toBe(200)
|
||||
expect(applyRegionalRehomeControl).toHaveBeenCalledOnce()
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'monitor-token',
|
||||
{ v: 1, action: 'inspect' }
|
||||
)).status).toBe(200)
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'monitor-token',
|
||||
apply
|
||||
)).status).toBe(403)
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
|
||||
)).status).toBe(400)
|
||||
expect(
|
||||
(
|
||||
await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', {
|
||||
v: 1,
|
||||
action: 'inspect'
|
||||
})
|
||||
).status
|
||||
).toBe(200)
|
||||
expect(
|
||||
(await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status
|
||||
).toBe(403)
|
||||
expect(
|
||||
(
|
||||
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
|
||||
...apply,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
).status
|
||||
).toBe(400)
|
||||
// The per-host cooldown is part of the durable shape an operator must state.
|
||||
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
withoutCooldown
|
||||
)).status).toBe(400)
|
||||
expect(
|
||||
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown))
|
||||
.status
|
||||
).toBe(400)
|
||||
})
|
||||
|
||||
it('probes dedicated trust twice and returns only aggregate proof', async () => {
|
||||
@@ -382,12 +500,11 @@ describe('regional rehome director controls', () => {
|
||||
}) as typeof fetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: cellIncarnation
|
||||
})
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
const responseBody = await response.json()
|
||||
@@ -448,12 +565,11 @@ describe('regional rehome director controls', () => {
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c27',
|
||||
sourceCellIncarnation: cellIncarnation
|
||||
})
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({ proven: true })
|
||||
@@ -481,12 +597,11 @@ describe('regional rehome director controls', () => {
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c27',
|
||||
sourceCellIncarnation: cellIncarnation
|
||||
})
|
||||
|
||||
expect(response.status).toBe(409)
|
||||
expect(sourceFetch).not.toHaveBeenCalled()
|
||||
@@ -504,18 +619,17 @@ describe('regional rehome director controls', () => {
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: cellIncarnation
|
||||
}
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'monitor-token',
|
||||
body
|
||||
)).status).toBe(401)
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ ...body, unexpected: true }
|
||||
)).status).toBe(400)
|
||||
expect(
|
||||
(await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status
|
||||
).toBe(401)
|
||||
expect(
|
||||
(
|
||||
await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
|
||||
...body,
|
||||
unexpected: true
|
||||
})
|
||||
).status
|
||||
).toBe(400)
|
||||
})
|
||||
|
||||
it('fails closed when the source rejects the dedicated identity', async () => {
|
||||
@@ -529,9 +643,9 @@ describe('regional rehome director controls', () => {
|
||||
regionalRehomeProtocol: 1
|
||||
}
|
||||
})
|
||||
const sourceFetch = vi.fn<typeof fetch>().mockResolvedValue(
|
||||
Response.json({ error: 'invalid_token' }, { status: 401 })
|
||||
)
|
||||
const sourceFetch = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 }))
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: { cellDeploymentStatus } as never,
|
||||
@@ -540,12 +654,11 @@ describe('regional rehome director controls', () => {
|
||||
regionalRehomeFetch: sourceFetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
|
||||
v: 1,
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: cellIncarnation
|
||||
})
|
||||
|
||||
expect(response.status).toBe(409)
|
||||
expect(sourceFetch).toHaveBeenCalledOnce()
|
||||
|
||||
@@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
|
||||
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
|
||||
}
|
||||
await primary.query(
|
||||
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
|
||||
)
|
||||
@@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
}
|
||||
}
|
||||
|
||||
it('defaults to a closed correction cohort even with enabled durable control', async () => {
|
||||
const context = await fixture()
|
||||
const closed = new RelayAssignmentStore(primary, context.now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
expect(await cutover(closed, context.now())).toBeNull()
|
||||
const preview = await closed.previewRegionalRehomeEligibility({
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
expect(preview.cohortPercent).toBe(0)
|
||||
expect(preview.counts['outside-cohort']).toBe(1)
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('counts existing generic migrations against the optimization cap and preview', async () => {
|
||||
const context = await fixture()
|
||||
const safety = {
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
const before = await context.store.previewRegionalRehomeEligibility(safety)
|
||||
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
|
||||
for (let index = 0; index < 8; index++) {
|
||||
const identity = {
|
||||
userId: `pg-rehome-user-budget-${sequence}-${index}`,
|
||||
relayHostId: `budgethost${String(index).padStart(6, '0')}`
|
||||
}
|
||||
await context.store.assign(identity, undefined, 'us-central1')
|
||||
await context.store.startEvacuation(identity, context.target.id)
|
||||
}
|
||||
const preview = await context.store.previewRegionalRehomeEligibility(safety)
|
||||
expect(preview.openMigrations).toBe(8)
|
||||
expect(preview.availableMigrationSlots).toBe(0)
|
||||
expect(preview.counts['concurrent-migration-cap']).toBe(1)
|
||||
expect(await cutover(context.store, context.now())).toBeNull()
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('preview excludes request capacity exhaustion before a claim', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
|
||||
[context.target.id]
|
||||
)
|
||||
const preview = await context.store.previewRegionalRehomeEligibility({
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
expect(preview.counts['no-target-headroom']).toBe(1)
|
||||
expect(await cutover(context.store, context.now())).toBeNull()
|
||||
})
|
||||
|
||||
it('claims through ambient per-cell sql retry noise', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
@@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
[context.source.id, context.target.id]
|
||||
)
|
||||
|
||||
expect(await context.store.claimRegionalRehome()).not.toBeNull()
|
||||
expect(await cutover(context.store, context.now())).not.toBeNull()
|
||||
})
|
||||
|
||||
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
|
||||
const context = await fixture()
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'asia-east2',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'asia-east2',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
preferred_region: 'asia-east2',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
|
||||
@@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
targetRegion: 'us-central1'
|
||||
})
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
// The durable attempt row must accept the reverse direction too.
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
expect(await primary.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ cell_id: context.target.id }])
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}
|
||||
])
|
||||
expect(
|
||||
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
|
||||
context.identity.userId
|
||||
])
|
||||
).toEqual([{ cell_id: context.target.id }])
|
||||
})
|
||||
|
||||
it('leaves a host whose preference already matches its own region', async () => {
|
||||
const context = await fixture({ preferredRegion: 'us-central1' })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(cutover(context.store, context.now())).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
@@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
it('leaves a host whose preference is older than the configured max age', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
`UPDATE relay_assignment_region_preferences SET observed_at = ?
|
||||
`UPDATE relay_region_decisions SET observed_at = ?
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
context.now() - 24 * 60 * 60_000 - 1,
|
||||
context.identity.userId,
|
||||
context.identity.relayHostId
|
||||
]
|
||||
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(cutover(context.store, context.now())).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
@@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(cutover(context.store, context.now())).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true,
|
||||
@@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
|
||||
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
|
||||
)
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
|
||||
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
@@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
// where no later rehome could move it out again.
|
||||
const context = await fixture({ targetProtocol: 0 })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(cutover(context.store, context.now())).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
@@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
[context.target.id]
|
||||
)
|
||||
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
expect(await cutover(context.store, context.now())).toBeNull()
|
||||
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
|
||||
)).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }])
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('lets only one director claim a host', async () => {
|
||||
const context = await fixture()
|
||||
const claims = await Promise.all([
|
||||
context.store.claimRegionalRehome(),
|
||||
context.competingStore.claimRegionalRehome()
|
||||
cutover(context.store, context.now()),
|
||||
cutover(context.competingStore, context.now())
|
||||
])
|
||||
|
||||
expect(claims.filter(Boolean)).toHaveLength(1)
|
||||
expect(await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
|
||||
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
|
||||
WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ count: '1' }])
|
||||
expect(await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '1' }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ count: '1' }])
|
||||
})
|
||||
|
||||
it('serializes an enable with a budget-exhausting failure without retries', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
|
||||
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
|
||||
const locked = Promise.withResolvers<void>()
|
||||
const release = Promise.withResolvers<void>()
|
||||
const primaryTransaction = primary.transaction.bind(primary)
|
||||
const secondaryTransaction = secondary.transaction.bind(secondary)
|
||||
let enableTransactions = 0
|
||||
let failureTransactions = 0
|
||||
let enablePid = 0
|
||||
let failurePid = 0
|
||||
const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) =>
|
||||
primaryTransaction(async (transaction) => {
|
||||
enableTransactions++
|
||||
enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
|
||||
return await operation({
|
||||
dialect: 'postgres',
|
||||
query: transaction.query.bind(transaction),
|
||||
queryLocked: async (sql, params, lockOptions) => {
|
||||
const rows = await transaction.queryLocked(sql, params, lockOptions)
|
||||
if (sql.includes('FROM relay_region_rehome_control')) {
|
||||
locked.resolve()
|
||||
await release.promise
|
||||
}
|
||||
return rows
|
||||
},
|
||||
transaction: transaction.transaction.bind(transaction),
|
||||
close: transaction.close.bind(transaction)
|
||||
})
|
||||
}, options)
|
||||
)
|
||||
const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) =>
|
||||
secondaryTransaction(async (transaction) => {
|
||||
failureTransactions++
|
||||
failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
|
||||
return await operation(transaction)
|
||||
}, options)
|
||||
)
|
||||
const enable = context.store.applyRegionalRehomeControl({
|
||||
expectedGeneration: 1,
|
||||
enabled: true,
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
let failure: Promise<void> | undefined
|
||||
let outcomes: PromiseSettledResult<unknown>[] = []
|
||||
try {
|
||||
await Promise.race([
|
||||
locked.promise,
|
||||
enable.then(() => {
|
||||
throw new Error('enable completed before the control lock')
|
||||
})
|
||||
])
|
||||
failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
|
||||
// Observe the actual PostgreSQL wait before letting enable acquire the worker row.
|
||||
await vi.waitFor(async () => {
|
||||
expect(failurePid).not.toBe(0)
|
||||
const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid])
|
||||
expect(rows[0]!.blockers).toContain(enablePid)
|
||||
}, { interval: 10, timeout: 800 })
|
||||
} finally {
|
||||
release.resolve()
|
||||
outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])])
|
||||
enableSpy.mockRestore()
|
||||
failureSpy.mockRestore()
|
||||
}
|
||||
expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled'])
|
||||
expect({ enableTransactions, failureTransactions }).toEqual({
|
||||
enableTransactions: 1,
|
||||
failureTransactions: 1
|
||||
})
|
||||
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
|
||||
generation: 2,
|
||||
enabled: true
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state`
|
||||
)).toEqual([{ consecutive_failures: '1', paused_until: '0' }])
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '1' }])
|
||||
})
|
||||
|
||||
it('increments the disable generation once across competing directors', async () => {
|
||||
@@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('records one receipt across competing directors', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const receipts = await Promise.all([
|
||||
context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'),
|
||||
context.competingStore.recordRegionalRehomeDrainReceipt(
|
||||
attempt!.attemptId,
|
||||
'accepted'
|
||||
)
|
||||
])
|
||||
|
||||
expect(receipts.sort()).toEqual([false, true])
|
||||
expect(await primary.query(
|
||||
`SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
|
||||
[attempt!.attemptId]
|
||||
)).toEqual([{ drain_outcome: 'accepted' }])
|
||||
})
|
||||
|
||||
it('rechecks a preference changed while the assignment row is locked', async () => {
|
||||
const context = await fixture()
|
||||
let unlock!: () => void
|
||||
@@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
await unlockPromise
|
||||
})
|
||||
await lockedPromise
|
||||
const claim = context.store.claimRegionalRehome()
|
||||
const claim = cutover(context.store, context.now())
|
||||
await primary.query(
|
||||
`UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1',
|
||||
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
|
||||
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[context.now(), context.identity.userId, context.identity.relayHostId]
|
||||
)
|
||||
@@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
await held
|
||||
|
||||
await expect(claim).resolves.toBeNull()
|
||||
expect(await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ count: '0' }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '0' }])
|
||||
})
|
||||
|
||||
it('rechecks fleet safety under locks before mutating a candidate', async () => {
|
||||
const context = await fixture()
|
||||
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
|
||||
expect(request).toBeDefined()
|
||||
let unlock!: () => void
|
||||
let locked!: () => void
|
||||
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
|
||||
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
|
||||
const held = secondary.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`,
|
||||
[context.target.id]
|
||||
)
|
||||
await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [
|
||||
context.target.id
|
||||
])
|
||||
await transaction.query(
|
||||
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
|
||||
[context.target.id]
|
||||
@@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
await unlockPromise
|
||||
})
|
||||
await lockedPromise
|
||||
const claim = context.store.claimRegionalRehome()
|
||||
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
|
||||
unlock()
|
||||
await held
|
||||
|
||||
await expect(claim).resolves.toBeNull()
|
||||
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
|
||||
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
|
||||
generation: 2,
|
||||
enabled: false
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ count: '0' }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '0' }])
|
||||
})
|
||||
|
||||
it('pauses when one required cell exceeds the reconnect limit', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`,
|
||||
[REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id]
|
||||
)
|
||||
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
|
||||
expect(request).toBeDefined()
|
||||
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
|
||||
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
|
||||
context.source.id
|
||||
])
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(
|
||||
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
|
||||
).resolves.toEqual({ outcome: 'deferred' })
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 2,
|
||||
enabled: false
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ count: '0' }])
|
||||
})
|
||||
|
||||
it('does not retry a drain against a replacement source incarnation', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
context.advance(31_000)
|
||||
await heartbeat(
|
||||
context.store,
|
||||
context.source,
|
||||
'33333333-3333-4333-8333-333333333333',
|
||||
1,
|
||||
context.now()
|
||||
)
|
||||
|
||||
await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull()
|
||||
expect(await primary.query(
|
||||
`SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
|
||||
[attempt!.attemptId]
|
||||
)).toEqual([{ send_attempts: '1' }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '0' }])
|
||||
})
|
||||
|
||||
it('makes concurrent completion and expiry cleanup idempotent', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
const targetControl = await context.store.activateControl(context.identity, {
|
||||
cellId: context.target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch,
|
||||
@@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.competingStore.abortExpiredRegionalRehomes()
|
||||
])
|
||||
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
|
||||
expect(await primary.query(
|
||||
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
|
||||
FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ completed: true, aborted: false }])
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ completed: true, aborted: false }])
|
||||
})
|
||||
|
||||
it('will not complete against a replacement target incarnation', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
await context.store.activateControl(context.identity, {
|
||||
cellId: context.target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch,
|
||||
@@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
)
|
||||
|
||||
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
|
||||
expect(await primary.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
})
|
||||
|
||||
it('does not roll an unregistered target back to a stale regional source', async () => {
|
||||
const context = await fixture()
|
||||
await context.store.claimRegionalRehome()
|
||||
await cutover(context.store, context.now())
|
||||
context.advance(6 * 60_000)
|
||||
await heartbeat(
|
||||
context.store,
|
||||
@@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
|
||||
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
|
||||
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
expect(await primary.query(
|
||||
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
|
||||
})
|
||||
|
||||
it('completes after the drained host re-resolves through the director', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
// The drain recovery lands while both controls are still live.
|
||||
await context.store.assign(context.identity, 'asia-east2')
|
||||
expect(await controlAccounting(context.identity)).toEqual({
|
||||
@@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
await context.store.releaseActivity(context.identity, context.sourceControl)
|
||||
|
||||
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
|
||||
expect(await primary.query(
|
||||
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
|
||||
expect(
|
||||
await primary.query(
|
||||
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
|
||||
WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ completed: true }])
|
||||
[context.identity.userId]
|
||||
)
|
||||
).toEqual([{ completed: true }])
|
||||
expect(await controlAccounting(context.identity)).toEqual({
|
||||
reservedControls: 1,
|
||||
controlLeases: 1
|
||||
@@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
|
||||
it('repairs a skewed control counter before completing the rehome', async () => {
|
||||
const context = await fixture()
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const attempt = await cutover(context.store, context.now())
|
||||
await context.store.activateControl(context.identity, {
|
||||
cellId: context.target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch,
|
||||
@@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
await context.store.releaseActivity(context.identity, context.sourceControl)
|
||||
// Damage already written by a pre-fix sticky grant.
|
||||
await primary.query(
|
||||
`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)
|
||||
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
|
||||
context.identity.userId
|
||||
])
|
||||
|
||||
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
|
||||
expect(await controlAccounting(context.identity)).toEqual({
|
||||
@@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
})
|
||||
|
||||
async function cutover(store: RelayAssignmentStore, now: number) {
|
||||
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
|
||||
if (!request) return null
|
||||
const result = await store.commitIdleRegionalRehome(request, safety(now))
|
||||
if (result.outcome !== 'committed') return null
|
||||
const [attempt] = await primary.query(
|
||||
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
|
||||
[request.attemptId]
|
||||
)
|
||||
return {
|
||||
...request,
|
||||
preferredRegion: String(attempt!.preferred_region),
|
||||
assignmentEpoch: Number(attempt!.assignment_epoch)
|
||||
}
|
||||
}
|
||||
|
||||
async function attemptAndMigrationCounts(identity: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
@@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
await store.reconcileCells([source, target])
|
||||
await heartbeat(
|
||||
store,
|
||||
source,
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
1,
|
||||
900_000
|
||||
)
|
||||
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
|
||||
await heartbeat(
|
||||
store,
|
||||
target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
options.targetProtocol ?? 1,
|
||||
options.targetProtocol ?? 3,
|
||||
900_000
|
||||
)
|
||||
const identity = {
|
||||
@@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
const sourceControl = await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
generation: 1,
|
||||
idleRegionalRehome: true,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
await store.assign(identity, preferredRegion)
|
||||
const issued = await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: issued.window!.generation,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: {
|
||||
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
|
||||
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
|
||||
}
|
||||
},
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
return {
|
||||
preferredRegion,
|
||||
store,
|
||||
@@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
|
||||
const storeOptions = {
|
||||
regionalRehomeCohortPercent: 100,
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
}
|
||||
@@ -816,3 +831,15 @@ async function heartbeat(
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function safety(now: number) {
|
||||
return {
|
||||
observedAt: now,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ async function setup() {
|
||||
let clock = 1_000_000
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayAssignmentStore(database, () => clock, {
|
||||
regionalRehomeCohortPercent: 100,
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
@@ -83,11 +84,40 @@ async function setup() {
|
||||
await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
generation: 1,
|
||||
idleRegionalRehome: true,
|
||||
cellIncarnation: incarnation(1)
|
||||
})
|
||||
await store.assign(identity, 'asia-east2')
|
||||
const { window } = await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
expect(window).toBeDefined()
|
||||
await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: window!.generation,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 180, 'asia-east2': 40 }
|
||||
},
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
}
|
||||
return { database, store, beat, activatePreferredSource }
|
||||
const safety = () => ({
|
||||
observedAt: clock,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
return { database, store, beat, activatePreferredSource, safety }
|
||||
}
|
||||
|
||||
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
|
||||
@@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
|
||||
describe('regional rehome target selection', () => {
|
||||
it('never selects a target without connection headroom, even at lowest load', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 1, {
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
// Lowest load but the connection hard cap is exhausted.
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 1, {
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(lowLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('falls to the next clean target when the load winner goes unclean', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 1, {
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 1, {
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(highLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -11,148 +11,12 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
|
||||
describe('regional rehome worker', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => {
|
||||
let now = 0
|
||||
const attempt = {
|
||||
attemptId: '11111111-1111-4111-8111-111111111111',
|
||||
userId: 'private-user',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
preferredRegion: 'asia-east2',
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellUrl: 'https://c7.relay.example.test',
|
||||
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
|
||||
targetCellId: 'production-gce-c27',
|
||||
targetCellIncarnation: '33333333-3333-4333-8333-333333333333',
|
||||
previousEpoch: 7,
|
||||
assignmentEpoch: 8,
|
||||
drainGraceMs: 60_000,
|
||||
sendAttempts: 1
|
||||
}
|
||||
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
|
||||
const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true)
|
||||
const assignments = {
|
||||
claimRegionalRehome,
|
||||
recordRegionalRehomeDrainReceipt
|
||||
} as unknown as RelayAssignmentStore
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = []
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
now: () => now,
|
||||
safetySnapshot: () => safety(now),
|
||||
intervalMs: 60_000,
|
||||
identityToken: async (audience) => {
|
||||
expect(audience).toBe('https://relay.example.test/v1/admin/host-drain')
|
||||
return 'secret-token'
|
||||
},
|
||||
fetch: (async (url, init) => {
|
||||
requests.push({ url: String(url), init })
|
||||
return Response.json({ v: 1, outcome: 'accepted' })
|
||||
}) as typeof fetch
|
||||
})!
|
||||
await settleWorker()
|
||||
now = 1_000
|
||||
await worker.run()
|
||||
worker.stop()
|
||||
|
||||
expect(requests).toHaveLength(1)
|
||||
expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain')
|
||||
expect(requests[0]!.url).not.toContain('secret-token')
|
||||
expect(requests[0]!.init?.headers).toMatchObject({
|
||||
authorization: 'Bearer secret-token'
|
||||
})
|
||||
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
|
||||
v: 1,
|
||||
attemptId: '11111111-1111-4111-8111-111111111111',
|
||||
userId: 'private-user',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'production-gce-c7',
|
||||
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
|
||||
sourceAssignmentEpoch: 7,
|
||||
graceMs: 60_000
|
||||
})
|
||||
expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith(
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'accepted'
|
||||
)
|
||||
const logs = warn.mock.calls.map((call) => String(call[0])).join('\n')
|
||||
expect(logs).not.toContain('private-user')
|
||||
expect(logs).not.toContain('abcdefghijklmnop')
|
||||
})
|
||||
|
||||
it('fails closed before the observation gate and records bounded dispatch failures', async () => {
|
||||
let now = 0
|
||||
const attempt = {
|
||||
attemptId: '11111111-1111-4111-8111-111111111111',
|
||||
userId: 'private-user',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'source',
|
||||
sourceCellUrl: 'https://source.example.test',
|
||||
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
|
||||
targetCellId: 'target',
|
||||
previousEpoch: 1,
|
||||
assignmentEpoch: 2,
|
||||
drainGraceMs: 60_000,
|
||||
sendAttempts: 1
|
||||
}
|
||||
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
|
||||
const assignments = {
|
||||
claimRegionalRehome,
|
||||
recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined)
|
||||
} as unknown as RelayAssignmentStore
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
now: () => now,
|
||||
safetySnapshot: () => safety(now),
|
||||
intervalMs: 60_000,
|
||||
identityToken: async () => {
|
||||
throw new Error('token unavailable')
|
||||
}
|
||||
})!
|
||||
await settleWorker()
|
||||
claimRegionalRehome.mockClear()
|
||||
now = 100
|
||||
await worker.run()
|
||||
worker.stop()
|
||||
expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith(
|
||||
'11111111-1111-4111-8111-111111111111'
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps a failed poll out of the durable dispatch-failure budget', async () => {
|
||||
let now = 0
|
||||
const claimRegionalRehome = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockRejectedValue(new Error('Connection terminated due to connection timeout'))
|
||||
const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined)
|
||||
const assignments = {
|
||||
claimRegionalRehome,
|
||||
recordRegionalRehomeDispatchFailure
|
||||
} as unknown as RelayAssignmentStore
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
now: () => now,
|
||||
safetySnapshot: () => safety(now),
|
||||
intervalMs: 60_000
|
||||
})!
|
||||
await settleWorker()
|
||||
now = 1_000
|
||||
await expect(worker.run()).resolves.toBeUndefined()
|
||||
worker.stop()
|
||||
|
||||
// The poll never claimed an attempt, so nothing was drained and nothing may
|
||||
// be charged to the budget that latches the durable control off.
|
||||
expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
|
||||
expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([
|
||||
'orca_relay_regional_rehome_poll_failed'
|
||||
])
|
||||
})
|
||||
|
||||
it('passes unsafe process telemetry to the durable claim gate', async () => {
|
||||
let now = 0
|
||||
let sqlFailures = 0
|
||||
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
|
||||
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
|
||||
const assignments = {
|
||||
claimRegionalRehome
|
||||
selectIdleRegionalRehomeCandidates
|
||||
} as unknown as RelayAssignmentStore
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
now: () => now,
|
||||
@@ -160,46 +24,40 @@ describe('regional rehome worker', () => {
|
||||
intervalMs: 60_000
|
||||
})!
|
||||
await settleWorker()
|
||||
claimRegionalRehome.mockClear()
|
||||
selectIdleRegionalRehomeCandidates.mockClear()
|
||||
now = 100
|
||||
sqlFailures = 1
|
||||
await worker.run()
|
||||
worker.stop()
|
||||
|
||||
expect(claimRegionalRehome).toHaveBeenCalledWith(
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
|
||||
)
|
||||
})
|
||||
|
||||
it('starts inert on directors so durable control can enable without a restart', async () => {
|
||||
let now = 0
|
||||
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
|
||||
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
|
||||
const assignments = {
|
||||
claimRegionalRehome
|
||||
selectIdleRegionalRehomeCandidates
|
||||
} as unknown as RelayAssignmentStore
|
||||
const worker = startRegionalRehomeWorker(
|
||||
config(),
|
||||
assignments,
|
||||
{
|
||||
now: () => now,
|
||||
safetySnapshot: () => safety(now),
|
||||
intervalMs: 60_000
|
||||
}
|
||||
)
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
now: () => now,
|
||||
safetySnapshot: () => safety(now),
|
||||
intervalMs: 60_000
|
||||
})
|
||||
expect(worker).not.toBeNull()
|
||||
await settleWorker()
|
||||
claimRegionalRehome.mockClear()
|
||||
selectIdleRegionalRehomeCandidates.mockClear()
|
||||
now = 100
|
||||
await worker!.run()
|
||||
worker!.stop()
|
||||
expect(claimRegionalRehome).toHaveBeenCalledOnce()
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
|
||||
|
||||
expect(
|
||||
startRegionalRehomeWorker(
|
||||
config({ role: 'cell' }),
|
||||
{} as RelayAssignmentStore,
|
||||
{ safetySnapshot: () => safety(1) }
|
||||
)
|
||||
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
|
||||
safetySnapshot: () => safety(1)
|
||||
})
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
@@ -208,16 +66,20 @@ describe('regional rehome worker', () => {
|
||||
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
|
||||
const processSafety = { ...safety(100), reconnects: limit * 10 }
|
||||
const fleetSafety = { ...safety(100), reconnects: limit }
|
||||
expect(regionalRehomeSafetyFailure(
|
||||
combineRegionalRehomeSafety(processSafety, fleetSafety),
|
||||
100,
|
||||
cells
|
||||
)).toBeNull()
|
||||
expect(regionalRehomeSafetyFailure(
|
||||
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
|
||||
100,
|
||||
cells
|
||||
)).toBe('elevated_reconnects')
|
||||
expect(
|
||||
regionalRehomeSafetyFailure(
|
||||
combineRegionalRehomeSafety(processSafety, fleetSafety),
|
||||
100,
|
||||
cells
|
||||
)
|
||||
).toBeNull()
|
||||
expect(
|
||||
regionalRehomeSafetyFailure(
|
||||
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
|
||||
100,
|
||||
cells
|
||||
)
|
||||
).toBe('elevated_reconnects')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { z } from 'zod'
|
||||
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
|
||||
@@ -20,13 +20,6 @@ export type RegionalRehomeWorker = {
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
const RegionalHostDrainResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected'])
|
||||
})
|
||||
.strict()
|
||||
|
||||
export function startRegionalRehomeWorker(
|
||||
config: RelayConfig,
|
||||
assignments: RelayAssignmentStore,
|
||||
@@ -42,7 +35,6 @@ export function startRegionalRehomeWorker(
|
||||
}
|
||||
const audience = config.rehomeAudience
|
||||
const safetySnapshot = options.safetySnapshot
|
||||
const now = options.now ?? Date.now
|
||||
const fetchImpl = options.fetch ?? fetch
|
||||
const tokenProvider =
|
||||
options.identityToken ??
|
||||
@@ -52,64 +44,50 @@ export function startRegionalRehomeWorker(
|
||||
const run = async (): Promise<void> => {
|
||||
if (stopped || inFlight) return
|
||||
inFlight = true
|
||||
let attemptId: string | null = null
|
||||
try {
|
||||
const processSafety = safetySnapshot()
|
||||
const attempt = await assignments.claimRegionalRehome(processSafety)
|
||||
if (!attempt) return
|
||||
attemptId = attempt.attemptId
|
||||
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
|
||||
if (candidates.length === 0) return
|
||||
const token = await tokenProvider(audience)
|
||||
const response = await fetchImpl(
|
||||
new URL('/v1/admin/host-drain', attempt.sourceCellUrl),
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
attemptId: attempt.attemptId,
|
||||
userId: attempt.userId,
|
||||
relayHostId: attempt.relayHostId,
|
||||
sourceCellId: attempt.sourceCellId,
|
||||
sourceCellIncarnation: attempt.sourceCellIncarnation,
|
||||
sourceAssignmentEpoch: attempt.previousEpoch,
|
||||
graceMs: attempt.drainGraceMs
|
||||
}),
|
||||
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
|
||||
for (const candidate of candidates) {
|
||||
if (stopped) return
|
||||
const { sourceCellUrl, ...request } = candidate
|
||||
try {
|
||||
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
...request,
|
||||
cohortPercent: config.regionCorrectionCohortPercent ?? 0,
|
||||
directorSafety: safetySnapshot()
|
||||
}),
|
||||
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
|
||||
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
|
||||
if (body.outcome === 'committed') {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_idle_rehome_committed',
|
||||
sourceCellId: candidate.sourceCellId,
|
||||
targetCellId: candidate.targetCellId
|
||||
})
|
||||
)
|
||||
return
|
||||
}
|
||||
} catch (error) {
|
||||
// The source may have committed; its durable outcome owns recovery.
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_idle_rehome_request_failed',
|
||||
reason: error instanceof Error ? error.message : 'unknown'
|
||||
})
|
||||
)
|
||||
}
|
||||
)
|
||||
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
|
||||
const body = RegionalHostDrainResponseSchema.safeParse(await response.json())
|
||||
if (!body.success) throw new Error('regional_rehome_source_invalid_response')
|
||||
await assignments.recordRegionalRehomeDrainReceipt(
|
||||
attempt.attemptId,
|
||||
body.data.outcome
|
||||
)
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_regional_rehome_dispatched',
|
||||
sourceCellId: attempt.sourceCellId,
|
||||
targetCellId: attempt.targetCellId,
|
||||
outcome: body.data.outcome,
|
||||
sendAttempts: attempt.sendAttempts
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
// Only a claimed attempt was drained. A poll that failed before the claim
|
||||
// - a pool timeout on the once-a-second control read - dispatched nothing,
|
||||
// so it must not spend the budget that latches the durable control off.
|
||||
if (attemptId) {
|
||||
await assignments
|
||||
.recordRegionalRehomeDispatchFailure(attemptId)
|
||||
.catch(() => undefined)
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: attemptId
|
||||
? 'orca_relay_regional_rehome_dispatch_failed'
|
||||
: 'orca_relay_regional_rehome_poll_failed',
|
||||
event: 'orca_relay_regional_rehome_poll_failed',
|
||||
reason: error instanceof Error ? error.message : 'unknown'
|
||||
})
|
||||
)
|
||||
|
||||
@@ -40,6 +40,7 @@ describe('Relay region API', () => {
|
||||
)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.clone().json()).not.toHaveProperty('regionCorrection')
|
||||
expect(assign).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: 'asiahost00000001' },
|
||||
'asia-east2',
|
||||
@@ -83,6 +84,160 @@ describe('Relay region API', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves the cold-start hint and binds a negotiated window after placement', async () => {
|
||||
const assignment = {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
cellId: 'asia-c1',
|
||||
cellUrl: 'https://asia-c1.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
assignmentEpoch: 7,
|
||||
leaseExpiresAt: Date.now() + 300_000
|
||||
}
|
||||
const assign = vi.fn(async () => assignment)
|
||||
const window = {
|
||||
generation: 2,
|
||||
expiresAt: Date.now() + 86_400_000,
|
||||
assignmentEpoch: 7,
|
||||
incumbentRegion: 'asia-east2',
|
||||
policyVersion: 1
|
||||
}
|
||||
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window }))
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, exchangeRegionCorrection } as never,
|
||||
drain: vi.fn(),
|
||||
ready: async () => true
|
||||
})
|
||||
const regionCorrection = { v: 1, action: 'issue-window' }
|
||||
const response = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('abcdefghijklmnop', {
|
||||
preferredRegion: 'asia-east2',
|
||||
regionCorrection
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(assign).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
|
||||
'asia-east2',
|
||||
'asia-east2'
|
||||
)
|
||||
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
|
||||
regionCorrection,
|
||||
7
|
||||
)
|
||||
expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({
|
||||
v: 1,
|
||||
window
|
||||
})
|
||||
})
|
||||
|
||||
it('returns successful placement when optional window storage is unavailable', async () => {
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
assign: async () => ({
|
||||
cellId: 'asia-c1',
|
||||
region: 'asia-east2',
|
||||
cellUrl: 'https://asia-c1.relay.example.test',
|
||||
assignmentEpoch: 7
|
||||
}),
|
||||
exchangeRegionCorrection: async () => {
|
||||
throw new Error('database unavailable')
|
||||
}
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: async () => true
|
||||
})
|
||||
const response = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('abcdefghijklmnop', {
|
||||
preferredRegion: 'asia-east2',
|
||||
regionCorrection: { v: 1, action: 'issue-window' }
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({
|
||||
cellUrl: 'https://asia-c1.relay.example.test',
|
||||
assignmentEpoch: 7
|
||||
})
|
||||
})
|
||||
|
||||
it('does not place or write a legacy hint when reporting migration evidence', async () => {
|
||||
const current = {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
cellId: 'asia-c1',
|
||||
cellUrl: 'https://asia-c1.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
assignmentEpoch: 7,
|
||||
leaseExpiresAt: Date.now() + 300_000
|
||||
}
|
||||
const assign = vi.fn()
|
||||
const resolve = vi.fn(async () => current)
|
||||
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' }))
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve, exchangeRegionCorrection } as never,
|
||||
drain: vi.fn(),
|
||||
ready: async () => true
|
||||
})
|
||||
const regionCorrection = {
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: 2,
|
||||
assignmentEpoch: 7,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 40, 'asia-east2': 180 }
|
||||
}
|
||||
const response = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('abcdefghijklmnop', {
|
||||
preferredRegion: 'us-central1',
|
||||
regionCorrection
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
|
||||
regionCorrection,
|
||||
7
|
||||
)
|
||||
expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7)
|
||||
})
|
||||
|
||||
it('does not manufacture an assignment for a report whose assignment disappeared', async () => {
|
||||
const assign = vi.fn()
|
||||
const exchangeRegionCorrection = vi.fn()
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never,
|
||||
drain: vi.fn(),
|
||||
ready: async () => true
|
||||
})
|
||||
const response = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('abcdefghijklmnop', {
|
||||
regionCorrection: {
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: 2,
|
||||
assignmentEpoch: 7,
|
||||
policyVersion: 1,
|
||||
outcome: 'inconclusive',
|
||||
reason: 'timeout'
|
||||
}
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(409)
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
expect(exchangeRegionCorrection).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('exposes only the store-provided healthy catalog from directors', async () => {
|
||||
const regionCatalog = vi.fn(async () => [
|
||||
{ region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] }
|
||||
|
||||
@@ -20,14 +20,12 @@ import { createRelayApp } from './app.js'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { RelayCredentialStore } from './credential-store.js'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js'
|
||||
import { HostSessionRegistry } from './host-session-registry.js'
|
||||
import { observeRelayDatabase } from './observed-relay-database.js'
|
||||
import { RelayObservability } from './relay-observability.js'
|
||||
import {
|
||||
RelayConnectionLedger,
|
||||
type RelayConnectionUpgrade
|
||||
} from './relay-connection-ledger.js'
|
||||
import { combineRegionalRehomeSafety } from './regional-rehome-safety.js'
|
||||
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js'
|
||||
import { createRelayReadiness } from './relay-readiness.js'
|
||||
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
|
||||
import { closeRelayWebSocket } from './relay-websocket-close.js'
|
||||
@@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void {
|
||||
|
||||
function admissionSource(request: IncomingMessage): string {
|
||||
const forwarded = request.headers['x-forwarded-for']
|
||||
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '')
|
||||
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
|
||||
.split(',')
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
@@ -112,6 +110,7 @@ export function createRelayServer(
|
||||
const store = new RelayCredentialStore(observedDatabase, options.now)
|
||||
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
|
||||
requireLiveCells: config.role === 'director',
|
||||
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
|
||||
recordControlRenewal: (durationMs, outcome) =>
|
||||
observability.recordControlRenewal?.(durationMs, outcome)
|
||||
})
|
||||
@@ -127,17 +126,35 @@ export function createRelayServer(
|
||||
queuedBytes,
|
||||
observability,
|
||||
options.now,
|
||||
options.random
|
||||
options.random,
|
||||
cellIncarnation
|
||||
)
|
||||
const app = createRelayApp(config, {
|
||||
store,
|
||||
assignments,
|
||||
drain: (graceMs) => sessions.drain(graceMs),
|
||||
drainHost: (input) => sessions.drainHost(input),
|
||||
idleRehome: (input) => {
|
||||
const now = (options.now ?? Date.now)()
|
||||
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
|
||||
return Promise.resolve({ outcome: 'deferred' })
|
||||
}
|
||||
return sessions.idleRehome(input,
|
||||
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
|
||||
input.directorSafety,
|
||||
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
|
||||
), input.cohortPercent),
|
||||
() => assignments.reconcileIdleRegionalRehome(input)
|
||||
)
|
||||
},
|
||||
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
|
||||
cellIncarnation,
|
||||
isDraining: () => sessions.isDraining(),
|
||||
runtimeCounts: () => runtimeCounts(),
|
||||
regionalRehomeSafetySnapshot: () => ({
|
||||
...observability.regionalRehomeRuntimeSafety(),
|
||||
...readRelayDatabasePoolPressure(database)
|
||||
}),
|
||||
ready,
|
||||
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
|
||||
recordAssignmentRejectionReason: (lane, reason) =>
|
||||
@@ -339,7 +356,7 @@ export function createRelayServer(
|
||||
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
|
||||
// Released combined-service invites gain their first durable cell assignment here.
|
||||
const assignment = identity
|
||||
? (await assignments.resolve(identity)) ?? (await assignments.assign(identity))
|
||||
? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity)))
|
||||
: null
|
||||
if (!invite || !assignment) {
|
||||
phoneAdmission?.hostData.release()
|
||||
|
||||
@@ -35,7 +35,7 @@ describe('sweep schedule jitter', () => {
|
||||
rehomeAudience: 'https://rehome.example.test',
|
||||
rehomeDirectorServiceAccount: 'rehome@example.test'
|
||||
} as never,
|
||||
{ claimRegionalRehome: async () => null } as never,
|
||||
{ selectIdleRegionalRehomeCandidates: async () => [] } as never,
|
||||
{ random: () => 0.5, safetySnapshot: () => ({}) as never }
|
||||
)
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
|
||||
|
||||
```text
|
||||
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
|
||||
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
|
||||
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
|
||||
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
|
||||
```
|
||||
@@ -0,0 +1,146 @@
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
Base6432ByteSchema,
|
||||
Base64Raw24ByteSchema,
|
||||
Base64Url32ByteSchema,
|
||||
EpochMsSchema,
|
||||
GenerationSchema,
|
||||
OpaqueIdSchema,
|
||||
PositiveDurationMsSchema,
|
||||
RelayHostIdSchema
|
||||
} from './wire-scalars.js'
|
||||
|
||||
const AppVersionSchema = z.string().min(1).max(128)
|
||||
const BoundedCiphertextSchema = z
|
||||
.string()
|
||||
.min(1)
|
||||
.max(16 * 1024)
|
||||
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
|
||||
const ConnectionKindSchema = z.enum(['invite', 'resume'])
|
||||
|
||||
export const HostHelloSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
relayHostId: RelayHostIdSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
hostPublicKeyB64: Base6432ByteSchema,
|
||||
appVersion: AppVersionSchema,
|
||||
previousGeneration: GenerationSchema.optional(),
|
||||
controlResumeSecret: Base64Url32ByteSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostChallengeSchema = z
|
||||
.object({
|
||||
challengeId: OpaqueIdSchema,
|
||||
relayEphemeralPublicKeyB64: Base6432ByteSchema,
|
||||
nonceB64: Base64Raw24ByteSchema,
|
||||
ciphertextB64: BoundedCiphertextSchema,
|
||||
expiresAt: EpochMsSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostChallengeAckSchema = z
|
||||
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
|
||||
.strict()
|
||||
|
||||
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
|
||||
// is strict, so a new hello key is refused by every already-deployed cell.
|
||||
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
|
||||
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
|
||||
// not advertise this parses those entries strictly and would drop the whole ack.
|
||||
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
|
||||
|
||||
export function parseRelayHostCapabilities(
|
||||
header: string | string[] | undefined
|
||||
): ReadonlySet<string> {
|
||||
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
|
||||
return new Set(
|
||||
raw
|
||||
.split(',')
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length > 0 && token.length <= 64)
|
||||
.slice(0, 16)
|
||||
)
|
||||
}
|
||||
|
||||
// kind/relayDeviceId are optional so an entry stays readable by a host that
|
||||
// predates them; the cell only emits them to a host that advertised support.
|
||||
const PendingConnectionSchema = z
|
||||
.object({
|
||||
connId: OpaqueIdSchema,
|
||||
connTicket: Base64Url32ByteSchema,
|
||||
kind: ConnectionKindSchema.optional(),
|
||||
relayDeviceId: OpaqueIdSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostHelloAckSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
generation: GenerationSchema,
|
||||
controlResumeSecret: Base64Url32ByteSchema,
|
||||
leaseExpiresAt: EpochMsSchema,
|
||||
activeConnIds: z.array(OpaqueIdSchema).max(8),
|
||||
pendingConns: z.array(PendingConnectionSchema).max(8)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const ConnectionOpenSchema = z
|
||||
.object({
|
||||
connId: OpaqueIdSchema,
|
||||
connTicket: Base64Url32ByteSchema,
|
||||
kind: ConnectionKindSchema,
|
||||
relayDeviceId: OpaqueIdSchema,
|
||||
attachDeadlineMs: PositiveDurationMsSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostDataAuthSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
connTicket: Base64Url32ByteSchema,
|
||||
generation: GenerationSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const InviteCreateSchema = z
|
||||
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
|
||||
.strict()
|
||||
|
||||
export const InviteCreatedSchema = z
|
||||
.object({
|
||||
reqId: OpaqueIdSchema,
|
||||
inviteToken: Base64Url32ByteSchema,
|
||||
expiresAt: EpochMsSchema,
|
||||
maxAttempts: z.number().int().positive().max(16)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const DeviceRevokeSchema = z
|
||||
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
|
||||
.strict()
|
||||
|
||||
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
|
||||
|
||||
export const DrainSchema = z
|
||||
.object({
|
||||
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
|
||||
recovery: z.literal('resolve-director')
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
|
||||
|
||||
export type HostHello = z.infer<typeof HostHelloSchema>
|
||||
export type HostChallenge = z.infer<typeof HostChallengeSchema>
|
||||
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
|
||||
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
|
||||
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
|
||||
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
|
||||
export type InviteCreate = z.infer<typeof InviteCreateSchema>
|
||||
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
|
||||
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
|
||||
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
|
||||
export type Drain = z.infer<typeof DrainSchema>
|
||||
export type Heartbeat = z.infer<typeof HeartbeatSchema>
|
||||
@@ -0,0 +1,75 @@
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
Base64Url32ByteSchema,
|
||||
CanonicalHttpsOriginSchema,
|
||||
EpochMsSchema,
|
||||
GenerationSchema,
|
||||
RelayHostIdSchema
|
||||
} from './wire-scalars.js'
|
||||
import { RelayRegionSchema } from './relay-regions.js'
|
||||
|
||||
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
|
||||
|
||||
export const AssignmentRequestSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
relayHostId: RelayHostIdSchema,
|
||||
// Client-declared reconnection; the director verifies it against the
|
||||
// durable assignment before granting fast-lane admission.
|
||||
reconnect: z.boolean().optional(),
|
||||
preferredRegion: RelayRegionSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const AssignmentResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
cellUrl: CanonicalHttpsOriginSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
lease: SignedAssignmentLeaseSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const ResolveRequestSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
relayHostId: RelayHostIdSchema,
|
||||
resumeToken: Base64Url32ByteSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const ResolveResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
cellUrl: CanonicalHttpsOriginSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
leaseExpiresAt: EpochMsSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const RelayMovedSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
cellUrl: CanonicalHttpsOriginSchema,
|
||||
assignmentEpoch: GenerationSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export function isTrustedNewerMove(input: {
|
||||
sourceOrigin: string
|
||||
configuredDirectorOrigin: string
|
||||
currentAssignmentEpoch: number
|
||||
move: z.infer<typeof RelayMovedSchema>
|
||||
}): boolean {
|
||||
// Why: cells and stale director responses must never redirect a credential-bearing client.
|
||||
return (
|
||||
input.sourceOrigin === input.configuredDirectorOrigin &&
|
||||
input.move.assignmentEpoch > input.currentAssignmentEpoch
|
||||
)
|
||||
}
|
||||
|
||||
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
|
||||
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
|
||||
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
|
||||
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
|
||||
export type RelayMoved = z.infer<typeof RelayMovedSchema>
|
||||
@@ -0,0 +1,71 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
|
||||
|
||||
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
|
||||
|
||||
export type RelayRegion = z.infer<typeof RelayRegionSchema>
|
||||
|
||||
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
|
||||
|
||||
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
|
||||
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
|
||||
// new region a compile error here, which is the point: a region with no segment would silently
|
||||
// drop out of the region-skew alert's denominators.
|
||||
export const RELAY_REGION_METRIC_SEGMENTS = {
|
||||
'us-central1': 'UsCentral1',
|
||||
'asia-east2': 'AsiaEast2'
|
||||
} as const satisfies Record<RelayRegion, string>
|
||||
|
||||
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
|
||||
|
||||
export const RelayRegionCatalogResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
regions: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
region: RelayRegionSchema,
|
||||
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
|
||||
})
|
||||
.strict()
|
||||
)
|
||||
.max(RELAY_REGIONS.length)
|
||||
})
|
||||
.strict()
|
||||
.superRefine((catalog, context) => {
|
||||
const regions = new Set<RelayRegion>()
|
||||
const origins = new Set<string>()
|
||||
for (const [regionIndex, entry] of catalog.regions.entries()) {
|
||||
if (regions.has(entry.region)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'duplicate relay region',
|
||||
path: ['regions', regionIndex, 'region']
|
||||
})
|
||||
}
|
||||
regions.add(entry.region)
|
||||
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
|
||||
if (origins.has(origin)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'duplicate relay probe origin',
|
||||
path: ['regions', regionIndex, 'probeOrigins', originIndex]
|
||||
})
|
||||
}
|
||||
origins.add(origin)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
|
||||
|
||||
function isCanonicalHttpsOrigin(value: string): boolean {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
return url.protocol === 'https:' && url.origin === value
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
|
||||
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
|
||||
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
|
||||
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
|
||||
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
|
||||
export const OpaqueIdSchema = z.string().min(1).max(128)
|
||||
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
|
||||
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
|
||||
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
|
||||
|
||||
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}, 'must be a canonical HTTPS origin')
|
||||
@@ -6,5 +6,5 @@
|
||||
"outDir": "dist",
|
||||
"rootDir": "src"
|
||||
},
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
"exclude": ["src/**/*.test.ts", "src/test-fixtures/**"]
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
|
||||
'orca-cloud-relay-regional-placement-enabled'
|
||||
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
|
||||
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
||||
export const DIRECTOR_REHOME_IDENTITY_ENV =
|
||||
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
|
||||
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
|
||||
@@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) {
|
||||
return { changed: additions.length > 0, value: JSON.stringify(desired) }
|
||||
}
|
||||
|
||||
export function correctionCohortPercent(value) {
|
||||
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
|
||||
throw new Error('region correction cohort must be an integer from 0 to 100')
|
||||
}
|
||||
return String(value)
|
||||
}
|
||||
|
||||
export function directorDeploymentEnvironment(config) {
|
||||
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
|
||||
if (config.image !== undefined && imageDigest === undefined) {
|
||||
@@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) {
|
||||
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
|
||||
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
|
||||
}
|
||||
if (config['region-correction-cohort-percent'] !== undefined &&
|
||||
config['region-correction-cohort-percent'] !== 'preserve') {
|
||||
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
|
||||
}
|
||||
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
|
||||
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
|
||||
const rehomeDirectorServiceAccount = projectServiceAccount(
|
||||
@@ -302,7 +314,8 @@ export function parseArguments(argv) {
|
||||
values['rehome-director-service-account'] !== undefined ||
|
||||
values['rehome-audience'] !== undefined ||
|
||||
values['expected-rehome-generation'] !== undefined ||
|
||||
values['rehome-control-origin'] !== undefined
|
||||
values['rehome-control-origin'] !== undefined ||
|
||||
values['region-correction-cohort-percent'] !== undefined
|
||||
) {
|
||||
throw new Error('director configuration arguments require --role director')
|
||||
}
|
||||
@@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) {
|
||||
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
|
||||
const currentEnvironment = revisionEnvironment(servingRevision)
|
||||
const deploymentEnvironment = directorDeploymentEnvironment(config)
|
||||
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
|
||||
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
|
||||
)
|
||||
if (config['region-correction-cohort-percent'] !== undefined &&
|
||||
config['region-correction-cohort-percent'] !== 'preserve' &&
|
||||
config['expected-rehome-generation'] === undefined) {
|
||||
throw new Error('cohort changes require an exact disabled regional-rehome generation')
|
||||
}
|
||||
const mutableEnvironment = {
|
||||
...deploymentEnvironment,
|
||||
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
|
||||
|
||||
@@ -4,6 +4,8 @@ import { test } from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
activeRevision,
|
||||
correctionCohortPercent,
|
||||
DIRECTOR_CORRECTION_COHORT_ENV,
|
||||
cloudRunTrafficTag,
|
||||
DIRECTOR_ADMISSION_ENVIRONMENT,
|
||||
DIRECTOR_REGIONAL_PLACEMENT_ENV,
|
||||
@@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err
|
||||
/forbidden/
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
|
||||
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
|
||||
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
|
||||
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
|
||||
}
|
||||
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
|
||||
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
|
||||
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
|
||||
})
|
||||
|
||||
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
|
||||
const harness = directorHarness()
|
||||
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
|
||||
await deployDirector({}, 'candidate-new', harness.operations)
|
||||
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
|
||||
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
|
||||
}
|
||||
})
|
||||
|
||||
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
|
||||
const harness = directorHarness()
|
||||
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
|
||||
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
|
||||
assert.equal(harness.state.activeRevision, 'relay-00001-old')
|
||||
assert.equal(harness.state.nextRevision, 2)
|
||||
await deployDirector({}, 'candidate-new', harness.operations)
|
||||
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
|
||||
})
|
||||
|
||||
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
|
||||
const harness = directorHarness()
|
||||
let verified = 0
|
||||
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
|
||||
await deployDirector(config, 'candidate-new', { ...harness.operations,
|
||||
assertRegionalRehomeDisabled: async () => { verified++ } })
|
||||
assert.ok(verified >= 2)
|
||||
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
|
||||
})
|
||||
|
||||
@@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
try {
|
||||
service = run(gcloudArguments('services', input))
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
|
||||
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
|
||||
throw error
|
||||
}
|
||||
const serving = (service.status?.traffic ?? []).filter(
|
||||
@@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
throw new Error('Relay director must have exactly one revision serving 100% traffic')
|
||||
}
|
||||
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
|
||||
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter((environment) =>
|
||||
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
|
||||
)
|
||||
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
|
||||
(typeof cohortSettings[0].value !== 'string' ||
|
||||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
|
||||
throw new Error('serving region correction cohort is invalid')
|
||||
}
|
||||
const cohort_percent = cohortSettings[0]?.value ?? '0'
|
||||
const references = (revision.spec?.containers ?? []).flatMap((container) =>
|
||||
(container.env ?? []).filter(
|
||||
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
|
||||
)
|
||||
)
|
||||
if (references.length === 0) return { version: input.bootstrap_version }
|
||||
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
|
||||
const reference = normalizeSecretReference(references[0])
|
||||
if (
|
||||
references.length !== 1 ||
|
||||
@@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
|
||||
) {
|
||||
throw new Error('serving regional placement secret reference is invalid')
|
||||
}
|
||||
return { version: reference.version }
|
||||
return { version: reference.version, cohort_percent }
|
||||
}
|
||||
|
||||
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
|
||||
|
||||
@@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
|
||||
}
|
||||
})
|
||||
|
||||
assert.deepEqual(result, { version: '11' })
|
||||
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
|
||||
assert.equal(calls[1][3], 'relay-serving')
|
||||
})
|
||||
|
||||
@@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
|
||||
]) {
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
|
||||
}), { version: '1' })
|
||||
}), { version: '1', cohort_percent: '0' })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => {
|
||||
notFound.code = 'NOT_FOUND'
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: () => { throw notFound }
|
||||
}), { version: '7' })
|
||||
}), { version: '7', cohort_percent: '0' })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services'
|
||||
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
|
||||
: { spec: { containers: [{ env: [] }] } }
|
||||
}), { version: '7' })
|
||||
}), { version: '7', cohort_percent: '0' })
|
||||
})
|
||||
|
||||
test('classifies real absent-service stderr without weakening revision failures', () => {
|
||||
@@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () =>
|
||||
run: () => { throw denied }
|
||||
}), denied)
|
||||
})
|
||||
|
||||
|
||||
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
|
||||
for (const value of ['0', '1', '17', '100']) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
|
||||
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), { version: '11', cohort_percent: value })
|
||||
}
|
||||
})
|
||||
|
||||
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
|
||||
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
|
||||
const cases = [
|
||||
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
|
||||
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
|
||||
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
|
||||
[{ name, value: '1' }, { name, value: '2' }]
|
||||
]
|
||||
for (const settings of cases) {
|
||||
const servingRevision = revision()
|
||||
servingRevision.spec.containers[0].env.push(...settings)
|
||||
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
|
||||
run: (args) => args[1] === 'services' ? serving() : servingRevision
|
||||
}), /cohort is invalid/)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -491,3 +491,64 @@ Run and record each scenario in staging before launch:
|
||||
- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback.
|
||||
|
||||
The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them.
|
||||
|
||||
## Optional measured region correction (deployment gated)
|
||||
|
||||
New optimization claims require both the durable regional-rehome control and
|
||||
`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**).
|
||||
Turning either gate off stops new optional moves; ordinary migration cleanup and
|
||||
recovery continue. Legacy preferred-region hints do not certify a correction. Both
|
||||
cells must advertise regional protocol3 and the authenticated desktop control must
|
||||
advertise idle-regional-rehome-v1. The source must have no actual client sockets or
|
||||
pending admission/control work; a live control socket alone does not prevent a move.
|
||||
|
||||
The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**.
|
||||
It returns full-population eligibility/exclusion counts, open-migration capacity,
|
||||
process-safety gating and aggregate migration outcomes; it never claims a
|
||||
host or changes the failure budget. This is advisory, with separately read state:
|
||||
concurrent assignments, capacity changes, rate pauses and control changes can make
|
||||
the next claim differ. Inspect the durable control separately before enabling.
|
||||
Do not treat an unavailable/failed preview as zero eligible hosts.
|
||||
|
||||
`orca_relay_region_correction_outcomes` reports attempts by source/target,
|
||||
registration/completion/abort state, oldest open age and target
|
||||
reservation units every five minutes. `orca_relay_region_comparison` samples a
|
||||
stable 10% of accepted reports (including unchanged hosts), keyed by host digest,
|
||||
assignment epoch and decision generation. Existing control RTT and client-accept
|
||||
logs include assignment epoch, control generation and drain mode; join those for
|
||||
matched before/after and unchanged-cohort comparisons. Client accept latency is
|
||||
connection setup, not application command round trip. No application-latency
|
||||
improvement has been demonstrated by probe differences alone.
|
||||
|
||||
Quiet live connections count as work and defer optional correction indefinitely.
|
||||
A returning client may race with the short admission gate and retry normally. No
|
||||
optimization timer may close an established client. Investigate failed registration,
|
||||
ambiguous authority, stuck reservations and reconnect/failure rates against agreed
|
||||
limits. A database outage can keep the source fenced until locked reconciliation
|
||||
establishes its authority; timeout alone is not permission to reopen admissions.
|
||||
|
||||
All directors must run the reviewed idle worker before enabling. Record the tested
|
||||
immutable source and rollback revisions, then verify the ordinary migration recovery
|
||||
path before rollout. There is no retained-source table or renewal protocol. Deploying
|
||||
supporting cells/desktops and enabling a cohort require separate rollout authorization
|
||||
and explicit numerical stop criteria; this change enables neither.
|
||||
|
||||
### Setting the correction cohort during a reviewed director rollout
|
||||
|
||||
The existing **Deploy Relay Production Director** workflow accepts
|
||||
`region-correction-cohort-percent`: `preserve` (default) or an integer0–100.
|
||||
It carries the cohort onto both candidate and compatible rollback revisions and
|
||||
verifies the environment before promotion. If the predecessor has no setting,
|
||||
`preserve` stamps zero. An explicit change requires the exact disabled durable
|
||||
rehome generation; configuring a nonzero cohort does not itself enable the sweep.
|
||||
The usual image, identity, health and traffic checks remain in force. No workflow
|
||||
was dispatched as part of implementation.
|
||||
|
||||
Terraform reads the cohort from the same traffic-serving revision used to preserve
|
||||
regional placement. A later apply therefore preserves a workflow-set cohort,
|
||||
including explicit zero; only an absent service/setting bootstraps to0. Malformed
|
||||
or ambiguous live settings fail the plan instead of silently resetting the cohort.
|
||||
The audited director workflow owns subsequent changes.
|
||||
Before the first nonzero cohort, verify compatible protocol2 cells, updated
|
||||
cleanup workers, preview eligibility, both serving/rollback images and the
|
||||
explicitly approved observation/stop criteria.
|
||||
|
||||
@@ -169,6 +169,11 @@ resource "google_cloud_run_v2_service" "relay" {
|
||||
}
|
||||
}
|
||||
|
||||
env {
|
||||
name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT"
|
||||
value = data.external.relay_serving_regional_placement_version.result.cohort_percent
|
||||
}
|
||||
|
||||
ports {
|
||||
container_port = 8080
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
|
||||
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
|
||||
// not advertise this parses those entries strictly and would drop the whole ack.
|
||||
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
|
||||
export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1'
|
||||
|
||||
export function parseRelayHostCapabilities(
|
||||
header: string | string[] | undefined
|
||||
@@ -121,11 +122,22 @@ export const DeviceRevokeSchema = z
|
||||
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
|
||||
.strict()
|
||||
|
||||
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
|
||||
export const AuthRefreshSchema = z
|
||||
.object({
|
||||
relayJwt: z
|
||||
.string()
|
||||
.min(1)
|
||||
.max(8 * 1024)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const DrainSchema = z
|
||||
.object({
|
||||
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
|
||||
graceMs: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.max(60 * 60 * 1000),
|
||||
recovery: z.literal('resolve-director')
|
||||
})
|
||||
.strict()
|
||||
|
||||
@@ -7,8 +7,15 @@ import {
|
||||
RelayHostIdSchema
|
||||
} from './wire-scalars.js'
|
||||
import { RelayRegionSchema } from './relay-regions.js'
|
||||
import {
|
||||
RegionCorrectionRequestSchema,
|
||||
RegionCorrectionResponseSchema
|
||||
} from './region-correction.js'
|
||||
|
||||
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
|
||||
const SignedAssignmentLeaseSchema = z
|
||||
.string()
|
||||
.min(1)
|
||||
.max(8 * 1024)
|
||||
|
||||
export const AssignmentRequestSchema = z
|
||||
.object({
|
||||
@@ -17,7 +24,8 @@ export const AssignmentRequestSchema = z
|
||||
// Client-declared reconnection; the director verifies it against the
|
||||
// durable assignment before granting fast-lane admission.
|
||||
reconnect: z.boolean().optional(),
|
||||
preferredRegion: RelayRegionSchema.optional()
|
||||
preferredRegion: RelayRegionSchema.optional(),
|
||||
regionCorrection: RegionCorrectionRequestSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
@@ -26,7 +34,8 @@ export const AssignmentResponseSchema = z
|
||||
v: z.literal(1),
|
||||
cellUrl: CanonicalHttpsOriginSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
lease: SignedAssignmentLeaseSchema
|
||||
lease: SignedAssignmentLeaseSchema,
|
||||
regionCorrection: RegionCorrectionResponseSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { z } from 'zod'
|
||||
import { GenerationSchema, RelayHostIdSchema } from './wire-scalars.js'
|
||||
|
||||
export const IdleRegionalRehomeRequestSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
attemptId: z.string().uuid(),
|
||||
userId: z.string().min(1).max(256),
|
||||
relayHostId: RelayHostIdSchema,
|
||||
sourceCellId: z.string().min(1).max(128),
|
||||
sourceCellIncarnation: z.string().uuid(),
|
||||
sourceAssignmentEpoch: GenerationSchema.refine((value) => value > 0),
|
||||
sourceGeneration: GenerationSchema.refine((value) => value > 0),
|
||||
targetCellId: z.string().min(1).max(128)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const IdleRegionalRehomeResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
outcome: z.enum(['busy', 'committed', 'deferred', 'stale'])
|
||||
})
|
||||
.strict()
|
||||
|
||||
export type IdleRegionalRehomeRequest = z.infer<typeof IdleRegionalRehomeRequestSchema>
|
||||
export type IdleRegionalRehomeOutcome = z.infer<typeof IdleRegionalRehomeResponseSchema>['outcome']
|
||||
@@ -13,3 +13,5 @@ export * from './resume-confirmation-contract.js'
|
||||
export * from './relay-regions.js'
|
||||
export * from './splice-state-machine.js'
|
||||
export * from './wire-scalars.js'
|
||||
export * from './region-correction.js'
|
||||
export * from './idle-regional-rehome.js'
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { AssignmentRequestSchema, AssignmentResponseSchema } from './director-messages.js'
|
||||
import { DrainSchema } from './control-messages.js'
|
||||
import { RegionCorrectionRequestSchema } from './region-correction.js'
|
||||
|
||||
const report = {
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: 3,
|
||||
assignmentEpoch: 7,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 40, 'asia-east2': 180 }
|
||||
}
|
||||
const retention = {
|
||||
mode: 'finish-existing',
|
||||
attemptId: '11111111-1111-4111-8111-111111111111',
|
||||
sourceGeneration: 3,
|
||||
sourceAssignmentEpoch: 7
|
||||
}
|
||||
|
||||
describe('region correction wire boundaries', () => {
|
||||
it('keeps legacy assignment shapes readable without negotiated fields', () => {
|
||||
expect(
|
||||
AssignmentRequestSchema.parse({ v: 1, relayHostId: 'abcdefghijklmnop' })
|
||||
).not.toHaveProperty('regionCorrection')
|
||||
expect(
|
||||
AssignmentResponseSchema.parse({
|
||||
v: 1,
|
||||
cellUrl: 'https://cell.example',
|
||||
assignmentEpoch: 1,
|
||||
lease: 'synthetic-lease'
|
||||
})
|
||||
).not.toHaveProperty('regionCorrection')
|
||||
})
|
||||
|
||||
it('accepts complete comparison evidence and explicit inconclusive reports', () => {
|
||||
expect(RegionCorrectionRequestSchema.safeParse(report).success).toBe(true)
|
||||
const { measurements: _measurements, ...basis } = report
|
||||
expect(
|
||||
RegionCorrectionRequestSchema.safeParse({
|
||||
...basis,
|
||||
outcome: 'inconclusive',
|
||||
reason: 'probe-unavailable'
|
||||
}).success
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ measurements: { 'us-central1': 40 } },
|
||||
{ measurements: { 'us-central1': -1, 'asia-east2': 10 } },
|
||||
{ measurements: { 'us-central1': Infinity, 'asia-east2': 10 } },
|
||||
{ measurements: { 'us-central1': 120_001, 'asia-east2': 10 } },
|
||||
{ generation: Number.MAX_SAFE_INTEGER + 1 },
|
||||
{ assignmentEpoch: 1.2 },
|
||||
{ policyVersion: 2 },
|
||||
{ outcome: 'inconclusive', reason: 'timeout' }
|
||||
])('rejects ambiguous or unbounded evidence: %j', (override) => {
|
||||
expect(RegionCorrectionRequestSchema.safeParse({ ...report, ...override }).success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects reporting and issuing a window in the same request', () => {
|
||||
expect(
|
||||
RegionCorrectionRequestSchema.safeParse({
|
||||
...report,
|
||||
action: 'issue-window'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('uses ordinary drain and rejects the superseded retention extension', () => {
|
||||
const ordinary = { graceMs: 0, recovery: 'resolve-director' }
|
||||
expect(DrainSchema.parse(ordinary)).toEqual(ordinary)
|
||||
expect(DrainSchema.safeParse({ ...ordinary, retention }).success).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,60 @@
|
||||
import { z } from 'zod'
|
||||
import { EpochMsSchema, GenerationSchema } from './wire-scalars.js'
|
||||
import { RelayRegionSchema } from './relay-regions.js'
|
||||
|
||||
const RttSchema = z.number().finite().nonnegative().max(120_000)
|
||||
export const RegionMeasurementsSchema = z
|
||||
.object({
|
||||
'us-central1': RttSchema,
|
||||
'asia-east2': RttSchema
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const RegionMeasurementWindowSchema = z
|
||||
.object({
|
||||
generation: GenerationSchema,
|
||||
expiresAt: EpochMsSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
incumbentRegion: RelayRegionSchema,
|
||||
policyVersion: z.literal(1)
|
||||
})
|
||||
.strict()
|
||||
|
||||
const ReportBasis = {
|
||||
v: z.literal(1),
|
||||
action: z.literal('report'),
|
||||
generation: GenerationSchema,
|
||||
assignmentEpoch: GenerationSchema,
|
||||
policyVersion: z.literal(1)
|
||||
}
|
||||
|
||||
export const RegionCorrectionRequestSchema = z.union([
|
||||
z.object({ v: z.literal(1), action: z.literal('issue-window') }).strict(),
|
||||
z
|
||||
.object({
|
||||
...ReportBasis,
|
||||
outcome: z.literal('conclusive'),
|
||||
measurements: RegionMeasurementsSchema
|
||||
})
|
||||
.strict(),
|
||||
z
|
||||
.object({
|
||||
...ReportBasis,
|
||||
outcome: z.literal('inconclusive'),
|
||||
reason: z.string().min(1).max(64)
|
||||
})
|
||||
.strict()
|
||||
])
|
||||
|
||||
export const RegionCorrectionResponseSchema = z
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
window: RegionMeasurementWindowSchema.optional(),
|
||||
reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export type RegionMeasurements = z.infer<typeof RegionMeasurementsSchema>
|
||||
export type RegionMeasurementWindow = z.infer<typeof RegionMeasurementWindowSchema>
|
||||
export type RegionCorrectionRequest = z.infer<typeof RegionCorrectionRequestSchema>
|
||||
export type RegionCorrectionResponse = z.infer<typeof RegionCorrectionResponseSchema>
|
||||
Reference in New Issue
Block a user