Files
orca/cloud/apps/relay/src/reserve-assignment.test.ts
T
OrcaWinandOrcaWin 7d6603aa17 [comment audit] Complete repository-wide comment cleanup (#33249)
* [comment audit] Align legacy failure-read counts with cached display

* [comment audit] Condense shared provider lifecycle and browser recovery comments [skip ci]

* [comment audit] Condense shared generation, path and preview comments [skip ci]

* [comment audit] Condense host, file and Git contract rationale [skip ci]

* [comment audit] Tighten Git lifecycle and provider contract comments [skip ci]

* [comment audit] Clarify shared host authority and launch isolation [skip ci]

* [comment audit] simplify local access mobile and native chat contracts [skip ci]

* [comment audit] simplify native chat presentation and state contracts [skip ci]

* [comment audit] simplify native runtime and provider contracts [skip ci]

* [comment audit] simplify migration and orchestration contracts [skip ci]

* [comment audit] simplify plugin trust and process capture rationale [skip ci]

* [comment audit] simplify terminal delivery and remote compatibility rationale [skip ci]

* [comment audit] clarify remote recovery and launch wire rationale [skip ci]

* [comment audit] shorten shared RPC compatibility and runtime evidence comments [skip ci]

* [comment audit] simplify secure storage shell startup and sidebar rationale [skip ci]

* [comment audit] clarify sidebar, skill and SSH invariants [skip ci]

* [comment audit] condense structured chat delivery and timing rationale [skip ci]

* [comment audit] condense terminal replay and telemetry invariants [skip ci]

* [comment audit] clarify launch permission and terminal delivery rationale [skip ci]

* [comment audit] clarify workspace ownership and layout contracts [skip ci]

* [comment audit] wait for fitted and painted SSH terminal links

* [comment audit] clarify layout persistence and session ownership [skip ci]

* [comment audit] clarify host routing and workspace lifecycle contracts [skip ci]

* [comment audit] clarify startup persistence and app lifetime comments [skip ci]

* [comment audit] clarify recovery and activity boundaries [skip ci]

* [comment audit] condense activity and artifact invariants [skip ci]

* [comment audit] condense automation authority contracts [skip ci]

* [comment audit] simplify automation scheduling and history contracts [skip ci]

* [comment audit] clarify browser chrome and annotation lifecycles [skip ci]

* [comment audit] clarify browser routing and retained guest ownership [skip ci]

* [comment audit] clarify browser recovery and document ownership comments [skip ci]

* [comment audit] restore hosted runners required by native IME fixtures

* [comment audit] shorten preview, palette and dashboard invariants [skip ci]

* [comment audit] clarify terminal preview, dashboard and retention comments [skip ci]

* [comment audit] clarify diff-note recovery and editor lifecycle comments [skip ci]

* [comment audit] shorten rich-text and combined-diff ownership comments [skip ci]

* [comment audit] shorten autosave, draft recovery and model ownership comments [skip ci]

* [comment audit] shorten save-queue, notebook and Markdown constraints [skip ci]

* [comment audit] shorten Monaco, PDF and rich Markdown safeguards [skip ci]

* [comment audit] clarify rich Markdown source and selection safeguards [skip ci]

* [comment audit] clarify editor loading and emulator stream lifetimes [skip ci]

* [comment audit] trim emulator and feature tour narration [skip ci]

* [comment audit] clarify floating workspace and GitHub cache ownership [skip ci]

* [comment audit] trim provider controls and Linear filter explanations [skip ci]

* [comment audit] clarify pairing and chat resume lifetimes [skip ci]

* [comment audit] clarify chat delivery and transcript lifetimes [skip ci]

* [comment audit] clarify session options and host-owned status [skip ci]

* [comment audit] clarify draft durability and live-status precedence [skip ci]

* [comment audit] Trim chat send and model ownership explanations [skip ci]

* [comment audit] Shorten transcript ownership and send recovery comments [skip ci]

* [comment audit] Simplify native chat composer and transcript hook comments [skip ci]

* [comment audit] Shorten native chat scroll and structured session explanations [skip ci]

* [comment audit] Trim workspace picker and onboarding commentary [skip ci]

* [comment audit] Simplify pet rendering, review cache and quick open comments [skip ci]

* [comment audit] Trim sidebar and session history explanations [skip ci]

* [comment audit] Condense session resume and review checks comments [skip ci]

* [comment audit] Shorten checks state and file explorer refresh comments [skip ci]

* [comment audit] Trim plugin bridge and source-control action rationale [skip ci]

* [comment audit] Trim source-control listing and review rationale [skip ci]

* [comment audit] Shorten sidebar lifecycle and explorer safeguards [skip ci]

* [comment audit] Condense Settings account and appearance rationale [skip ci]

* [comment audit] Trim pairing and repository Settings explanations [skip ci]

* [comment audit] Condense Settings search SSH and terminal safeguards [skip ci]

* [comment audit] Shorten Settings voice and provider ownership comments [skip ci]

* [comment audit] Condense Settings draft persistence and host-state rationale [skip ci]

* [comment audit] Trim Settings hooks and sidebar setup rationale [skip ci]

* [comment audit] Condense sidebar deletion and board-filter safeguards [skip ci]

* [comment audit] Condense worktree cards, host ownership and Markdown contracts [skip ci]

* [comment audit] Condense sidebar drag, folder and filtering contracts [skip ci]

* [comment audit] Condense sidebar parity, sleep and host ownership contracts [skip ci]

* [comment audit] Prune focus-read baseline after comment cleanup

* [comment audit] Clarify sidebar visibility, board drops and agent evidence [skip ci]

* [comment audit] Tighten sidebar listing, drag and reveal contracts [skip ci]

* [comment audit] Clarify viewport geometry and skill update safeguards [skip ci]

* [comment audit] Tighten skill inventory, usage and host status comments [skip ci]

* [comment audit] Clarify usage ownership and resource kill safeguards [skip ci]

* [comment audit] Tighten tab selection, rename and search contracts [skip ci]

* [comment audit] Tighten tab-strip geometry and drag ownership comments [skip ci]

* [comment audit] Tighten Tasks recovery, pagination and mutation comments [skip ci]

* [comment audit] Tighten provider loading and terminal recovery comments [skip ci]

* [comment audit] Tighten completion, cadence and PTY ownership comments [skip ci]

* [comment audit] Clarify terminal restore and parked-pane boundaries [skip ci]

* [comment audit] Tighten PTY startup and replay explanations [skip ci]

* [comment audit] Clarify PTY identity and delivery ownership [skip ci]

* [comment audit] Shorten remote reconnect and input replay rationale [skip ci]

* [comment audit] Condense terminal parking clipboard and file link comments [skip ci]

* [comment audit] Clarify IME ownership and trim trace history [skip ci]

* [comment audit] Shorten pane recovery and parked watcher explanations [skip ci]

* [comment audit] Condense snapshot replay and visibility recovery rationale [skip ci]

* [comment audit] Shorten terminal parking and keyboard ownership rationale [skip ci]

* [comment audit] Trim renderer lifecycle and menu rationale [skip ci]

* [comment audit] Condense workspace cleanup ownership and settlement notes [skip ci]

* [comment audit] Clarify renderer hook evidence and source ownership [skip ci]

* [comment audit] Tighten IPC ownership and reconnect explanations [skip ci]

* [comment audit] Trim renderer focus, paste and attention comments [skip ci]

* [comment audit] Shorten scroll, locale and agent lifecycle comments [skip ci]

* [comment audit] Trim agent launch, resume and ownership commentary [skip ci]

* [comment audit] Shorten account, browser and editor comments [skip ci]

* [comment audit] Clarify host-mirror waits, previews and input ownership [skip ci]

* [comment audit] Condense launch recovery and editor tokenization comments [skip ci]

* [comment audit] Condense workspace selection and terminal pane rationale [skip ci]

* [comment audit] Condense terminal fitting and rendering recovery comments [skip ci]

* [comment audit] Condense terminal input and host routing explanations [skip ci]

* [comment audit] Condense ownership and sleeping-session recovery rationale [skip ci]

* [comment audit] Condense structured launch and terminal link explanations [skip ci]

* [comment audit] Condense terminal ownership and workspace file routing rationale [skip ci]

* [comment audit] Condense session persistence and workspace activation rationale [skip ci]

* [comment audit] Condense workspace routing and browser ownership rationale [skip ci]

* [comment audit] Condense mirror recovery and layout authority rationale [skip ci]

* [comment audit] Condense runtime transport and file safety rationale [skip ci]

* [comment audit] Condense runtime graph and structured session rationale [skip ci]

* [comment audit] Condense browser creation and remote tab recovery rationale [skip ci]

* [comment audit] Condense snapshot authority and publisher recovery rationale [skip ci]

* [comment audit] Condense frame, startup and provider cache rationale [skip ci]

* [comment audit] Condense host twin and agent lifecycle rationale [skip ci]

* [comment audit] Condense browser authority and editor cleanup rationale [skip ci]

* [comment audit] Condense editor buffer identity and review cache rationale [skip ci]

* [comment audit] Condense provider lifetime and repository ownership rationale [skip ci]

* [comment audit] Condense reconnect, hydration and terminal retirement rationale [skip ci]

* [comment audit] Condense recovery, reconciliation and persistence rationale [skip ci]

* [comment audit] Condense cleanup ownership and listing admission rationale [skip ci]

* [comment audit] Condense teardown ordering and terminal ownership rationale [skip ci]

* [comment audit] Condense recovery ledger and web host-boundary rationale [skip ci]

* [comment audit] Condense web reconnect and credential lifecycle rationale [skip ci]

* [comment audit] Condense cloud rollout and assignment safety rationale [skip ci]

* [comment audit] Condense relay session and database recovery rationale [skip ci]

* [comment audit] Condense relay placement and protocol compatibility rationale [skip ci]

* [comment audit] Condense build and packaging constraints [skip ci]

* [comment audit] Condense CI gates and validation isolation rationale [skip ci]

* [comment audit] Condense catalog identity and browser validation rationale [skip ci]

* [comment audit] Condense mobile browser and frame-budget rationale [skip ci]

* [comment audit] Shorten mobile route and browser test explanations [skip ci]

* [comment audit] Condense editor and session seam test rationale [skip ci]

* [comment audit] Trim session and terminal closure history [skip ci]

* [comment audit] Shorten terminal browser and census explanations [skip ci]

* [comment audit] Preserve root-only E2E diagnostic baseline

* [comment audit] Condense mobile seam and HTML preview test rationale [skip ci]

* [comment audit] Condense preview instrumentation and render harness rationale [skip ci]

* [comment audit] Condense bundle contracts and native-build constraints [skip ci]

* [comment audit] Shorten native packaging and CI routing rationale [skip ci]

* [comment audit] Condense release tooling and benchmark constraints [skip ci]

* [comment audit] Condense packaging and platform gate rationale [skip ci]

* [comment audit] Shorten mobile route and history rationale [skip ci]

* [comment audit] Condense mobile browser and component rationale [skip ci]

* [comment audit] Shorten drawer and workspace-form rationale [skip ci]

* [comment audit] Trim mobile review and editor-document explanations [skip ci]

* [comment audit] Condense editor lifecycle and Markdown explanations [skip ci]

* [comment audit] Trim mobile file, diagnostics and dictation commentary [skip ci]

* [comment audit] Simplify mobile host-list and workspace-view explanations [skip ci]

* [comment audit] Clarify mobile shell lifetime and bridge delivery constraints [skip ci]

* [comment audit] Condense bridge stream, native request and audio rationale [skip ci]

* [comment audit] Reduce bridge schema and page-client commentary [skip ci]

* [comment audit] Tighten mobile page bridge contracts [skip ci]

* [comment audit] Clarify mobile cache and session constraints [skip ci]

* [comment audit] Condense mobile route and lifecycle rationale [skip ci]

* [comment audit] Tighten mobile shell hook timing rationale [skip ci]

* [comment audit] Clarify route handoff and notification compatibility [skip ci]

* [comment audit] Clarify push registration and dictation cleanup constraints [skip ci]

* [comment audit] Condense keyboard geometry and native media ownership rationale [skip ci]

* [comment audit] Tighten mobile chat scroll and composer concurrency rationale [skip ci]

* [comment audit] Condense mobile reply compatibility and upload ownership rationale [skip ci]

* [comment audit] Condense optimistic echo and scroll ownership rationale [skip ci]

* [comment audit] Tighten stream replay and queued session ownership rationale [skip ci]

* [comment audit] Condense mobile composer and reply-schema constraints [skip ci]

* [comment audit] Clarify mobile transcript recovery and session ownership [skip ci]

* [comment audit] Clarify mobile queue and source-control compatibility [skip ci]

* [comment audit] Condense storage and task launch constraints [skip ci]

* [comment audit] Clarify task schemas and terminal document ownership [skip ci]

* [comment audit] Tighten terminal input and viewport rationale [skip ci]

* [comment audit] Shorten terminal and RPC recorder contracts [skip ci]

* [comment audit] Condense recorder engine and connection contracts [skip ci]

* [comment audit] Shorten host credential and relay recovery comments [skip ci]

* [comment audit] Tighten pairing recovery and bundle transport rationale [skip ci]

* [comment audit] Clarify RPC delivery and workspace catalog boundaries [skip ci]

* [comment audit] Condense catalog and relay entry contracts [skip ci]

* [comment audit] Shorten shutdown and SSH relay delivery contracts [skip ci]

* [comment audit] Condense WSL relay and E2E regression rationale [skip ci]

* [comment audit] Shorten cross-version harness and renderer fixture explanations [skip ci]

* [comment audit] Condense paired-host and E2E lifecycle rationale [skip ci]

* [comment audit] Shorten layout oracle and regression explanations [skip ci]

* [comment audit] Condense paired recovery and ownership regression contracts [skip ci]

* [comment audit] Shorten restart, sidebar and SSH lifecycle contracts [skip ci]

* [comment audit] Condense terminal rendering and IME regression rationale [skip ci]

* [comment audit] Shorten terminal recovery and benchmark contracts [skip ci]

* [comment audit] Condense tooling and Windows lifecycle proof rationale [skip ci]

* [comment audit] Condense workflow routing and release rationale [skip ci]

* [comment audit] Condense mobile security and installer contracts [skip ci]

* [comment audit] Shorten native rationale and stylesheet workarounds [skip ci]

* [comment audit] Shorten Windows visibility probe rationale [skip ci]

* [comment audit] Condense embedded release and signing rationale [skip ci]

* [comment audit] Retain current main test behavior in final reconciliation [skip ci]

* [comment audit] Complete final comment review across current source inventory

* [comment audit] Handle consolidated audit reports before final CI

* [comment audit] Preserve credential storage census detail [skip ci]

* [comment audit] Scale broad changed E2E selections with verified timing shards

* [comment audit] Align E2E assertions with browser close and settled session metadata

* [comment audit] Match existing E2Es to current resource and windowing contracts

* [comment audit] Observe the shared server outage notice in reconnect coverage

* [comment audit] Make existing renderer probes measure settled tab and PTY work

* [comment audit] Align palette E2E selector with task search

* [comment audit] Refresh remaining palette search selectors

* [comment audit] Keep selector correction within selected audit specs

* [comment audit] Await emoji fixture settings inside renderer evaluation

* [comment audit] Cancel fixture fit retries between terminal tests

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
2026-10-25 04:05:46 -07:00

1042 lines
47 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, it, vi } from 'vitest'
import type { RelayRegion } from '@orca-cloud/relay-contract'
import type { CellReserveClient } from './cell-reserve-client.js'
import {
ReserveAssignment,
ReservePaceError,
RESERVE_ROW_FLOOR_TTL_MS,
RESERVE_WRONG_CELL_ROW_READ_MS,
RESERVE_STARTUP_GATE_MS
} from './reserve-assignment.js'
import { ReservePlacer, type ReserveAttempt } from './reserve-placement.js'
import { ReserveRehome } from './reserve-rehome.js'
import { ShadowSeatDirectory, type SeatFeedCell, type SeatFeedResponse } from './shadow-seat-directory.js'
const HOST = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const US = 'us-central1' as const
function cellList(ids: string[], region: RelayRegion = US): SeatFeedCell[] {
return ids.map((cellId) => ({
cellId,
cellUrl: `https://${cellId}.example`,
region,
heartbeatExpiresAt: Number.MAX_SAFE_INTEGER,
requiredForComplete: true,
general: true
}))
}
function feed(
cellId: string,
overrides: Partial<SeatFeedResponse> & { admitMode?: 'db' | 'reserve' } = {}
): SeatFeedResponse {
const { admitMode = 'reserve', ...rest } = overrides
return {
v: 1,
cellId,
incarnation: `${cellId}-incarnation-1`,
seq: 1,
at: 0,
counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 500 },
intake: { perSec: 100, burst: 20, tokens: 20 },
flagsApplied: { generation: 3, flags: { admitMode } },
full: [],
...rest
}
}
function setup(
options: {
cells?: SeatFeedCell[]
feeds?: SeatFeedResponse[]
reserve?: (cellId: string, sticky: boolean) => ReserveAttempt
row?: (identity: { userId: string; relayHostId: string }) => Promise<{ cellId: string; assignmentEpoch: number } | null>
databaseBusy?: () => boolean
mode?: 'off' | 'dry-run' | 'on'
startedAt?: number
log?: (line: string) => void
demote?: (cellId: string) => string | null
rehome?: ReserveRehome
} = {}
) {
const now = { value: 100_000 }
const cells = options.cells ?? cellList(['c1', 'c2'])
const directory = new ShadowSeatDirectory()
directory.setCells(cells.map((cell) => cell.cellId))
for (const response of options.feeds ?? cells.map((cell) => feed(cell.cellId))) {
directory.apply(response.cellId, response, now.value - 1, now.value - 1)
}
const reserved: Array<{ cellId: string; epoch: number; sticky: boolean; dryRun: boolean }> = []
const demoted: Array<{ cellId: string; epoch: number; joinedAt: number }> = []
const client = {
reserve: vi.fn(async (target: { cellId: string }, item: { epoch: number; sticky?: boolean }, dryRun = false) => {
reserved.push({ cellId: target.cellId, epoch: item.epoch, sticky: item.sticky ?? false, dryRun })
return options.reserve?.(target.cellId, item.sticky ?? false) ?? { outcome: 'ok' }
}),
demote: vi.fn(async (target: { cellId: string }, request: { epoch: number; joinedAt: number }) => {
demoted.push({ cellId: target.cellId, epoch: request.epoch, joinedAt: request.joinedAt })
return options.demote ? options.demote(target.cellId) : 'demoted'
})
} satisfies Pick<CellReserveClient, 'reserve' | 'demote'>
// Each director's estimate starts empty; a second of refill gives it its share.
const placer = new ReservePlacer(() => now.value, () => 0.25)
const assignment = new ReserveAssignment({
mode: options.mode ?? 'on',
directory,
cells: () => cells,
startedAt: options.startedAt ?? 0,
placer,
// SAFETY: only reserve and demote are called, and both are implemented above.
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: see above.
client: client as unknown as CellReserveClient,
readRow: options.row ?? (async () => null),
...(options.databaseBusy ? { databaseBusy: options.databaseBusy } : {}),
...(options.rehome ? { rehome: options.rehome } : {}),
now: () => now.value,
random: () => 0.25,
log: options.log ?? (() => undefined)
})
// Starts each cell's refill clock, as a director's first look does.
for (const cell of assignment.placementCells()) placer.hasBudget(cell)
now.value += 1_000
for (const response of options.feeds ?? cells.map((cell) => feed(cell.cellId))) {
directory.apply(response.cellId, { ...response, full: response.full ?? [] }, now.value - 1, now.value - 1)
}
return { assignment, directory, reserved, demoted, now, placer }
}
function seated(epoch: number, joinedAt = 0) {
return { ...HOST, epoch, generation: 1, state: 'active', joinedAt }
}
describe('reserve assignment on a director', () => {
it('keeps today’s path when off, or when no cell reports reserve mode', async () => {
expect((await setup({ mode: 'off' }).assignment.plan(HOST, { reconnect: true, region: US })).kind).toBe(
'database'
)
const databaseOnly = setup({ feeds: [feed('c1', { admitMode: 'db' }), feed('c2', { admitMode: 'db' })] })
expect(await databaseOnly.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({
kind: 'database'
})
})
it('reads what each cell acts on: a tripped dead-man is db, a flip back still leasing is reserve', async () => {
const tripped = setup({
feeds: [feed('c1', { admitModeEffective: 'db' }), feed('c2', { admitModeEffective: 'db' })]
})
expect(await tripped.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
const leasing = setup({ feeds: [feed('c1', { admitMode: 'db', admitModeEffective: 'reserve' }), feed('c2')] })
expect(leasing.directory.admitModeOf('c1')).toBe('reserve')
// Raw db must immediately stop booking/pacing even while effective mode stays reserve during re-registration.
const reregistering = setup({
feeds: [
feed('c1', { admitModeEffective: 'reserve', admitModeRaw: 'db' }),
feed('c2', { admitModeEffective: 'reserve', admitModeRaw: 'db' })
]
})
expect(await reregistering.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
expect(reregistering.reserved).toEqual([])
expect(reregistering.directory.admitModeOf('c1')).toBe('reserve')
})
it('before every live cell has answered, holds back only hosts that may be on an unheard cell', async () => {
const cells = cellList(['c1', 'c2'])
const onUnheard = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
const gated = setup({
cells,
feeds: [feed('c1')],
startedAt: 100_500,
row: async (identity) =>
identity.relayHostId === onUnheard.relayHostId ? { cellId: 'c2', assignmentEpoch: 3 } : null
})
expect(await gated.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({
kind: 'database',
epochFloor: 0
})
expect(gated.reserved).toEqual([])
expect(await gated.assignment.plan(onUnheard, { reconnect: true, region: US })).toMatchObject({
kind: 'retry',
reason: 'map-incomplete'
})
gated.now.value = 100_500 + RESERVE_STARTUP_GATE_MS
expect((await gated.assignment.plan(onUnheard, { reconnect: true, region: US })).kind).not.toBe('retry')
})
it('answers a reconnect from memory at the same epoch with no booking and no database read', async () => {
const row = vi.fn(async () => null)
const { assignment, reserved } = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')],
row
})
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({ kind: 'answer', lane: 'sticky', assignment: { cellId: 'c1', assignmentEpoch: 7 } })
expect(reserved).toEqual([])
expect(row).not.toHaveBeenCalled()
})
it('re-books a host on a cell that restarted since its seat, and only then answers it', async () => {
const { assignment, directory, reserved, now } = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')]
})
directory.apply('c1', feed('c1', { incarnation: 'c1-incarnation-2' }), now.value, now.value)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(reserved[0]).toEqual({ cellId: 'c1', epoch: 7, sticky: true, dryRun: false })
expect(plan).toMatchObject({ kind: 'answer', lane: 'sticky', assignment: { cellId: 'c1', assignmentEpoch: 7 } })
})
it('after the reconcile demotes a seat behind its row, mints above the row and never answers that seat', async () => {
const { assignment, directory, reserved, now } = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')]
})
// The row names another cell at epoch 8; the mirror would refuse a re-assign at 8.
assignment.raiseEpochFloor(HOST, 8)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { assignmentEpoch: 9 }
})
expect(reserved.length).toBeGreaterThan(0)
expect(reserved.every((call) => !call.sticky && call.epoch === 9)).toBe(true)
now.value += RESERVE_ROW_FLOOR_TTL_MS + 1
for (const response of [feed('c1', { seq: 2, full: [seated(7)] }), feed('c2', { seq: 2 })]) {
directory.apply(response.cellId, response, now.value, now.value)
}
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
lane: 'sticky'
})
})
it('never sends a drained or demoted host back, and books it elsewhere above every known epoch', async () => {
const { assignment, directory, reserved, now } = setup()
directory.apply(
'c1',
feed('c1', {
full: undefined,
seq: 3,
changes: [
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4503, at: 2 }
]
}),
now.value,
now.value
)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { assignmentEpoch: 8 }
})
expect(reserved.length).toBeGreaterThan(0)
expect(reserved.every((call) => !call.sticky && call.epoch === 8)).toBe(true)
})
it('leaves a host the map last saw on a database-mode cell to today’s path, with the floor', async () => {
const { assignment } = setup({
feeds: [feed('c1', { admitMode: 'db', full: [seated(4)] }), feed('c2')]
})
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({ kind: 'database', epochFloor: 4 })
expect(plan.kind === 'database' && typeof plan.placeFresh).toBe('function')
})
it('leaves a cell without a feed (c17/c18) on today’s path without starving the reserve pool', async () => {
const cells = cellList(['c1', 'c2', 'c17'])
let rowOnOld = false
const { assignment, directory, reserved, now } = setup({
cells,
feeds: [feed('c1'), feed('c2', { admitMode: 'db' })],
row: async () => (rowOnOld ? { cellId: 'c17', assignmentEpoch: 5 } : null)
})
directory.markNoFeed('c17', now.value)
expect(directory.isComplete()).toBe(true)
// An old image reports no ceiling; counting it as a database cell would zero the share.
expect(await assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { cellId: 'c1' }
})
expect(reserved.map((call) => call.cellId)).not.toContain('c17')
rowOnOld = true
expect(
await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: true, region: US })
).toMatchObject({ kind: 'database', epochFloor: 5 })
directory.markNoFeed('c1', now.value)
expect(directory.cellState('c1')?.ceiling).toBeUndefined()
expect(assignment.placementCells().find((cell) => cell.cellId === 'c1')?.general).toBe(false)
})
it('reads the row once for a host the map never saw, and refuses it while the database is down', async () => {
const down = setup({ row: async () => Promise.reject(new Error('timeout')) })
expect(await down.assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
kind: 'retry',
reason: 'database'
})
const read = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 11 }))
const known = setup({ row: read })
expect(await known.assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
kind: 'answer',
assignment: { assignmentEpoch: 12 }
})
expect(read).toHaveBeenCalledExactlyOnceWith(HOST)
})
it('after a WRONG_CELL leave, any director reads the row and mints above it', async () => {
const leftWrongCell = (
row: () => Promise<{ cellId: string; assignmentEpoch: number } | null>,
databaseBusy?: () => boolean
) => {
const harness = setup({ row, ...(databaseBusy ? { databaseBusy } : {}) })
// Another director demoted the seat: this one only sees the leave in c1's feed.
harness.directory.apply(
'c1',
feed('c1', {
full: undefined,
seq: 3,
changes: [
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4409, at: 2 }
]
}),
harness.now.value,
harness.now.value
)
return harness
}
const row = vi.fn(async () => ({ cellId: 'c2', assignmentEpoch: 8 }))
const read = leftWrongCell(row)
expect(await read.assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { assignmentEpoch: 9 }
})
expect(row).toHaveBeenCalledTimes(1)
const down = leftWrongCell(async () => Promise.reject(new Error('timeout')))
expect(await down.assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
assignment: { assignmentEpoch: 8 }
})
// Pool waiters: the read is skipped outright, so a stall cannot stack timed-out reads.
const busyRow = vi.fn(async () => ({ cellId: 'c2', assignmentEpoch: 8 }))
const busy = leftWrongCell(busyRow, () => true)
expect(await busy.assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
assignment: { assignmentEpoch: 8 }
})
expect(busyRow).not.toHaveBeenCalled()
vi.useFakeTimers()
try {
const stalled = leftWrongCell(() => new Promise(() => undefined))
const plan = stalled.assignment.plan(HOST, { reconnect: true, region: US })
await vi.advanceTimersByTimeAsync(RESERVE_WRONG_CELL_ROW_READ_MS)
expect(await plan).toMatchObject({ kind: 'answer', assignment: { assignmentEpoch: 8 } })
} finally {
vi.useRealTimers()
}
})
describe('for a share-draw loser', () => {
const ASIA = 'asia-east2' as const
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) =>
setup({
...(reserve ? { reserve } : {}),
cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])],
feeds: [
reserveCell,
// Database room far above the reserve cell's, so the 0.25 draw loses.
feed('c2', {
admitMode: 'db',
counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 }
}),
feed('c3')
]
})
const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => {
const plan = await assignment.plan(identity, { reconnect: false, region: ASIA })
expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 })
return plan.kind === 'database' ? plan.regionReserve : undefined
}
it('offers the reserve cell in its region only, only while it has a seat', async () => {
const { assignment, reserved, directory, now } = setupAsia()
const regionReserve = await regionReserveOf(assignment)
expect(reserved).toEqual([])
await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA })
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } })
directory.apply('c1', full, now.value, now.value)
expect(await regionReserveOf(assignment, OTHER)).toBeUndefined()
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
})
it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => {
const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } }))
const regionReserve = await regionReserveOf(assignment)
await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError)
expect(reserved).toEqual([])
})
it('gives the region up when the reserve cell refuses with a seat still on the map', async () => {
for (const outcome of ['full', 'unreachable']) {
const { assignment, reserved } = setupAsia(feed('c1'), (cellId) =>
cellId === 'c1' ? { outcome } : { outcome: 'ok' }
)
const regionReserve = await regionReserveOf(assignment)
// Null sends the host to the database lane's cross-region fallback, not a 1 s retry.
await expect(regionReserve?.()).resolves.toBeNull()
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
}
})
})
describe('for a region-correction request', () => {
const request = { reconnect: true, region: US, regionCorrection: true }
it('leaves the answer to the database lane, offering only the in-region reserve fallback and the spill', async () => {
const row = vi.fn(async () => null)
// A memory answer would skip the window exchange; the share draw is not offered either.
const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row })
const plan = await assignment.plan(HOST, request)
expect(plan).toEqual({
kind: 'database',
regionReserve: expect.any(Function),
reserveSpill: expect.any(Function)
})
expect(reserved).toEqual([])
expect(row).not.toHaveBeenCalled()
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({
assignmentEpoch: 8
})
})
it('keeps today’s path for a host the map never saw, without reading its row, but may spill', async () => {
const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 }))
const { assignment, reserved } = setup({ row })
const plan = await assignment.plan(HOST, request)
expect(plan).toEqual({ kind: 'database', reserveSpill: expect.any(Function) })
expect(row).not.toHaveBeenCalled()
expect(reserved).toEqual([])
await expect(
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 3 }) : null
).resolves.toMatchObject({ assignmentEpoch: 4 })
})
it('keeps today’s path while the map is incomplete, and in dry run', async () => {
const gated = setup({ cells: cellList(['c1', 'c2']), feeds: [feed('c1', { full: [seated(4)] })], startedAt: 100_500 })
expect(await gated.assignment.plan(HOST, request)).toEqual({ kind: 'database' })
const dryRun = setup({ mode: 'dry-run', feeds: [feed('c1', { full: [seated(4)] }), feed('c2')] })
expect(await dryRun.assignment.plan(HOST, request)).toEqual({ kind: 'database' })
expect([...gated.reserved, ...dryRun.reserved]).toEqual([])
})
})
describe('after the database lane found no cell anywhere', () => {
const ASIA = 'asia-east2' as const
const paced = { intake: { perSec: 1, burst: 20, tokens: 0 } }
const fullCounts = { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } }
// The host sits on an Asia database cell, so the plan goes to the database lane.
const setupSpill = (asiaReserve: Partial<SeatFeedResponse>, usReserve: Partial<SeatFeedResponse> = {}, log?: (line: string) => void) =>
setup({
...(log ? { log } : {}),
cells: [...cellList(['asia-r', 'asia-db'], ASIA), ...cellList(['us-r'])],
feeds: [
feed('asia-r', asiaReserve),
feed('asia-db', { admitMode: 'db', full: [seated(4)] }),
feed('us-r', usReserve)
]
})
const spillOf = async (harness: ReturnType<typeof setup>, request: { regionCorrection?: boolean } = {}) => {
const plan = await harness.assignment.plan(HOST, { reconnect: true, region: ASIA, ...request })
expect(plan.kind).toBe('database')
return plan.kind === 'database' ? plan.reserveSpill : undefined
}
it('books its own region first, minting above the row the database lane read', async () => {
const harness = setupSpill({})
const spill = await spillOf(harness)
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 9 })).resolves.toMatchObject({
cellId: 'asia-r',
region: ASIA,
assignmentEpoch: 10
})
expect(harness.reserved).toEqual([{ cellId: 'asia-r', epoch: 10, sticky: false, dryRun: false }])
})
it('books the other region when its own is full, or already tried', async () => {
const full = setupSpill(fullCounts)
await expect((await spillOf(full))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
cellId: 'us-r',
region: US,
assignmentEpoch: 5
})
const tried = setupSpill({})
await expect((await spillOf(tried))?.({ skipOwnRegion: true, rowEpoch: 0 })).resolves.toMatchObject({
cellId: 'us-r'
})
expect(tried.reserved.map((call) => call.cellId)).toEqual(['us-r'])
})
it('tries the other region after a pace, and answers the pace only if nothing places', async () => {
const abroad = setupSpill(paced)
await expect((await spillOf(abroad))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
cellId: 'us-r'
})
const nowhere = setupSpill(paced, fullCounts)
await expect((await spillOf(nowhere))?.({ skipOwnRegion: false, rowEpoch: 0 })).rejects.toBeInstanceOf(
ReservePaceError
)
})
it('returns null with no reserve seat anywhere, so the refusal stands', async () => {
const harness = setupSpill(fullCounts, fullCounts)
await expect((await spillOf(harness))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toBeNull()
expect(harness.reserved).toEqual([])
})
it('is not offered while the map is incomplete', async () => {
const gated = setup({
cells: [...cellList(['asia-r', 'asia-db'], ASIA), ...cellList(['us-r'])],
feeds: [feed('asia-r'), feed('asia-db', { admitMode: 'db', full: [seated(4)] })],
startedAt: 100_500
})
const plan = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA })
expect(plan).toMatchObject({ kind: 'database' })
expect(plan.kind === 'database' && plan.reserveSpill).toBeUndefined()
const correction = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA, regionCorrection: true })
expect(correction).toEqual({ kind: 'database' })
})
it('answers the shortest pace when every region paces', async () => {
const harness = setupSpill(paced, { intake: { perSec: 2, burst: 20, tokens: 0 } })
const spill = await spillOf(harness)
// Asia refills its token in 4 s, US in 2 s.
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 0 })).rejects.toSatisfy(
(error) => error instanceof ReservePaceError && error.retryAfterSeconds === 2
)
})
it('counts a spill once, not as a database fallback or a pace per region', async () => {
for (const [asiaReserve, label] of [
[fullCounts, 'full'],
[paced, 'paced']
] as const) {
const lines: Record<string, unknown>[] = []
const harness = setupSpill(asiaReserve, {}, (line) => lines.push(JSON.parse(line)))
await (await spillOf(harness))?.({ skipOwnRegion: false, rowEpoch: 0 })
harness.now.value += 60_000
await harness.assignment.plan(HOST, { reconnect: true, region: ASIA })
expect(lines.find((line) => line.event === 'orca_relay_reserve_placement_summary'), label).toMatchObject({
spill: 1,
placed: 1,
database: 0,
retries: { paced: 0 }
})
expect(lines.filter((line) => line.event === 'orca_relay_reserve_spill'), label).toEqual([
{ event: 'orca_relay_reserve_spill', fromRegion: ASIA, toRegion: US, count: 1 }
])
}
})
})
describe('rehome on reconnect', () => {
const ASIA = 'asia-east2' as const
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
const ask = { reconnect: true, region: ASIA, preferred: true }
// 2,000 controls on the US cell refill its move budget once a minute.
const usCounts = { counts: { seats: 1, controls: 2_000, bookings: 0, units: 1, ceiling: 4_000 } }
const asiaFull = (share: number) => ({
counts: { seats: Math.round(share * 1_000), controls: 0, bookings: 0, units: Math.round(share * 1_000), ceiling: 1_000 }
})
const setupRehome = (
options: {
asia?: Partial<SeatFeedResponse>
usSeats?: ReturnType<typeof seated>[]
reserve?: (cellId: string) => ReserveAttempt
log?: (line: string) => void
percent?: number
switchOpen?: boolean
} = {}
) => {
const feeds = () => [
feed('us-r', { ...usCounts, full: options.usSeats ?? [seated(5, 1_000)] }),
feed('asia-r', { ...asiaFull(0.5), ...options.asia })
]
const harness = setup({
cells: [...cellList(['us-r']), ...cellList(['asia-r'], ASIA)],
feeds: feeds(),
rehome: new ReserveRehome({ percent: options.percent ?? 100, switchOpen: () => options.switchOpen ?? true }),
...(options.reserve ? { reserve: options.reserve } : {}),
...(options.log ? { log: options.log } : {})
})
// The first ask starts the US cell's empty budget; a minute later it holds one move.
const warm = async () => {
expect(await harness.assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
harness.now.value += 60_000
for (const response of feeds()) {
harness.directory.apply(response.cellId, { ...response, seq: 2 }, harness.now.value, harness.now.value)
}
}
return { ...harness, warm }
}
it('books an Asia-preferring host on its own region, above every known epoch, and supersedes the US seat', async () => {
const { assignment, demoted, warm } = setupRehome()
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({
kind: 'answer',
lane: 'rehome',
assignment: { cellId: 'asia-r', region: ASIA, assignmentEpoch: 6 }
})
expect(demoted).toEqual([{ cellId: 'us-r', epoch: 5, joinedAt: 1_000 }])
})
it('answers a region-correction request on the rehome lane', async () => {
const { assignment, warm } = setupRehome()
await warm()
expect(await assignment.plan(HOST, { ...ask, regionCorrection: true })).toMatchObject({
kind: 'answer',
lane: 'rehome'
})
})
it('moves no one into Asia at 98% full, or anywhere under 10% free', async () => {
for (const share of [0.98, 0.91]) {
const { assignment, reserved, warm } = setupRehome({ asia: asiaFull(share) })
await warm()
expect(await assignment.plan(HOST, ask), String(share)).toMatchObject({
kind: 'answer',
lane: 'sticky',
assignment: { cellId: 'us-r' }
})
expect(reserved.filter((call) => call.cellId === 'asia-r')).toEqual([])
}
const { assignment, warm } = setupRehome({ asia: asiaFull(0.9) })
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
})
// 10% rehome headroom above the 0% spill line prevents spilled hosts bouncing back.
it('keeps a gap above the spill line: 5% free takes placements, not rehomes', async () => {
const { assignment, warm } = setupRehome({ asia: asiaFull(0.95) })
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky', assignment: { cellId: 'us-r' } })
expect(await assignment.plan(OTHER, { reconnect: false, region: ASIA })).toMatchObject({
lane: 'placement',
assignment: { cellId: 'asia-r' }
})
})
it('keeps today’s answer when any rule fails', async () => {
const sticky = { kind: 'answer', lane: 'sticky', assignment: { cellId: 'us-r' } }
const unhinted = setupRehome()
await unhinted.warm()
expect(await unhinted.assignment.plan(HOST, { ...ask, preferred: false })).toMatchObject(sticky)
const off = setupRehome({ percent: 0 })
await off.warm()
expect(await off.assignment.plan(HOST, ask)).toMatchObject(sticky)
const closed = setupRehome({ switchOpen: false })
await closed.warm()
expect(await closed.assignment.plan(HOST, ask)).toMatchObject(sticky)
const cold = setupRehome()
expect(await cold.assignment.plan(HOST, ask)).toMatchObject(sticky)
const sameRegion = setupRehome()
await sameRegion.warm()
expect(await sameRegion.assignment.plan(HOST, { ...ask, region: US })).toMatchObject(sticky)
})
it('never runs for a fresh ask, an unknown host, a database-mode source, or an incomplete map', async () => {
const fresh = setupRehome()
await fresh.warm()
expect(await fresh.assignment.plan(HOST, { ...ask, reconnect: false })).toMatchObject({ lane: 'placement' })
expect(await fresh.assignment.plan(OTHER, ask)).toMatchObject({ kind: 'answer', lane: 'placement' })
const onDatabase = setup({
cells: [...cellList(['us-db']), ...cellList(['asia-r'], ASIA)],
feeds: [feed('us-db', { admitMode: 'db', ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
})
expect(await onDatabase.assignment.plan(HOST, ask)).toMatchObject({ kind: 'database' })
const gated = setup({
cells: [...cellList(['us-r']), ...cellList(['asia-r', 'asia-unheard'], ASIA)],
feeds: [feed('us-r', { ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
startedAt: 100_500,
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
})
expect(await gated.assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
expect(gated.reserved).toEqual([])
})
it('passes a region-correction request to the database lane when rehome has no room', async () => {
const { assignment, warm } = setupRehome({ asia: asiaFull(0.98) })
await warm()
expect(await assignment.plan(HOST, { ...ask, regionCorrection: true })).toEqual({
kind: 'database',
regionReserve: expect.any(Function),
reserveSpill: expect.any(Function)
})
})
it('falls through to the sticky answer when the rehome booking is paced', async () => {
const { assignment, reserved, warm } = setupRehome({
reserve: (cellId) => (cellId === 'asia-r' ? { outcome: 'intake' } : { outcome: 'ok' })
})
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ kind: 'answer', lane: 'sticky' })
expect(reserved.map((call) => call.cellId)).toEqual(['asia-r'])
})
it('leaves a host another install keeps dropping', async () => {
const { assignment, directory, now, warm } = setupRehome()
await warm()
directory.apply(
'us-r',
feed('us-r', {
...usCounts,
full: undefined,
seq: 3,
changes: [{ seq: 3, kind: 'leave', ...HOST, epoch: 4, generation: 1, closeCode: 4408, at: now.value - 1 }]
}),
now.value,
now.value
)
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
})
it('falls through to the sticky answer when the booking is refused, and gives the move back', async () => {
let refuse = true
const { assignment, warm } = setupRehome({
reserve: (cellId) => (cellId === 'asia-r' && refuse ? { outcome: 'full' } : { outcome: 'ok' })
})
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
refuse = false
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
})
it('moves one host per source budget a minute, and logs moves and skips per region pair', async () => {
const lines: Record<string, unknown>[] = []
const { assignment, now, warm } = setupRehome({
usSeats: [seated(5, 1_000), { ...OTHER, epoch: 5, generation: 1, state: 'active', joinedAt: 1_000 }],
log: (line) => lines.push(JSON.parse(line))
})
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
expect(await assignment.plan(OTHER, ask)).toMatchObject({ lane: 'sticky' })
now.value += 60_000
await assignment.plan({ ...HOST, relayHostId: 'flushflushflush1' }, ask)
expect(lines.filter((line) => line.event === 'orca_relay_reserve_rehome')).toEqual([
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 0, skippedByRule: { 'source-budget': 1 } },
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 1, skippedByRule: { 'source-budget': 1 } }
])
})
})
it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => {
const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) })
await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toMatchObject({
message: 'reserve_cells_paced',
retryAfterSeconds: expect.any(Number)
})
})
it('tells the old reserve seat to go when it books the host elsewhere, naming that seat exactly', async () => {
const { assignment, directory, demoted, now } = setup({
feeds: [feed('c1', { full: [seated(5, 1_000)] }), feed('c2')],
reserve: (cellId) => (cellId === 'c1' ? { outcome: 'full' } : { outcome: 'ok' })
})
// The host left c1 for a reason that moves it: its next placement is fresh, above 5.
directory.apply(
'c1',
feed('c1', {
full: undefined,
seq: 2,
changes: [{ seq: 2, kind: 'drain-only', ...HOST, epoch: 5, generation: 1, at: 2 }]
}),
now.value,
now.value
)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({ kind: 'answer', lane: 'placement', assignment: { assignmentEpoch: 6 } })
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
})
// A half-open seat missed by the booking map must be demoted by the feed, before its 75s silence close.
it('tells an old reserve seat the booking missed to go once a newer join shows in the feed', () => {
const { assignment, directory, demoted, now } = setup()
directory.onOutranked = (seat) => assignment.supersedeOutranked(seat)
directory.apply('c2', feed('c2', { full: [seated(6, 3_000)] }), now.value, now.value)
expect(demoted).toEqual([])
directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
})
it('answers a reconnect at the newest epoch even when an older epoch joined later', async () => {
const { assignment, reserved } = setup({
feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
})
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
lane: 'sticky',
assignment: { cellId: 'c1', assignmentEpoch: 6 }
})
expect(reserved).toEqual([])
})
it('in dry run only asks a cell to check a booking, and never changes the answer', async () => {
const { assignment, reserved } = setup({ mode: 'dry-run' })
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
expect(plan.kind).toBe('database')
expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
expect(reserved).toMatchObject([{ dryRun: true }])
})
it('in dry run counts outcomes per picked cell, and a `full` refusal while the map showed room', async () => {
const summaries: Array<{ dryRun: Record<string, unknown> }> = []
const dryRunOn = async (units: number, outcome: string) => {
const { assignment, now } = setup({
mode: 'dry-run',
cells: cellList(['c1']),
feeds: [feed('c1', { counts: { seats: units, controls: 0, bookings: 0, units, ceiling: 500 } })],
reserve: () => ({ outcome }),
log: (line) => summaries.push(JSON.parse(line))
})
for (let index = 0; index < 2; index += 1) {
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
if (plan.kind === 'database') await plan.placeFresh?.()
}
// Lets the fire-and-forget dry-run reserves settle before the summary flushes.
await Promise.resolve()
now.value += 60_000
await assignment.plan(HOST, { reconnect: false, region: US })
}
// 1 free seat of 500 is under the 1% bound; 250 free is far over it.
await dryRunOn(499, 'full')
await dryRunOn(250, 'full')
await dryRunOn(250, 'ok')
expect(summaries.map((summary) => summary.dryRun)).toMatchObject([
{
outcomes: { full: 2 },
count: 2,
cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 0, minFullFreeShare: 0.002 } }
},
{ outcomes: { full: 2 }, cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 2, minFullFreeShare: 0.5 } } },
{ outcomes: { ok: 2 }, cells: { c1: { outcomes: { ok: 2 }, fullWithRoom: 0, minFullFreeShare: null } } }
])
})
describe('when a cell answers a demotion', () => {
const outcomes = (lines: string[]) =>
lines
.map((line) => JSON.parse(line))
.filter((entry) => entry.event === 'orca_relay_reserve_demote_outcome')
it('logs the answer by host digest and leaves the seat to the feed', async () => {
const lines: string[] = []
const logged = Promise.withResolvers<void>()
const { assignment, directory } = setup({
feeds: [feed('c1', { full: [seated(10, 1_000)] }), feed('c2')],
demote: () => 'not-seated',
log: (line) => {
lines.push(line)
if (JSON.parse(line).event === 'orca_relay_reserve_demote_outcome') {
logged.resolve()
}
}
})
assignment.demoteSeat({ ...HOST, cellId: 'c1', epoch: 10, joinedAt: 1_000 }, 'reconcile')
await logged.promise
expect(outcomes(lines)).toMatchObject([
{ via: 'reconcile', cellId: 'c1', epoch: 10, outcome: 'not-seated' }
])
expect(outcomes(lines)[0].relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(lines.join('\n')).not.toContain(HOST.relayHostId)
expect(directory.seatsOnCell('c1')).toHaveLength(1)
})
it('records a cell it could not reach as unreachable', async () => {
const lines: string[] = []
const logged = Promise.withResolvers<void>()
const { assignment } = setup({
feeds: [feed('c1', { full: [seated(10, 1_000)] }), feed('c2')],
demote: () => null,
log: (line) => {
lines.push(line)
if (JSON.parse(line).event === 'orca_relay_reserve_demote_outcome') {
logged.resolve()
}
}
})
assignment.demoteSeat({ ...HOST, cellId: 'c1', epoch: 10, joinedAt: 1_000 }, 'reconcile')
await logged.promise
expect(outcomes(lines)).toMatchObject([{ outcome: 'unreachable' }])
})
})
describe('for a host re-asking off a draining cell', () => {
const events = (lines: string[], event: string) =>
lines.map((line) => JSON.parse(line)).filter((entry) => entry.event === event)
const flushedSummary = async (assignment: ReserveAssignment, now: { value: number }, lines: string[]) => {
now.value += 60_001
await assignment.plan({ ...HOST, relayHostId: 'zzzzzzzzzzzzzzzz' }, { reconnect: false, region: US })
return events(lines, 'orca_relay_reserve_placement_summary')[0]
}
// graceMs 0: the host re-asks while its seat still shows, but the cell's feed already says draining.
it('counts the placement by source cell before any leave lands', async () => {
const lines: string[] = []
const { assignment, now } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
log: (line) => lines.push(line)
})
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { cellId: 'c2', assignmentEpoch: 8 }
})
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([
{ event: 'orca_relay_reserve_drain_replaced', fromCellId: 'c1', toCellId: 'c2', region: US }
])
expect((await flushedSummary(assignment, now, lines)).drainReplaced).toEqual({ c1: 1 })
})
it('counts a drain-only seat, and a newest leave on a draining cell', async () => {
const lines: string[] = []
const drainOnly = setup({
feeds: [feed('c1', { full: [{ ...seated(7), state: 'drain-only' }] }), feed('c2')],
log: (line) => lines.push(line)
})
await drainOnly.assignment.plan(HOST, { reconnect: true, region: US })
const left = setup({ feeds: [feed('c1', { draining: true }), feed('c2')], log: (line) => lines.push(line) })
left.directory.apply(
'c1',
feed('c1', {
draining: true,
full: undefined,
seq: 3,
changes: [
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4503, at: 2 }
]
}),
left.now.value,
left.now.value
)
await left.assignment.plan(HOST, { reconnect: true, region: US })
expect(events(lines, 'orca_relay_reserve_drain_replaced').map((entry) => entry.fromCellId)).toEqual([
'c1',
'c1'
])
})
it('counts a host once: after its re-booking, a second ask is not a drain re-ask', async () => {
const lines: string[] = []
const { assignment, directory, now } = setup({
feeds: [feed('c1', { draining: true, full: [{ ...seated(7), state: 'drain-only' }] }), feed('c2')],
log: (line) => lines.push(line)
})
await assignment.plan(HOST, { reconnect: true, region: US })
// c2 reports the booking before the host's join: the map still holds only the drain-only seat.
directory.apply(
'c2',
feed('c2', {
full: undefined,
seq: 2,
changes: [{ seq: 2, kind: 'reserve', ...HOST, epoch: 8, generation: 0, reservedBy: 'director', at: now.value }]
}),
now.value,
now.value
)
expect(directory.bookingOf(HOST.userId, HOST.relayHostId)).toMatchObject({ cellId: 'c2', epoch: 8 })
await assignment.plan(HOST, { reconnect: false, region: US })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toHaveLength(1)
})
it('counts a host once when it asks again before the new cell reports the booking', async () => {
const lines: string[] = []
const { assignment } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
log: (line) => lines.push(line)
})
await assignment.plan(HOST, { reconnect: true, region: US })
await assignment.plan(HOST, { reconnect: false, region: US })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toHaveLength(1)
})
// Placement memories are clock-free: a host that never reached c2 is not counted again later.
it('counts a host once per drain, also when it asks again after 30 s', async () => {
const lines: string[] = []
const { assignment, directory, now } = setup({
feeds: [feed('c1', { draining: true, full: [{ ...seated(7), state: 'drain-only' }] }), feed('c2')],
log: (line) => lines.push(line)
})
await assignment.plan(HOST, { reconnect: true, region: US })
now.value += 31_000
for (const cellId of ['c1', 'c2']) {
directory.apply(cellId, feed(cellId, { draining: cellId === 'c1', full: undefined, seq: 1, changes: [] }), now.value, now.value)
}
await assignment.plan(HOST, { reconnect: true, region: US })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toHaveLength(1)
})
it('counts afresh once the drained cell restarts', async () => {
const lines: string[] = []
const { assignment, directory, now } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
log: (line) => lines.push(line)
})
await assignment.plan(HOST, { reconnect: true, region: US })
directory.apply(
'c1',
feed('c1', { incarnation: 'c1-incarnation-2', draining: true, full: [seated(20)] }),
now.value,
now.value
)
await assignment.plan(HOST, { reconnect: true, region: US })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toHaveLength(2)
})
it('marks its pace as the drain’s, on the reserve lane, in the region fallback and in the spill', async () => {
const { assignment } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
reserve: () => ({ outcome: 'intake' })
})
await expect(assignment.plan(HOST, { reconnect: true, region: US })).rejects.toMatchObject({ drained: true })
const plan = await assignment.plan(HOST, { reconnect: true, region: US, regionCorrection: true })
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).rejects.toMatchObject({ drained: true })
await expect(
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 0 }) : null
).rejects.toMatchObject({ drained: true })
})
it('leaves every other host as today: no drain count, and a plain pace', async () => {
const lines: string[] = []
const { assignment } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], log: (line) => lines.push(line) })
const paced = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2', { draining: true, full: [seated(6)] })],
reserve: () => ({ outcome: 'intake' })
})
expect(await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: false, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement'
})
await expect(
paced.assignment.plan(HOST, { reconnect: false, region: US })
).rejects.toMatchObject({ drained: false })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([])
})
// The database lane logs orca_relay_sticky_replaced_off_isolated_cell for these; counting both would double the canary's sum.
it('does not count a placement the database lane makes through its region fallback', async () => {
const lines: string[] = []
const { assignment } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
log: (line) => lines.push(line)
})
const plan = await assignment.plan(HOST, { reconnect: true, region: US, regionCorrection: true })
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({ cellId: 'c2' })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([])
})
})
})