test: remove 145 low-value tests and nine suites (#37982)

* test: remove 145 low-value test declarations

* test: clean up remaining type-only relay imports

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-11-03 16:59:13 -08:00
committed by GitHub
co-authored by OrcaWin
parent c332002953
commit 249faa46c3
48 changed files with 185 additions and 2761 deletions
@@ -10,7 +10,7 @@ import {
assignmentLeaseKeyId,
classifyAssignmentLease
} from './assignment-lease.js'
import { ASSIGNMENT_LEASE_SHADOW_EVENT, AssignmentLeaseShadow } from './assignment-lease-shadow.js'
import { AssignmentLeaseShadow } from './assignment-lease-shadow.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from '../config.js'
import type { RelayCredentialStore } from '../credential-store.js'
@@ -61,14 +61,6 @@ const classify = async (value: string | undefined, helloEpoch = 4) =>
})
describe('assignment lease classification', () => {
it('names the key by a short hash of it', () => {
expect(assignmentLeaseKeyId(KEY)).toMatch(/^k1-[0-9a-f]{8}$/)
expect(assignmentLeaseKeyId(KEY)).toBe(
`k1-${createHash('sha256').update(KEY).digest('hex').slice(0, 8)}`
)
expect(assignmentLeaseKeyId(OTHER_KEY)).not.toBe(assignmentLeaseKeyId(KEY))
})
it('sorts every lease into one class', async () => {
expect(await classify(undefined)).toBe('absent')
expect(await classify(await lease())).toBe('valid')
@@ -90,58 +82,6 @@ describe('assignment lease classification', () => {
})
})
describe('assignment lease shadow', () => {
it('does nothing while the switch is off', () => {
const shadow = new AssignmentLeaseShadow({ enabled: () => false, key: KEY, cellId: CELL_ID })
expect(shadow.check({ lease: undefined, ...HOST, helloEpoch: 4 })).toBeNull()
})
it('counts each class against the database answer and logs one line per window', async () => {
let now = 0
const log = vi.fn()
const shadow = new AssignmentLeaseShadow({
enabled: () => true,
key: KEY,
cellId: CELL_ID,
now: () => now,
log
})
const valid = await lease()
const record = async (value: string | undefined, dbValid: boolean, dbReadMs: number) => {
const check = shadow.check({ lease: value, ...HOST, helloEpoch: 4 })
expect(check).not.toBeNull()
if (check) shadow.record(check, dbValid, dbReadMs)
await check
await Promise.resolve()
}
await record(valid, true, 2)
await record(valid, false, 4)
await record(undefined, true, 6)
expect(log).not.toHaveBeenCalled()
now = 60_000
await record(valid, true, 8)
expect(log).toHaveBeenCalledTimes(1)
expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toEqual({
event: ASSIGNMENT_LEASE_SHADOW_EVENT,
cellId: CELL_ID,
windowMs: 60_000,
classes: {
agree: 2,
disagree: 1,
absent: 1,
expired: 0,
badSignature: 0,
wrongHost: 0,
wrongCell: 0,
epochBehind: 0,
epochAhead: 0
},
dbRefused: 1,
dbReadMs: { samples: 4, p50: 4, p99: 8, max: 8 }
})
})
})
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSED = 3
@@ -181,7 +121,13 @@ describe('host hello with the lease shadow on', () => {
{} as RelayCredentialStore,
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
new ProcessQueuedByteBudget(),
{ recordAuth: vi.fn(), recordForwardedBytes: vi.fn(), recordHttp: vi.fn(), recordReconnect: vi.fn(), recordSql: vi.fn() },
{
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn()
},
Date.now,
Math.random,
'incarnation-1',
@@ -189,7 +135,13 @@ describe('host hello with the lease shadow on', () => {
)
const socket = new FakeSocket()
const hostLease = input.withLease ? await lease({ relayHostId }) : undefined
registry.acceptControl(socket as unknown as WebSocket, identity, undefined, undefined, hostLease)
registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
undefined,
hostLease
)
socket.emit(
'message',
Buffer.from(
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayHomeCellUnavailableError, type RelayAssignment } from './assignment-store.js'
import type { RelayAssignment } from './assignment-store.js'
import type { RelayConfig } from '../config.js'
const fakes = vi.hoisted(() => ({
@@ -42,9 +42,9 @@ describe('assignment rejection logging', () => {
expect(hinted.status).toBe(503)
expect(await hinted.json()).toEqual({ error: 'relay_capacity_exhausted' })
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment rejected')
)
const line = warn.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('assignment rejected'))
expect(line).toContain('route=assign')
expect(line).toContain('lane=sticky')
expect(line).toContain('hinted=true')
@@ -53,67 +53,6 @@ describe('assignment rejection logging', () => {
expect(line).not.toContain(host)
})
it('separates an unavailable home cell from capacity and names its cause', async () => {
const host = 'cccccccccccccccc'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => {
throw new RelayHomeCellUnavailableError('cell-asia-1', 'not_ready')
}),
// The sticky lane refuses a host whose home cell is not live, so this
// arrives hinted on the placement lane.
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(503)
expect(await response.json()).toEqual({ error: 'relay_home_cell_unavailable' })
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment rejected')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('reason=relay_home_cell_unavailable')
expect(line).toContain('cause=not_ready')
expect(line).toContain('cell=cell-asia-1')
expect(line).not.toContain('relay_capacity_exhausted')
expect(line).not.toContain(host)
})
it('logs an unhinted placement rejection without the raw host id', async () => {
const host = 'gggggggggggggggg'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => {
throw new Error('relay_connection_headroom_exhausted')
}),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host))
expect(response.status).toBe(503)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment rejected')
)
expect(line).toContain('route=assign')
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=false')
expect(line).toContain('reason=relay_connection_headroom_exhausted')
expect(line).not.toContain(host)
})
it('names the throttled host once per window and closes it with the suppressed count', async () => {
vi.useFakeTimers()
const host = 'mmmmmmmmmmmmmmmm'
@@ -148,27 +87,6 @@ describe('assignment rejection logging', () => {
expect(throttled()[1]).toContain('suppressed=1')
expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`)
})
it('stays silent for successful assignments', async () => {
const host = 'ssssssssssssssss'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-r', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
expect(
warn.mock.calls.map((call) => String(call[0])).filter((entry) =>
entry.includes('assignment rejected')
)
).toEqual([])
})
})
function assignmentRequest(
@@ -224,80 +142,3 @@ function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
...overrides
}
}
describe('assignment grant logging', () => {
afterEach(() => {
vi.restoreAllMocks()
})
it('logs sticky grants with cell id and host digest only', async () => {
const host = 'gggggggggggggggg'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-r', host)),
resolve: vi.fn(async () => assignment('cell-r', host))
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(200)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment granted')
)
expect(line).toContain('lane=sticky')
expect(line).toContain('cell=cell-r')
expect(line).toContain(`host=${relayHostLogDigest(host)}`)
expect(line).not.toContain(host)
})
it('logs a hinted grant served by the placement lane', async () => {
const host = 'rrrrrrrrrrrrrrrr'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-new', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(200)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment granted')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('cell=cell-new')
expect(line).not.toContain(host)
})
it('does not log unhinted placement grants', async () => {
const host = 'pppppppppppppppp'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-r', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
expect(
warn.mock.calls.map((call) => String(call[0])).filter((entry) =>
entry.includes('assignment granted')
)
).toEqual([])
})
})
@@ -34,11 +34,15 @@ describe('assignment reserve-mode behavior', () => {
// A store that has never read the set: the table is unreadable.
async function unknownSet(now: () => number) {
const unknown = new RelayAssignmentStore(database!, now)
await database!.query('ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone')
await database!.query(
'ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone'
)
return {
unknown,
restore: async () =>
await database!.query('ALTER TABLE relay_cell_admit_modes_gone RENAME TO relay_cell_admit_modes')
await database!.query(
'ALTER TABLE relay_cell_admit_modes_gone RENAME TO relay_cell_admit_modes'
)
}
}
@@ -61,55 +65,6 @@ describe('assignment reserve-mode behavior', () => {
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
})
it('logs an unreadable reserve-mode set at most once a minute', async () => {
let now = 1_000
await setup(() => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const clock = vi.spyOn(performance, 'now')
let at = 1_000_000
clock.mockImplementation(() => at)
const { unknown, restore } = await unknownSet(() => now)
try {
for (let read = 0; read < 3; read += 1) {
expect(await unknown.reserveModeCells()).toBeNull()
at += 10_000
}
at += 60_000
expect(await unknown.reserveModeCells()).toBeNull()
const lines = warn.mock.calls.filter(([line]) => String(line).includes('orca_relay_reserve_mode_read_failed'))
expect(lines).toHaveLength(2)
} finally {
await restore()
clock.mockRestore()
warn.mockRestore()
}
})
it('tells break glass whether a cell is still heartbeating', async () => {
let now = 1_000
const store = await setup(() => now)
expect(await store.cellHeartbeatFresh('cell-a')).toBe(false)
await store.reconcileCells([{ ...CELLS[0]!, connectionHardCap: 600, connectionUnobservedBound: 50 }, CELLS[1]!])
await store.recordCellHeartbeat({
cellId: 'cell-a',
cellUrl: CELLS[0]!.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: false,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionHardCap: 600,
connectionUnobservedBound: 50
})
// Not ready is still alive: break glass is for a cell that has gone silent.
expect(await store.cellHeartbeatFresh('cell-a')).toBe(true)
now += 45_001
expect(await store.cellHeartbeatFresh('cell-a')).toBe(false)
})
it('keeps an idle host sticky on a reserve-mode cell where today it would be re-placed', async () => {
let now = 1_000
const store = await setup(() => now)
@@ -136,8 +91,13 @@ describe('assignment reserve-mode behavior', () => {
await setReserve(store, [first.cellId])
// The reserve ledger stamps last_activity_at and the cell keeps no activity rows.
now += 5 * 60_000
await database!.query('DELETE FROM relay_assignment_activity_leases WHERE user_id = ?', [identity.userId])
await database!.query('UPDATE relay_assignments SET last_activity_at = ? WHERE user_id = ?', [now, identity.userId])
await database!.query('DELETE FROM relay_assignment_activity_leases WHERE user_id = ?', [
identity.userId
])
await database!.query('UPDATE relay_assignments SET last_activity_at = ? WHERE user_id = ?', [
now,
identity.userId
])
await store.setCellAdmissionState(first.cellId, 'existing-only')
now += 2 * 60_000
expect(await store.assign(identity)).toMatchObject({
@@ -221,12 +181,16 @@ describe('assignment reserve-mode behavior', () => {
await store.changeActivity(identity, 'control', -1)
now += ASSIGNMENT_LIMITS.activityLeaseMs + ASSIGNMENT_LIMITS.dormantTtlMs + 1
expect([...((await store.reserveModeCells()) ?? [])]).toEqual([])
await database!.query('ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone')
await database!.query(
'ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone'
)
at += 61_000
expect(await store.reserveModeCells()).toBeNull()
expect((await store.assign(identity)).assignmentEpoch).toBeGreaterThan(first.assignmentEpoch)
} finally {
await database!.query('ALTER TABLE relay_cell_admit_modes_gone RENAME TO relay_cell_admit_modes').catch(() => undefined)
await database!
.query('ALTER TABLE relay_cell_admit_modes_gone RENAME TO relay_cell_admit_modes')
.catch(() => undefined)
clock.mockRestore()
}
})
@@ -277,11 +241,16 @@ describe('assignment reserve-mode behavior', () => {
source_request_units, target_reserved_units, expires_at, created_at, updated_at)
VALUES ('user-a', 'host000000000001', 'cell-b', 'cell-a', 1, 2, 0, 1, 9999, 1000, 1000)`
)
await expect(store.setCellAdmitMode('cell-a', 'reserve')).rejects.toThrow('cell_has_open_migration')
await expect(store.setCellAdmitMode('cell-a', 'reserve')).rejects.toThrow(
'cell_has_open_migration'
)
expect(await store.cellAdmitMode('cell-a')).toMatchObject({ admitMode: 'db' })
await store.setCellAdmitMode('cell-a', 'db')
await database!.query(`UPDATE relay_assignment_migrations SET aborted_at = 1001`)
await store.setCellAdmitMode('cell-a', 'reserve')
expect(await store.cellAdmitMode('cell-a')).toMatchObject({ admitMode: 'reserve', updatedAt: 1_000 })
expect(await store.cellAdmitMode('cell-a')).toMatchObject({
admitMode: 'reserve',
updatedAt: 1_000
})
})
})
+3 -67
View File
@@ -4,7 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { z } from 'zod'
import type { RelayAssignmentStore } from './assignment/assignment-store.js'
import { CELL_FLAG_DEFAULTS, type CellFlags } from './cell-flags.js'
import type { CellFlags } from './cell-flags.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { HostSessionRegistry, type HostSession } from './host-session-registry.js'
@@ -227,26 +227,6 @@ describe('cell seat feed', () => {
vi.useRealTimers()
})
it('answers a first poll with every seated host, matching the controls count', async () => {
const { connect, poll, read } = createCell()
await connect(1)
await connect(2, { epoch: 4 })
expect((await poll()).status).toBe(200)
const body = await read()
expect(body).toMatchObject({
v: 1,
cellId: 'production-gce-c7',
incarnation,
seq: 2,
draining: false,
counts: { controls: 2, seats: 2 }
})
expect(body.full).toHaveLength(body.counts.seats)
expect(body.full).toContainEqual(
expect.objectContaining({ userId: 'user-2', epoch: 4, generation: 1, state: 'active' })
)
})
it('keeps a closing socket seated until its close lands, as the change log does', async () => {
const { connect, read } = createCell()
const socket = await connect(1)
@@ -270,10 +250,7 @@ describe('cell seat feed', () => {
const body = await read(`${incarnation}:${seq}`)
expect(body.full).toBeUndefined()
expect(body.draining).toBe(true)
expect(body.changes?.map((change) => change.kind)).toEqual([
'drain-only',
'leave'
])
expect(body.changes?.map((change) => change.kind)).toEqual(['drain-only', 'leave'])
expect(body.changes?.[1]).toMatchObject({
userId: 'user-1',
closeCode: RELAY_CLOSE_CODE.DRAINING,
@@ -322,9 +299,7 @@ describe('cell seat feed', () => {
await connect(1, { generation: 2 })
const body = await read(`${incarnation}:${seq}`)
// Readers drop the leave: its generation is older than the seat's.
expect(
body.changes?.map((change) => [change.kind, change.generation])
).toEqual([
expect(body.changes?.map((change) => [change.kind, change.generation])).toEqual([
['leave', 1],
['join', 2]
])
@@ -338,25 +313,6 @@ describe('cell seat feed', () => {
expect((await poll('not a cursor')).status).toBe(400)
})
it('reports the applied cell flags, so a flip can be read back through the feed and runtime status', async () => {
let applied: AppliedControlFlags<CellFlags> = { generation: 0, flags: CELL_FLAG_DEFAULTS }
const { read, app } = createCell(config(), () => applied)
expect((await read()).flagsApplied).toEqual({ generation: 0, flags: CELL_FLAG_DEFAULTS })
applied = { generation: 12, flags: { ...CELL_FLAG_DEFAULTS, readinessLocal: true, ticketCheck: 'shadow' } }
expect((await read()).flagsApplied).toEqual(applied)
const runtime = await app.request('/v1/admin/runtime-status', {
method: 'POST',
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
body: JSON.stringify({ v: 1 })
})
expect(await runtime.json()).toMatchObject({
flagsApplied: applied,
// Not reserve mode, so db whatever the switch says.
phonePathEffective: 'db',
databasePoolMax: expect.any(Number)
})
})
it('accepts only the directors rehome identity, verified once per poll', async () => {
const { poll } = createCell()
adminTokenChecks.count = 0
@@ -377,24 +333,4 @@ describe('cell seat feed', () => {
const director = createCell(config({ role: 'director', cellId: 'director' }))
expect((await director.poll()).status).toBe(404)
})
// Review B1: the feed must not need the pair on more cells, and must not change what a cell
// with or without it reports as its rehome protocol.
it('leaves the reported rehome protocol to the existing env', async () => {
const rehomeProtocol = async (relayConfig: RelayConfig) => {
const response = await createCell(relayConfig).app.request('/v1/admin/runtime-status', {
method: 'POST',
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
body: JSON.stringify({ v: 1 })
})
return z.object({ regionalRehomeProtocol: z.number() }).parse(await response.json())
.regionalRehomeProtocol
}
expect(await rehomeProtocol(config())).toBe(3)
expect(
await rehomeProtocol(
config({ rehomeAudience: undefined, rehomeDirectorServiceAccount: undefined })
)
).toBe(0)
})
})
@@ -21,7 +21,11 @@ import type { CellReserveClient } from './cell-reserve-client.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
import { ReserveAssignment, type ReservePlacementMode } from './reserve-assignment.js'
import { ReservePlacer } from './reserve-placement.js'
import { ShadowSeatDirectory, type SeatFeedCell, type SeatFeedResponse } from './shadow-seat-directory.js'
import {
ShadowSeatDirectory,
type SeatFeedCell,
type SeatFeedResponse
} from './shadow-seat-directory.js'
const INCARNATION = '11111111-1111-4111-8111-111111111111'
const CELLS = [
@@ -66,7 +70,9 @@ describe('reserve-aware cell status', () => {
database = undefined
})
async function setup(options: { mode?: ReservePlacementMode; startedAt?: number; placement?: boolean } = {}) {
async function setup(
options: { mode?: ReservePlacementMode; startedAt?: number; placement?: boolean } = {}
) {
const now = { value: 100_000 }
database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => now.value, { heartbeatTtlMs: 45_000 })
@@ -98,7 +104,10 @@ describe('reserve-aware cell status', () => {
directory,
cells: () => cells,
startedAt: options.startedAt ?? 0,
placer: new ReservePlacer(() => now.value, () => 0.25),
placer: new ReservePlacer(
() => now.value,
() => 0.25
),
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status route never books or demotes.
client: {} as CellReserveClient,
readRow: async () => null,
@@ -130,12 +139,19 @@ describe('reserve-aware cell status', () => {
const reserve = await store.reserveAwareCellDeploymentStatus('cell-a')
expect(reserve).toMatchObject({
admitMode: 'reserve',
status: { cellId: 'cell-a', runtime: { cellIncarnation: INCARNATION, ready: true, heartbeatFresh: true } }
status: {
cellId: 'cell-a',
runtime: { cellIncarnation: INCARNATION, ready: true, heartbeatFresh: true }
}
})
expect(Object.keys(reserve.status).filter((key) => PER_HOST_FIELDS.includes(key))).toEqual([])
await database!.query('ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone')
await database!.query(
'ALTER TABLE relay_cell_admit_modes RENAME TO relay_cell_admit_modes_gone'
)
const unknown = new RelayAssignmentStore(database!, () => 100_000)
await expect(unknown.reserveAwareCellDeploymentStatus('cell-b')).rejects.toThrow('reserve_mode_unknown')
await expect(unknown.reserveAwareCellDeploymentStatus('cell-b')).rejects.toThrow(
'reserve_mode_unknown'
)
})
it('keeps the 404 for a reserve cell without the opt-in, and 400s an unknown view', async () => {
@@ -146,18 +162,15 @@ describe('reserve-aware cell status', () => {
expect((await status({ cellId: 'cell-a', view: 'all' })).status).toBe(400)
})
it('answers a database cell with the opt-in as today, plus reserve: null', async () => {
const { status } = await setup()
const today = await statusBody(await status({ cellId: 'cell-b' }))
const response = await status({ cellId: 'cell-b', view: 'reserve-aware' })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ...today, status: { ...today.status, reserve: null } })
})
it('answers a reserve cell from the map, with perHost null and never zeros', async () => {
const { directory, now, status } = await setup()
directory.apply('cell-a', feed('cell-a', { draining: true }), now.value, now.value - 1_500)
directory.apply('cell-b', feed('cell-b', { admitModeEffective: 'db', admitModeRaw: 'db' }), now.value, now.value)
directory.apply(
'cell-b',
feed('cell-b', { admitModeEffective: 'db', admitModeRaw: 'db' }),
now.value,
now.value
)
now.value += 20_000
const response = await status({ cellId: 'cell-a', view: 'reserve-aware' })
expect(response.status).toBe(200)
@@ -233,19 +246,25 @@ describe('reserve-aware cell status', () => {
// A starting director with one cell unheard.
const starting = await setup({ startedAt: 99_000 })
starting.directory.apply('cell-a', feed('cell-a'), starting.now.value, starting.now.value)
expect(await unavailable(await starting.status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(
expected('map-incomplete')
)
expect(
await unavailable(await starting.status({ cellId: 'cell-a', view: 'reserve-aware' }))
).toEqual(expected('map-incomplete'))
await database?.close()
const { directory, now, status } = await setup()
directory.apply('cell-b', feed('cell-b'), now.value, now.value)
directory.markNoFeed('cell-a', now.value)
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(expected('no-feed'))
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(
expected('no-feed')
)
// A feed without the drain flag or counts is an older image: no feed, never an empty cell.
directory.apply('cell-a', feed('cell-a', { draining: undefined }), now.value, now.value)
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(expected('no-feed'))
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(
expected('no-feed')
)
directory.apply('cell-a', feed('cell-a', { counts: undefined }), now.value, now.value)
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(expected('no-feed'))
expect(await unavailable(await status({ cellId: 'cell-a', view: 'reserve-aware' }))).toEqual(
expected('no-feed')
)
// The process that answered the feed is not the one heartbeating the runtime row.
directory.apply(
'cell-a',
@@ -257,41 +276,6 @@ describe('reserve-aware cell status', () => {
expected('incarnation-mismatch')
)
})
it('probes rehome trust on a reserve cell from its runtime rows', async () => {
const { store } = await setup()
await database!.query(
`INSERT INTO relay_cell_capabilities (cell_id, cell_incarnation, regional_rehome_protocol, last_heartbeat_at)
VALUES ('cell-a', ?, 1, 100000)`,
[INCARNATION]
)
const sourceFetch = vi.fn(async () =>
Response.json({ v: 1, outcome: 'host-not-connected', sharedRuntimeIdentityRejected: true })
)
const app = createRelayApp(
config({
rehomeDirectorServiceAccount: 'relay-director@example.test',
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain'
}),
{
store: {} as never,
assignments: store,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the probe only awaits a Response.
regionalRehomeFetch: sourceFetch as unknown as typeof fetch,
ready: vi.fn(async () => true)
}
)
const response = await app.request('/v1/admin/regional-rehome-trust-probe', {
method: 'POST',
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, sourceCellId: 'cell-a', sourceCellIncarnation: INCARNATION })
})
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
expect(sourceFetch).toHaveBeenCalledTimes(2)
})
})
async function statusBody(response: Response) {
@@ -2,11 +2,7 @@ import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment/assignment-store.js'
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type SqlRow
} from './database.js'
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js'
const now = 1_900_000_000_000
const expiresAt = now + 105_000
@@ -57,53 +53,6 @@ class RenewalStatementProbe implements RelayDatabase {
}
describe('batched control renewals on PostgreSQL', () => {
it('spends one statement on every host that came due', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000003')
])
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
expect(probe.statements).toHaveLength(1)
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
})
it('locks assignment rows in primary-key order and still answers in input order', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-c', 'host000000000003'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000001'),
renewal('user-a', 'host000000000001')
])
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
// statement's ORDER BY repeats it: no batch can queue against another in a
// different sequence.
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
expect(probe.statements[0]!.params[1]).toEqual([
'host000000000001',
'host000000000002',
'host000000000001',
'host000000000003'
])
expect(outcomes).toEqual([
'renewed',
'renewed',
'control_activity_moved',
'renewed'
])
})
it('keeps a malformed request out of the statement and fails only that row', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
@@ -148,39 +97,6 @@ describe('batched control renewals on PostgreSQL', () => {
}
})
it('runs the batch on the priority lane and the per-host fallback on the general one', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const lanes: string[] = []
const database: RelayDatabase = {
dialect: 'postgres',
query: async (sql, params) => {
lanes.push('general')
return await probe.query(sql, params)
},
queryPriority: async (sql, params) => {
lanes.push('priority')
return await probe.query(sql, params)
},
queryLocked: async () => await probe.queryLocked(),
transaction: async () => await probe.transaction(),
close: async () => undefined
}
const store = new RelayAssignmentStore(database, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
// A 200-row fallback would otherwise jump the whole general queue.
expect(lanes).toEqual(['priority', 'general', 'general'])
} finally {
warn.mockRestore()
}
})
it('does not fan out per host when the batch never got a connection', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
@@ -216,59 +132,6 @@ describe('batched control renewals on PostgreSQL', () => {
warn.mockRestore()
}
})
it('counts a lone renewal that threw before rethrowing it', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
// still owes an outcome for the attempt.
await expect(
store.renewControlActivities([renewal('user-a', 'host000000000001')])
).rejects.toThrow('statement timeout')
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
})
it('counts a batch in which every row was rejected before the statement', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
const outcomes = await store.renewControlActivities([
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
renewal('user-a', 'host000000000002', now - 1)
])
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
expect(probe.statements).toHaveLength(0)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'invalid_activity_id',
'invalid_activity_expiry'
])
})
it('counts one renewal outcome metric per row', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
)
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'renewed',
'assignment_not_found'
])
})
})
describe('batched control renewals on SQLite', () => {
@@ -210,20 +210,6 @@ describe('host pre-auth latency', () => {
}
})
it('records a hello the database refuses on the db path', async () => {
const h = setup({ admitMode: 'db', databaseSays: false })
const { keyPair, identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
h.advance(120)
sendHello(socket, keyPair, identity.relayHostId)
await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
expect(h.take).not.toHaveBeenCalled()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'refused', 120)
})
it('records a hello timeout as a refusal before any check, once', () => {
vi.useFakeTimers()
const h = setup({ admitMode: 'db', databaseSays: true })
@@ -240,17 +226,6 @@ describe('host pre-auth latency', () => {
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 2_000)
})
it('records a host that hangs up before its hello as abandoned', () => {
const h = setup({ admitMode: 'db', databaseSays: true })
const { identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
h.advance(300)
socket.close()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'abandoned', 300)
})
it.each([true, false])(
'records a database answer (%s) that arrives after the host hung up as abandoned',
async (valid) => {
@@ -270,33 +245,6 @@ describe('host pre-auth latency', () => {
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'abandoned', 900)
}
)
it('records a host that hangs up during its proof as abandoned on its path', async () => {
const h = setup({ admitMode: 'reserve', databaseSays: false })
const { keyPair, identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
sendHello(socket, keyPair, identity.relayHostId)
await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
h.advance(700)
socket.close()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'abandoned', 700)
})
it('records the pool-pressure shed as a refusal before any check', async () => {
const h = setup({ admitMode: 'reserve', databaseSays: true, databaseShedding: true })
h.take.mockReturnValue(null)
const { keyPair, identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
h.advance(40)
sendHello(socket, keyPair, identity.relayHostId)
await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
expect(h.verifyCellAssignment).not.toHaveBeenCalled()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 40)
})
})
function config(): RelayConfig {
@@ -127,41 +127,6 @@ describe('POST /v1/admin/host-whereabouts', () => {
expect(capacity.status).toBe(401)
})
it('returns the database row, the map seats, the verdict and the named cells', async () => {
const lookup = vi.fn(async () => ROW)
const app = appWith(lookup, { shadowSeats: directory() })
const response = await app.request(
'/v1/admin/host-whereabouts',
request({ v: 1, relayHostId: HOST, userId: 'user-a' })
)
expect(response.status).toBe(200)
expect(lookup).toHaveBeenCalledWith({ userId: 'user-a', relayHostId: HOST })
expect(await response.json()).toMatchObject({
v: 1,
directorIncarnation: 'director-incarnation',
relayHostId: HOST,
shadow: { state: 'on', complete: true },
users: [
{
userId: 'user-a',
database: ROW,
seats: [{ cellId: 'cell-a', epoch: 3, state: 'active' }],
recentlyLeft: [],
verdict: { class: 'agree', explained: true }
}
],
cells: {
'cell-a': {
status: 'live',
lastFailure: null,
flagsApplied: { generation: '7', flags: { readinessLocal: false } }
}
}
})
})
it('names the user from the map when the request has only a host id', async () => {
const lookup = vi.fn(async () => ROW)
const app = appWith(lookup, { shadowSeats: directory() })
@@ -196,44 +161,6 @@ describe('POST /v1/admin/host-whereabouts', () => {
users: [{ database: ROW, verdict: { class: 'map-only-cell-unlive', explained: true } }]
})
})
it('answers an unknown host with an empty result, not a 404', async () => {
const lookup = vi.fn(async () => null)
const shadowOff = appWith(lookup)
const unknown = await shadowOff.request(
'/v1/admin/host-whereabouts',
request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu' })
)
expect(unknown.status).toBe(200)
expect(await unknown.json()).toEqual({
v: 1,
directorIncarnation: 'director-incarnation',
relayHostId: 'uuuuuuuuuuuuuuuu',
shadow: { state: 'off' },
users: [],
cells: {}
})
expect(lookup).not.toHaveBeenCalled()
const named = await shadowOff.request(
'/v1/admin/host-whereabouts',
request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu', userId: 'user-z' })
)
expect(await named.json()).toMatchObject({
users: [{ userId: 'user-z', database: null, seats: [], recentlyLeft: [], verdict: null }]
})
})
it('answers a briefly unreachable database with a retryable 503', async () => {
const app = appWith(async () => {
throw new Error('Connection terminated due to connection timeout')
})
const response = await app.request(
'/v1/admin/host-whereabouts',
request({ v: 1, relayHostId: HOST, userId: 'user-a' })
)
expect(response.status).toBe(503)
})
})
describe('RelayAssignmentStore.hostWhereabouts', () => {
@@ -1,10 +1,5 @@
import { describe, expect, it } from 'vitest'
import {
PLACEMENT_INTAKE_FLOOR,
PLACEMENT_INTAKE_SHARE,
PLACEMENT_INTAKE_WINDOW_MS,
PlacementLoadBand
} from './placement-load-band.js'
import { PLACEMENT_INTAKE_SHARE, PlacementLoadBand } from './placement-load-band.js'
import { RelayAssignmentStore } from './assignment/assignment-store.js'
import { openInMemoryRelayDatabase } from './database.js'
@@ -36,8 +31,7 @@ function simulate(
.filter((cell) => cell.reserved < cell.capacity)
.map((cell) => ({ cellId: cell.cellId, loadRatio: cell.reserved / cell.capacity, cell }))
.sort(
(left, right) =>
left.loadRatio - right.loadRatio || left.cellId.localeCompare(right.cellId)
(left, right) => left.loadRatio - right.loadRatio || left.cellId.localeCompare(right.cellId)
)
const director = directors[Math.floor(route() * directors.length)]
const picked = director?.pick(ranked, at(index))
@@ -73,26 +67,6 @@ function drainShare(paceMs: number, seed: number) {
}
describe('PlacementLoadBand', () => {
it('spreads a drain over an empty cell and four fuller cells within one window', () => {
const cells: SimulatedCell[] = [
{ cellId: 'cell-empty', reserved: 0, capacity: 6000 },
...['cell-b', 'cell-c', 'cell-d', 'cell-e'].map((cellId) => ({
cellId,
reserved: 2100,
capacity: 6000
}))
]
const director = new PlacementLoadBand(seededRandom(7), quiet)
const counts = simulate([director], cells, 2000, () => 1_000)
expect([...counts.values()].reduce((sum, count) => sum + count, 0)).toBe(2000)
expect(counts.get('cell-empty')).toBeLessThanOrEqual(PLACEMENT_INTAKE_SHARE * 2000 + 1)
for (const cellId of ['cell-b', 'cell-c', 'cell-d', 'cell-e']) {
expect(counts.get(cellId)).toBeGreaterThan(200)
}
for (const cell of cells) expect(cell.reserved).toBeLessThanOrEqual(cell.capacity)
})
it.each([
['1,200 s', 1_200_000],
['300 s', 300_000]
@@ -105,74 +79,6 @@ describe('PlacementLoadBand', () => {
for (const cell of cells) expect(cell.reserved).toBeLessThanOrEqual(cell.capacity)
}
})
it('gives the empty cell about 60% when only one other cell is eligible', () => {
const cells: SimulatedCell[] = [
{ cellId: 'cell-empty', reserved: 0, capacity: 6000 },
{ cellId: 'cell-b', reserved: 2100, capacity: 6000 }
]
const counts = simulate([new PlacementLoadBand(seededRandom(4), quiet)], cells, 1000, () => 1)
const share = (counts.get('cell-empty') ?? 0) / 1000
expect(share).toBeGreaterThan(0.55)
expect(share).toBeLessThan(0.65)
})
it('logs one summary a minute with skips, fallbacks and band size', () => {
const lines: Record<string, unknown>[] = []
const band = new PlacementLoadBand(seededRandom(6), (line) => lines.push(JSON.parse(line)))
const ranked = [
{ cellId: 'cell-a', loadRatio: 0 },
{ cellId: 'cell-b', loadRatio: 0.5 }
]
for (let second = 0; second <= 60; second += 1) band.pick(ranked, second * 1_000)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
event: 'orca_relay_placement_load_band',
windowMs: 60_000,
picks: 61,
bandSizeMax: 1
})
expect(lines[0]?.budgetSkips).toBeGreaterThan(0)
expect(lines[0]?.fallbacks).toBeGreaterThan(0)
})
it('falls back to the least-loaded cell when every candidate is over budget', () => {
const band = new PlacementLoadBand(seededRandom(3), quiet)
const only = [{ cellId: 'cell-a', loadRatio: 0.5 }]
for (let index = 0; index < 50; index += 1) {
expect(band.pick(only, 1_000)?.cellId).toBe('cell-a')
}
expect(band.pick([], 1_000)).toBeUndefined()
})
it('keeps quiet placement on the least-loaded cell outside the band', () => {
let now = 0
const band = new PlacementLoadBand(seededRandom(5), quiet)
const ranked = [
{ cellId: 'cell-a', loadRatio: 0.1 },
{ cellId: 'cell-b', loadRatio: 0.3 }
]
for (let index = 0; index < 20; index += 1) {
now += PLACEMENT_INTAKE_WINDOW_MS / PLACEMENT_INTAKE_FLOOR
expect(band.pick(ranked, now)?.cellId).toBe('cell-a')
}
})
it('picks at random among cells inside the band', () => {
const band = new PlacementLoadBand(seededRandom(9), quiet)
const ranked = [
{ cellId: 'cell-a', loadRatio: 0.3 },
{ cellId: 'cell-b', loadRatio: 0.32 },
{ cellId: 'cell-c', loadRatio: 0.4 }
]
const picked = new Set<string>()
let now = 0
for (let index = 0; index < 40; index += 1) {
now += PLACEMENT_INTAKE_WINDOW_MS
picked.add(band.pick(ranked, now)?.cellId ?? '')
}
expect([...picked].sort()).toEqual(['cell-a', 'cell-b'])
})
})
describe('leastLoadedCell with the load band', () => {
@@ -10,8 +10,6 @@ import {
SLOW_CONNECTION_WINDOW,
SlowConnectionRetirement
} from './postgres-connection-latency.js'
import { DIRECTOR_SLOW_CONNECTION_STALLS } from './test-fixtures/director-slow-connection-stalls.js'
import { DIRECTOR_TAIL_CONNECTION_STALLS } from './test-fixtures/director-tail-connection-stalls.js'
let clock = 0
@@ -19,7 +17,10 @@ type Slot = { name: string; statementMs: () => number }
// Hands out its connections in turn; a client released with a truthy value is replaced by a fresh
// 3 ms one, as pg-pool would dial a new connection on the next ask.
function pool(slots: Slot[], options: { retiresSlowConnections?: boolean } = { retiresSlowConnections: true }) {
function pool(
slots: Slot[],
options: { retiresSlowConnections?: boolean } = { retiresSlowConnections: true }
) {
const destroyed: string[] = []
let dialed = 0
let turn = 0
@@ -167,18 +168,6 @@ describe('slow connection retirement', () => {
expect(tx.destroyed).toEqual(['slow'])
})
it('opens no connection itself, so the pool never grows', async () => {
const slow = pool([
{ name: 'a', statementMs: steady(3) },
{ name: 'slow', statementMs: steady(55) }
])
await runStatements(slow.database, SLOW_CONNECTION_WINDOW * 2)
expect(slow.destroyed).toEqual(['slow'])
// Two original connections plus the one redial that replaced the retired client.
expect(slow.dialed()).toBe(3)
})
it('retires at most one connection per pool per interval', async () => {
const two = pool([
{ name: 'a', statementMs: steady(3) },
@@ -221,7 +210,9 @@ describe('slow connection retirement', () => {
expect(retiredAt.length).toBeLessThanOrEqual(3)
expect(retiredAt.length).toBeGreaterThan(0)
expect(Math.max(...retiredAt)).toBeLessThan(SLOW_CONNECTION_PEER_SLACK_MS + SLOW_CONNECTION_RETIRE_INTERVAL_MS)
expect(Math.max(...retiredAt)).toBeLessThan(
SLOW_CONNECTION_PEER_SLACK_MS + SLOW_CONNECTION_RETIRE_INTERVAL_MS
)
})
// Every 50th statement takes `outlierMs`: a 2% tail, like the ~200/400/800 ms retransmit stalls.
@@ -239,7 +230,11 @@ describe('slow connection retirement', () => {
await runStatements(lossy.database, 600)
expect(lossy.destroyed).toEqual(['lossy'])
expect(JSON.parse(warnings[0] ?? '{}')).toMatchObject({ shape: 'tail', tailMs: 415, peerTailMs: 3 })
expect(JSON.parse(warnings[0] ?? '{}')).toMatchObject({
shape: 'tail',
tailMs: 415,
peerTailMs: 3
})
})
it('retires nothing when every connection has the same tail, as a whole-database spike does', async () => {
@@ -256,13 +251,6 @@ describe('slow connection retirement', () => {
expect(far.destroyed).toEqual([])
})
it.each([
['14:58', 320, '14:58:22'],
['15:24', 367, '15:24:22']
] as const)('replays the %s tail stall of 2026-10-26 and retires c%i in the window ending %s', (stall, id, windowEnd) => {
expect(replayTail(stall)).toEqual([{ connectionId: id, windowEnd }])
})
it('lets one peer with a slow statement of its own block a tail retirement', async () => {
const noisy = pool([
{ name: 'a', statementMs: steady(3) },
@@ -307,88 +295,9 @@ describe('slow connection retirement', () => {
retirement.shouldRetire(live, 51)
expect(retirement.trackedClients).toBe(1)
})
it('replays the 09:38 stall of 2026-10-26 and retires only its slow connection', () => {
expect(replay('09:38')).toEqual([{ connectionId: 11, windowEnd: '09:38:21' }])
})
it('replays the 09:51 stall, where a second connection (c31, 38 ms) also turned slow', () => {
expect(replay('09:51')).toEqual([
{ connectionId: 19, windowEnd: '09:51:51' },
{ connectionId: 31, windowEnd: '09:52:51' }
])
})
})
function fakeClient(): pg.PoolClient {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retirement rule only uses clients as identity keys.
return {} as pg.PoolClient
}
// Spreads each 30 s window's statements evenly at its median and releases after every statement,
// as autocommit queries do. Stops judging a connection once it is retired.
function replay(stall: '09:38' | '09:51'): { connectionId: number; windowEnd: string }[] {
const at = (time: string) => Date.parse(`2026-10-26T${time}Z`)
const samples = DIRECTOR_SLOW_CONNECTION_STALLS.filter(([name]) => name === stall).flatMap(
([, windowEnd, connectionId, statements, medianMs]) =>
Array.from({ length: statements }, (_, index) => ({
at: at(windowEnd) - 30_000 + ((index + 0.5) * 30_000) / statements,
windowEnd,
connectionId,
medianMs
}))
)
samples.sort((a, b) => a.at - b.at)
const clients = new Map<number, pg.PoolClient>()
const retired: { connectionId: number; windowEnd: string }[] = []
const retirement = new SlowConnectionRetirement(() => clock)
for (const sample of samples) {
if (retired.some((entry) => entry.connectionId === sample.connectionId)) {
continue
}
clock = sample.at
const client = clients.get(sample.connectionId) ?? fakeClient()
clients.set(sample.connectionId, client)
retirement.track(client)
noteConnectionStatement(client, sample.medianMs)
if (retirement.shouldRetire(client, sample.connectionId)) {
retired.push({ connectionId: sample.connectionId, windowEnd: sample.windowEnd })
}
}
return retired
}
// Spreads each window's statements evenly at its median, with ceil(1%) of them at its p99 and one at
// its max, and releases after every statement. Stops judging a connection once it is retired.
function replayTail(stall: '14:58' | '15:24'): { connectionId: number; windowEnd: string }[] {
const at = (time: string) => Date.parse(`2026-10-26T${time}Z`)
const samples = DIRECTOR_TAIL_CONNECTION_STALLS.filter(([name]) => name === stall).flatMap(
([, windowEnd, connectionId, statements, medianMs, p99Ms, maxMs]) => {
const tailEvery = Math.floor(statements / Math.ceil(statements * 0.01))
return Array.from({ length: statements }, (_, index) => ({
at: at(windowEnd) - 30_000 + ((index + 0.5) * 30_000) / statements,
windowEnd,
connectionId,
ms: index === statements - 1 ? maxMs : (index + 1) % tailEvery === 0 ? p99Ms : medianMs
}))
}
)
samples.sort((a, b) => a.at - b.at)
const clients = new Map<number, pg.PoolClient>()
const retired: { connectionId: number; windowEnd: string }[] = []
const retirement = new SlowConnectionRetirement(() => clock)
for (const sample of samples) {
if (retired.some((entry) => entry.connectionId === sample.connectionId)) {
continue
}
clock = sample.at
const client = clients.get(sample.connectionId) ?? fakeClient()
clients.set(sample.connectionId, client)
retirement.track(client)
noteConnectionStatement(client, sample.ms)
if (retirement.shouldRetire(client, sample.connectionId)) {
retired.push({ connectionId: sample.connectionId, windowEnd: sample.windowEnd })
}
}
return retired
}
@@ -1,11 +1,6 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentRowBusyError, type RelayAssignment } from './assignment/assignment-store.js'
import type { RelayConfig } from './config.js'
import type { StickySlotSample } from './sticky-slot-statements.js'
import {
ScriptedPoolClient,
scriptedPostgresDatabase
} from './test-fixtures/scripted-postgres-pool.js'
const fakes = vi.hoisted(() => ({
verifyRelayToken: vi.fn(async (token: string) => ({
@@ -105,26 +100,6 @@ describe('public assignment reconnect lane', () => {
expect(assign).toHaveBeenCalledTimes(1)
})
it('sends an unverified reconnect hint through the placement lane unchanged', async () => {
const host = 'nnnnnnnnnnnnnnnn'
const assign = vi.fn(async () => assignment('cell-n', host))
const resolve = vi.fn(async () => null)
const outcomes: string[] = []
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
recordAssignmentAdmission: (outcome) => outcomes.push(outcome)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(200)
expect(assign).toHaveBeenCalledTimes(1)
expect(outcomes).toEqual(['placement'])
})
it('rejects on the fast lane when the verification probe fails transiently', async () => {
const host = 'tttttttttttttttt'
const assign = vi.fn()
@@ -208,70 +183,6 @@ describe('public assignment reconnect lane', () => {
expect(await response.json()).toEqual({ error: 'relay_assignment_row_busy' })
warn.mockRestore()
})
it('reports what each sticky slot spent its hold on', async () => {
vi.useFakeTimers({ toFake: ['performance'] })
const first = 'ffffffffffffffff'
const queued = 'qqqqqqqqqqqqqqqq'
const newcomer = 'nnnnnnnnnnnnnnnn'
const firstHost = new ScriptedPoolClient()
const queuedHost = new ScriptedPoolClient()
// first: verify, assign; newcomer: assign; queued: verify, assign.
const database = scriptedPostgresDatabase([
firstHost,
firstHost,
new ScriptedPoolClient(),
queuedHost,
queuedHost
])
const firstAssign = deferred<RelayAssignment>()
const firstReady = deferred<void>()
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
await database.query('SELECT verify')
return assignment('cell-r', relayHostId)
})
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
await database.transaction(async (transaction) => {
await transaction.query('SELECT pin')
await transaction.query('UPDATE lease')
})
if (relayHostId === first) {
firstReady.resolve()
return await firstAssign.promise
}
return assignment('cell-r', relayHostId)
})
const samples: StickySlotSample[] = []
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
recordStickySlot: (sample) => samples.push(sample)
})
const holding = app.request('/v1/assign', assignmentRequest(first, { reconnect: true }))
await firstReady.promise
expect(firstHost.statements).toHaveLength(5)
const waiting = app.request('/v1/assign', assignmentRequest(queued, { reconnect: true }))
// A placement request's statements are its own, even while a sticky slot is held.
expect((await app.request('/v1/assign', assignmentRequest(newcomer))).status).toBe(200)
await vi.advanceTimersByTimeAsync(60)
firstAssign.resolve(assignment('cell-r', first))
expect((await holding).status).toBe(200)
expect((await waiting).status).toBe(200)
// Verify, then BEGIN, two statements and COMMIT, each on that host's own connection.
const roundTrips = (sample: StickySlotSample | undefined) =>
[...(sample?.statementMsByConnection ?? [])].map(([id, ms]) => [id, ms.length])
expect(samples).toHaveLength(2)
expect(samples[0]).toMatchObject({ statements: 5 })
expect(roundTrips(samples[0])).toEqual([[1, 5]])
expect(samples[0]?.queueWaitMs).toBe(0)
expect(samples[1]).toMatchObject({ statements: 5 })
expect(roundTrips(samples[1])).toEqual([[3, 5]])
expect(samples[1]?.queueWaitMs).toBe(60)
})
})
function assignmentRequest(relayHostId: string, extra: { reconnect?: boolean } = {}): RequestInit {
@@ -1,8 +1,6 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment/assignment-store.js'
import { rehomeSourceRegionAllowed } from './idle-regional-rehome-selection.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
import { REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS } from './regional-rehome-poll-telemetry.js'
type Region = 'us-central1' | 'asia-east2'
const cell = (id: string, region: Region) => ({
@@ -141,12 +139,6 @@ afterEach(() => {
})
describe('rehome source-region filter', () => {
it('keys on the source region, never on the target region', () => {
expect(rehomeSourceRegionAllowed('us-central1', 'us-central1')).toBe(true)
expect(rehomeSourceRegionAllowed('asia-east2', 'us-central1')).toBe(false)
expect(rehomeSourceRegionAllowed('asia-east2', undefined)).toBe(true)
})
it('keeps US to Asia moves and drops Asia to US moves', async () => {
const context = await setup('us-central1')
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
@@ -168,23 +160,6 @@ describe('rehome source-region filter', () => {
await context.database.close()
})
it('counts the skipped source cells in the poll summary line', async () => {
const context = await setup('us-central1')
const lines: string[] = []
vi.spyOn(console, 'warn').mockImplementation((line: unknown) => {
lines.push(String(line))
})
await context.store.selectIdleRegionalRehomeCandidates(context.safety())
context.advance(REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS)
await context.beatAll()
await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const summary = lines
.map((line) => JSON.parse(line) as Record<string, unknown>)
.find(({ event }) => event === 'orca_relay_regional_rehome_poll_summary')
expect(summary).toMatchObject({ open: 2, skippedOffRegionSourceCells: 2 })
await context.database.close()
})
it('reports the filtered hosts in the preview, agreeing with selection', async () => {
const context = await setup('us-central1')
const preview = await context.store.previewRegionalRehomeEligibility(context.safety())
@@ -1,11 +1,6 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from './assignment/assignment-store.js'
import type { RelayConfig } from './config.js'
import {
combineRegionalRehomeSafety,
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
regionalRehomeSafetyFailure
} from './regional-rehome-safety.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
@@ -14,12 +9,16 @@ describe('regional rehome worker', () => {
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
vi.useFakeTimers()
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const worker = startRegionalRehomeWorker(config(), {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore, {
safetySnapshot: () => safety(Date.now()),
random: () => 0
})!
const worker = startRegionalRehomeWorker(
config(),
{
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => safety(Date.now()),
random: () => 0
}
)!
try {
await vi.advanceTimersByTimeAsync(0)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
@@ -36,56 +35,6 @@ describe('regional rehome worker', () => {
}
})
it('passes unsafe process telemetry to the durable claim gate', async () => {
let now = 0
let sqlFailures = 0
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => ({ ...safety(now), sqlFailures }),
intervalMs: 60_000
})!
await settleWorker()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
sqlFailures = 1
await worker.run()
worker.stop()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
)
})
it('starts on directors with no candidates and stays disabled on cells', async () => {
let now = 0
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})
expect(worker).not.toBeNull()
await settleWorker()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
await worker!.run()
worker!.stop()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
expect(
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
safetySnapshot: () => safety(1)
})
).toBeNull()
})
it('stops walking the page when the source names a deferral no candidate can pass', async () => {
const fetchImpl = respondWith([{ outcome: 'deferred', reason: 'concurrency-limit' }])
const summaries = collectSummaries()
@@ -143,39 +92,6 @@ describe('regional rehome worker', () => {
outcomes: { 'deferred:host-unsupported': 1, busy: 1, committed: 1 }
})
})
it('counts a source that answers with an error in the same summary', async () => {
const fetchImpl = vi.fn(async () => new Response('nope', { status: 503 }))
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 2, summaries)
} finally {
summaries.restore()
}
expect(summaries.entries[0]).toMatchObject({ dispatched: 2, outcomes: { failed: 2 } })
})
it('treats the reconnect threshold as per-cell and excludes the director', () => {
const cells = 2
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
const processSafety = { ...safety(100), reconnects: limit * 10 }
const fleetSafety = { ...safety(100), reconnects: limit }
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)
).toBeNull()
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)
).toBe('elevated_reconnects')
})
})
function respondWith(bodies: { outcome: string; reason?: string }[]) {
@@ -271,8 +187,3 @@ function safety(observedAt: number) {
databasePoolWaitMsMax: 0
}
}
async function settleWorker(): Promise<void> {
await Promise.resolve()
await Promise.resolve()
}
@@ -45,15 +45,6 @@ describe('Relay regional placement', () => {
afterEach(async () => await database.close())
it('prefers the requested region and keeps unhinted placement US-first', async () => {
await expect(
store.assign({ userId: 'asia-user', relayHostId: 'asiahost00000001' }, 'asia-east2')
).resolves.toMatchObject({ cellId: 'asia-c1', region: 'asia-east2' })
await expect(
store.assign({ userId: 'us-user', relayHostId: 'ushost0000000001' })
).resolves.toMatchObject({ cellId: 'us-c1', region: 'us-central1' })
})
it('falls back globally only when the target region has no safe general cell', async () => {
await store.configureCell(CELLS[1]!, 'migration-only')
await store.configureCell(CELLS[2]!, 'migration-only')
@@ -89,8 +80,9 @@ describe('Relay regional placement', () => {
'us-central1'
)
).resolves.toMatchObject({ cellId: 'us-c1', region: 'us-central1' })
expect(await database.query(`SELECT preferred_region FROM relay_assignment_region_preferences`))
.toEqual([{ preferred_region: 'asia-east2' }])
expect(
await database.query(`SELECT preferred_region FROM relay_assignment_region_preferences`)
).toEqual([{ preferred_region: 'asia-east2' }])
})
it('returns at most two deterministic healthy general probe origins per region', async () => {
@@ -203,14 +195,6 @@ describe('Relay regional placement', () => {
for (const cell of CELLS.slice(1)) await store.configureCell(cell, 'migration-only')
}
it('books the reserve cell rather than spill out of a full region', async () => {
await fillAsia()
const place = vi.fn(async () => onReserve)
await expect(assignAsia(place)).resolves.toEqual({ ...onReserve, placed: true })
expect(place).toHaveBeenCalledOnce()
expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
})
it('answers a throttled reserve cell with its pace, not a cell abroad', async () => {
await fillAsia()
await expect(
@@ -233,15 +217,6 @@ describe('Relay regional placement', () => {
store.assign({ ...identity, relayHostId: 'spillhost0000002' }, 'asia-east2', 'asia-east2')
).resolves.toMatchObject({ cellId: 'us-c1' })
})
it('leaves a region with database room to the database lane', async () => {
const place = vi.fn(async () => onReserve)
await expect(assignAsia(place)).resolves.toMatchObject({
cellId: 'asia-c1',
region: 'asia-east2'
})
expect(place).not.toHaveBeenCalled()
})
})
it('expires identity-linked region preferences after 30 days', async () => {
+43 -34
View File
@@ -2,7 +2,6 @@ import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment/assignment-store.js'
import { openInMemoryRelayDatabase } from './database.js'
import { hostCohortBucket } from './host-cohort-bucket.js'
import {
RESERVE_REHOME_HOST_COOLDOWN_MS,
RESERVE_REHOME_SWITCH_STALE_MS,
@@ -16,7 +15,10 @@ const ROOM: RehomeTargetRoom = { free: 500, ceiling: 1_000, bookable: true }
// 2,000 controls refill one token a minute.
const CONTROLS = 2_000
function admitted(rehome: ReserveRehome, overrides: Partial<Parameters<ReserveRehome['admit']>[0]> = {}) {
function admitted(
rehome: ReserveRehome,
overrides: Partial<Parameters<ReserveRehome['admit']>[0]> = {}
) {
return rehome.admit({
identity: HOST,
sourceCellId: 'us-r',
@@ -37,25 +39,14 @@ function warmed(percent = 100) {
}
describe('rehome on reconnect rules', () => {
it('is off at 0%, and admits a host whose every rule holds', () => {
expect(new ReserveRehome({ percent: 0, switchOpen: () => true }).enabled).toBe(false)
expect(admitted(warmed())).toBeNull()
})
it('stays off while the operator switch is closed', () => {
expect(admitted(new ReserveRehome({ percent: 100, switchOpen: () => false }))).toBe('switch')
})
it('moves only the cohort below the percent', () => {
const bucket = hostCohortBucket(HOST)
expect(admitted(new ReserveRehome({ percent: bucket, switchOpen: () => true }))).toBe('cohort')
expect(admitted(warmed(bucket + 1))).toBeNull()
})
it('leaves a host that another install dropped in the last 10 minutes', () => {
const peerDropped = { region: 'us-central1' as const, closeCode: RELAY_CLOSE_CODE.PEER_DROPPED }
expect(admitted(warmed(), { leaves: [{ ...peerDropped, at: 60_000 - 10 * 60_000 }] })).toBe('peer-dropped')
expect(admitted(warmed(), { leaves: [{ ...peerDropped, at: 60_000 - 10 * 60_000 - 1 }] })).toBeNull()
expect(admitted(warmed(), { leaves: [{ ...peerDropped, at: 60_000 - 10 * 60_000 }] })).toBe(
'peer-dropped'
)
expect(
admitted(warmed(), { leaves: [{ ...peerDropped, at: 60_000 - 10 * 60_000 - 1 }] })
).toBeNull()
})
it('leaves a host whose remembered leaves span regions, or that it moved within a day', () => {
@@ -64,7 +55,9 @@ describe('rehome on reconnect rules', () => {
{ region: 'asia-east2' as const, at: 2 }
]
expect(admitted(warmed(), { leaves })).toBe('moved-recently')
expect(admitted(warmed(), { leaves: [{ region: undefined, at: 1 }] })).toBe('unknown-leave-region')
expect(admitted(warmed(), { leaves: [{ region: undefined, at: 1 }] })).toBe(
'unknown-leave-region'
)
const rehome = warmed()
rehome.recordMoved(HOST, 0)
expect(admitted(rehome)).toBe('moved-recently')
@@ -72,17 +65,27 @@ describe('rehome on reconnect rules', () => {
})
it('moves no one into a region under 10% free, however full the source is', () => {
expect(admitted(warmed(), { target: { free: 20, ceiling: 1_000, bookable: true } })).toBe('target-room')
expect(admitted(warmed(), { target: { free: 99, ceiling: 1_000, bookable: true } })).toBe('target-room')
expect(admitted(warmed(), { target: { free: 20, ceiling: 1_000, bookable: true } })).toBe(
'target-room'
)
expect(admitted(warmed(), { target: { free: 99, ceiling: 1_000, bookable: true } })).toBe(
'target-room'
)
expect(admitted(warmed(), { target: { free: 100, ceiling: 1_000, bookable: true } })).toBeNull()
expect(admitted(warmed(), { target: { free: 500, ceiling: 1_000, bookable: false } })).toBe('target-room')
expect(admitted(warmed(), { target: { free: 0, ceiling: 0, bookable: false } })).toBe('target-room')
expect(admitted(warmed(), { target: { free: 500, ceiling: 1_000, bookable: false } })).toBe(
'target-room'
)
expect(admitted(warmed(), { target: { free: 0, ceiling: 0, bookable: false } })).toBe(
'target-room'
)
})
it('takes a source cell at 0.05% of its controls a minute, burst one, and refunds an unmade move', () => {
const rehome = warmed()
expect(admitted(rehome)).toBeNull()
expect(admitted(rehome, { identity: { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } })).toBe('source-budget')
expect(admitted(rehome, { identity: { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } })).toBe(
'source-budget'
)
expect(admitted(rehome, { sourceCellId: 'us-r2', now: 0 })).toBe('source-budget')
rehome.refund('us-r')
expect(admitted(rehome, { identity: { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } })).toBeNull()
@@ -100,7 +103,10 @@ describe('rehome switch', () => {
enabled: true,
notBefore: 0
})
const control = new ReserveRehomeSwitch(async () => await answer(), () => now)
const control = new ReserveRehomeSwitch(
async () => await answer(),
() => now
)
expect(control.isOpen(now)).toBe(false)
await control.refresh()
expect(control.isOpen(now)).toBe(true)
@@ -129,14 +135,17 @@ describe('the rehome switch under a stalled read', () => {
let now = 1_000
let release!: (value: { enabled: boolean; notBefore: number }) => void
const reads: ('stalled' | 'off')[] = []
const control = new ReserveRehomeSwitch(async () => {
if (reads.length === 0) {
reads.push('stalled')
return await new Promise((resolve) => (release = resolve))
}
reads.push('off')
return { enabled: false, notBefore: 0 }
}, () => now)
const control = new ReserveRehomeSwitch(
async () => {
if (reads.length === 0) {
reads.push('stalled')
return await new Promise((resolve) => (release = resolve))
}
reads.push('off')
return { enabled: false, notBefore: 0 }
},
() => now
)
const stalled = control.refresh()
await control.refresh()
expect(reads).toEqual(['stalled'])
+1 -62
View File
@@ -5,11 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RawData } from 'ws'
import { RELAY_ADMISSION_BUDGETS } from '@orca-cloud/relay-contract'
import {
ProcessQueuedByteBudget,
wireSplice,
type SpliceCloseInfo
} from './splice-forwarder.js'
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
// SAFETY: ws exports Receiver at runtime; its published types omit this Writable constructor.
const { Receiver } = createRequire(import.meta.url)('ws') as {
@@ -39,29 +35,6 @@ class FakeSocket extends EventEmitter {
describe('splice forwarder', () => {
afterEach(() => vi.useRealTimers())
it('preserves text/binary opcodes and propagates peer close', () => {
const client = new FakeSocket()
const host = new FakeSocket()
const onClose = vi.fn()
const onForwardedBytes = vi.fn()
wireSplice({
client: client as unknown as WebSocket,
host: host as unknown as WebSocket,
budget: new ProcessQueuedByteBudget(),
onClose,
onForwardedBytes
})
client.emit('message', Buffer.from('text'), false)
client.emit('message', Buffer.from([1, 2]), true)
expect(host.sent).toEqual([
{ data: Buffer.from('text'), binary: false },
{ data: Buffer.from([1, 2]), binary: true }
])
expect(onForwardedBytes.mock.calls).toEqual([[4], [2]])
client.emit('close', 1000, Buffer.alloc(0))
expect(host.closes[0]?.code).toBe(4408)
expect(onClose).toHaveBeenCalledOnce()
})
it('hard-closes a splice exceeding the queue limit and releases its global queued-byte reservation', () => {
const client = new FakeSocket()
@@ -82,40 +55,6 @@ describe('splice forwarder', () => {
expect(budget.current()).toBe(0)
})
it('reports the close trigger so limit and oversize kills are attributable', () => {
const wire = (): { client: FakeSocket; host: FakeSocket; closes: SpliceCloseInfo[] } => {
const client = new FakeSocket()
const host = new FakeSocket()
const closes: SpliceCloseInfo[] = []
wireSplice({
client: client as unknown as WebSocket,
host: host as unknown as WebSocket,
budget: new ProcessQueuedByteBudget(),
onClose: vi.fn(),
onClosed: (closeInfo) => closes.push(closeInfo)
})
return { client, host, closes }
}
const queueLimit = wire()
queueLimit.host.bufferedAmount = 1024 * 1024
queueLimit.client.emit('message', Buffer.alloc(8 * 1024 * 1024), true)
queueLimit.client.emit('message', Buffer.alloc(512 * 1024), true)
expect(queueLimit.closes).toEqual([
{ code: 4429, reason: 'relay queue limit exceeded', trigger: 'queue-limit' }
])
// The ws receiver error for a frame above maxPayload names the payload cap.
const oversize = wire()
oversize.host.emit('error', new RangeError('Max payload size exceeded'))
expect(oversize.closes[0]?.trigger).toBe('host-oversize-frame')
const peerClose = wire()
peerClose.client.emit('close', 1001, Buffer.alloc(0))
expect(peerClose.closes[0]?.trigger).toBe('client-closed')
expect(peerClose.closes).toHaveLength(1)
})
it('queues one full catalog-sized frame for a backpressured peer without closing', () => {
// Large workspace catalogs exceed 1MiB; one maxFrameBytes frame must survive backpressure.
vi.useFakeTimers()
@@ -1,41 +0,0 @@
// Director instance ...48a393ae on 2026-10-26, from its runtime lines' stickySqlByConnection: per 30 s
// window ending at `windowEnd` (UTC), one pooled connection's statement count and median ms.
// Columns: stall, windowEnd, connectionId, statements, medianMs.
type StallWindow = readonly [
stall: '09:38' | '09:51',
windowEnd: string,
connectionId: number,
statements: number,
medianMs: number
]
export const DIRECTOR_SLOW_CONNECTION_STALLS: readonly StallWindow[] = [
['09:38', '09:37:21', 7, 1216, 3.536],
['09:38', '09:37:21', 10, 10, 3.595],
['09:38', '09:37:21', 11, 322, 3.796],
['09:38', '09:37:51', 7, 184, 3.535],
['09:38', '09:37:51', 11, 733, 3.592],
['09:38', '09:37:51', 12, 616, 3.435],
['09:38', '09:38:21', 11, 488, 52.398],
['09:38', '09:38:21', 13, 237, 3.481],
['09:38', '09:38:51', 11, 495, 58.993],
['09:38', '09:39:21', 11, 533, 56.129],
['09:38', '09:39:21', 12, 10, 4.379],
['09:38', '09:39:51', 11, 1052, 8.312],
['09:38', '09:39:51', 12, 213, 3.549],
['09:38', '09:40:21', 11, 713, 3.62],
['09:38', '09:40:21', 15, 868, 3.405],
['09:51', '09:50:51', 19, 654, 3.388],
['09:51', '09:50:51', 29, 829, 3.121],
['09:51', '09:51:21', 19, 835, 3.536],
['09:51', '09:51:21', 29, 382, 3.205],
['09:51', '09:51:51', 19, 498, 58.791],
['09:51', '09:52:21', 19, 536, 55.549],
['09:51', '09:52:51', 19, 19, 57.906],
['09:51', '09:52:51', 29, 991, 3.375],
['09:51', '09:52:51', 31, 354, 37.754],
['09:51', '09:53:21', 19, 506, 3.272],
['09:51', '09:53:21', 31, 784, 19.04],
['09:51', '09:53:51', 19, 1262, 3.304],
['09:51', '09:53:51', 31, 400, 3.205],
]
@@ -1,52 +0,0 @@
// Director instance ...1dc2e1c3 on 2026-10-26, from its runtime lines' stickySqlByConnection: per 30 s
// window ending at `windowEnd` (UTC), one pooled connection's statement count, median, p99 and max ms.
type TailStallWindow = readonly [
stall: '14:58' | '15:24',
windowEnd: string,
connectionId: number,
statements: number,
medianMs: number,
p99Ms: number,
maxMs: number
]
export const DIRECTOR_TAIL_CONNECTION_STALLS: readonly TailStallWindow[] = [
['14:58', '14:56:52', 315, 1053, 3.481, 10.815, 65.702],
['14:58', '14:56:52', 320, 727, 3.474, 11.571, 18.8],
['14:58', '14:56:52', 321, 10, 3.598, 6.737, 6.737],
['14:58', '14:57:22', 315, 741, 3.594, 11.245, 19.562],
['14:58', '14:57:22', 320, 584, 3.461, 8.032, 9.57],
['14:58', '14:57:52', 315, 1205, 3.462, 7.312, 20.124],
['14:58', '14:57:52', 320, 425, 3.523, 13.859, 18.517],
['14:58', '14:58:22', 315, 418, 3.395, 7.049, 17.142],
['14:58', '14:58:22', 320, 413, 4.854, 416.306, 815.523],
['14:58', '14:58:52', 315, 780, 3.506, 8.739, 51.87],
['14:58', '14:58:52', 320, 914, 4.543, 215.253, 418.243],
['14:58', '14:59:22', 315, 790, 3.453, 7.602, 16.841],
['14:58', '14:59:22', 320, 968, 3.517, 10.595, 214.811],
['14:58', '14:59:22', 324, 10, 3.296, 7.945, 7.945],
['14:58', '14:59:22', 325, 22, 3.277, 7.373, 7.373],
['14:58', '14:59:52', 315, 921, 3.425, 11.826, 52.059],
['14:58', '14:59:52', 325, 958, 3.422, 7.779, 17.631],
['15:24', '15:22:52', 363, 162, 3.531, 6.968, 8.816],
['15:24', '15:22:52', 365, 852, 3.307, 7.833, 13.862],
['15:24', '15:22:52', 366, 417, 3.645, 10.906, 17.662],
['15:24', '15:23:22', 365, 770, 3.341, 8.067, 20.72],
['15:24', '15:23:22', 366, 154, 3.492, 7.03, 7.406],
['15:24', '15:23:22', 367, 513, 3.654, 11.077, 211.46],
['15:24', '15:23:52', 365, 847, 3.438, 9.728, 29.608],
['15:24', '15:23:52', 367, 629, 3.476, 9.464, 17.448],
['15:24', '15:24:22', 365, 522, 3.309, 12.595, 25.105],
['15:24', '15:24:22', 367, 687, 4.551, 414.215, 417.657],
['15:24', '15:24:52', 367, 604, 4.967, 414.827, 418.756],
['15:24', '15:24:52', 368, 48, 3.75, 10.407, 10.407],
['15:24', '15:24:52', 370, 6, 4.097, 9.925, 9.925],
['15:24', '15:25:22', 367, 845, 4.71, 415.764, 818.912],
['15:24', '15:25:22', 370, 30, 3.399, 7.265, 7.265],
['15:24', '15:25:52', 367, 1220, 4.383, 213.454, 216.581],
['15:24', '15:25:52', 370, 196, 3.363, 13.136, 17.227],
['15:24', '15:25:52', 371, 420, 3.445, 18.831, 42.126],
['15:24', '15:26:22', 367, 570, 3.259, 13.859, 39.525],
['15:24', '15:26:22', 371, 792, 3.295, 8.899, 44.169],
['15:24', '15:26:22', 372, 286, 3.255, 6.977, 13.688],
]
@@ -1,110 +1,9 @@
import { describe, expect, it } from 'vitest'
import {
ConnectorDeliverySchema,
ConnectorPollResponseSchema,
ConnectorAcksRequestSchema
} from './delivery-messages.js'
import {
ConnectorEventSchema,
ConnectorEventsRequestSchema,
ConnectorPendingPromptSchema,
ConnectorReplyResponseSchema
} from './event-messages.js'
import { ConnectorHelloRequestSchema, ConnectorHelloResponseSchema } from './hello-messages.js'
import { ConnectorEventSchema } from './event-messages.js'
import { ConnectorHelloRequestSchema } from './hello-messages.js'
import { CONNECTOR_LIMITS } from './connector-limits.js'
const conversation = { source: 'slack', externalId: 'T1:C9:1792.9:U5' } as const
const base = { deliveryId: 'T1:C9:1793.0001', seq: 41, expiresAt: 1_793_000_300_000, conversation }
const start = { ...base, kind: 'start', agent: 'claude', placeId: 'pl_1', prompt: 'fix it' }
const poll = (deliveries: unknown[], extra: object = {}) =>
ConnectorPollResponseSchema.safeParse({
v: 1,
serverNow: 1_793_000_000_000,
holdMs: 50_000,
deliveries,
...extra
})
describe('poll response', () => {
it('parses every known delivery kind and drops unknown fields', () => {
const result = poll([
{ ...start, addedLater: true },
{ ...base, kind: 'send', text: 'more' },
{ ...base, kind: 'answer', promptId: 'p7', optionId: 'allow_once' },
{ ...base, kind: 'cancel' }
])
expect(result.success && result.data.deliveries.map((d) => d.kind)).toEqual([
'start',
'send',
'answer',
'cancel'
])
expect(result.data?.deliveries[0]).not.toHaveProperty('addedLater')
expect(result.data?.control).toBeUndefined()
})
it('rejects a delivery the cloud must never send', () => {
for (const delivery of [
{ ...base, kind: 'rewind' },
{ ...start, prompt: '' },
{ deliveryId: base.deliveryId },
{ ...base, conversation: { source: 'github', externalId: 'G1' }, kind: 'cancel' }
]) {
expect(ConnectorDeliverySchema.safeParse(delivery).success).toBe(false)
}
})
it('carries one control at a time', () => {
expect(poll([], { control: { stop: 'client_too_old' } }).data?.control).toEqual({
stop: 'client_too_old'
})
expect(poll([], { control: { pausedForMs: 0 } }).success).toBe(false)
expect(poll([], { control: { stop: 'client_too_old', reconnect: true } }).success).toBe(false)
expect(poll([], { control: { reconnect: true, addedLater: 1 } }).data?.control).toEqual({
reconnect: true
})
})
it('bounds the batch and the hold', () => {
const tooMany = Array.from({ length: CONNECTOR_LIMITS.maxDeliveriesPerPoll + 1 }, () => start)
expect(poll(tooMany).success).toBe(false)
expect(poll([], { holdMs: CONNECTOR_LIMITS.pollTimeoutMs }).success).toBe(false)
})
})
describe('acks', () => {
it('accepts received and done with each result outcome', () => {
const results = [
{ outcome: 'accepted', reopened: true },
{ outcome: 'refused', reason: 'prompt_not_pending' },
{ outcome: 'unknown', reason: 'no_local_record' },
{ outcome: 'expired' },
{ outcome: 'unsupported' }
]
expect(
ConnectorAcksRequestSchema.safeParse({
v: 1,
helloRev: 7,
acks: [
{ deliveryId: 'd1', step: 'received' },
...results.map((result) => ({ deliveryId: 'd1', step: 'done', result }))
]
}).success
).toBe(true)
expect(
ConnectorAcksRequestSchema.safeParse({
v: 1,
helloRev: 7,
acks: [{ deliveryId: 'd1', step: 'done' }]
}).success
).toBe(false)
})
it('names the hello they belong to', () => {
const acks = [{ deliveryId: 'd1', step: 'received' }]
expect(ConnectorAcksRequestSchema.safeParse({ v: 1, acks }).success).toBe(false)
})
})
describe('events', () => {
const turnEnded = {
@@ -116,58 +15,18 @@ describe('events', () => {
stopReason: null
}
it('names the turn that ended by its start', () => {
const { turnId: _, ...withoutTurnId } = turnEnded
expect(ConnectorEventSchema.safeParse(turnEnded).success).toBe(true)
expect(ConnectorEventSchema.safeParse(withoutTurnId).success).toBe(false)
})
it('caps the last assistant message in UTF-8 bytes', () => {
const limit = CONNECTOR_LIMITS.maxLastAssistantMessageBytes
expect(
ConnectorEventSchema.safeParse({ ...turnEnded, lastAssistantMessage: 'a'.repeat(limit) })
.success
).toBe(true)
expect(
ConnectorEventSchema.safeParse({ ...turnEnded, lastAssistantMessage: 'é'.repeat(limit / 2 + 1) })
.success
).toBe(false)
})
it('carries the pending prompt in snapshots', () => {
expect(
ConnectorEventSchema.safeParse({
kind: 'snapshot',
conversations: [
{
conversation,
paneKey: 'tab:leaf',
agent: 'codex',
working: false,
prompt: { kind: 'approval', promptId: 'p1', tool: 'Bash', options: [] }
}
]
...turnEnded,
lastAssistantMessage: 'é'.repeat(limit / 2 + 1)
}).success
).toBe(true)
})
it('numbers every events batch within the hello it belongs to', () => {
const events = [{ kind: 'turn-started', conversation, paneKey: 'tab:leaf', at: 1 }]
const parse = (body: object) => ConnectorEventsRequestSchema.safeParse({ v: 1, events, ...body })
expect(parse({ helloRev: 7, seq: 0 }).success).toBe(true)
expect(parse({ helloRev: 7 }).success).toBe(false)
expect(parse({ seq: 0 }).success).toBe(false)
expect(parse({ helloRev: 7, seq: -1 }).success).toBe(false)
})
it('carries a question the agent left without text, as the seam reports it', () => {
const prompt = {
kind: 'question',
promptId: 'p2',
question: '',
options: [{ id: '1', label: '' }]
}
expect(ConnectorPendingPromptSchema.safeParse(prompt).success).toBe(true)
).toBe(false)
})
})
@@ -214,37 +73,12 @@ describe('hello and replies', () => {
projects
}).success
const longest = 'x'.repeat(CONNECTOR_LIMITS.maxDisplayTextLength)
expect(hello(Array.from({ length: CONNECTOR_LIMITS.maxHelloProjects }, () => project(longest)))).toBe(true)
expect(
hello(Array.from({ length: CONNECTOR_LIMITS.maxHelloProjects }, () => project(longest)))
).toBe(true)
expect(hello([project(`${longest}x`)])).toBe(false)
expect(
hello(Array.from({ length: CONNECTOR_LIMITS.maxHelloProjects + 1 }, () => project('orca')))
).toBe(false)
})
it('answers a hello with the revision the cloud assigned', () => {
expect(ConnectorHelloResponseSchema.safeParse({ helloRev: 3 }).success).toBe(true)
expect(ConnectorHelloResponseSchema.safeParse({}).success).toBe(false)
})
it('reads a reply as posted or queued', () => {
expect(ConnectorReplyResponseSchema.parse({ status: 'queued' })).toEqual({ status: 'queued' })
expect(
ConnectorReplyResponseSchema.parse({ status: 'posted', messageId: '1793.0001' })
).toEqual({
status: 'posted',
messageId: '1793.0001'
})
})
it('carries a conversation from any connected platform', () => {
for (const source of ['slack', 'discord', 'linear']) {
expect(
ConnectorDeliverySchema.safeParse({
...base,
conversation: { source, externalId: 'x1' },
kind: 'cancel'
}).success
).toBe(true)
}
})
})