test: remove low-value host control and connection test inventories (#38578)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-11-05 07:12:53 -08:00
committed by GitHub
co-authored by OrcaWin
parent 167cb81e60
commit 4fee14f89b
10 changed files with 1 additions and 1077 deletions
@@ -8,29 +8,6 @@ function memoryAt(clock: { now: number }): HostCloseReasonMemory {
}
describe('HostCloseReasonMemory', () => {
it('remembers only reasons it knows', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('b', 'quitting')
memory.record('c', Buffer.alloc(0))
memory.record('d', undefined)
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(memory.read('b')).toBeNull()
expect(memory.read('c')).toBeNull()
expect(memory.read('d')).toBeNull()
})
it('accepts the reason as the Buffer a ws close delivers', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('expires an entry once its host may have been rebalanced away', () => {
const clock = { now: 1_000 }
@@ -45,16 +22,6 @@ describe('HostCloseReasonMemory', () => {
expect(memory.size()).toBe(0)
})
it('forgets on demand', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.forget('a')
expect(memory.read('a')).toBeNull()
})
it('drops the oldest survivors rather than growing without bound', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
@@ -67,16 +34,4 @@ describe('HostCloseReasonMemory', () => {
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('re-recording refreshes recency so a live host is not evicted first', () => {
const memory = memoryAt({ now: 1_000 })
for (let index = 0; index < 50_000; index++) {
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
}
memory.record('host-0', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('overflow', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(memory.size()).toBe(50_000)
expect(memory.read('host-0')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(memory.read('host-1')).toBeNull()
expect(memory.read('overflow')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
})
@@ -1,4 +1,3 @@
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import nacl from 'tweetnacl'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
@@ -16,49 +15,6 @@ afterEach(() => {
})
describe('host control proof cleanup', () => {
it('allocates no hello stage for an already closed peer', () => {
const h = fixture()
const socket = new ProofSocket()
socket.peerClose()
h.registry.acceptControl(socket.registrySocket(), h.identity)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('message')).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
expect(h.verify).not.toHaveBeenCalled()
})
it('releases the host hello timer and listeners when its peer closes early', () => {
const h = fixture()
const socket = new ProofSocket()
h.registry.acceptControl(socket.registrySocket(), h.identity)
expect(vi.getTimerCount()).toBe(1)
expect(socket.listenerCount('message')).toBe(1)
socket.peerClose()
expect({
timers: vi.getTimerCount(),
message: socket.listenerCount('message'),
close: socket.listenerCount('close')
}).toEqual({ timers: 0, message: 0, close: 0 })
vi.advanceTimersByTime(2000)
expect(socket.close).not.toHaveBeenCalled()
expect(h.verify).not.toHaveBeenCalled()
expect(h.database.query).not.toHaveBeenCalled()
})
it('preserves the exact hello deadline and refusal while releasing its message listener', () => {
const h = fixture()
const socket = new ProofSocket()
h.registry.acceptControl(socket.registrySocket(), h.identity)
vi.advanceTimersByTime(1999)
expect(socket.close).not.toHaveBeenCalled()
vi.advanceTimersByTime(1)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'host hello timeout'
)
expect(socket.listenerCount('message')).toBe(0)
expect(vi.getTimerCount()).toBe(0)
})
it('releases the challenge timer and listeners when its peer closes before proof', async () => {
const h = fixture()
@@ -77,21 +33,6 @@ describe('host control proof cleanup', () => {
expect(h.database.query).not.toHaveBeenCalled()
})
it('preserves the exact proof deadline and refusal with no leftover listener', async () => {
const h = fixture()
const socket = await openProof(h)
await vi.advanceTimersByTimeAsync(9999)
expect(socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'host proof timeout'
)
expect(socket.listenerCount('message')).toBe(0)
expect(h.activate).not.toHaveBeenCalled()
expect(h.recordAuth).not.toHaveBeenCalled()
})
it('does no challenge crypto, send, timer or registration after a closed peer finishes verification', async () => {
const h = fixture()
let finish!: (valid: boolean) => void
@@ -119,86 +60,6 @@ describe('host control proof cleanup', () => {
).toBeNull()
})
it.each([false, true])('preserves invalid first-frame refusal (binary=%s)', (binary) => {
const h = fixture()
const socket = new ProofSocket()
h.registry.acceptControl(socket.registrySocket(), h.identity)
socket.emit('message', Buffer.from('{}'), binary)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
binary ? 'host hello must be text' : 'invalid host hello'
)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
expect(h.activate).not.toHaveBeenCalled()
})
it.each([false, true])(
'preserves invalid proof authentication failure (binary=%s)',
async (binary) => {
const h = fixture()
const socket = await openProof(h)
socket.emit('message', Buffer.from('{}'), binary)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host proof'
)
expect(h.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
expect(h.activate).not.toHaveBeenCalled()
}
)
it('allocates no proof wait when sending the challenge closes its peer', async () => {
const h = fixture()
const socket = new ProofSocket()
socket.send.mockImplementation(() => socket.peerClose())
await openProof(h, socket)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('message')).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
expect(h.activate).not.toHaveBeenCalled()
})
it('keeps the existing diagnostic and refusal when challenge send throws', async () => {
const h = fixture()
const socket = new ProofSocket()
socket.send.mockImplementation(() => {
throw new Error('synthetic send failure')
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
await openProof(h, socket)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
'relay temporarily unavailable'
)
expect(warn).toHaveBeenCalledExactlyOnceWith(
'[orca-relay] host hello proof failed: synthetic send failure'
)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
})
it('contains assignment lookup rejection with its existing close and diagnostic', async () => {
const h = fixture()
h.verify.mockRejectedValueOnce(new Error('synthetic lookup failure'))
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const socket = new ProofSocket()
h.registry.acceptControl(socket.registrySocket(), h.identity)
socket.emit('message', Buffer.from(h.hello), false)
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
'relay temporarily unavailable'
)
expect(warn).toHaveBeenCalledExactlyOnceWith(
'[orca-relay] host hello proof failed: synthetic lookup failure'
)
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('close')).toBe(0)
})
it('keeps a newer same-host peer live when the old proof peer closes', async () => {
const h = fixture()
const oldPeer = await openProof(h)
@@ -144,23 +144,6 @@ describe('a resumed control adopts the token its hello proved', () => {
})
describe('an auth refresh', () => {
it('never lowers the expiry the session already holds', async () => {
const verify = vi.fn<(token: string) => Promise<RelayTokenClaims | null>>()
const { h, first } = await connectedHost(600, verify)
const session = h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })!
const held = session.identity.exp
verify.mockResolvedValueOnce({ ...h.identity, exp: held - 300 })
sendRefresh(first)
await vi.advanceTimersByTimeAsync(0)
expect(session.identity.exp).toBe(held)
verify.mockResolvedValueOnce({ ...h.identity, exp: held + 300 })
sendRefresh(first)
await vi.advanceTimersByTimeAsync(0)
expect(session.identity.exp).toBe(held + 300)
expect(first.close).not.toHaveBeenCalled()
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
})
it('closes the socket a refused refresh arrived on, not the control that replaced it', async () => {
let finish!: (claims: RelayTokenClaims | null) => void
@@ -1,14 +1,11 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract/protocol-limits'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
import type { RelayConfig } from '../config.js'
import type { CredentialReservation, RelayCredentialStore } from '../credentials/credential-store.js'
import {
CONTROL_LEASE_JITTER_MS,
CONTROL_LEASE_MS,
HostSessionRegistry
} from './host-session-registry.js'
import type { RelayRuntimeObserver } from '../observability/relay-observability.js'
@@ -298,240 +295,8 @@ describe('client accept abandoned mid-DB-phase', () => {
}
})
it('stops after a slow resume lookup before starting the invite and assignment lookups', async () => {
const h = harness()
await activeHost(h)
const store = h.store as typeof h.store & { resolveInviteForMove: ReturnType<typeof vi.fn> }
store.resolveInviteForMove = vi.fn().mockResolvedValue(null)
const slowResume = deferred<null>()
h.store.resolveResume.mockReturnValueOnce(slowResume.promise)
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
.resolve
resolveAssignment.mockClear()
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowResume.resolve(null)
await accepting
expect(store.resolveInviteForMove).not.toHaveBeenCalled()
expect(resolveAssignment).not.toHaveBeenCalled()
expect(h.store.reserveCredential).not.toHaveBeenCalled()
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'assignment',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('stops after a slow same-cell assignment resolve, before reserving a credential', async () => {
const h = harness()
await activeHost(h)
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
.resolve
const slowResolve = deferred<{ cellId: string }>()
resolveAssignment.mockReturnValueOnce(slowResolve.promise)
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
// A correct, same-cell assignment: only the closed socket stops the accept.
slowResolve.resolve({ cellId: config.cellId })
await accepting
// Proves the accept reached the third guard, not the first.
expect(resolveAssignment).toHaveBeenCalled()
expect(h.store.reserveCredential).not.toHaveBeenCalled()
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'assignment',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('logs and counts an attach deadline with what the cell knew of the control', async () => {
let now = 1_700_000_000_000
const h = harness({ now: () => now })
await activeHost(h)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
now += RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs
vi.advanceTimersByTime(RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs)
// The phone-only reason; relay-hello itself is unchanged for strict old parsers.
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, 'attach-timeout')
expect(client.send).toHaveBeenCalledWith(
JSON.stringify({ type: 'relay-hello', ok: false, code: 4404 })
)
expect(h.observer.recordPhoneAttachDeadline).toHaveBeenCalledOnce()
expect(h.observer.recordPhoneRejection).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE)
const lines = warn.mock.calls
.map(([line]) => String(line))
.filter((line) => line.includes('orca_relay_attach_deadline_offline'))
expect(lines.map((line) => JSON.parse(line))).toEqual([
{
event: 'orca_relay_attach_deadline_offline',
relayHostIdDigest: expect.any(String),
credentialKind: 'resume',
msSinceLastPong: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
pingOutstanding: false,
controlAgeMs: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
controlRttMs: null,
pong: 'fresh',
cellId: config.cellId
}
])
expect(lines[0]).not.toContain(identity.relayHostId)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('still opens the connection when the phone is holding on', async () => {
const h = harness()
const control = await activeHost(h)
const capacity = { bind: vi.fn(), release: vi.fn() }
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
expect(capacity.bind).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptAbandoned).not.toHaveBeenCalled()
expect(client.close).not.toHaveBeenCalled()
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('successful client accept timing', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
let now = 1_700_000_000_000
const h = harness({ now: () => now })
const control = await activeHost(h)
h.store.resolveResume.mockImplementationOnce(async () => {
now += 5
return { userId: identity.sub }
})
h.store.reserveCredential.mockImplementationOnce(async () => {
now += 7
return reservation
})
h.acquireActivity.mockImplementationOnce(async () => {
now += 11
})
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
now += 3
})
const client = new FakeSocket()
const hostData = new FakeSocket()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
const connOpen = JSON.parse(
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
connOpen.connTicket,
1
)
expect(accepted).toBe(true)
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 },
phonePath: 'db'
})
const line = log.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
expect(line).toBeDefined()
const event = JSON.parse(line!) as {
role: string
cellId: string
region: string
credentialKind: string
stageMs: Record<string, number>
totalMs: number
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
'activity',
'assignment',
'attach',
'basis',
'credential'
])
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
// The stages tile the accept end to end: every millisecond is attributed.
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
expect(summed).toBe(event.totalMs)
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(line).not.toContain(identity.relayHostId)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
// echo the registry will time.
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const ping = control.send.mock.calls
.filter((call) => String(call[0]).includes('"type":"ping"'))
.at(-1)!
return (JSON.parse(String(ping[0])) as { t: number }).t
}
// The attach resolves its owning session once and hands it to the unfenced leg;
// these hold the session it must be and the order the client hears about it.
describe('host data attach ownership', () => {
@@ -606,147 +371,6 @@ describe('host data attach ownership', () => {
})
})
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('logs a host once at the fourth sample and not again within the hour', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const rttLines = (): string[] =>
log.mock.calls
.map((call) => String(call[0]))
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
const roundTrip = async (): Promise<void> => {
const pingAt = await advanceToPing(control, clock)
clock.now += 40
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
}
try {
for (let round = 0; round < 3; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
expect(rttLines()).toHaveLength(0)
await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
expect(rttLines()).toHaveLength(1)
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
event: 'orca_relay_host_control_rtt',
role: 'cell',
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
assignmentEpoch: 1,
controlGeneration: 1,
drainMode: 'none',
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
// Later samples keep feeding the fleet metric, but stay silent for an hour.
for (let round = 0; round < 8; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
expect(rttLines()).toHaveLength(1)
const elapsedStart = clock.now
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
expect(rttLines()).toHaveLength(2)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('ignores a pong that answers no outstanding ping', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
try {
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
control.emit('message', JSON.stringify({ type: 'pong' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
const pingAt = await advanceToPing(control, clock)
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
clock.now += 10
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
} finally {
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('records one sample per ping however many pongs a host floods', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
const pingAt = await advanceToPing(control, clock)
clock.now += 12
for (let flood = 0; flood < 5_000; flood++) {
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
}
// One answered ping is one process-wide sample and one per-session sample, so
// neither the metric window nor the hourly log line can be flooded.
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
expect(
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
).toHaveLength(0)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
describe('control lease jitter', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('rebinds re-roll the jitter instead of pinning the cohort phase', async () => {
const now = 1_700_000_000_000
let roll = 0
const h = harness({ now: () => now, random: () => roll })
const first = await activeHost(h)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const firstLease = session.leaseExpiresAt
roll = 0.75
const rebind = new FakeSocket()
await (
h.registry as unknown as {
activate: (...args: unknown[]) => Promise<void>
}
).activate(rebind as unknown as WebSocket, identity, session, 1, true, 1, '1.4.197')
expect(session.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS / 2)
expect(session.leaseExpiresAt).not.toBe(firstLease)
expect(first.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound')
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('paced drain and the phones of a host not yet told', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -809,12 +433,4 @@ describe('paced drain and the phones of a host not yet told', () => {
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
})
it('keeps an unpaced drain refusing every phone at once', async () => {
const { h } = await twoHostCell()
h.registry.drain(0)
for (const hostId of [identity.relayHostId, laterHostId]) {
const refused = await dial(h, hostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
}
})
})
@@ -93,60 +93,6 @@ describe('a zero-grace drain releases the host before telling it', () => {
expect(slow[0]).not.toContain(h.identity.relayHostId)
})
it("starts the host's splice releases before its control release", async () => {
// The store runs one host's releases in the order they start, so waiting on the
// control release covers the splices the drain closed ahead of it.
const { h, key } = await connectedHost()
const order: string[] = []
vi.spyOn(h.assignments, 'releaseActivity').mockImplementation(async (_identity, id) => {
order.push(id)
return true
})
const session = h.registry.get(key)!
session.activeSplices.set('conn-a', () => {
session.activeSplices.delete('conn-a')
void h.assignments.releaseActivity(key, 'splice:conn-a')
})
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(1)
expect(order).toEqual(['splice:conn-a', 'control:1'])
})
it('closes at once when the release fails, so a database outage never holds the host', async () => {
const { h, socket } = await connectedHost()
const release = pendingRelease()
vi.spyOn(h.assignments, 'releaseActivity').mockReturnValue(release.promise)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(1)
release.settle('failed')
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
})
it('does not wait for a host that holds no lease here', async () => {
// A reserve-mode cell admits hosts from memory, with no activity row to release.
const { h, socket, key } = await connectedHost()
h.registry.get(key)!.controlActivityId = null
const releaseActivity = vi.spyOn(h.assignments, 'releaseActivity')
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(1)
expect(releaseActivity).not.toHaveBeenCalled()
expect(frames(socket)).toEqual([{ type: 'drain', graceMs: 0, recovery: 'resolve-director' }])
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
})
it('still tells a host with grace at once, and releases it before the closing deadline', async () => {
const { h, socket } = await connectedHost()
const release = pendingRelease()
@@ -1,273 +0,0 @@
import { createHash, createHmac } from 'node:crypto'
import { EventEmitter } from 'node:events'
import { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } from '@orca-cloud/relay-contract/host-proof-transcript'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import nacl from 'tweetnacl'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
import type { CellReserveBook } from '../cell/reservations/cell-reserve-book.js'
import type { RelayConfig } from '../config.js'
import type { RelayCredentialStore } from '../credentials/credential-store.js'
import { HostSessionRegistry, type CellReserveAdmission } from './host-session-registry.js'
import type { RelayRuntimeObserver } from '../observability/relay-observability.js'
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
const CELL_ID = 'staging-gce-c1'
class FakeSocket extends EventEmitter {
readonly CLOSING = 2
readonly OPEN = 1
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(''))
})
asWebSocket(): WebSocket {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: FakeSocket implements the registry's WebSocket event and lifecycle surface.
return this as unknown as WebSocket
}
// The peer's close frame: ws leaves OPEN at once and emits 'close' later.
hangUp(): void {
this.readyState = this.CLOSING
}
}
function hostKey(): { keyPair: nacl.BoxKeyPair; identity: RelayTokenClaims } {
const keyPair = nacl.box.keyPair()
const relayHostId = createHash('sha256')
.update(keyPair.publicKey)
.digest('base64url')
.slice(0, 16)
return {
keyPair,
identity: {
sub: 'user-1',
prof: 'profile-1',
relayHostId,
purpose: 'host-control',
exp: 4_102_444_800
}
}
}
function setup(input: {
admitMode: 'db' | 'reserve'
databaseSays: boolean
databaseShedding?: boolean
}) {
let clock = 1_000_000
const recordHostPreAuth = vi.fn<NonNullable<RelayRuntimeObserver['recordHostPreAuth']>>()
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordHostPreAuth
} satisfies RelayRuntimeObserver
const take = vi.fn<CellReserveBook['take']>((userId, relayHostId, epoch) => ({
userId,
relayHostId,
epoch,
directorId: 'director-1',
expiresAt: clock + 60_000
}))
const reserveAdmission: CellReserveAdmission = {
mode: () => input.admitMode,
ticketEnforce: () => false,
dryRunEnabled: () => false,
databaseShedding: () => input.databaseShedding ?? false,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture books controls with take() and drains them with clear().
book: { take, clear: vi.fn() } as unknown as CellReserveBook,
verifyLease: vi.fn(async () => 'absent' as const)
}
let answerDatabase: (valid: boolean) => void = () => {}
const databaseAnswered = new Promise<boolean>((resolve) => {
answerDatabase = resolve
})
// Answers at once unless the test holds it with holdDatabase().
let held = false
const verifyCellAssignment = vi.fn(async () => (held ? databaseAnswered : input.databaseSays))
const registry = new HostSessionRegistry(
config(),
vi.fn(),
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path never reads credentials.
{} as RelayCredentialStore,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path calls only verifyCellAssignment().
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
new ProcessQueuedByteBudget(),
observer,
() => clock,
Math.random,
'incarnation-1',
undefined,
reserveAdmission
)
return {
registry,
recordHostPreAuth,
verifyCellAssignment,
take,
holdDatabase: () => {
held = true
},
answerDatabase: (valid: boolean) => answerDatabase(valid),
advance: (ms: number) => {
clock += ms
}
}
}
function sendHello(socket: FakeSocket, keyPair: nacl.BoxKeyPair, relayHostId: string): void {
socket.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'host-hello',
v: 1,
relayHostId,
assignmentEpoch: 4,
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
appVersion: '1.4.200'
})
),
false
)
}
function answerChallenge(socket: FakeSocket, keyPair: nacl.BoxKeyPair): void {
const challenge: unknown = JSON.parse(String(socket.send.mock.calls[0]?.[0]))
const field = (name: string): string => {
const value: unknown =
typeof challenge === 'object' && challenge !== null ? Reflect.get(challenge, name) : null
if (typeof value !== 'string') throw new Error(`host-challenge has no ${name}`)
return value
}
const plaintext = nacl.box.open(
Buffer.from(field('ciphertextB64'), 'base64'),
Buffer.from(field('nonceB64'), 'base64'),
Buffer.from(field('relayEphemeralPublicKeyB64'), 'base64'),
keyPair.secretKey
)
if (!plaintext) throw new Error('host-challenge did not open')
const domainLength = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`).length
const transcriptLength = new DataView(
plaintext.buffer,
plaintext.byteOffset + domainLength,
4
).getUint32(0, false)
const transcriptStart = domainLength + 4
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
const secret = plaintext.slice(transcriptStart + transcriptLength)
socket.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'host-challenge-ack',
challengeId: field('challengeId'),
proofB64: createHmac('sha256', secret)
.update(buildHostProofMacInput(transcript))
.digest('base64')
})
),
false
)
}
describe('host pre-auth latency', () => {
afterEach(() => vi.useRealTimers())
it('records a booked hello on the reserve path, from accept to a verified proof', async () => {
vi.useFakeTimers()
const h = setup({ admitMode: 'reserve', databaseSays: false })
try {
const { keyPair, identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
h.advance(30)
sendHello(socket, keyPair, identity.relayHostId)
await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
expect(h.recordHostPreAuth).not.toHaveBeenCalled()
h.advance(45)
answerChallenge(socket, keyPair)
await vi.advanceTimersByTimeAsync(0)
expect(h.take).toHaveBeenCalledOnce()
expect(h.verifyCellAssignment).not.toHaveBeenCalled()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'admitted', 75)
} finally {
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
}
})
it('records a hello timeout as a refusal before any check, once', () => {
vi.useFakeTimers()
const h = setup({ admitMode: 'db', databaseSays: true })
const { identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
h.advance(2_000)
vi.advanceTimersByTime(2_000)
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'host hello timeout'
)
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 2_000)
})
it.each([true, false])(
'records a database answer (%s) that arrives after the host hung up as abandoned',
async (valid) => {
const h = setup({ admitMode: 'db', databaseSays: valid })
h.holdDatabase()
const { keyPair, identity } = hostKey()
const socket = new FakeSocket()
h.registry.acceptControl(socket.asWebSocket(), identity)
sendHello(socket, keyPair, identity.relayHostId)
await vi.waitFor(() => expect(h.verifyCellAssignment).toHaveBeenCalled())
socket.hangUp()
h.advance(900)
h.answerDatabase(valid)
await vi.waitFor(() => expect(h.recordHostPreAuth).toHaveBeenCalled())
expect(socket.send).not.toHaveBeenCalled()
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'abandoned', 900)
}
)
})
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://c1.relay.example.test',
cellUrl: 'https://c1.relay.example.test',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new TextEncoder().encode('0123456789abcdef0123456789abcdef'),
role: 'cell',
cellId: CELL_ID,
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data'
}
}
@@ -8,7 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } fr
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
import type { RelayConfig } from '../config.js'
import { RelayStoreError, type RelayCredentialStore } from '../credentials/credential-store.js'
import type { RelayCredentialStore } from '../credentials/credential-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from '../control-leases/control-renewal-batch.js'
import type {
ControlRenewalOutcome,
@@ -906,51 +906,6 @@ describe('host session cleanup races', () => {
vi.advanceTimersByTime(0)
})
it('re-acquires a missing control activity lease', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
const { registry, activate, acquireActivity, renewControlActivities } =
createRegistry(activateControl)
renewControlActivities.mockResolvedValueOnce(['control_activity_not_found'])
const socket = new FakeSocket()
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
{
activityId: 'control:production-gce-c3:1',
kind: 'control',
cellId: config.cellId
}
)
expect(socket.close).not.toHaveBeenCalled()
registry.drain(0)
vi.advanceTimersByTime(0)
})
it('closes a control whose activity lease moved to another cell', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
const { registry, activate, acquireActivity, renewControlActivities } =
createRegistry(activateControl)
renewControlActivities.mockResolvedValueOnce(['control_activity_moved'])
const socket = new FakeSocket()
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).not.toHaveBeenCalled()
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
registry.drain(0)
vi.advanceTimersByTime(0)
})
it('closes when a missing control activity cannot be re-acquired on this cell', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
@@ -1675,45 +1630,6 @@ describe('host data attach owner lookup', () => {
return { ...h, owner, ordered, counts, client, control, sessions: internals.sessions }
}
it.each([
{
placement: 'first',
after: { visits: 1, membership: 1 }
},
{
placement: 'middle',
after: { visits: 500, membership: 500 }
},
{
placement: 'last',
after: { visits: 1000, membership: 1000 }
},
{
placement: 'absent',
after: { visits: 1000, membership: 1000 }
}
] as const)(
'visits the inventory once, not twice, for a $placement owner',
async ({ placement, after }) => {
const h = await attachRegistry(placement)
expect(h.sessions.size).toBe(SESSION_COUNT)
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
// An unusable ticket stops the attach immediately after the lookup, so the
// counts below belong to the lookup alone.
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'wrong', 1)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host data ticket'
)
expect({ ...h.counts }).toEqual(after)
}
)
it.each([
{ reason: 'ticket', ticket: 'wrong', generation: 1, state: 'active' },
{ reason: 'generation', ticket: 'ticket', generation: 2, state: 'active' },
@@ -1739,38 +1655,6 @@ describe('host data attach owner lookup', () => {
expect(h.owner.activeConnIds.size).toBe(0)
})
it('rejects on the earlier duplicate owner rather than the later live one', async () => {
const h = await attachRegistry('middle')
h.ordered[0]!.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'stale-ticket') as never)
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
})
it('consumes the earlier duplicate owner on basis failure and leaves the later one untouched', async () => {
const basis = vi.fn().mockRejectedValue(new Error('basis failed'))
const h = await attachRegistry('first', { recordConnectionBasis: basis })
const duplicate = h.ordered[3]!
duplicate.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'ticket') as never)
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.observer.recordAuth).toHaveBeenCalledWith(true)
expect(basis).toHaveBeenCalledOnce()
// The first owner's entry was consumed; the later duplicate never was.
expect(h.owner.pendingConns.has(CONN_ID)).toBe(false)
expect(duplicate.pendingConns.has(CONN_ID)).toBe(true)
expect(h.owner.activeConnIds.size).toBe(0)
})
})
describe('paced drain', () => {
@@ -2007,13 +1891,4 @@ describe('control error codes', () => {
])
})
it.each([
['a store code', new RelayStoreError('reject-revoked'), 'reject-revoked'],
['an invariant slug', new Error('authorization_expired'), 'authorization_expired']
])('keeps %s on the wire', async (_kind, error, code) => {
const { sent, warned } = await revokeFailure(error)
expect(controlErrors(sent)).toEqual([{ type: 'control-error', reqId: 'revoke-1', code }])
expect(warned.join('\n')).not.toContain('control device-revoke failed')
})
})
@@ -228,23 +228,6 @@ it('delivers the rejection code and a graceful FIN to a peer that keeps reading'
const BINARY_FIRST_FRAME = Buffer.from([0x82, 0x80, 0, 0, 0, 0])
const CONCURRENT_PEERS_PER_SOURCE = 4
it('returns capacity to its exact baseline after repeated bursts of rejections', async () => {
const { relay, port } = await fixture({ hardCap: 2 * CONCURRENT_PEERS_PER_SOURCE + 1 })
for (let wave = 0; wave < 3; wave++) {
const peers = await Promise.all(
Array.from({ length: CONCURRENT_PEERS_PER_SOURCE }, () =>
silentUpgrade(port, PHONE_TARGET)
)
)
expect(relay.connectionSnapshot()).toMatchObject({
enforcedConnectionUnits: 2 * CONCURRENT_PEERS_PER_SOURCE
})
for (const peer of peers) peer.socket.write(BINARY_FIRST_FRAME)
await expectIdleLedger(relay, 3_000)
expect(relay.runtimeCounts().preAuthConnections).toBe(0)
}
})
// Ledger does not clamp at zero, so a disconnect racing rejection exposes double release as a negative count.
it('releases exactly once when a rejected peer disconnects at the same moment', async () => {
const { relay, port } = await fixture({ hardCap: 2 * CONCURRENT_PEERS_PER_SOURCE + 1 })
@@ -7,7 +7,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
import WebSocket from 'ws'
import type { RelayConfig } from '../config.js'
import { PostgresDatabase } from '../database/database.js'
import { HostSessionRegistry } from './host-session-registry.js'
import { createRelayServer, HOST_HELLO_SHED_OLDEST_WAIT_MS } from '../relay-server.js'
async function unusedPort(): Promise<number> {
@@ -176,11 +175,4 @@ describe('host hello shedding under database pool pressure', () => {
expect(error.message).toBe('Unexpected server response: 503')
})
it('never refuses a rebind over a live control', async () => {
vi.spyOn(HostSessionRegistry.prototype, 'hasActiveControl').mockReturnValue(true)
const cell = await startCell(2)
cell.advance(HOST_HELLO_SHED_OLDEST_WAIT_MS * 2)
expect(await dial(cell)).toEqual({ status: 101 })
})
})
@@ -107,20 +107,6 @@ describe('control accept cell-row lock span', () => {
return Number(row.units)
}
it('writes the shared cell row once, last, and never takes it as a read lock', async () => {
const control = await accept(1)
await store.releaseActivity(host, control)
await accept(2)
const cells = cellStatements()
expect(cells.map((statement) => statement.locked)).toEqual([false])
expect(cells[0]!.sql).toContain('RETURNING cell_id')
// The contended row is written by the last statement of the transaction, so
// its write lock is held across the commit alone, not the whole accept.
expect(recorder.statements.at(-1)).toBe(cells[0])
})
it('leaves the cell row untouched when a rebind retires and installs one control', async () => {
await accept(1)