mirror of
https://github.com/stablyai/orca.git
synced 2026-11-13 09:27:29 +00:00
test: remove low-value host control and connection test inventories (#38578)
Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
@@ -8,29 +8,6 @@ function memoryAt(clock: { now: number }): HostCloseReasonMemory {
|
||||
}
|
||||
|
||||
describe('HostCloseReasonMemory', () => {
|
||||
it('remembers only reasons it knows', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
memory.record('b', 'quitting')
|
||||
memory.record('c', Buffer.alloc(0))
|
||||
memory.record('d', undefined)
|
||||
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
expect(memory.read('b')).toBeNull()
|
||||
expect(memory.read('c')).toBeNull()
|
||||
expect(memory.read('d')).toBeNull()
|
||||
})
|
||||
|
||||
it('accepts the reason as the Buffer a ws close delivers', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
|
||||
memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
|
||||
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
|
||||
it('expires an entry once its host may have been rebalanced away', () => {
|
||||
const clock = { now: 1_000 }
|
||||
@@ -45,16 +22,6 @@ describe('HostCloseReasonMemory', () => {
|
||||
expect(memory.size()).toBe(0)
|
||||
})
|
||||
|
||||
it('forgets on demand', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
memory.forget('a')
|
||||
|
||||
expect(memory.read('a')).toBeNull()
|
||||
})
|
||||
|
||||
it('drops the oldest survivors rather than growing without bound', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
@@ -67,16 +34,4 @@ describe('HostCloseReasonMemory', () => {
|
||||
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
|
||||
it('re-recording refreshes recency so a live host is not evicted first', () => {
|
||||
const memory = memoryAt({ now: 1_000 })
|
||||
for (let index = 0; index < 50_000; index++) {
|
||||
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
}
|
||||
memory.record('host-0', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
memory.record('overflow', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
expect(memory.size()).toBe(50_000)
|
||||
expect(memory.read('host-0')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
expect(memory.read('host-1')).toBeNull()
|
||||
expect(memory.read('overflow')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
@@ -16,49 +15,6 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('host control proof cleanup', () => {
|
||||
it('allocates no hello stage for an already closed peer', () => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
socket.peerClose()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('message')).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
expect(h.verify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('releases the host hello timer and listeners when its peer closes early', () => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
expect(socket.listenerCount('message')).toBe(1)
|
||||
socket.peerClose()
|
||||
expect({
|
||||
timers: vi.getTimerCount(),
|
||||
message: socket.listenerCount('message'),
|
||||
close: socket.listenerCount('close')
|
||||
}).toEqual({ timers: 0, message: 0, close: 0 })
|
||||
vi.advanceTimersByTime(2000)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
expect(h.verify).not.toHaveBeenCalled()
|
||||
expect(h.database.query).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves the exact hello deadline and refusal while releasing its message listener', () => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
vi.advanceTimersByTime(1999)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
vi.advanceTimersByTime(1)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'host hello timeout'
|
||||
)
|
||||
expect(socket.listenerCount('message')).toBe(0)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
|
||||
it('releases the challenge timer and listeners when its peer closes before proof', async () => {
|
||||
const h = fixture()
|
||||
@@ -77,21 +33,6 @@ describe('host control proof cleanup', () => {
|
||||
expect(h.database.query).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves the exact proof deadline and refusal with no leftover listener', async () => {
|
||||
const h = fixture()
|
||||
const socket = await openProof(h)
|
||||
await vi.advanceTimersByTimeAsync(9999)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'host proof timeout'
|
||||
)
|
||||
expect(socket.listenerCount('message')).toBe(0)
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
expect(h.recordAuth).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does no challenge crypto, send, timer or registration after a closed peer finishes verification', async () => {
|
||||
const h = fixture()
|
||||
let finish!: (valid: boolean) => void
|
||||
@@ -119,86 +60,6 @@ describe('host control proof cleanup', () => {
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it.each([false, true])('preserves invalid first-frame refusal (binary=%s)', (binary) => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
socket.emit('message', Buffer.from('{}'), binary)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
binary ? 'host hello must be text' : 'invalid host hello'
|
||||
)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([false, true])(
|
||||
'preserves invalid proof authentication failure (binary=%s)',
|
||||
async (binary) => {
|
||||
const h = fixture()
|
||||
const socket = await openProof(h)
|
||||
socket.emit('message', Buffer.from('{}'), binary)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'invalid host proof'
|
||||
)
|
||||
expect(h.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it('allocates no proof wait when sending the challenge closes its peer', async () => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
socket.send.mockImplementation(() => socket.peerClose())
|
||||
await openProof(h, socket)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('message')).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the existing diagnostic and refusal when challenge send throws', async () => {
|
||||
const h = fixture()
|
||||
const socket = new ProofSocket()
|
||||
socket.send.mockImplementation(() => {
|
||||
throw new Error('synthetic send failure')
|
||||
})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
await openProof(h, socket)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
|
||||
'relay temporarily unavailable'
|
||||
)
|
||||
expect(warn).toHaveBeenCalledExactlyOnceWith(
|
||||
'[orca-relay] host hello proof failed: synthetic send failure'
|
||||
)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
})
|
||||
|
||||
it('contains assignment lookup rejection with its existing close and diagnostic', async () => {
|
||||
const h = fixture()
|
||||
h.verify.mockRejectedValueOnce(new Error('synthetic lookup failure'))
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const socket = new ProofSocket()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
socket.emit('message', Buffer.from(h.hello), false)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(socket.close).toHaveBeenCalledExactlyOnceWith(
|
||||
RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
|
||||
'relay temporarily unavailable'
|
||||
)
|
||||
expect(warn).toHaveBeenCalledExactlyOnceWith(
|
||||
'[orca-relay] host hello proof failed: synthetic lookup failure'
|
||||
)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('close')).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps a newer same-host peer live when the old proof peer closes', async () => {
|
||||
const h = fixture()
|
||||
const oldPeer = await openProof(h)
|
||||
|
||||
@@ -144,23 +144,6 @@ describe('a resumed control adopts the token its hello proved', () => {
|
||||
})
|
||||
|
||||
describe('an auth refresh', () => {
|
||||
it('never lowers the expiry the session already holds', async () => {
|
||||
const verify = vi.fn<(token: string) => Promise<RelayTokenClaims | null>>()
|
||||
const { h, first } = await connectedHost(600, verify)
|
||||
const session = h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })!
|
||||
const held = session.identity.exp
|
||||
verify.mockResolvedValueOnce({ ...h.identity, exp: held - 300 })
|
||||
sendRefresh(first)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(session.identity.exp).toBe(held)
|
||||
verify.mockResolvedValueOnce({ ...h.identity, exp: held + 300 })
|
||||
sendRefresh(first)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(session.identity.exp).toBe(held + 300)
|
||||
expect(first.close).not.toHaveBeenCalled()
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
it('closes the socket a refused refresh arrived on, not the control that replaced it', async () => {
|
||||
let finish!: (claims: RelayTokenClaims | null) => void
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract/protocol-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import type { CredentialReservation, RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import {
|
||||
CONTROL_LEASE_JITTER_MS,
|
||||
CONTROL_LEASE_MS,
|
||||
HostSessionRegistry
|
||||
} from './host-session-registry.js'
|
||||
import type { RelayRuntimeObserver } from '../observability/relay-observability.js'
|
||||
@@ -298,240 +295,8 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('stops after a slow resume lookup before starting the invite and assignment lookups', async () => {
|
||||
const h = harness()
|
||||
await activeHost(h)
|
||||
const store = h.store as typeof h.store & { resolveInviteForMove: ReturnType<typeof vi.fn> }
|
||||
store.resolveInviteForMove = vi.fn().mockResolvedValue(null)
|
||||
const slowResume = deferred<null>()
|
||||
h.store.resolveResume.mockReturnValueOnce(slowResume.promise)
|
||||
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
|
||||
.resolve
|
||||
resolveAssignment.mockClear()
|
||||
const client = new FakeSocket()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const accepting = h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
client.close(1000, 'client bound')
|
||||
slowResume.resolve(null)
|
||||
await accepting
|
||||
|
||||
expect(store.resolveInviteForMove).not.toHaveBeenCalled()
|
||||
expect(resolveAssignment).not.toHaveBeenCalled()
|
||||
expect(h.store.reserveCredential).not.toHaveBeenCalled()
|
||||
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
|
||||
'assignment',
|
||||
expect.any(Number)
|
||||
)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('stops after a slow same-cell assignment resolve, before reserving a credential', async () => {
|
||||
const h = harness()
|
||||
await activeHost(h)
|
||||
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
|
||||
.resolve
|
||||
const slowResolve = deferred<{ cellId: string }>()
|
||||
resolveAssignment.mockReturnValueOnce(slowResolve.promise)
|
||||
const client = new FakeSocket()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const accepting = h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
client.close(1000, 'client bound')
|
||||
// A correct, same-cell assignment: only the closed socket stops the accept.
|
||||
slowResolve.resolve({ cellId: config.cellId })
|
||||
await accepting
|
||||
|
||||
// Proves the accept reached the third guard, not the first.
|
||||
expect(resolveAssignment).toHaveBeenCalled()
|
||||
expect(h.store.reserveCredential).not.toHaveBeenCalled()
|
||||
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
|
||||
'assignment',
|
||||
expect.any(Number)
|
||||
)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('logs and counts an attach deadline with what the cell knew of the control', async () => {
|
||||
let now = 1_700_000_000_000
|
||||
const h = harness({ now: () => now })
|
||||
await activeHost(h)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const client = new FakeSocket()
|
||||
await h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
now += RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs
|
||||
vi.advanceTimersByTime(RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs)
|
||||
// The phone-only reason; relay-hello itself is unchanged for strict old parsers.
|
||||
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, 'attach-timeout')
|
||||
expect(client.send).toHaveBeenCalledWith(
|
||||
JSON.stringify({ type: 'relay-hello', ok: false, code: 4404 })
|
||||
)
|
||||
expect(h.observer.recordPhoneAttachDeadline).toHaveBeenCalledOnce()
|
||||
expect(h.observer.recordPhoneRejection).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE)
|
||||
const lines = warn.mock.calls
|
||||
.map(([line]) => String(line))
|
||||
.filter((line) => line.includes('orca_relay_attach_deadline_offline'))
|
||||
expect(lines.map((line) => JSON.parse(line))).toEqual([
|
||||
{
|
||||
event: 'orca_relay_attach_deadline_offline',
|
||||
relayHostIdDigest: expect.any(String),
|
||||
credentialKind: 'resume',
|
||||
msSinceLastPong: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
|
||||
pingOutstanding: false,
|
||||
controlAgeMs: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
|
||||
controlRttMs: null,
|
||||
pong: 'fresh',
|
||||
cellId: config.cellId
|
||||
}
|
||||
])
|
||||
expect(lines[0]).not.toContain(identity.relayHostId)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('still opens the connection when the phone is holding on', async () => {
|
||||
const h = harness()
|
||||
const control = await activeHost(h)
|
||||
const capacity = { bind: vi.fn(), release: vi.fn() }
|
||||
const client = new FakeSocket()
|
||||
await h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential',
|
||||
capacity
|
||||
)
|
||||
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
|
||||
expect(capacity.bind).toHaveBeenCalledOnce()
|
||||
expect(h.observer.recordClientAcceptAbandoned).not.toHaveBeenCalled()
|
||||
expect(client.close).not.toHaveBeenCalled()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('successful client accept timing', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
|
||||
let now = 1_700_000_000_000
|
||||
const h = harness({ now: () => now })
|
||||
const control = await activeHost(h)
|
||||
h.store.resolveResume.mockImplementationOnce(async () => {
|
||||
now += 5
|
||||
return { userId: identity.sub }
|
||||
})
|
||||
h.store.reserveCredential.mockImplementationOnce(async () => {
|
||||
now += 7
|
||||
return reservation
|
||||
})
|
||||
h.acquireActivity.mockImplementationOnce(async () => {
|
||||
now += 11
|
||||
})
|
||||
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
|
||||
now += 3
|
||||
})
|
||||
const client = new FakeSocket()
|
||||
const hostData = new FakeSocket()
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
|
||||
const connOpen = JSON.parse(
|
||||
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
|
||||
) as { connId: string; connTicket: string }
|
||||
// The desktop's data leg is the attach window this is meant to expose.
|
||||
now += 23
|
||||
const accepted = await h.registry.acceptHostData(
|
||||
hostData as unknown as WebSocket,
|
||||
connOpen.connId,
|
||||
connOpen.connTicket,
|
||||
1
|
||||
)
|
||||
|
||||
expect(accepted).toBe(true)
|
||||
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
|
||||
totalMs: 49,
|
||||
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 },
|
||||
phonePath: 'db'
|
||||
})
|
||||
const line = log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
|
||||
expect(line).toBeDefined()
|
||||
const event = JSON.parse(line!) as {
|
||||
role: string
|
||||
cellId: string
|
||||
region: string
|
||||
credentialKind: string
|
||||
stageMs: Record<string, number>
|
||||
totalMs: number
|
||||
relayHostIdDigest: string
|
||||
}
|
||||
expect(event.credentialKind).toBe('resume')
|
||||
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
|
||||
// Joins the line back to the emitting process, like the runtime metrics event.
|
||||
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
|
||||
expect(Object.keys(event.stageMs).sort()).toEqual([
|
||||
'activity',
|
||||
'assignment',
|
||||
'attach',
|
||||
'basis',
|
||||
'credential'
|
||||
])
|
||||
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
|
||||
// The stages tile the accept end to end: every millisecond is attributed.
|
||||
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
|
||||
expect(summed).toBe(event.totalMs)
|
||||
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
|
||||
expect(line).not.toContain(identity.relayHostId)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
|
||||
// echo the registry will time.
|
||||
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
|
||||
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
const ping = control.send.mock.calls
|
||||
.filter((call) => String(call[0]).includes('"type":"ping"'))
|
||||
.at(-1)!
|
||||
return (JSON.parse(String(ping[0])) as { t: number }).t
|
||||
}
|
||||
|
||||
// The attach resolves its owning session once and hands it to the unfenced leg;
|
||||
// these hold the session it must be and the order the client hears about it.
|
||||
describe('host data attach ownership', () => {
|
||||
@@ -606,147 +371,6 @@ describe('host data attach ownership', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('control round-trip sampling', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('logs a host once at the fourth sample and not again within the hour', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
const rttLines = (): string[] =>
|
||||
log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
|
||||
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
|
||||
const roundTrip = async (): Promise<void> => {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 40
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
}
|
||||
try {
|
||||
for (let round = 0; round < 3; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
|
||||
expect(rttLines()).toHaveLength(0)
|
||||
|
||||
await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
role: 'cell',
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
rttMsMedian: 40,
|
||||
assignmentEpoch: 1,
|
||||
controlGeneration: 1,
|
||||
drainMode: 'none',
|
||||
sampleCount: 4
|
||||
})
|
||||
expect(rttLines()[0]).not.toContain(identity.relayHostId)
|
||||
|
||||
// Later samples keep feeding the fleet metric, but stay silent for an hour.
|
||||
for (let round = 0; round < 8; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
|
||||
const elapsedStart = clock.now
|
||||
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
|
||||
expect(rttLines()).toHaveLength(2)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores a pong that answers no outstanding ping', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
try {
|
||||
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
|
||||
control.emit('message', JSON.stringify({ type: 'pong' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
clock.now += 10
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
|
||||
} finally {
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('records one sample per ping however many pongs a host floods', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 12
|
||||
for (let flood = 0; flood < 5_000; flood++) {
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
}
|
||||
// One answered ping is one process-wide sample and one per-session sample, so
|
||||
// neither the metric window nor the hourly log line can be flooded.
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
|
||||
expect(
|
||||
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
|
||||
).toHaveLength(0)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('control lease jitter', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('rebinds re-roll the jitter instead of pinning the cohort phase', async () => {
|
||||
const now = 1_700_000_000_000
|
||||
let roll = 0
|
||||
const h = harness({ now: () => now, random: () => roll })
|
||||
const first = await activeHost(h)
|
||||
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
const firstLease = session.leaseExpiresAt
|
||||
roll = 0.75
|
||||
const rebind = new FakeSocket()
|
||||
await (
|
||||
h.registry as unknown as {
|
||||
activate: (...args: unknown[]) => Promise<void>
|
||||
}
|
||||
).activate(rebind as unknown as WebSocket, identity, session, 1, true, 1, '1.4.197')
|
||||
expect(session.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS / 2)
|
||||
expect(session.leaseExpiresAt).not.toBe(firstLease)
|
||||
expect(first.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound')
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('paced drain and the phones of a host not yet told', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
@@ -809,12 +433,4 @@ describe('paced drain and the phones of a host not yet told', () => {
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
})
|
||||
|
||||
it('keeps an unpaced drain refusing every phone at once', async () => {
|
||||
const { h } = await twoHostCell()
|
||||
h.registry.drain(0)
|
||||
for (const hostId of [identity.relayHostId, laterHostId]) {
|
||||
const refused = await dial(h, hostId)
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -93,60 +93,6 @@ describe('a zero-grace drain releases the host before telling it', () => {
|
||||
expect(slow[0]).not.toContain(h.identity.relayHostId)
|
||||
})
|
||||
|
||||
it("starts the host's splice releases before its control release", async () => {
|
||||
// The store runs one host's releases in the order they start, so waiting on the
|
||||
// control release covers the splices the drain closed ahead of it.
|
||||
const { h, key } = await connectedHost()
|
||||
const order: string[] = []
|
||||
vi.spyOn(h.assignments, 'releaseActivity').mockImplementation(async (_identity, id) => {
|
||||
order.push(id)
|
||||
return true
|
||||
})
|
||||
const session = h.registry.get(key)!
|
||||
session.activeSplices.set('conn-a', () => {
|
||||
session.activeSplices.delete('conn-a')
|
||||
void h.assignments.releaseActivity(key, 'splice:conn-a')
|
||||
})
|
||||
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
|
||||
expect(order).toEqual(['splice:conn-a', 'control:1'])
|
||||
})
|
||||
|
||||
it('closes at once when the release fails, so a database outage never holds the host', async () => {
|
||||
const { h, socket } = await connectedHost()
|
||||
const release = pendingRelease()
|
||||
vi.spyOn(h.assignments, 'releaseActivity').mockReturnValue(release.promise)
|
||||
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
release.settle('failed')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
'resolve configured director'
|
||||
)
|
||||
})
|
||||
|
||||
it('does not wait for a host that holds no lease here', async () => {
|
||||
// A reserve-mode cell admits hosts from memory, with no activity row to release.
|
||||
const { h, socket, key } = await connectedHost()
|
||||
h.registry.get(key)!.controlActivityId = null
|
||||
const releaseActivity = vi.spyOn(h.assignments, 'releaseActivity')
|
||||
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
|
||||
expect(releaseActivity).not.toHaveBeenCalled()
|
||||
expect(frames(socket)).toEqual([{ type: 'drain', graceMs: 0, recovery: 'resolve-director' }])
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
'resolve configured director'
|
||||
)
|
||||
})
|
||||
|
||||
it('still tells a host with grace at once, and releases it before the closing deadline', async () => {
|
||||
const { h, socket } = await connectedHost()
|
||||
const release = pendingRelease()
|
||||
|
||||
@@ -1,273 +0,0 @@
|
||||
import { createHash, createHmac } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } from '@orca-cloud/relay-contract/host-proof-transcript'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
|
||||
import type { CellReserveBook } from '../cell/reservations/cell-reserve-book.js'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import type { RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import { HostSessionRegistry, type CellReserveAdmission } from './host-session-registry.js'
|
||||
import type { RelayRuntimeObserver } from '../observability/relay-observability.js'
|
||||
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
|
||||
|
||||
const CELL_ID = 'staging-gce-c1'
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly CLOSING = 2
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(''))
|
||||
})
|
||||
|
||||
asWebSocket(): WebSocket {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: FakeSocket implements the registry's WebSocket event and lifecycle surface.
|
||||
return this as unknown as WebSocket
|
||||
}
|
||||
|
||||
// The peer's close frame: ws leaves OPEN at once and emits 'close' later.
|
||||
hangUp(): void {
|
||||
this.readyState = this.CLOSING
|
||||
}
|
||||
}
|
||||
|
||||
function hostKey(): { keyPair: nacl.BoxKeyPair; identity: RelayTokenClaims } {
|
||||
const keyPair = nacl.box.keyPair()
|
||||
const relayHostId = createHash('sha256')
|
||||
.update(keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
return {
|
||||
keyPair,
|
||||
identity: {
|
||||
sub: 'user-1',
|
||||
prof: 'profile-1',
|
||||
relayHostId,
|
||||
purpose: 'host-control',
|
||||
exp: 4_102_444_800
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setup(input: {
|
||||
admitMode: 'db' | 'reserve'
|
||||
databaseSays: boolean
|
||||
databaseShedding?: boolean
|
||||
}) {
|
||||
let clock = 1_000_000
|
||||
const recordHostPreAuth = vi.fn<NonNullable<RelayRuntimeObserver['recordHostPreAuth']>>()
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn(),
|
||||
recordHostPreAuth
|
||||
} satisfies RelayRuntimeObserver
|
||||
const take = vi.fn<CellReserveBook['take']>((userId, relayHostId, epoch) => ({
|
||||
userId,
|
||||
relayHostId,
|
||||
epoch,
|
||||
directorId: 'director-1',
|
||||
expiresAt: clock + 60_000
|
||||
}))
|
||||
const reserveAdmission: CellReserveAdmission = {
|
||||
mode: () => input.admitMode,
|
||||
ticketEnforce: () => false,
|
||||
dryRunEnabled: () => false,
|
||||
databaseShedding: () => input.databaseShedding ?? false,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The fixture books controls with take() and drains them with clear().
|
||||
book: { take, clear: vi.fn() } as unknown as CellReserveBook,
|
||||
verifyLease: vi.fn(async () => 'absent' as const)
|
||||
}
|
||||
let answerDatabase: (valid: boolean) => void = () => {}
|
||||
const databaseAnswered = new Promise<boolean>((resolve) => {
|
||||
answerDatabase = resolve
|
||||
})
|
||||
// Answers at once unless the test holds it with holdDatabase().
|
||||
let held = false
|
||||
const verifyCellAssignment = vi.fn(async () => (held ? databaseAnswered : input.databaseSays))
|
||||
const registry = new HostSessionRegistry(
|
||||
config(),
|
||||
vi.fn(),
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path never reads credentials.
|
||||
{} as RelayCredentialStore,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path calls only verifyCellAssignment().
|
||||
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
|
||||
new ProcessQueuedByteBudget(),
|
||||
observer,
|
||||
() => clock,
|
||||
Math.random,
|
||||
'incarnation-1',
|
||||
undefined,
|
||||
reserveAdmission
|
||||
)
|
||||
return {
|
||||
registry,
|
||||
recordHostPreAuth,
|
||||
verifyCellAssignment,
|
||||
take,
|
||||
holdDatabase: () => {
|
||||
held = true
|
||||
},
|
||||
answerDatabase: (valid: boolean) => answerDatabase(valid),
|
||||
advance: (ms: number) => {
|
||||
clock += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sendHello(socket: FakeSocket, keyPair: nacl.BoxKeyPair, relayHostId: string): void {
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId,
|
||||
assignmentEpoch: 4,
|
||||
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
|
||||
appVersion: '1.4.200'
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
function answerChallenge(socket: FakeSocket, keyPair: nacl.BoxKeyPair): void {
|
||||
const challenge: unknown = JSON.parse(String(socket.send.mock.calls[0]?.[0]))
|
||||
const field = (name: string): string => {
|
||||
const value: unknown =
|
||||
typeof challenge === 'object' && challenge !== null ? Reflect.get(challenge, name) : null
|
||||
if (typeof value !== 'string') throw new Error(`host-challenge has no ${name}`)
|
||||
return value
|
||||
}
|
||||
const plaintext = nacl.box.open(
|
||||
Buffer.from(field('ciphertextB64'), 'base64'),
|
||||
Buffer.from(field('nonceB64'), 'base64'),
|
||||
Buffer.from(field('relayEphemeralPublicKeyB64'), 'base64'),
|
||||
keyPair.secretKey
|
||||
)
|
||||
if (!plaintext) throw new Error('host-challenge did not open')
|
||||
const domainLength = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`).length
|
||||
const transcriptLength = new DataView(
|
||||
plaintext.buffer,
|
||||
plaintext.byteOffset + domainLength,
|
||||
4
|
||||
).getUint32(0, false)
|
||||
const transcriptStart = domainLength + 4
|
||||
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
|
||||
const secret = plaintext.slice(transcriptStart + transcriptLength)
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-challenge-ack',
|
||||
challengeId: field('challengeId'),
|
||||
proofB64: createHmac('sha256', secret)
|
||||
.update(buildHostProofMacInput(transcript))
|
||||
.digest('base64')
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
describe('host pre-auth latency', () => {
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
it('records a booked hello on the reserve path, from accept to a verified proof', async () => {
|
||||
vi.useFakeTimers()
|
||||
const h = setup({ admitMode: 'reserve', databaseSays: false })
|
||||
try {
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(30)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
|
||||
expect(h.recordHostPreAuth).not.toHaveBeenCalled()
|
||||
h.advance(45)
|
||||
answerChallenge(socket, keyPair)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(h.take).toHaveBeenCalledOnce()
|
||||
expect(h.verifyCellAssignment).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'admitted', 75)
|
||||
} finally {
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('records a hello timeout as a refusal before any check, once', () => {
|
||||
vi.useFakeTimers()
|
||||
const h = setup({ admitMode: 'db', databaseSays: true })
|
||||
const { identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(2_000)
|
||||
vi.advanceTimersByTime(2_000)
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'host hello timeout'
|
||||
)
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 2_000)
|
||||
})
|
||||
|
||||
it.each([true, false])(
|
||||
'records a database answer (%s) that arrives after the host hung up as abandoned',
|
||||
async (valid) => {
|
||||
const h = setup({ admitMode: 'db', databaseSays: valid })
|
||||
h.holdDatabase()
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(h.verifyCellAssignment).toHaveBeenCalled())
|
||||
socket.hangUp()
|
||||
h.advance(900)
|
||||
h.answerDatabase(valid)
|
||||
await vi.waitFor(() => expect(h.recordHostPreAuth).toHaveBeenCalled())
|
||||
|
||||
expect(socket.send).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'abandoned', 900)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://c1.relay.example.test',
|
||||
cellUrl: 'https://c1.relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new TextEncoder().encode('0123456789abcdef0123456789abcdef'),
|
||||
role: 'cell',
|
||||
cellId: CELL_ID,
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } fr
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from '../assignment/assignment-store.js'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import { RelayStoreError, type RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import type { RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from '../control-leases/control-renewal-batch.js'
|
||||
import type {
|
||||
ControlRenewalOutcome,
|
||||
@@ -906,51 +906,6 @@ describe('host session cleanup races', () => {
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
|
||||
it('re-acquires a missing control activity lease', async () => {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValue('control:production-gce-c3:1')
|
||||
const { registry, activate, acquireActivity, renewControlActivities } =
|
||||
createRegistry(activateControl)
|
||||
renewControlActivities.mockResolvedValueOnce(['control_activity_not_found'])
|
||||
const socket = new FakeSocket()
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(acquireActivity).toHaveBeenCalledWith(
|
||||
{ userId: identity.sub, relayHostId: identity.relayHostId },
|
||||
{
|
||||
activityId: 'control:production-gce-c3:1',
|
||||
kind: 'control',
|
||||
cellId: config.cellId
|
||||
}
|
||||
)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
|
||||
it('closes a control whose activity lease moved to another cell', async () => {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValue('control:production-gce-c3:1')
|
||||
const { registry, activate, acquireActivity, renewControlActivities } =
|
||||
createRegistry(activateControl)
|
||||
renewControlActivities.mockResolvedValueOnce(['control_activity_moved'])
|
||||
const socket = new FakeSocket()
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(acquireActivity).not.toHaveBeenCalled()
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
|
||||
registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
|
||||
it('closes when a missing control activity cannot be re-acquired on this cell', async () => {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
@@ -1675,45 +1630,6 @@ describe('host data attach owner lookup', () => {
|
||||
return { ...h, owner, ordered, counts, client, control, sessions: internals.sessions }
|
||||
}
|
||||
|
||||
it.each([
|
||||
{
|
||||
placement: 'first',
|
||||
after: { visits: 1, membership: 1 }
|
||||
},
|
||||
{
|
||||
placement: 'middle',
|
||||
after: { visits: 500, membership: 500 }
|
||||
},
|
||||
{
|
||||
placement: 'last',
|
||||
after: { visits: 1000, membership: 1000 }
|
||||
},
|
||||
{
|
||||
placement: 'absent',
|
||||
after: { visits: 1000, membership: 1000 }
|
||||
}
|
||||
] as const)(
|
||||
'visits the inventory once, not twice, for a $placement owner',
|
||||
async ({ placement, after }) => {
|
||||
const h = await attachRegistry(placement)
|
||||
expect(h.sessions.size).toBe(SESSION_COUNT)
|
||||
h.counts.visits = 0
|
||||
h.counts.membership = 0
|
||||
const host = new FakeSocket()
|
||||
// An unusable ticket stops the attach immediately after the lookup, so the
|
||||
// counts below belong to the lookup alone.
|
||||
expect(
|
||||
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'wrong', 1)
|
||||
).toBe(false)
|
||||
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
|
||||
expect(host.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'invalid host data ticket'
|
||||
)
|
||||
expect({ ...h.counts }).toEqual(after)
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
{ reason: 'ticket', ticket: 'wrong', generation: 1, state: 'active' },
|
||||
{ reason: 'generation', ticket: 'ticket', generation: 2, state: 'active' },
|
||||
@@ -1739,38 +1655,6 @@ describe('host data attach owner lookup', () => {
|
||||
expect(h.owner.activeConnIds.size).toBe(0)
|
||||
})
|
||||
|
||||
it('rejects on the earlier duplicate owner rather than the later live one', async () => {
|
||||
const h = await attachRegistry('middle')
|
||||
h.ordered[0]!.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'stale-ticket') as never)
|
||||
h.counts.visits = 0
|
||||
h.counts.membership = 0
|
||||
const host = new FakeSocket()
|
||||
expect(
|
||||
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
|
||||
).toBe(false)
|
||||
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
|
||||
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
|
||||
})
|
||||
|
||||
it('consumes the earlier duplicate owner on basis failure and leaves the later one untouched', async () => {
|
||||
const basis = vi.fn().mockRejectedValue(new Error('basis failed'))
|
||||
const h = await attachRegistry('first', { recordConnectionBasis: basis })
|
||||
const duplicate = h.ordered[3]!
|
||||
duplicate.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'ticket') as never)
|
||||
h.counts.visits = 0
|
||||
h.counts.membership = 0
|
||||
const host = new FakeSocket()
|
||||
expect(
|
||||
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
|
||||
).toBe(false)
|
||||
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
|
||||
expect(h.observer.recordAuth).toHaveBeenCalledWith(true)
|
||||
expect(basis).toHaveBeenCalledOnce()
|
||||
// The first owner's entry was consumed; the later duplicate never was.
|
||||
expect(h.owner.pendingConns.has(CONN_ID)).toBe(false)
|
||||
expect(duplicate.pendingConns.has(CONN_ID)).toBe(true)
|
||||
expect(h.owner.activeConnIds.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('paced drain', () => {
|
||||
@@ -2007,13 +1891,4 @@ describe('control error codes', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['a store code', new RelayStoreError('reject-revoked'), 'reject-revoked'],
|
||||
['an invariant slug', new Error('authorization_expired'), 'authorization_expired']
|
||||
])('keeps %s on the wire', async (_kind, error, code) => {
|
||||
const { sent, warned } = await revokeFailure(error)
|
||||
|
||||
expect(controlErrors(sent)).toEqual([{ type: 'control-error', reqId: 'revoke-1', code }])
|
||||
expect(warned.join('\n')).not.toContain('control device-revoke failed')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -228,23 +228,6 @@ it('delivers the rejection code and a graceful FIN to a peer that keeps reading'
|
||||
const BINARY_FIRST_FRAME = Buffer.from([0x82, 0x80, 0, 0, 0, 0])
|
||||
const CONCURRENT_PEERS_PER_SOURCE = 4
|
||||
|
||||
it('returns capacity to its exact baseline after repeated bursts of rejections', async () => {
|
||||
const { relay, port } = await fixture({ hardCap: 2 * CONCURRENT_PEERS_PER_SOURCE + 1 })
|
||||
for (let wave = 0; wave < 3; wave++) {
|
||||
const peers = await Promise.all(
|
||||
Array.from({ length: CONCURRENT_PEERS_PER_SOURCE }, () =>
|
||||
silentUpgrade(port, PHONE_TARGET)
|
||||
)
|
||||
)
|
||||
expect(relay.connectionSnapshot()).toMatchObject({
|
||||
enforcedConnectionUnits: 2 * CONCURRENT_PEERS_PER_SOURCE
|
||||
})
|
||||
for (const peer of peers) peer.socket.write(BINARY_FIRST_FRAME)
|
||||
await expectIdleLedger(relay, 3_000)
|
||||
expect(relay.runtimeCounts().preAuthConnections).toBe(0)
|
||||
}
|
||||
})
|
||||
|
||||
// Ledger does not clamp at zero, so a disconnect racing rejection exposes double release as a negative count.
|
||||
it('releases exactly once when a rejected peer disconnects at the same moment', async () => {
|
||||
const { relay, port } = await fixture({ hardCap: 2 * CONCURRENT_PEERS_PER_SOURCE + 1 })
|
||||
|
||||
@@ -7,7 +7,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import WebSocket from 'ws'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import { PostgresDatabase } from '../database/database.js'
|
||||
import { HostSessionRegistry } from './host-session-registry.js'
|
||||
import { createRelayServer, HOST_HELLO_SHED_OLDEST_WAIT_MS } from '../relay-server.js'
|
||||
|
||||
async function unusedPort(): Promise<number> {
|
||||
@@ -176,11 +175,4 @@ describe('host hello shedding under database pool pressure', () => {
|
||||
expect(error.message).toBe('Unexpected server response: 503')
|
||||
})
|
||||
|
||||
it('never refuses a rebind over a live control', async () => {
|
||||
vi.spyOn(HostSessionRegistry.prototype, 'hasActiveControl').mockReturnValue(true)
|
||||
const cell = await startCell(2)
|
||||
cell.advance(HOST_HELLO_SHED_OLDEST_WAIT_MS * 2)
|
||||
|
||||
expect(await dial(cell)).toEqual({ status: 101 })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -107,20 +107,6 @@ describe('control accept cell-row lock span', () => {
|
||||
return Number(row.units)
|
||||
}
|
||||
|
||||
it('writes the shared cell row once, last, and never takes it as a read lock', async () => {
|
||||
const control = await accept(1)
|
||||
await store.releaseActivity(host, control)
|
||||
|
||||
await accept(2)
|
||||
|
||||
const cells = cellStatements()
|
||||
expect(cells.map((statement) => statement.locked)).toEqual([false])
|
||||
expect(cells[0]!.sql).toContain('RETURNING cell_id')
|
||||
// The contended row is written by the last statement of the transaction, so
|
||||
// its write lock is held across the commit alone, not the whole accept.
|
||||
expect(recorder.statements.at(-1)).toBe(cells[0])
|
||||
})
|
||||
|
||||
it('leaves the cell row untouched when a rebind retires and installs one control', async () => {
|
||||
await accept(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user