Merge main into brennanb2025/main-terminal-replay

Resolve conflicts onto main's reorganized modules: provider imports move
to providers/pty and providers/working-directory; the terminal CLI
recovery hints move to src/cli/terminals beside main's cli-error layout.
Accept main's deletion of two low-value tests whose only changes here
adapted them to this branch's APIs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Brennan Benson
2026-11-13 12:12:09 -08:00
co-authored by Claude Opus 5.5
8245 changed files with 87957 additions and 169826 deletions
+1 -1
View File
@@ -4,7 +4,7 @@
/skill-stubs/*.md text eol=lf
/skill-stubs/_shared/*.md text eol=lf
/skills/*/SKILL.md text eol=lf
/src/cli/bundled-skill-guides.ts text eol=lf
/src/cli/skills/bundled-skill-guides.ts text eol=lf
# SVG bytes feed embedded data URLs and asset hashes on every build host.
*.svg text eol=lf
# Compiler input line breaks must not depend on the checkout OS.
+35 -2
View File
@@ -12,24 +12,57 @@ body:
Please write your issue in English.
- type: dropdown
id: where
attributes:
label: Where were you using Orca?
description: Where you saw the bug.
options:
- Desktop only
- Desktop + SSH host
- Desktop + Orca server
- Desktop + WSL
- Mobile app
- Web browser
- Other
validations:
required: true
- type: dropdown
id: os
attributes:
label: Operating system
description: 'The device you were using. WSL: pick Windows. Phone: pick iOS or Android.'
options:
- macOS
- Windows
- Linux
- iOS
- Android
- Other
validations:
required: true
- type: dropdown
id: remote_os
attributes:
label: Remote host operating system
description: Only for SSH, Orca server or web.
options:
- Linux
- macOS
- Windows
- Other
- Not sure
validations:
required: false
- type: input
id: orca_version
attributes:
label: Orca version
description: Include the exact Orca version if you can. You can find it in `Orca > About Orca`.
placeholder: v0.14.2
description: 'macOS: Orca > About Orca. Windows/Linux: Help > About Orca. Mobile: Settings > About.'
placeholder: v1.4.2
validations:
required: false
@@ -1,28 +1,16 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
CloudSqlRolloutLease,
LEASE_TTL_MS,
LeaseConflict,
LeaseUnreadable
} from './storage-lease.mjs'
import { CloudSqlRolloutLease, LeaseConflict, LeaseUnreadable } from './storage-lease.mjs'
const BUCKET = 'onorca-cloud-terraform-state'
const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock'
const NOW = 1_756_000_000_000
/** Model real GCS generation increments, ifGenerationMatch and 412 conflicts; faults inject failures. */
function fakeStorage({ object = null, faults = [] } = {}) {
const state = { object, requests: [] }
function fakeStorage({ object = null } = {}) {
const state = { object }
const fetcher = async (rawUrl, init = {}) => {
const url = new URL(rawUrl)
const method = init.method ?? 'GET'
const record = { method, url, path: url.pathname, search: url.searchParams }
state.requests.push(record)
const fault = faults.find((candidate) => candidate.when(record))
if (fault) {
return json(fault.status, fault.body ?? {})
}
if (url.pathname.startsWith('/upload/')) {
const want = url.searchParams.get('ifGenerationMatch')
@@ -63,15 +51,10 @@ function json(status, body) {
}
}
function storedRecord({
holderKey,
expiresAt,
repository = 'stablyai/orca',
workflow = 'Deploy Relay Production'
}) {
function storedRecord({ holderKey, expiresAt }) {
return {
repository,
workflow,
repository: 'stablyai/orca',
workflow: 'Deploy Relay Production',
run_id: '9001',
run_url: 'https://github.com/stablyai/orca/actions/runs/9001',
run_attempt: '1',
@@ -81,14 +64,14 @@ function storedRecord({
}
}
function leaseFor(storage, { warn = () => {} } = {}) {
function leaseFor(storage) {
return new CloudSqlRolloutLease({
bucket: BUCKET,
objectName: OBJECT,
accessToken: 'test-token',
fetcher: storage.fetcher,
now: () => NOW,
warn
warn: () => {}
})
}
@@ -114,77 +97,24 @@ test('refuses a live lease held by another run and never writes', async () => {
.catch((thrown) => thrown)
assert.ok(error instanceof LeaseConflict)
assert.equal(error.holder.repository, 'stablyai/orca')
assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001')
assert.equal(
storage.state.requests.filter((request) => request.method !== 'GET').length,
0,
'a foreign live lease must not be written'
)
assert.equal(storage.state.object.generation, '1500')
})
test('re-enters a live lease this run already holds and extends it', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 })
}
})
const warnings = []
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
HOLDER
)
assert.equal(claim.state, 'reentrant')
assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover')
assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved')
assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS)
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
assert.equal(storage.state.object.generation, '1501')
})
test('takes over an expired lease and warns naming the stale holder', async () => {
test('takes over an expired lease', async () => {
const storage = fakeStorage({
object: {
generation: '1500',
body: storedRecord({
holderKey: 'stablyai/orca/9001',
expiresAt: NOW - 1,
repository: 'stablyai/orca',
workflow: 'Deploy Relay Production Capacity'
expiresAt: NOW - 1
})
}
})
const warnings = []
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
HOLDER
)
await leaseFor(storage).acquire(HOLDER)
assert.equal(claim.state, 'takeover')
assert.equal(warnings.length, 1)
assert.match(warnings[0], /stablyai\/orca/)
assert.match(warnings[0], /Deploy Relay Production Capacity/)
assert.match(warnings[0], /actions\/runs\/9001/)
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
})
test('releases with a generation match and leaves the object gone', async () => {
const storage = fakeStorage()
const lease = leaseFor(storage)
const claim = await lease.acquire(HOLDER)
const released = await lease.release(HOLDER.holderKey)
assert.deepEqual(released, { released: true, generation: claim.generation })
const remove = storage.state.requests.find((request) => request.method === 'DELETE')
assert.equal(remove.search.get('ifGenerationMatch'), claim.generation)
assert.equal(storage.state.object, null)
})
test('refuses to release a lease another run now holds', async () => {
const storage = fakeStorage({
object: {
@@ -212,17 +142,3 @@ test('treats an unreadable record as held, not free', async () => {
assert.ok(error instanceof LeaseUnreadable)
assert.equal(storage.state.object.generation, '1500')
})
test('renewal rewrites only expires_at on the observed generation', async () => {
const storage = fakeStorage()
const lease = leaseFor(storage)
await lease.acquire(HOLDER)
storage.state.object.body.expires_at = NOW - 1
const renewed = await lease.renew(HOLDER.holderKey)
assert.equal(renewed.renewed, true)
assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS)
assert.equal(storage.state.object.body.acquired_at, NOW)
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
})
+16 -3
View File
@@ -62,11 +62,24 @@ jobs:
current.closed_at === pr.closed_at
}
if (!Number.isSafeInteger(pr.number) || pr.number <= 0 ||
(superseded ? !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? '') :
!Number.isFinite(Date.parse(pr.closed_at)))) {
(superseded && !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? ''))) {
throw new Error('Missing PR identity')
}
if (!await stillCurrent()) return
if (!superseded && !Number.isFinite(Date.parse(pr.closed_at))) {
// A close quickly followed by a reopen can deliver a payload rendered after the reopen.
const { data: current } = await github.rest.pulls.get({
...context.repo, pull_number: pr.number
})
if (current.state !== 'closed' || current.merged_at !== null) {
core.info(`PR #${pr.number} changed after this close event; skipping cancellation`)
return
}
throw new Error('Missing PR identity')
}
if (!await stillCurrent()) {
core.info(`PR #${pr.number} changed after this event; skipping cancellation`)
return
}
for (const [prefix, workflow, suffix = ''] of targets) {
const group = prefix === 'cloud-verify-refs/pull'
? `${prefix}/${pr.number}${suffix}` : `${prefix}-${pr.number}${suffix}`
+8 -8
View File
@@ -38,8 +38,8 @@ on:
- '!src/main/daemon/executable-host/windows/testing/daemon-host-relocation.test-fixture.ts'
- '!src/main/daemon/endpoint/publication/testing/daemon-endpoint-reachability-test-harness.ts'
- 'src/main/providers/local-pty/local-pty-provider.ts'
- 'src/main/providers/windows-shell-args.ts'
- 'src/main/providers/windows-shell-preflight-runtime.windows.test.ts'
- 'src/main/providers/shell-launch/windows/shell-args.ts'
- 'src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts'
- 'native/computer-use-macos/**'
- 'native/computer-use-linux/**'
- 'native/computer-use-windows/**'
@@ -53,7 +53,8 @@ on:
- 'src/shared/computer-use-*.ts'
- 'tests/e2e/computer-linux.e2e.ts'
- 'tests/e2e/computer-windows.e2e.ts'
- 'tests/e2e/computer-windows-store.e2e.ts'
- 'tests/e2e/computer-windows-background.e2e.ts'
- 'tests/e2e/helpers/windows-semantic-controls-fixture.*'
- 'tests/e2e/helpers/computer-cli-driver.ts'
- 'tests/e2e/helpers/computer-driver.ts'
- 'tests/e2e/vitest.config.ts'
@@ -95,11 +96,10 @@ jobs:
config/scripts/performance/macos-computer-owner/macos-computer-helper-owner-loss-processes.test.mjs
src/main/computer/sidecar-transport/sidecar-client.test.ts
src/main/computer/macos-native/macos-native-provider-client.test.ts
src/main/providers/windows-shell-preflight-runtime.windows.test.ts
src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts
src/main/computer/desktop-script/actions/desktop-script-provider-actions.test.ts
src/cli/format.test.ts
src/cli/handlers/computer-action-validation.test.ts
src/cli/handlers/computer-state-formatting.test.ts
src/cli/output/command-result-output.test.ts
src/cli/computer/computer-state-formatting.test.ts
src/cli/index.test.ts
src/cli/runtime/envelope-schema.test.ts
src/shared/remote-runtime/remote-runtime-client.test.ts
@@ -152,4 +152,4 @@ jobs:
- run: pnpm verify:computer-native
- run: pnpm build:cli
- run: pnpm run build:electron-vite:parallel
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-background.e2e.ts
@@ -38,6 +38,8 @@ jobs:
mac-native-owner-smoke:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: macos-15
env:
ORCA_BACKGROUND_LAUNCH: '1'
timeout-minutes: 180
permissions:
contents: read
+231 -31
View File
@@ -14,48 +14,248 @@ jobs:
apply-os-label:
runs-on: ubuntu-slim
timeout-minutes: 5
# Serialize runs per issue; each run reconciles from current state, so order does not matter.
concurrency:
group: ${{ github.workflow }}-issue-${{ github.event.issue.number }}
cancel-in-progress: false
steps:
- name: Apply OS label from issue form
- name: Reconcile OS and mobile labels from issue form
uses: actions/github-script@v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const issue = context.payload.issue;
const body = issue.body || "";
const osMatch = body.match(/^### Operating system\s+(.+)$/m);
// The only labels this workflow may add or remove.
const MANAGED_LABELS = ["os:macos", "os:Windows", "os:linux", "os:mobile", "mobile"];
const MAX_PASSES = 3;
if (!osMatch) {
core.info("No operating system field found. Skipping.");
return;
const NOTE_PREFIX =
"> \u{1F310} **Translation note (machine-translated by an assistant on ";
const NOTE_SUFFIX =
"):** This issue's title was originally written in Chinese and has been translated to English (the body is unchanged). The original, untranslated text is preserved intact in the collapsible section below.";
const NOTE_DATE = /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/;
// Metadata headings in accepted order; equal rank = current/legacy names of one field.
const HEADING_RANKS = new Map([
["Summary", 1],
["Where were you using Orca?", 2],
["Client", 2],
["Operating system", 3],
["Host operating system", 3],
["Remote host operating system", 4],
["Orca version", 5],
["Coding harness and version", 6],
["Details", 7],
["Reproduction steps", 7],
]);
const FREE_TEXT_RANK = 7;
const OS_LABELS = new Map([
["macos", "os:macos"],
["windows", "os:Windows"],
["linux", "os:linux"],
["ios", "os:mobile"],
["android", "os:mobile"],
["other", null],
]);
// Value: whether the choice means the mobile app. Retired option strings stay for open issues.
const WHERE_IS_MOBILE = new Map([
["desktop only", false],
["desktop + ssh host", false],
["desktop + orca server", false],
["desktop + wsl", false],
["mobile app", true],
["web browser", false],
["other", false],
["desktop (this computer)", false],
["desktop + ssh", false],
["desktop + orca remote server", false],
["mobile (ios)", true],
["mobile (android)", true],
["web / browser", false],
]);
const CLIENT_IS_MOBILE = new Map([
...WHERE_IS_MOBILE,
["desktop", false],
["mobile", true],
["web", false],
]);
const REMOTE_OS_VALUES = new Set(["linux", "macos", "windows", "other", "not sure"]);
const isBlank = (line) => /^[ \t]*$/.test(line);
const isHeading = (line) => /^###(?:[ \t]|$)/.test(line);
const isUnselected = (value) => value === "" || value === "_no response_";
const isTranslationNote = (line) =>
line.length === NOTE_PREFIX.length + 10 + NOTE_SUFFIX.length &&
line.startsWith(NOTE_PREFIX) &&
line.endsWith(NOTE_SUFFIX) &&
NOTE_DATE.test(line.slice(NOTE_PREFIX.length, NOTE_PREFIX.length + 10));
// Reads only the leading metadata prefix; null means "not a recognizable bug form".
function readFormFields(body) {
if (typeof body !== "string") {
return null;
}
const lines = body.replace(/\r\n/g, "\n").split("\n");
let index = 0;
const skipBlankLines = () => {
while (index < lines.length && isBlank(lines[index])) {
index += 1;
}
};
skipBlankLines();
if (index < lines.length && isTranslationNote(lines[index])) {
index += 1;
if (index >= lines.length || !isBlank(lines[index])) {
return null;
}
skipBlankLines();
}
const fields = new Map();
let lastRank = 0;
while (index < lines.length) {
const line = lines[index];
if (!isHeading(line)) {
return null;
}
// Exact "### <heading>" only; any other heading shape is unknown.
const name = line.startsWith("### ") ? line.slice(4) : null;
const rank = HEADING_RANKS.get(name);
if (rank === undefined || rank < lastRank || fields.has(name)) {
return null;
}
if (rank === FREE_TEXT_RANK) {
// Free text starts here and is never inspected.
return fields;
}
lastRank = rank;
index += 1;
let value = "";
let valueLines = 0;
while (index < lines.length && !isHeading(lines[index])) {
if (!isBlank(lines[index])) {
value = lines[index];
valueLines += 1;
}
index += 1;
}
if (valueLines > 1) {
return null;
}
fields.set(name, value.trim().toLowerCase());
}
return null;
}
const selectedOs = osMatch[1].trim().toLowerCase();
const osLabelMap = {
macos: "os:macos",
windows: "os:Windows",
linux: "os:linux",
};
// Returns the managed labels the body asks for, or null to leave the issue untouched.
function desiredLabelsFor(body) {
const fields = readFormFields(body);
if (!fields || !fields.has("Orca version")) {
return null;
}
const osHeading = ["Operating system", "Host operating system"].find((name) =>
fields.has(name),
);
if (!osHeading) {
return null;
}
const desired = new Set();
const desiredLabel = osLabelMap[selectedOs];
if (!desiredLabel) {
core.info(`No OS label configured for "${selectedOs}". Skipping.`);
return;
const os = fields.get(osHeading);
if (!isUnselected(os)) {
if (!OS_LABELS.has(os)) {
return null;
}
const osLabel = OS_LABELS.get(os);
if (osLabel) {
desired.add(osLabel);
}
}
// The current heading wins by presence, even when blank.
const whereChoices = fields.has("Where were you using Orca?")
? [fields.get("Where were you using Orca?"), WHERE_IS_MOBILE]
: fields.has("Client")
? [fields.get("Client"), CLIENT_IS_MOBILE]
: null;
if (whereChoices && !isUnselected(whereChoices[0])) {
const [where, isMobileByChoice] = whereChoices;
if (!isMobileByChoice.has(where)) {
return null;
}
if (isMobileByChoice.get(where)) {
desired.add("mobile");
}
}
// Validated only; the remote host never contributes a label.
const remoteOs = fields.get("Remote host operating system") ?? "";
if (!isUnselected(remoteOs) && !REMOTE_OS_VALUES.has(remoteOs)) {
return null;
}
return desired;
}
const managedLabels = Object.values(osLabelMap);
const existingLabels = issue.labels.map((label) =>
typeof label === "string" ? label : label.name,
);
const labelsToKeep = existingLabels.filter(
(label) => !managedLabels.includes(label),
);
const nextLabels = [...new Set([...labelsToKeep, desiredLabel])];
await github.rest.issues.setLabels({
const issueRef = {
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
labels: nextLabels,
});
issue_number: context.payload.issue.number,
};
core.info(`Applied label ${desiredLabel} to issue #${issue.number}.`);
// The event is only a signal: decisions come from the issue as it is now.
async function readCurrentIssue() {
const { data } = await github.rest.issues.get(issueRef);
const labelNames = (data.labels ?? []).map((label) =>
typeof label === "string" ? label : label.name,
);
const desired = desiredLabelsFor(data.body);
const present = MANAGED_LABELS.filter((label) => labelNames.includes(label));
return {
desired,
toAdd: desired ? MANAGED_LABELS.filter((l) => desired.has(l) && !present.includes(l)) : [],
toRemove: desired ? present.filter((l) => !desired.has(l)) : [],
};
}
let state = await readCurrentIssue();
let hasWritten = false;
for (let pass = 1; ; pass += 1) {
if (!state.desired) {
if (hasWritten) {
throw new Error(
`Issue #${issueRef.issue_number} stopped matching the bug form while labels were being updated. Managed labels may be incomplete; rerun this job once the issue body is settled.`,
);
}
core.info("No recognizable bug form at the start of the issue body. Labels left untouched.");
return;
}
if (state.toAdd.length === 0 && state.toRemove.length === 0) {
core.info(
`Issue #${issueRef.issue_number} managed labels match the form: [${[...state.desired].join(", ")}].`,
);
return;
}
if (pass > MAX_PASSES) {
throw new Error(
`Issue #${issueRef.issue_number} labels still differ from the form after ${MAX_PASSES} passes (to add: [${state.toAdd.join(", ")}], to remove: [${state.toRemove.join(", ")}]). The issue is probably being edited; rerun this job.`,
);
}
hasWritten = true;
if (state.toAdd.length > 0) {
await github.rest.issues.addLabels({ ...issueRef, labels: state.toAdd });
}
for (const name of state.toRemove) {
try {
await github.rest.issues.removeLabel({ ...issueRef, name });
} catch (error) {
// Already gone is the desired outcome; the re-read below verifies real state.
if (error?.status !== 404) {
throw error;
}
}
}
state = await readCurrentIssue();
}
+5 -5
View File
@@ -11,12 +11,12 @@ on:
- 'native/macos-shipit-scheduling/**'
- 'config/scripts/macos-shipit-scheduling*'
- 'config/electron-builder.config.cjs'
- 'src/main/startup/main-process-quit*'
- 'src/main/window/main-window-state-lifecycle*'
- 'src/main/startup/quit/main-process-quit*'
- 'src/main/window/main-window/main-window-state-lifecycle*'
- 'src/main/window/dashboard-popout-window*'
- 'src/packages/process-host/**'
- 'src/main/runtime/agent-session/lifecycle/structured-agent-cli-writers*'
- 'src/main/providers/macos-tcc-login-shell*'
- 'src/main/providers/shell-launch/macos/tcc-login-shell*'
- 'src/shared/process-table-snapshot*'
- 'src/shared/darwin-process-table-capture*'
- 'src/shared/update-status-types.ts'
@@ -55,8 +55,8 @@ jobs:
src/main/updater/mac-install/updater.mac-install.test.ts
src/main/updater/updater.headless-serve-install.test.ts
src/main/updater/mac-install/updater-mac-quit-guard.test.ts
src/main/startup/main-process-quit-update-veto.test.ts
src/main/window/main-window-state-lifecycle.test.ts
src/main/startup/quit/main-process-quit-update-veto.test.ts
src/main/window/main-window/main-window-state-lifecycle.test.ts
- name: Prove login-backed terminal writer census on the disposable runner
env:
ORCA_MACOS_LOGIN_CENSUS_CI: '1'
+14 -22
View File
@@ -34,13 +34,19 @@ on:
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
- 'src/main/github/issue-field-options.ts'
- 'src/main/github/issues/issue-field-options.ts'
- 'config/tsconfig.test.mobile-host-declarations.json'
- 'config/tsconfig.node.json'
- 'src/types/build-constants.d.ts'
- 'src/main/native-chat/agent-session-wire/**'
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
- 'src/main/native-chat/*transcript*.ts'
- 'src/main/native-chat/transcript-discovery/**'
- 'src/main/native-chat/transcript-filesystem/**'
- 'src/main/native-chat/transcript-decoding/**'
- 'src/main/native-chat/transcript-streaming/**'
- 'src/main/native-chat/transcript-watching/**'
- 'src/main/native-chat/transcript-history/**'
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
@@ -58,13 +64,11 @@ on:
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
- 'src/main/ai-vault/discovery/session-json-file.ts'
- 'src/main/ai-vault/discovery/session-path-absence.ts'
- 'src/main/ai-vault/discovery/session-scanner-agent-root-overrides.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
- 'src/main/ai-vault/discovery/session-scanner-roots.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
- 'src/main/ai-vault/parsing/session-json-object.ts'
@@ -73,7 +77,6 @@ on:
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
- 'src/main/ai-vault/parsing/session-scanner-conversation-name-requests.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
@@ -81,24 +84,17 @@ on:
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-parser-stream-cleanup.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
- 'src/main/ai-vault/parsing/session-scanner-timeline.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-document-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-every-agent-fixture.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-log-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-vault-roots.ts'
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
@@ -180,13 +176,19 @@ on:
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
- 'src/main/github/issue-field-options.ts'
- 'src/main/github/issues/issue-field-options.ts'
- 'config/tsconfig.test.mobile-host-declarations.json'
- 'config/tsconfig.node.json'
- 'src/types/build-constants.d.ts'
- 'src/main/native-chat/agent-session-wire/**'
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
- 'src/main/native-chat/*transcript*.ts'
- 'src/main/native-chat/transcript-discovery/**'
- 'src/main/native-chat/transcript-filesystem/**'
- 'src/main/native-chat/transcript-decoding/**'
- 'src/main/native-chat/transcript-streaming/**'
- 'src/main/native-chat/transcript-watching/**'
- 'src/main/native-chat/transcript-history/**'
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
@@ -204,13 +206,11 @@ on:
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
- 'src/main/ai-vault/discovery/session-json-file.ts'
- 'src/main/ai-vault/discovery/session-path-absence.ts'
- 'src/main/ai-vault/discovery/session-scanner-agent-root-overrides.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
- 'src/main/ai-vault/discovery/session-scanner-roots.test.ts'
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
- 'src/main/ai-vault/parsing/session-json-object.ts'
@@ -219,7 +219,6 @@ on:
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
- 'src/main/ai-vault/parsing/session-scanner-conversation-name-requests.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
@@ -227,24 +226,17 @@ on:
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-parser-stream-cleanup.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
- 'src/main/ai-vault/parsing/session-scanner-timeline.test.ts'
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-document-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-every-agent-fixture.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-log-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
- 'src/main/ai-vault/scanner-fixtures/session-scanner-vault-roots.ts'
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
+1 -1
View File
@@ -247,7 +247,7 @@ jobs:
run: node "$(if [ -f config/scripts/orcad-distribution/template/build-orcad-template.mjs ]; then printf %s config/scripts/orcad-distribution/template/build-orcad-template.mjs; else printf %s config/scripts/build-orcad-template.mjs; fi)" --targets linux-${{ runner.arch == 'ARM64' && 'arm64' || 'x64' }}-glibc
- name: Verify a structured chat from the desktop server template
if: runner.os == 'Linux'
run: pnpm test:node-server --artifact src/main/orcad/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
run: pnpm test:node-server --artifact src/main/orcad/testing/structured-chat/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
# Only a Windows runner compiles it; arm64 cross-compiles here.
# Before the Node 18 check below: the build script imports TypeScript, which Node 18 cannot load.
# orcad's instance lock and stop proof read a PID's creation time through it; the x64 binary this
+2 -3
View File
@@ -853,8 +853,8 @@ jobs:
src/main/pty/shell-startup/zsh-startup-hook-user-config-equivalence.live-shell.test.ts \
src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts \
src/main/runtime/agent-session/environment/structured-session-cli-login-shell.live-shell.test.ts \
src/main/startup/shell-probe-terminal-detach.live-shell.test.ts \
src/main/startup/shell-probe-lifetime.live-shell.test.ts \
src/main/startup/shell-environment/shell-probe-terminal-detach.live-shell.test.ts \
src/main/startup/shell-environment/shell-probe-lifetime.live-shell.test.ts \
src/renderer/src/components/terminal-pane/pty-connection/emulation/fish-color-scheme-child-stdin.node-pty.test.ts \
src/shared/fish-wsl-cwd-report.node-pty.test.ts \
src/main/wsl/filesystem/wsl-sh-cwd-report.node-pty.test.ts \
@@ -1117,7 +1117,6 @@ jobs:
src/main/browser/cookies/browser-google-cookie-clear.electron.test.ts
src/main/browser/guest-lifecycle/popups/presentation/popup-origin-chrome.electron.test.ts
src/main/browser/guest-lifecycle/navigation/browser-link-materialization.electron.test.ts
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-compatibility.electron.test.ts
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-native-compatibility.electron.test.ts
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-background-compatibility.electron.test.ts
src/main/browser/extensions/api-runtime/browser-extension-api.electron.test.ts
@@ -47,8 +47,8 @@ jobs:
src/main/ipc/preflight-runnable-local-cli.test.ts
src/main/ipc/preflight-wsl-agent-detection.test.ts
src/main/ipc/preflight-windows-path-refresh.repro.test.ts
src/main/startup/hydrate-shell-path.windows.test.ts
src/main/startup/login-shell-environment.test.ts
src/main/startup/shell-environment/hydrate-shell-path.windows.test.ts
src/main/startup/shell-environment/login-shell-environment.test.ts
src/packages/process-host/src/process-tree-termination.test.ts
src/packages/process-host/src/run-process.test.ts
src/packages/process-host/src/run-process-termination-failure.test.ts
+2 -2
View File
@@ -15,8 +15,8 @@ on:
- 'src/main/managed-server/**'
- 'src/main/ssh/migration-source/**'
- 'src/main/persistence/migrating-orcad-catalog/**'
- 'src/main/orcad/orcad-artifact-materializer.ts'
- 'src/main/orcad/orcad-host-unavailable.ts'
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
- 'src/main/node-runtime/**'
- 'src/main/ssh/remote-exec/**'
- 'src/main/ssh/remote-host/**'
+3 -3
View File
@@ -12,8 +12,8 @@ on:
- 'src/main/managed-server/**'
- 'src/main/ssh/migration-source/**'
- 'src/main/persistence/migrating-orcad-catalog/**'
- 'src/main/orcad/orcad-artifact-materializer.ts'
- 'src/main/orcad/orcad-host-unavailable.ts'
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
- 'src/main/node-runtime/**'
- 'src/main/ssh/remote-exec/**'
- 'src/main/ssh/remote-host/**'
@@ -38,7 +38,7 @@ on:
- 'config/scripts/native-build/windows/build-windows-cli-launcher.mjs'
- 'config/scripts/native-build/windows/windows-rust-executable-build.mjs'
- 'native/windows-cli-launcher/**'
- 'src/main/orcad/orcad-cli-launcher.ts'
- 'src/main/orcad/serve-startup/orcad-cli-launcher.ts'
- '.github/actions/prepare-windows-cli-launcher/**'
- 'config/patches/node-pty*'
- 'config/patches/@vscode__windows-process-tree*'
+2 -3
View File
@@ -221,8 +221,8 @@ jobs:
pnpm-lock.yaml
cloud/pnpm-lock.yaml
# These two tests import the cloud relay workspace directly. Keeping them in one job
# avoids installing and building the same workspace once per unit-test shard.
# This test imports the cloud relay workspace directly. Its dedicated job avoids
# installing and building the same workspace once per unit-test shard.
- name: Install relay integration dependencies
id: relay_dependencies
if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }}
@@ -237,5 +237,4 @@ jobs:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
tests/e2e/relay-region-compatibility.unit.test.ts
tests/e2e/relay-region-correction.unit.test.ts
@@ -7,8 +7,7 @@ on:
- 'src/main/ai-vault/scanners/opencode/**'
- 'src/main/ai-vault/scanners/opencode2/**'
- 'src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime*'
- 'src/main/native-chat/transcript-opencode*'
- 'src/main/native-chat/transcript-zcode*'
- 'src/main/native-chat/transcript-history/opencode/**'
- 'src/shared/native-chat/native-chat-types.ts'
- 'src/shared/native-chat/transcript/native-chat-transcript-projection*'
- '.github/workflows/zcode-transcript-contract.yml'
@@ -41,4 +40,4 @@ jobs:
bun-version-file: config/.bun-version
no-cache: true
- name: Verify SQLite ordering, legacy readers, and host routing
run: pnpm test src/main/native-chat/transcript-zcode.test.ts src/main/native-chat/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
run: pnpm test src/main/native-chat/transcript-history/opencode/transcript-zcode.test.ts src/main/native-chat/transcript-history/opencode/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
+1 -1
View File
@@ -308,7 +308,7 @@
"ignorePatterns": [
"src/main/acp/generated/acp-protocol.generated.ts",
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
"src/cli/bundled-skill-guides.ts",
"src/cli/skills/bundled-skill-guides.ts",
"resources/plugins/declarative/*.mjs",
"**/node_modules",
"**/dist",
@@ -1,6 +1,6 @@
import { afterEach, expect, it } from 'vitest'
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
import { DELIVERY_LEASE_MS, DurablePushStore } from './durable-push-store.js'
import { DurablePushStore } from './durable-push-store.js'
import {
cleanupDurablePushFixtures,
@@ -63,20 +63,6 @@ it.skipIf(!durablePushTestDatabaseUrl)(
}
)
it('leases exactly one delivery per device across concurrent claimers', async () => {
const { db, store, clock, advance } = await fixture()
for (let seq = 1; seq <= 5; seq++) {
await store.accept('host', 'phone', notification(seq))
advance(1)
}
const claims = await Promise.all(
Array.from({ length: 8 }, () => new DurablePushStore(db, clock).claim())
)
const leased = claims.filter((claim) => claim !== null)
expect(leased).toHaveLength(1)
expect(leased[0]?.notification.notificationSeq).toBe(1)
})
it.skipIf(!durablePushTestDatabaseUrl)(
'does not lease the row behind a head another claimer holds',
async () => {
@@ -138,21 +124,3 @@ it.skipIf(!durablePushTestDatabaseUrl)(
expect((await first)?.notification.notificationSeq).toBe(1)
}
)
// Only the TTL term keeps the candidate scan off an unpruned backlog; expires_at is not indexed first.
it('frees a device held by a crashed worker once its lease lapses', async () => {
const { db, store, clock, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
advance(1)
await store.accept('host', 'phone', notification(2))
const crashed = (await new DurablePushStore(db, clock).claim())!
advance(DELIVERY_LEASE_MS - 1)
expect(await store.claim()).toBeNull()
advance(1)
const reclaimed = (await store.claim())!
expect(reclaimed.id).toBe(crashed.id)
expect(reclaimed.attempts).toBe(2)
await store.finish(reclaimed)
expect((await store.claim())?.notification.notificationSeq).toBe(2)
})
@@ -1,9 +1,7 @@
import { afterEach, describe, expect, it } from 'vitest'
import { openPushDatabase } from '../database/push-database.js'
import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js'
import {
cleanupDurablePushFixtures,
durablePushTestDatabaseUrl,
fixture,
notification
} from './durable-push-store.test-fixture.js'
@@ -11,20 +9,6 @@ import {
afterEach(cleanupDurablePushFixtures)
describe('durable push acceptance', () => {
it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => {
const { store, advance } = await fixture()
for (let i = 0; i < 300; i++) {
expect(await store.accept('host', 'phone1', notification(i))).toBe('queued')
expect(await store.accept('host', 'phone2', notification(i))).toBe('queued')
expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued')
}
expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited')
expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited')
expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued')
advance(15 * 60_000)
expect(await store.accept('host', 'phone1', notification(301))).toBe('queued')
})
it('queues one delivery per event and recovers work across service instances', async () => {
const { db, store, clock, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
@@ -32,16 +16,6 @@ describe('durable push acceptance', () => {
await restarted.accept('host', 'phone', notification(1))
advance(1)
await restarted.accept('host', 'phone', notification(2))
const rows = await db.query(
"SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id",
['phone']
)
expect(rows).toHaveLength(2)
expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([
notification(1),
notification(2)
])
expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true)
const delivery = await restarted.claim()
expect(delivery?.notification.notificationSeq).toBe(1)
expect(await store.claim()).toBeNull()
@@ -93,18 +67,9 @@ describe('durable push acceptance', () => {
const { db, store } = await fixture()
await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable')
try {
const databaseUrl = durablePushTestDatabaseUrl
if (databaseUrl) {
const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' })
try {
await expect(
concurrent.query('SELECT COUNT(*) FROM push_delivery_batches')
).resolves.toHaveLength(1)
} finally {
await concurrent.close()
}
}
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow()
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow(
/push_delivery_batches/
)
expect(await db.query('SELECT * FROM push_events')).toEqual([])
expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([])
} finally {
@@ -160,38 +125,3 @@ it('does not resurrect an in-flight alert after a dismissal and transient provid
expect(await store.claim()).toBeNull()
expect(await store.pendingCount('phone')).toBe(0)
})
it('fences late renew and finish after an expired claim is dismissed', async () => {
const { db, store, advance } = await fixture()
const alert = notification(1)
await store.accept('host', 'phone', alert)
const stale = (await store.claim())!
advance(DELIVERY_LEASE_MS)
await store.accept('host', 'phone', {
...notification(2, 'dismiss'),
notificationId: alert.notificationId
})
const read = async () =>
(
await db.query(
'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?',
[stale.id]
)
)[0]
expect(await read()).toBeUndefined()
await store.renew(stale)
await store.finish(stale, 1000)
await store.finish(stale)
expect(await read()).toBeUndefined()
const dismissal = (await store.claim())!
expect(dismissal.notification.kind).toBe('dismiss')
await store.finish(dismissal)
await store.accept('host', 'phone', notification(3))
const fresh = (await store.claim())!
await store.finish(fresh, 1000)
advance(1000)
const retry = (await store.claim())!
expect(retry.id).toBe(fresh.id)
await store.finish(retry)
expect(await store.claim()).toBeNull()
})
@@ -13,7 +13,6 @@ import { pauseAfter, within } from './durable-push-store.test-fixture.js'
const cleanups: (() => Promise<void>)[] = []
afterEach(async () => {
for (const cleanup of cleanups.splice(0)) await cleanup()
vi.useRealTimers()
vi.restoreAllMocks()
})
const note = (seq: number, overrides: Partial<PushNotification> = {}): PushNotification => ({
@@ -42,9 +41,8 @@ async function fixture() {
const send = vi.fn(async (_delivery: PushDelivery): Promise<PushProviderOutcome> => ({
status: 'sent'
}))
const onRetry = vi.fn()
const dispatcher = new PushDispatcher({ devices, fcm: { send } as never })
const worker = new DurablePushWorker(store, dispatcher, { now, onRetry })
const worker = new DurablePushWorker(store, dispatcher, { now })
cleanups.push(async () => {
await worker.stop()
await db.close()
@@ -57,7 +55,6 @@ async function fixture() {
worker,
dispatcher,
send,
onRetry,
now,
registrationId: device.registrationId,
accept: (notification: PushNotification) =>
@@ -102,7 +99,6 @@ it('stops queued claim admissions without leasing the remaining device', async (
paused.release()
await within(Promise.all([pending, stopping]), 2_000)
expect(h.send).toHaveBeenCalledOnce()
expect(h.send.mock.calls[0]?.[0].orca.notificationSeq).toBe(1)
const [remaining] = await h.db.query(
'SELECT attempts, lease_until FROM push_delivery_batches WHERE registration_id = ?',
[other.registrationId]
@@ -26,18 +26,10 @@ describe('push host challenge store', () => {
await database.close()
})
it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => {
it('rejects a proof issued for another challenge', async () => {
const owner = createPushHostKeypair(6)
const intruder = createPushHostKeypair(7)
const ownerChallenge = await store.issue(hostPublicKeyB64(owner))
expect(
answerPushHostChallenge(ownerChallenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: intruder,
now: () => clock
})
).toBeNull()
const intruderChallenge = await store.issue(hostPublicKeyB64(intruder))
const intruderProof = answerPushHostChallenge(intruderChallenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
@@ -19,11 +19,6 @@ async function concurrentSessions(db: PushDatabase) {
sessions.map((session) => store.resolve(session.sessionToken))
)
expect(decisions.filter((decision) => decision.ok)).toHaveLength(1)
const [row] = await db.query(
'SELECT COUNT(*) AS count FROM push_sessions WHERE host_fingerprint = ?',
[host]
)
expect(Number(row?.count)).toBe(1)
} finally {
await db.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [host])
}
@@ -1,36 +0,0 @@
import { afterEach, expect, it } from 'vitest'
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
const harnesses: Awaited<ReturnType<typeof createPushServerHarness>>[] = []
afterEach(async () => {
await Promise.all(harnesses.splice(0).map((h) => h.close()))
})
it('returns queued for concurrent retries without double quota or delivery', async () => {
const h = await createPushServerHarness()
harnesses.push(h)
const token = await h.signIn(createPushHostKeypair(2))
const registrationId = await h.registerAndroid(token)
const body = { v: 1, registrationIds: [registrationId], notification: notification() }
const responses = await Promise.all(
Array.from({ length: 10 }, () => h.post('/v1/send', body, token))
)
for (const response of responses)
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1)
await h.flushDeliveries()
await h.post('/v1/send', body, token)
await h.flushDeliveries()
expect(h.fcmRequests).toHaveLength(1)
expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined()
expect(
Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count)
).toBe(1)
await h.post(
'/v1/send',
{ ...body, notification: notification({ notificationEpoch: 'new-epoch' }) },
token
)
await h.flushDeliveries()
expect(h.fcmRequests).toHaveLength(2)
})
@@ -2,7 +2,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
import {
createPushServerHarness,
FCM_TOKEN,
notification
} from './push-server-harness.test-fixture.js'
@@ -34,12 +33,6 @@ describe('push gateway send route', () => {
})
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: {
token: FCM_TOKEN,
data: { title: 'Agent needs input', tag: expect.stringMatching(/^[a-f0-9]{64}$/) }
}
})
const afterDeath = await harness.post(
'/v1/send',
@@ -47,11 +40,6 @@ describe('push gateway send route', () => {
sessionToken
)
expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] })
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
expect(await listed.json()).toEqual({
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }]
})
})
it('reports an error for a registration the host does not own', async () => {
@@ -36,7 +36,6 @@ it('delivers again after a topic error without re-registering the phone', async
expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
await h.flushDeliveries()
if (seq === 1) {
expect(h.server.observability.consume().delivery_error).toBe(1)
h.setApnsResponse({ status: 200, body: '' })
}
}
@@ -67,11 +67,10 @@ describe('fcm client', () => {
error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' }
})
})
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toMatchObject({
status: 'error',
reason: 'INVALID_ARGUMENT',
retryable: false,
retryAfterMs: 10000
retryable: false
})
})
})
+1 -1
View File
@@ -5,7 +5,7 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs
import { SignJWT, exportJWK, generateKeyPair, importJWK, type JWK } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
@@ -2,7 +2,7 @@
// client wrapper, the echo service, phone sockets and workerd memory sampling.
import { execSync } from 'node:child_process'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const PORT = 8787
@@ -39,7 +39,7 @@ export async function runLifecycleChecks(ctx: Ctx, main: Control, mainAck: Ack):
mute.on('close', (c) => (muteClose = c))
await new Promise((res) => mute.on('open', res))
// Complete the handshake by hand with the shipped proof answerer, then never pong.
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/relay-host-proof')
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-host-proof')
mute.send(JSON.stringify({ type: 'host-hello', v: 1, relayHostId: ctx.relayHostId, assignmentEpoch: 7,
hostPublicKeyB64: ctx.keypair.publicKeyB64, appVersion: 'spike' }))
mute.on('message', (raw) => {
@@ -12,7 +12,7 @@ const ERASER = 'https://erasure.test.invalid'
const BODY = JSON.stringify({ relayHostId: 'AAAAAAAAAAAAAAAA' })
const JTI = 'jti-0123456789abcdef'
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string; iat?: number; exp?: number }
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string }
async function sign(opts: Sign = {}): Promise<string> {
const claims: Record<string, unknown> = {
@@ -26,8 +26,8 @@ async function sign(opts: Sign = {}): Promise<string> {
.setProtectedHeader({ alg: 'HS256' })
.setIssuer(opts.issuer ?? ISSUER)
.setAudience('orca-relay-edge-admin')
.setIssuedAt(opts.iat ?? now)
.setExpirationTime(opts.exp ?? now + 60)
.setIssuedAt(now)
.setExpirationTime(now + 60)
.sign(new TextEncoder().encode(opts.key ?? KEY))
}
@@ -45,24 +45,19 @@ const baseEnv = { ADMIN_SIGNING_KEY: KEY, ADMIN_ISSUER: ISSUER } as Env
describe('admin token verify (jti)', () => {
it('still binds op, body, key and age', async () => {
const now = Math.floor(Date.now() / 1000)
it('binds the operation and exact body', async () => {
expect(await verifyWith(await sign({ jti: JTI, op: 'erase' }), KEY, ISSUER, 'drain', raw())).toBeNull()
expect(await verifyWith(await sign({ jti: JTI, body: '{}' }), KEY, ISSUER, 'drain', raw())).toBeNull()
expect(await verifyWith(await sign({ jti: JTI, key: NEXT }), KEY, ISSUER, 'drain', raw())).toBeNull()
expect(await verifyWith(await sign({ jti: JTI, iat: now - 180, exp: now + 60 }), KEY, ISSUER, 'drain', raw())).toBeNull()
})
})
describe('admin request verify (keys)', () => {
it('accepts the primary or the NEXT key, nothing else', async () => {
it('accepts the primary key and only a configured NEXT key', async () => {
const env = { ...baseEnv, ADMIN_SIGNING_KEY_NEXT: NEXT } as Env
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI })), 'drain')).not.toBeNull()
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: NEXT })), 'drain')).not.toBeNull()
expect(await verifyAdminRequest(baseEnv, request(await sign({ jti: JTI, key: NEXT })), 'drain')).toBeNull()
const wrong = randomBytes(36).toString('base64url')
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: wrong })), 'drain')).toBeNull()
})
it('the erasure keys (primary or NEXT) verify only erase, under their own issuer', async () => {
@@ -5,7 +5,7 @@ import { readFileSync } from 'node:fs'
import { SignJWT, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const PORT = 8789
@@ -8,7 +8,7 @@ import { join } from 'node:path'
import { SignJWT, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
+3 -7
View File
@@ -28,7 +28,7 @@ const mint = (key: CryptoKey = privateKey, kid = 'k1') =>
.sign(key)
// A Cache API stand-in shared by every "isolate" (module instance) in the test.
function stubCache(): Map<string, Response> {
function stubCache(): void {
const store = new Map<string, Response>()
vi.stubGlobal('caches', {
default: {
@@ -38,7 +38,6 @@ function stubCache(): Map<string, Response> {
match: async (request: Request) => store.get(request.url)?.clone()
}
})
return store
}
// Each call is a fresh isolate: a new module instance with an empty in-memory key cache.
@@ -64,24 +63,21 @@ afterEach(() => {
describe('relay JWT key availability', () => {
it('still says invalid for a bad token when keys are available', async () => {
it('rejects an unknown key when the key set is available', async () => {
stubCache()
authUp(() => true)
const verify = await coldVerifier()
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
expect(await verify(env, await mint(otherKey))).toEqual({ ok: false, reason: 'invalid' })
expect(await verify(env, 'not-a-jwt')).toEqual({ ok: false, reason: 'invalid' })
expect(await verify(env, await mint(privateKey, 'unknown-kid'))).toEqual({ ok: false, reason: 'invalid' })
})
it('a cold isolate verifies from the persisted set while auth is down', async () => {
const store = stubCache()
stubCache()
let up = true
authUp(() => up)
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
const warm = await coldVerifier()
expect((await warm(env, await mint())).ok).toBe(true)
expect(store.size).toBe(1)
up = false
const cold = await coldVerifier()
const result = await cold(env, await mint())
+1 -1
View File
@@ -15,7 +15,7 @@ import { SignJWT, exportJWK, generateKeyPair, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import pg from 'pg'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
@@ -10,4 +10,3 @@ vi.mock('../../relay/src/database/database.js', async (importOriginal) => ({
}))
await import('../../relay/src/credentials/credential-store.test.ts')
await import('../../relay/src/credentials/credential-store-cleanup.test.ts')
@@ -1,140 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const cell = {
id: 'control-supersession-postgres',
url: 'https://control-supersession-postgres.example.com',
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
const identity = {
userId: 'control-supersession-postgres-user',
relayHostId: 'supersedehost001'
}
describePostgres('PostgreSQL control supersession', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
databases.push(
await openRelayDatabase({ databaseUrl, dataDir: '' }),
await openRelayDatabase({ databaseUrl, dataDir: '' })
)
})
afterAll(async () => {
const database = databases[0]
if (database) {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
[identity.userId]
)
await database.query(
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
[identity.userId]
)
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
// A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
cell.id
])
await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id])
}
for (const connection of databases) await connection.close()
})
it('serializes parallel generations into one durable control', async () => {
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await stores[0]!.reconcileCells([cell])
await stores[0]!.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600,
connectionUnobservedBound: 50
})
const assignment = await stores[0]!.assign(identity)
await Promise.all([
stores[0]!.activateControl(identity, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1,
connectionInclusionWatermark: 10
}),
stores[1]!.activateControl(identity, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 2,
connectionInclusionWatermark: 11
})
])
await databases[0]!.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'control', ?, 1, 90100, 100),
(?, ?, ?, 'control', ?, 1, 90100, 100)`,
[
identity.userId,
identity.relayHostId,
`control:${cell.id}:100`,
cell.id,
identity.userId,
identity.relayHostId,
`control:${cell.id}:101`,
cell.id
]
)
// The store reserves a unit per control lease, so a hand-written pair has to
// carry its own reservation or the fixture starts out of balance.
await databases[0]!.query(
`UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`,
[cell.id]
)
await stores[0]!.activateControl(identity, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 102,
connectionInclusionWatermark: 12
})
const controls = await databases[0]!.query(
`SELECT COUNT(*) AS count FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_kind = 'control'`,
[identity.userId]
)
expect(Number(controls[0]!.count)).toBe(1)
const assignments = await databases[0]!.query(
`SELECT reserved_controls FROM relay_assignments WHERE user_id = ?`,
[identity.userId]
)
expect(Number(assignments[0]!.reserved_controls)).toBe(1)
const cells = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cell.id]
)
expect(Number(cells[0]!.reserved_requests)).toBe(1)
const claims = await databases[0]!.query(
`SELECT claim_activity_id FROM relay_control_connection_reservations
WHERE user_id = ? AND state = 'claimed'`,
[identity.userId]
)
expect(claims).toEqual([{ claim_activity_id: `control:${cell.id}:102` }])
}, 15_000)
})
@@ -169,7 +169,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
const ledger = new ReserveLedgerWriter(database, { now: () => now, log: () => undefined })
for (const epoch of [6, 7]) ledger.enqueue({ userId: USER, relayHostId: host, cellId: dead.id, epoch })
await ledger.flush()
expect(ledger.pending()).toBe(0)
expect(await rowOf(host)).toEqual({ cell_id: live.id, assignment_epoch: 7 })
})
@@ -181,63 +180,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
await expect(hostAssign(host, true, null)).rejects.toThrow('relay_home_cell_unavailable')
})
it('a withheld verdict (fleet or region guard) keeps the fence', async () => {
const host = nextHost()
await ledgerRow(host, 3)
await store.setCellAdmitMode(dead.id, 'reserve')
await killDeadCell()
store = new RelayAssignmentStore(database, () => now, {
requireLiveCells: true,
heartbeatTtlMs: HEARTBEAT_TTL_MS,
deadCellVerdict: () => ({
dead: false,
withheld: 'region',
fleetFailing: 6,
fleetPolled: 34,
regionFailing: 6,
regionPolled: 6
})
})
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
})
// Only a heartbeat that stopped (cause 'unheard') is death; polls failing for 45 s add nothing else.
it('a cell heartbeating but not ready keeps its hosts, however long its polls fail', async () => {
const host = nextHost()
await ledgerRow(host, 3)
await store.setCellAdmitMode(dead.id, 'reserve')
now += HEARTBEAT_TTL_MS + 15_000
await heartbeat(dead, false)
await heartbeat(live)
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
})
it('drain then death keeps the fence: a disabled cell is draining, not unheard', async () => {
const host = nextHost()
await ledgerRow(host, 3)
await store.setCellAdmitMode(dead.id, 'reserve')
await database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [dead.id])
try {
await killDeadCell()
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
} finally {
await database.query(`UPDATE relay_cells SET enabled = 1 WHERE cell_id = ?`, [dead.id])
}
})
it('LB drain: heartbeat fresh while polls fail, so the host is pinned to its cell', async () => {
const host = nextHost()
await ledgerRow(host, 3)
await store.setCellAdmitMode(dead.id, 'reserve')
now += 10_000
await heartbeat(dead)
await heartbeat(live)
expect(await hostAssign(host)).toMatchObject({ cellId: dead.id, assignmentEpoch: 3 })
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
})
it('DB outage or director freeze: heartbeat stale but polls answered (no verdict), so 503', async () => {
const host = nextHost()
await ledgerRow(host, 3)
@@ -84,27 +84,4 @@ describePostgres('PostgreSQL deployment status', () => {
restartBlockingReservedRequests: -1
})
})
it('reads the same counts through the reserve-aware status, and none for a reserve-mode cell', async () => {
const store = new RelayAssignmentStore(database!, () => 100)
const reserveCell = { ...cell, id: `${cell.id}-reserve`, url: 'https://deployment-status-reserve.example.com' }
await store.reconcileCells([cell, reserveCell])
await store.assign({ userId: 'aware-user', relayHostId: 'd1e2f3a4b5c6' })
expect(await store.reserveAwareCellDeploymentStatus(cell.id)).toEqual({
admitMode: 'db',
status: await store.cellDeploymentStatus(cell.id)
})
await store.setCellAdmitMode(reserveCell.id, 'reserve')
await expect(store.cellDeploymentStatus(reserveCell.id)).rejects.toThrow('cell_in_reserve_mode')
const reserve = await store.reserveAwareCellDeploymentStatus(reserveCell.id)
expect(reserve).toMatchObject({
admitMode: 'reserve',
status: { cellId: reserveCell.id, cellUrl: reserveCell.url, enabled: true, capacityRequests: 20 }
})
for (const field of ['assignments', 'activityLeases', 'reservedRequests', 'restartBlockingReservedRequests']) {
expect(reserve.status).not.toHaveProperty(field)
}
await expect(store.reserveAwareCellDeploymentStatus('missing-cell')).rejects.toThrow('cell_not_found')
})
})
@@ -3,42 +3,27 @@ import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
const NOW = 100_000
const CELL_COUNT = 32
const identity = { userId: 'headroom-user', relayHostId: 'headroomhost0001' }
function observeHeadroom(database: RelayDatabase, rowCounts: number[]): RelayDatabase {
return {
dialect: database.dialect,
async query(sql, params) {
const rows = await database.query(sql, params)
if (sql.includes('FROM relay_cell_connection_limits limits')) rowCounts.push(rows.length)
return rows
},
queryLocked: (sql, params, options) => database.queryLocked(sql, params, options),
transaction: (operation, options) =>
database.transaction((transaction) => operation(observeHeadroom(transaction, rowCounts)), options),
close: () => database.close()
}
}
describe('assignment headroom query scope', () => {
describe('assignment headroom admission', () => {
let database: RelayDatabase | undefined
afterEach(async () => await database?.close())
async function setup() {
database = await openInMemoryRelayDatabase()
const rowCounts: number[] = []
const store = new RelayAssignmentStore(observeHeadroom(database, rowCounts), () => NOW, {
const store = new RelayAssignmentStore(database, () => NOW, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
const cells = Array.from({ length: CELL_COUNT }, (_, index) => ({
id: `cell-${String(index).padStart(2, '0')}`,
url: `https://cell-${index}.example.com`,
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}))
const cells = [
{
id: 'cell-00',
url: 'https://cell-0.example.com',
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
]
await store.reconcileCells(cells)
for (const cell of cells) {
await store.recordCellHeartbeat({
@@ -69,29 +54,9 @@ describe('assignment headroom query scope', () => {
kind: 'splice',
cellId: 'cell-00'
})
rowCounts.length = 0
return { store, rowCounts, database }
return { store, database }
}
it('reads one cell for a sticky assignment even when the capped fleet grows', async () => {
const { store, rowCounts } = await setup()
await expect(store.assign(identity)).resolves.toMatchObject({
...identity,
cellId: 'cell-00',
assignmentEpoch: 1
})
console.info('sticky headroom rows', { fleetCells: CELL_COUNT, rowCounts })
expect(rowCounts).toEqual([1])
})
it('still considers the full fleet for a new placement', async () => {
const { store, rowCounts } = await setup()
await expect(
store.assign({ userId: 'new-user', relayHostId: 'newheadroomhost1' })
).resolves.toMatchObject({ cellId: 'cell-01', assignmentEpoch: 1 })
expect(rowCounts).toEqual([CELL_COUNT])
})
it.each([
['missing snapshot', `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = 'cell-00'`],
['expired snapshot', `UPDATE relay_cell_connection_snapshots SET snapshot_at = ${NOW - 45_000} WHERE cell_id = 'cell-00'`],
@@ -104,10 +69,9 @@ describe('assignment headroom query scope', () => {
})
it('preserves admission for cells without a connection limit', async () => {
const { store, rowCounts, database } = await setup()
const { store, database } = await setup()
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = 'cell-00'`)
await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'cell-00' })
expect(rowCounts).toEqual([0])
})
it('counts outstanding reservations at the admission boundary', async () => {
@@ -106,15 +106,6 @@ describe('home cell unavailable', () => {
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
})
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
await store.assign(IDENTITY)
await expect(
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
).rejects.toThrow('relay_capacity_exhausted')
})
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
const home: RelayCellConfig = {
id: 'home',
@@ -1,108 +0,0 @@
import { describe, expect, it } from 'vitest'
import {
formatAssignmentInventorySnapshot,
readAssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
import { openInMemoryRelayDatabase } from '../database/database.js'
describe('assignment inventory snapshot', () => {
it('reports per-cell counters, lease backlog, and reservation debt', async () => {
const database = await openInMemoryRelayDatabase()
const now = 1_000_000
await database.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 3999, 5, ?, ?)`,
[now, now]
)
await database.query(
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
VALUES ('cell-a', 'general', ?)`,
[now]
)
await database.query(
`INSERT INTO relay_cell_runtime
(cell_id, cell_url, cell_incarnation, started_at, ready, observed_requests,
last_heartbeat_at, updated_at)
VALUES ('cell-a', 'https://a.example.test', 'inc-1', ?, 1, 5, ?, ?)`,
[now - 60_000, now - 10_000, now]
)
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units,
expires_at, updated_at)
VALUES
('user-1', 'host-1', 'control:1', 'control', 'cell-a', 1, ?, ?),
('user-1', 'host-2', 'control:1', 'control', 'cell-a', 3, ?, ?)`,
[now - 1, now, now + 90_000, now]
)
await database.query(
`INSERT INTO relay_control_connection_reservations
(reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch,
cell_id, state, created_at, timeout_at, updated_at)
VALUES
('r1', 'k1', 'user-1', 'host-1', 1, 'cell-a', 'late-arrival-debt', ?, ?, ?),
('r2', 'k2', 'user-1', 'host-2', 1, 'cell-a', 'reserved', ?, ?, ?),
('r3', 'k3', 'user-1', 'host-3', 1, 'cell-a', 'claimed', ?, ?, ?),
('r4', 'k4', 'user-1', 'host-4', 1, 'cell-a', 'released', ?, ?, ?)`,
[now, now, now, now, now, now, now, now, now, now, now, now]
)
const snapshot = await readAssignmentInventorySnapshot(database, now)
expect(snapshot.cells).toEqual([
{
cellId: 'cell-a',
region: 'us-central1',
admissionState: 'general',
enabled: true,
capacityRequests: 4000,
reservedRequests: 3999,
runtimeReady: true,
heartbeatAgeMs: 10_000
}
])
expect(snapshot.activityLeases).toEqual({ total: 2, expired: 1, requestUnits: 4 })
expect(snapshot.connectionReservations).toEqual({ outstanding: 3, lateArrivalDebt: 1 })
expect(snapshot.regionalRehomes).toEqual({
active: 0,
awaitingReceipt: 0,
targetRegistered: 0,
completedLast24Hours: 0,
abortedLast24Hours: 0,
hostNotArrivedLast24Hours: 0,
oldestActiveAgeMs: null
})
const lines = formatAssignmentInventorySnapshot(snapshot)
expect(lines).toHaveLength(3)
expect(lines[0]).toContain('cellId=cell-a')
expect(lines[0]).toContain('reserved=3999')
expect(lines[1]).toContain('expiredLeases=1')
expect(lines[1]).toContain('lateArrivalDebt=1')
expect(lines[2]).toContain('active=0')
})
it('reports cells missing runtime and admission rows without failing', async () => {
const database = await openInMemoryRelayDatabase()
await database.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 0)`
)
const snapshot = await readAssignmentInventorySnapshot(database, 5_000)
expect(snapshot.cells[0]).toMatchObject({
cellId: 'cell-b',
admissionState: 'unset',
enabled: false,
runtimeReady: null,
heartbeatAgeMs: null
})
expect(snapshot.activityLeases).toEqual({ total: 0, expired: 0, requestUnits: 0 })
expect(snapshot.regionalRehomes.active).toBe(0)
})
})
@@ -302,7 +302,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
).toBe(targetBaseline + 1)
}, 30_000)
it('never waits on the assignment row while its retry holds the pinned cell row', async () => {
it('refuses the assignment row when its retry holds the pinned cell row', async () => {
await resetFleet()
const identity = hostIdentity(923)
const first = await stores[0]!.assign(identity, 'us-central1')
@@ -324,10 +324,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
[identity.userId, identity.relayHostId]
)
await releaseCell()
const cellFreedAt = performance.now()
expect(await regrant).toBeInstanceOf(RelayAssignmentRowBusyError)
// NOWAIT once the cell row is held: nowhere near the 1s bounded wait.
expect(performance.now() - cellFreedAt).toBeLessThan(300)
} finally {
await releaseAssignment?.()
await releaseCell()
@@ -1,22 +1,10 @@
import { createHash } from 'node:crypto'
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import { SignJWT } from 'jose'
import nacl from 'tweetnacl'
import { describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { describe, expect, it } from 'vitest'
import {
ASSIGNMENT_LEASE_AUDIENCE,
assignmentLeaseKeyId,
classifyAssignmentLease
} from './assignment-lease.js'
import { AssignmentLeaseShadow } from './assignment-lease-shadow.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from '../config.js'
import type { RelayCredentialStore } from '../credentials/credential-store.js'
import { HostSessionRegistry } from '../connections/host-session-registry.js'
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
import { ProcessQueuedByteBudget } from '../connections/splice-forwarder.js'
const KEY = new TextEncoder().encode('0123456789abcdef0123456789abcdef')
const OTHER_KEY = new TextEncoder().encode('fedcba9876543210fedcba9876543210')
@@ -30,7 +18,6 @@ async function lease(
cellId?: string
assignmentEpoch?: number
sub?: string
relayHostId?: string
expiresAt?: string | number
} = {}
): Promise<string> {
@@ -40,7 +27,7 @@ async function lease(
cellId: overrides.cellId ?? CELL_ID,
cellUrl: 'https://c7.relay.example.test',
assignmentEpoch: overrides.assignmentEpoch ?? 4,
relayHostId: overrides.relayHostId ?? HOST.relayHostId
relayHostId: HOST.relayHostId
})
.setProtectedHeader(kid === null ? { alg: 'HS256' } : { alg: 'HS256', kid })
.setIssuer('https://relay.example.test')
@@ -73,7 +60,6 @@ describe('assignment lease classification', () => {
expect(await classify(await lease({ kid: assignmentLeaseKeyId(OTHER_KEY) }))).toBe(
'bad-signature'
)
expect(await classify('not.a.jwt')).toBe('bad-signature')
expect(await classify('x'.repeat(5_000))).toBe('bad-signature')
expect(await classify(await lease({ sub: 'user-2' }))).toBe('wrong-host')
expect(await classify(await lease({ cellId: 'production-gce-c8' }))).toBe('wrong-cell')
@@ -81,127 +67,3 @@ describe('assignment lease classification', () => {
expect(await classify(await lease({ assignmentEpoch: 5 }))).toBe('epoch-ahead')
})
})
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(''))
})
}
describe('host hello with the lease shadow on', () => {
async function hello(input: { withLease: boolean; databaseSays: boolean }) {
const keyPair = nacl.box.keyPair()
const relayHostId = createHash('sha256')
.update(keyPair.publicKey)
.digest('base64url')
.slice(0, 16)
const identity: RelayTokenClaims = {
sub: HOST.userId,
prof: 'profile-1',
relayHostId,
purpose: 'host-control',
exp: 4_102_444_800
}
const recorded: string[] = []
const shadow = new AssignmentLeaseShadow({ enabled: () => true, key: KEY, cellId: CELL_ID })
const record = shadow.record.bind(shadow)
vi.spyOn(shadow, 'record').mockImplementation((check, dbValid, dbReadMs) => {
void check.then((leaseClass) => recorded.push(`${leaseClass}:${dbValid}`))
record(check, dbValid, dbReadMs)
})
const verifyCellAssignment = vi.fn(async () => input.databaseSays)
const registry = new HostSessionRegistry(
config(),
vi.fn(),
{} as RelayCredentialStore,
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
new ProcessQueuedByteBudget(),
{
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn()
},
Date.now,
Math.random,
'incarnation-1',
shadow
)
const socket = new FakeSocket()
const hostLease = input.withLease ? await lease({ relayHostId }) : undefined
registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
undefined,
hostLease
)
socket.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'host-hello',
v: 1,
relayHostId,
assignmentEpoch: 4,
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
appVersion: '1.4.200'
})
),
false
)
await vi.waitFor(() => expect(recorded).toHaveLength(1))
return { socket, recorded, verifyCellAssignment }
}
it('still refuses a host the database refuses, whatever its lease says', async () => {
const { socket, recorded } = await hello({ withLease: true, databaseSays: false })
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
expect(recorded).toEqual(['valid:false'])
})
it('still admits a host the database admits, with no lease at all', async () => {
const { socket, recorded, verifyCellAssignment } = await hello({
withLease: false,
databaseSays: true
})
expect(verifyCellAssignment).toHaveBeenCalledTimes(1)
expect(socket.close).not.toHaveBeenCalled()
expect(String(socket.send.mock.calls[0]?.[0])).toContain('"type":"host-challenge"')
expect(recorded).toEqual(['absent:true'])
})
})
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://c7.relay.example.test',
cellUrl: 'https://c7.relay.example.test',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: KEY,
role: 'cell',
cellId: CELL_ID,
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data'
}
}
@@ -1,48 +0,0 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
const CELL = { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' }
describe('assign marks only the answers it minted as placed', () => {
let database: RelayDatabase | undefined
afterEach(async () => await database?.close())
async function setup(now: () => number): Promise<RelayAssignmentStore> {
database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, now)
await store.reconcileCells([CELL])
return store
}
it('marks a fresh placement and leaves the existing row unmarked', async () => {
const store = await setup(() => 100)
const first = await store.assign(IDENTITY)
const again = await store.assign(IDENTITY)
expect(first.placed).toBe(true)
expect(again).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
expect(again.placed).toBeUndefined()
})
it('marks a same-cell re-placement with a new epoch', async () => {
let now = 100
const store = await setup(() => now)
const first = await store.assign(IDENTITY)
await store.changeActivity(IDENTITY, 'control', -1)
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
await store.releaseExpiredActivityLeases()
await store.releaseExpiredActivity()
now += ASSIGNMENT_LIMITS.dormantTtlMs
const replaced = await store.assign(IDENTITY)
expect(replaced).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch + 1
})
expect(replaced.placed).toBe(true)
})
})
@@ -36,28 +36,8 @@ describe('assignment rejection log window', () => {
expect(window.admit('assign:placement:host-rate-limited', 'host-a')).toBe(true)
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(true)
expect(window.admit('assign:sticky:host-rate-limited', 'host-a')).toBe(true)
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(false)
})
it('bounds the tracked keys and reports what an evicted window suppressed', () => {
const timers = manualTimers()
const closed: { key: string; suppressed: number }[] = []
const window = new AssignmentRejectionLogWindow<string>({
windowMs: 10_000,
schedule: timers.schedule,
onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed })
})
window.admit('key-0', 'host-a')
window.admit('key-0', 'host-b')
for (let index = 1; index < 200; index++) window.admit(`key-${index}`, 'host-a')
expect(closed[0]).toEqual({ key: 'key-0', suppressed: 1 })
// Evicted keys emit again rather than staying silently suppressed.
expect(window.admit('key-0', 'host-a')).toBe(true)
expect(window.admit('key-199', 'host-a')).toBe(false)
})
})
function manualTimers(): {
@@ -15,11 +15,9 @@ vi.mock('../authentication/relay-token-verifier.js', () => ({
}))
import { createRelayApp } from '../app.js'
import { relayHostLogDigest } from '../observability/relay-host-log-digest.js'
describe('assignment rejection logging', () => {
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
@@ -49,44 +47,9 @@ describe('assignment rejection logging', () => {
expect(line).toContain('lane=sticky')
expect(line).toContain('hinted=true')
expect(line).toContain('reason=relay_capacity_exhausted')
expect(line).toContain(`host=${relayHostLogDigest(host)}`)
expect(line).toContain('host=af342ad00ee4')
expect(line).not.toContain(host)
})
it('names the throttled host once per window and closes it with the suppressed count', async () => {
vi.useFakeTimers()
const host = 'mmmmmmmmmmmmmmmm'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-r', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
const throttled = (): string[] =>
warn.mock.calls
.map((call) => String(call[0]))
.filter((entry) => entry.includes('reason=host-rate-limited'))
expect(throttled()).toHaveLength(1)
expect(throttled()[0]).toContain('route=assign')
expect(throttled()[0]).toContain('lane=placement')
expect(throttled()[0]).toContain(`host=${relayHostLogDigest(host)}`)
expect(throttled()[0]).not.toContain('suppressed=')
expect(throttled()[0]).not.toContain(host)
await vi.advanceTimersByTimeAsync(10_000)
expect(throttled()).toHaveLength(2)
expect(throttled()[1]).toContain('suppressed=1')
expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`)
})
})
function assignmentRequest(
@@ -202,49 +202,5 @@ describePostgres('reserve-mode census on PostgreSQL', () => {
await holder.close()
}
}, 20_000)
// A stall must not double connection demand: concurrent assigns share one reserve-set read,
// and after it fails the last answer stands for the backoff without another query.
it('reads the reserve set once for concurrent assigns during a stall, and not again in the backoff', async () => {
const timed = await openRelayDatabase({ databaseUrl, dataDir: '/tmp', statementTimeoutMs: 300 })
const holder = await openRelayDatabase({ databaseUrl, dataDir: '/tmp' })
let release!: () => void
const released = new Promise<void>((resolve) => (release = resolve))
let held!: () => void
const holding = new Promise<void>((resolve) => (held = resolve))
const lock = holder.transaction(async (transaction) => {
await transaction.query(`LOCK TABLE relay_cell_admit_modes IN ACCESS EXCLUSIVE MODE`)
held()
await released
})
try {
await holding
const store = new RelayAssignmentStore(timed, () => 500_000)
await store.reconcileCells([cellA, cellB], false)
await heartbeat(store, cellA)
await heartbeat(store, cellB)
const query = timed.query.bind(timed)
let reads = 0
timed.query = async (sql, params) => {
if (sql.includes('FROM relay_cell_admit_modes m')) reads += 1
return await query(sql, params)
}
const placed = await Promise.all(
Array.from({ length: 6 }, async (_, index) =>
await store.assign({ userId: USER, relayHostId: `hoststall0000${String(index).padStart(3, '0')}` })
)
)
expect(placed).toHaveLength(6)
expect(reads).toBe(1)
await store.assign({ userId: USER, relayHostId: 'hoststall0000100' })
expect(await store.reserveModeCells()).toBeNull()
expect(reads).toBe(1)
} finally {
release()
await lock
await timed.close()
await holder.close()
}
}, 20_000)
})
@@ -59,7 +59,6 @@ describe('assignment reserve-mode behavior', () => {
const { unknown, restore } = await unknownSet(() => now)
expect(await unknown.reserveModeCells()).toBeNull()
expect(await unknown.releaseExpiredActivityLeases()).toBe(0)
expect(await unknown.evacuateDeadCells()).toBe(0)
await restore()
await setReserve(store, [])
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
@@ -125,7 +124,7 @@ describe('assignment reserve-mode behavior', () => {
}
return await query(sql, params)
}
return { counter, release, restore: () => (database!.query = query) }
return { release, restore: () => (database!.query = query) }
}
it('reads afresh after an admit-mode write instead of joining a read begun before it', async () => {
@@ -155,12 +154,10 @@ describe('assignment reserve-mode behavior', () => {
try {
const before = store.reserveModeCells()
await store.setCellAdmitMode('cell-a', 'reserve')
const reads = held.counter.reads
held.release()
await before
// A write cleared the cache: the next caller reads now, not after a 5 s backoff.
expect([...((await store.reserveModeCells()) ?? [])]).toEqual(['cell-a'])
expect(held.counter.reads).toBeGreaterThan(reads)
} finally {
held.release()
held.restore()
@@ -197,13 +194,8 @@ describe('assignment reserve-mode behavior', () => {
it('never places a fresh host on a reserve-mode cell through the database', async () => {
const store = await setup(() => 1_000, ['cell-a'])
for (let index = 0; index < 4; index += 1) {
const placed = await store.assign({
userId: `user-${index}`,
relayHostId: `host${String(index).padStart(12, '0')}`
})
expect(placed.cellId).toBe('cell-b')
}
const placed = await store.assign({ userId: 'user-0', relayHostId: 'host000000000000' })
expect(placed.cellId).toBe('cell-b')
})
it('refuses drains, evacuations and status reads that would treat a reserve-mode cell as empty', async () => {
@@ -1,152 +0,0 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
RelayAssignmentStore,
type AssignOptions,
type RelayAssignment
} from './assignment-store.js'
import type { RelayCellConfig } from '../config.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type RelayLockOptions,
type RelayTransactionOptions,
type SqlRow
} from '../database/database.js'
import { ReservePaceError } from './reserve-placement/reserve-assignment.js'
const CELLS: RelayCellConfig[] = [
{ id: 'us-c1', url: 'https://us-c1.relay.example.com', region: 'us-central1', capacityRequests: 100 },
{ id: 'asia-c1', url: 'https://asia-c1.relay.example.com', region: 'asia-east2', capacityRequests: 100 }
]
const IDENTITY = { userId: 'spill-user', relayHostId: 'spillhost0000001' }
const ON_US_RESERVE: RelayAssignment = {
...IDENTITY,
cellId: 'us-reserve',
cellUrl: 'https://us-reserve.relay.example.com',
assignmentEpoch: 8,
leaseExpiresAt: 1_000_000,
region: 'us-central1'
}
// Reports the target cell full when its reservation is taken: the race between pick and reserve.
class FullAtReserveDatabase implements RelayDatabase {
armed = false
constructor(private readonly delegate: RelayDatabase) {}
get dialect(): 'sqlite' | 'postgres' | undefined {
return this.delegate.dialect
}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
return await this.delegate.query(sql, params)
}
async queryLocked(sql: string, params?: unknown[], options?: RelayLockOptions): Promise<SqlRow[]> {
const rows = await this.delegate.queryLocked(sql, params, options)
if (!this.armed || sql.trim() !== 'SELECT * FROM relay_cells WHERE cell_id = ?') {
return rows
}
return rows.map((row) => ({ ...row, reserved_requests: row.capacity_requests }))
}
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>, options?: RelayTransactionOptions): Promise<T> {
return await this.delegate.transaction(async (inner) => {
const wrapped = new FullAtReserveDatabase(inner)
wrapped.armed = this.armed
return await operation(wrapped)
}, options)
}
async close(): Promise<void> {
await this.delegate.close()
}
}
describe('reserve spill before relay_capacity_exhausted', () => {
let database: FullAtReserveDatabase
let store: RelayAssignmentStore
let now: number
beforeEach(async () => {
now = 100
database = new FullAtReserveDatabase(await openInMemoryRelayDatabase())
store = new RelayAssignmentStore(database, () => now)
await store.reconcileCells(CELLS)
})
afterEach(async () => {
vi.restoreAllMocks()
await database.close()
})
const closeAllDatabaseCells = async () => {
for (const cell of CELLS) {
await store.configureCell(cell, 'migration-only')
}
}
const assignAsia = async (step5: AssignOptions) =>
await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, step5)
it('books a reserve seat when no database cell has room anywhere', async () => {
await closeAllDatabaseCells()
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
await expect(assignAsia({ reserveSpill })).resolves.toEqual({ ...ON_US_RESERVE, placed: true })
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: 0 })
expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
})
it('skips the own region after the in-region reserve fallback already tried it', async () => {
await closeAllDatabaseCells()
const regionReserve = vi.fn(async () => null)
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
await expect(assignAsia({ regionReserve, reserveSpill })).resolves.toMatchObject({ cellId: 'us-reserve' })
expect(regionReserve).toHaveBeenCalledOnce()
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: true, rowEpoch: 0 })
})
it('hands the spill the epoch of the host row it read', async () => {
const first = await assignAsia({})
// Dormant: its next assign is a placement, which reads the row and finds no cell.
await store.changeActivity(IDENTITY, 'control', -1)
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
await store.releaseExpiredActivityLeases()
await store.releaseExpiredActivity()
now += ASSIGNMENT_LIMITS.dormantTtlMs
await closeAllDatabaseCells()
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
await assignAsia({ reserveSpill })
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: first.assignmentEpoch })
})
it('refuses with relay_capacity_exhausted when no reserve seat exists either', async () => {
await closeAllDatabaseCells()
await expect(assignAsia({ reserveSpill: async () => null })).rejects.toThrow('relay_capacity_exhausted')
})
it('answers a paced spill with its pace, not a refusal', async () => {
await closeAllDatabaseCells()
const reserveSpill = vi.fn(async () => {
throw new ReservePaceError(4)
})
await expect(assignAsia({ reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
})
it('answers an in-region reserve pace before any spill', async () => {
await closeAllDatabaseCells()
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
const regionReserve = async () => {
throw new ReservePaceError(2)
}
await expect(assignAsia({ regionReserve, reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
expect(reserveSpill).not.toHaveBeenCalled()
})
it('does not spill a reservation race: the host retries and re-picks', async () => {
database.armed = true
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
await expect(assignAsia({ reserveSpill })).rejects.toThrow('relay_capacity_exhausted')
expect(reserveSpill).not.toHaveBeenCalled()
})
})
@@ -143,18 +143,12 @@ async function setup(cells: RelayCellConfig[] = [SOURCE, TARGET]) {
// The old process stopped heartbeating at START_MS, so it is unheard from here.
const OLD_UNHEARD_AT = START_MS + HEARTBEAT_TTL_MS + 1_000
// Both routes that refused in production: a hinted assign (placement lane after its
// verification read finds the cell not live) and resolve, which then assigns.
async function hintedAssign(store: RelayAssignmentStore) {
return await store.assign(IDENTITY, 'us-central1', 'us-central1', undefined, {
hostRequested: true
})
}
async function resolveRoute(store: RelayAssignmentStore) {
return (await store.resolve(IDENTITY)) ?? (await store.assign(IDENTITY))
}
async function expectRefused(
attempt: Promise<unknown>,
cause: 'not_ready' | 'unheard'
@@ -164,10 +158,7 @@ async function expectRefused(
expect(error).toMatchObject({ unavailableCause: cause })
}
describe.each([
['hinted assign', hintedAssign],
['resolve', resolveRoute]
] as const)('a host offline through a roll redials via %s', (_route, redial) => {
describe('a host offline through a roll redials via hinted assign', () => {
describe('while the replacement process boots (not ready)', () => {
it('is re-placed when that process started after the roll isolated the cell', async () => {
const { store, first, heartbeat, at, isolateForRoll } = await setup()
@@ -175,7 +166,7 @@ describe.each([
await at(20_000)
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
const moved = await redial(store)
const moved = await hintedAssign(store)
expect(moved).toMatchObject({ cellId: TARGET.id })
expect(moved.assignmentEpoch).toBeGreaterThan(first.assignmentEpoch)
})
@@ -186,7 +177,7 @@ describe.each([
await at(20_000)
await heartbeat(SOURCE, { ready: false })
await expectRefused(redial(store), 'not_ready')
await expectRefused(hintedAssign(store), 'not_ready')
})
it('is refused while the new process reports holding connections', async () => {
@@ -200,7 +191,7 @@ describe.each([
totalConnections: 1
})
await expectRefused(redial(store), 'not_ready')
await expectRefused(hintedAssign(store), 'not_ready')
})
it('is refused once the new process itself goes unheard', async () => {
@@ -210,7 +201,7 @@ describe.each([
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
await at(20_000 + HEARTBEAT_TTL_MS + 1_000)
await expectRefused(redial(store), 'unheard')
await expectRefused(hintedAssign(store), 'unheard')
})
it('is refused on an ordinary readiness dip with no roll stamp', async () => {
@@ -218,7 +209,7 @@ describe.each([
await at(20_000)
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
await expectRefused(redial(store), 'not_ready')
await expectRefused(hintedAssign(store), 'not_ready')
})
it('is refused on a migration-only cell parked without a roll stamp', async () => {
@@ -227,7 +218,7 @@ describe.each([
await at(20_000)
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
await expectRefused(redial(store), 'not_ready')
await expectRefused(hintedAssign(store), 'not_ready')
})
})
@@ -238,7 +229,7 @@ describe.each([
await isolateForRoll()
await at(OLD_UNHEARD_AT)
await expectRefused(redial(store), 'unheard')
await expectRefused(hintedAssign(store), 'unheard')
})
it('is refused when the roll stamp is stale, even with a new process', async () => {
@@ -248,7 +239,7 @@ describe.each([
await at(stale)
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: stale - 1, ready: false })
await expectRefused(redial(store), 'not_ready')
await expectRefused(hintedAssign(store), 'not_ready')
})
})
@@ -1,134 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const cells = [
{
id: 'row-lock-order-a',
url: 'https://row-lock-order-a.example.com',
capacityRequests: 200,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
},
{
id: 'row-lock-order-b',
url: 'https://row-lock-order-b.example.com',
capacityRequests: 200,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
]
const userId = 'row-lock-order-user'
const hosts = ['rowlockhost00001', 'rowlockhost00002', 'rowlockhost00003', 'rowlockhost00004'].map(
(relayHostId) => ({ userId, relayHostId })
)
// pg_stat_database exposes cycles hidden by three 40P01 retries; eventual success alone misses deadlocks that exhaust retries under load.
describePostgres('PostgreSQL row lock order', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
for (let index = 0; index < 4; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
[userId]
)
for (const table of [
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [userId])
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
async function deadlockCount(): Promise<number> {
const rows = await databases[0]!.query(
`SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()`
)
return Number(rows[0]!.deadlocks)
}
it('runs the cell accept and the placement retry concurrently without a cycle', async () => {
await removeTestRows(databases[0]!)
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await stores[0]!.reconcileCells(cells)
for (const cell of cells) {
await stores[0]!.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600,
connectionUnobservedBound: 50
})
}
const placements = new Map<string, { cellId: string; assignmentEpoch: number }>()
for (const identity of hosts) {
const assignment = await stores[0]!.assign(identity)
placements.set(identity.relayHostId, assignment)
}
const before = await deadlockCount()
// The accept path (host rows, then the cell row last) against the paths
// that must read the inventory first: placement and evacuation.
for (let round = 0; round < 12; round++) {
await Promise.allSettled(
hosts.flatMap((identity, index) => {
const placement = placements.get(identity.relayHostId)!
const store = stores[index % stores.length]!
const other = stores[(index + 1) % stores.length]!
return [
store.activateControl(identity, {
cellId: placement.cellId,
assignmentEpoch: placement.assignmentEpoch,
generation: round + 2
}),
other.assign(identity),
other.startEvacuation(
identity,
placement.cellId === cells[0]!.id ? cells[1]!.id : cells[0]!.id
)
]
})
)
}
const after = await deadlockCount()
expect(after - before).toBe(0)
}, 120_000)
})
@@ -5,46 +5,16 @@ import type { RelayCellConfig } from '../../config.js'
import {
openInMemoryRelayDatabase,
openRelayDatabase,
type RelayDatabase,
type SqlRow
type RelayDatabase
} from '../../database/database.js'
// Both dialects must skip host scans with no actionable cells; stale existing-only rows previously scanned every assignment for nothing.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const schema = 'relay_dead_cell_precheck_test'
const HOST_QUERY = 'FROM relay_assignments assignment'
const CELLS: RelayCellConfig[] = [
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
]
class RecordingDatabase implements RelayDatabase {
readonly statements: string[] = []
constructor(private readonly delegate: RelayDatabase) {}
get dialect() {
return this.delegate.dialect
}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push(sql)
return await this.delegate.query(sql, params)
}
async queryLocked(...args: Parameters<RelayDatabase['queryLocked']>): Promise<SqlRow[]> {
return await this.delegate.queryLocked(...args)
}
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
return await this.delegate.transaction(operation)
}
async close(): Promise<void> {
await this.delegate.close()
}
}
function scopedUrl(): string {
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
@@ -91,13 +61,12 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
async function setup(now: () => number) {
database = await open()
const recording = new RecordingDatabase(database)
const store = new RelayAssignmentStore(recording, now, {
const store = new RelayAssignmentStore(database, now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
await store.reconcileCells(CELLS)
return { store, recording }
return { store }
}
async function heartbeat(store: RelayAssignmentStore, cell: RelayCellConfig): Promise<void> {
@@ -111,31 +80,9 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
})
}
it('skips the host query when the only dead cell is unfenced and existing-only', async () => {
let now = 100
const { store, recording } = await setup(() => now)
for (const cell of CELLS) await heartbeat(store, cell)
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
await store.setCellEnabled('cell-b', false)
expect(await store.assign(identity)).toMatchObject({ cellId: 'cell-a' })
await store.setCellEnabled('cell-b', true)
await store.setCellEnabled('cell-a', false)
now += 45_001
await heartbeat(store, CELLS[1]!)
recording.statements.length = 0
expect(await store.evacuateDeadCells()).toBe(0)
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(false)
expect(
await database!.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
identity.userId
])
).toEqual([{ cell_id: 'cell-a' }])
})
it('still evacuates hosts from a dead uncapped cell that admits', async () => {
let now = 100
const { store, recording } = await setup(() => now)
const { store } = await setup(() => now)
for (const cell of CELLS) await heartbeat(store, cell)
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
await store.setCellEnabled('cell-b', false)
@@ -143,10 +90,8 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
await store.setCellEnabled('cell-b', true)
now += 45_001
await heartbeat(store, CELLS[1]!)
recording.statements.length = 0
expect(await store.evacuateDeadCells()).toBe(1)
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(true)
expect((await store.resolve(identity))?.cellId).toBe('cell-b')
})
})
@@ -1,54 +0,0 @@
import { expect, it } from 'vitest'
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import type { SqlRow } from '../../database/database.js'
function text(row: SqlRow, field: string): string {
const value = row[field]
if (typeof value !== 'string') {
throw new Error(`invalid_${field}`)
}
return value
}
it('keeps first assignment matches, lease order, row identity, and separate identity components', () => {
const rows = [
{ user_id: 'a:b', relay_host_id: 'c', value: 1 },
{ user_id: 'a', relay_host_id: 'b:c', value: 2 },
{ user_id: 'a:b', relay_host_id: 'c', value: 3 },
{ user_id: '', relay_host_id: '', value: 4 }
]
const lookup = createDrainMigrationRowLookup(rows, text)
for (const identity of [
{ userId: 'a:b', relayHostId: 'c' },
{ userId: 'a', relayHostId: 'b:c' },
{ userId: '', relayHostId: '' },
{ userId: 'missing', relayHostId: 'c' }
]) {
const expected = rows.filter(
(row) =>
row.user_id === identity.userId && row.relay_host_id === identity.relayHostId
)
expect(lookup.first(identity)).toBe(expected[0])
expect(lookup.all(identity)).toEqual(expected)
lookup.all(identity).forEach((row, index) => expect(row).toBe(expected[index]))
}
})
it('retains lazy validation and short circuiting when an inventory is malformed', () => {
const first = { user_id: 'user', relay_host_id: 'host' }
const identity = { userId: 'user', relayHostId: 'host' }
const lookup = createDrainMigrationRowLookup(
[first, { user_id: null, relay_host_id: 'bad' }],
text
)
expect(lookup.first(identity)).toBe(first)
expect(() => lookup.all(identity)).toThrow('invalid_user_id')
const unrelated = createDrainMigrationRowLookup(
[first, { user_id: 'other', relay_host_id: null }],
text
)
expect(unrelated.all(identity)).toEqual([first])
expect(() => unrelated.first({ userId: 'other', relayHostId: 'host' })).toThrow(
'invalid_relay_host_id'
)
})
@@ -1,122 +0,0 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
import { afterEach, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../assignment-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from '../../database/database.js'
let database: RelayDatabase | undefined
afterEach(async () => await database?.close())
it.each([false, true])(
'looks up a whole drain inventory with linear identity reads (expired: %s)',
async (expired) => {
database = await openInMemoryRelayDatabase()
let measuring = false
let identityReads = 0
let indexedRows = 0
const instrument = (delegate: RelayDatabase): RelayDatabase => ({
dialect: delegate.dialect,
query: (sql, params) => delegate.query(sql, params),
queryLocked: async (sql, params, options) => {
const rows = await delegate.queryLocked(sql, params, options)
if (
!measuring ||
!sql.includes('WHERE EXISTS') ||
!(sql.includes('SELECT assignment.*') || sql.includes('SELECT lease.*'))
) {
return rows
}
indexedRows += rows.length
return rows.map(
(row): SqlRow =>
new Proxy(row, {
get(target, key) {
if (key === 'user_id' || key === 'relay_host_id') {
identityReads++
}
return Reflect.get(target, key)
}
})
)
},
transaction: (operation, options) =>
delegate.transaction((tx) => operation(instrument(tx)), options),
close: () => delegate.close()
})
let now = 100
const store = new RelayAssignmentStore(instrument(database), () => now, {
requireLiveCells: true
})
const cells = ['a', 'b'].map((id) => ({
id: `cell-${id}`,
url: `https://relay-${id}.example.com`,
capacityRequests: 500
}))
await store.reconcileCells(cells)
const incarnation = '11111111-1111-4111-8111-111111111111'
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
await store.setCellEnabled('cell-b', false)
const identities = Array.from({ length: 50 }, (_, index) => ({
userId: `user-${index % 5}`,
relayHostId: `host${String(index).padStart(12, '0')}`
}))
for (const identity of identities) {
await store.assign(identity)
}
await store.setCellEnabled('cell-b', true)
await store.setCellEnabled('cell-a', false)
for (const identity of identities) {
const migration = await store.startEvacuation(identity, 'cell-b')
await store.markMigrationTargetRegistered(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch
})
}
const attempt = {
attemptId: '55555555-5555-4555-8555-555555555555',
cellId: 'cell-a',
cellIncarnation: incarnation,
traceValue: '66666666-6666-4666-8666-666666666666',
plannedGraceMs: 120_000
}
await store.prepareCellDrainAttempt(attempt)
if (expired) {
now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
await store.releaseExpiredActivityLeases()
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
}
measuring = true
await expect(store.beginCellDrainSend(attempt)).resolves.toMatchObject({
state: 'send-may-have-started',
shouldSend: true
})
expect(indexedRows).toBeGreaterThanOrEqual(100)
expect(identityReads).toBeLessThanOrEqual(indexedRows * 2)
const migrations = await database.query(
'SELECT expires_at FROM relay_assignment_migrations'
)
expect(migrations).toHaveLength(50)
expect(
migrations.every(
(row) => row.expires_at === now + ASSIGNMENT_LIMITS.migrationLeaseMs
)
).toBe(true)
}
)
@@ -1,282 +0,0 @@
import { createHash } from 'node:crypto'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../assignment-store.js'
import {
encodeMembership,
type CellAdmissionMembership,
type CellAdmissionState
} from '../../cell/admission/cell-admission-selector.js'
import type { RelayCellConfig } from '../../config.js'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const DIRECTOR_INSTANCES = 5
const DIRECTOR_POOL_MAX = 3
const DIRECTOR_APPLICATION = 'drain-release-contention-director'
const USER_PREFIX = 'drain-release-contention'
const NOW = 1_000_000
const CAPPED = {
capacityRequests: 6_000,
connectionHardCap: 3_000,
connectionUnobservedBound: 60
} as const
const SOURCE: RelayCellConfig = {
id: 'drain-release-contention-a-source',
url: 'https://drain-release-contention-source.example.test',
region: 'asia-east2',
...CAPPED
}
const TARGETS: RelayCellConfig[] = ['b', 'c', 'd'].map((suffix) => ({
id: `drain-release-contention-${suffix}-target`,
url: `https://drain-release-contention-${suffix}-target.example.test`,
region: 'asia-east2',
...CAPPED
}))
// Next to the database. Taken only once every Asia neighbour is at its cap.
const US_CELL: RelayCellConfig = {
id: 'drain-release-contention-e-us',
url: 'https://drain-release-contention-us.example.test',
region: 'us-central1',
...CAPPED
}
const CELLS = [SOURCE, ...TARGETS, US_CELL]
type Identity = { userId: string; relayHostId: string }
type Instance = { database: RelayDatabase; store: RelayAssignmentStore }
function hostIdentity(index: number): Identity {
return {
userId: `${USER_PREFIX}-${index}`,
// Relay host ids are fixed-width opaque ids.
relayHostId: `drainrel${String(index).padStart(8, '0')}`
}
}
describePostgres('PostgreSQL drain releases against director placement', () => {
const storeOptions = { requireLiveCells: true, heartbeatTtlMs: 45_000 }
const directors: Instance[] = []
let observer: RelayDatabase
function admin(): Instance {
return directors[0]!
}
async function applySelector(
states: Record<string, CellAdmissionState>,
rollIsolatedCells?: string[]
): Promise<void> {
const current = await admin().store.inspectCellAdmissionSelector()
// Built from relay_cells: the apply requires exact coverage of a shared fleet.
const fleet = await admin().database.query(
`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`
)
const membership: CellAdmissionMembership = {
existingOnly: [],
migrationOnly: [],
general: []
}
for (const row of fleet) {
const cellId = String(row['cell_id'])
const state =
states[cellId] ??
(current.selector.membership.existingOnly.includes(cellId)
? 'existing-only'
: current.selector.membership.migrationOnly.includes(cellId)
? 'migration-only'
: 'general')
if (state === 'existing-only') membership.existingOnly.push(cellId)
else if (state === 'migration-only') membership.migrationOnly.push(cellId)
else membership.general.push(cellId)
}
await admin().store.applyCellAdmissionSelector({
attemptId: `drain-release-${current.selector.generation}`,
expectedGeneration: current.selector.generation,
...(current.selector.generation === 0
? {
expectedMembershipSha256: createHash('sha256')
.update(encodeMembership(current.selector.membership))
.digest('hex')
}
: {}),
membership,
...(rollIsolatedCells ? { rollIsolatedCells } : {})
})
}
// Other Postgres files write admission through the generation-0 helpers, so
// the selector goes back to generation 0 exactly as found.
async function resetSelectorBoundary(): Promise<void> {
await admin().database.query(
`UPDATE relay_admission_selectors SET generation = 0, attempt_id = NULL
WHERE selector_id = 'general'`
)
await admin().database.query(
`DELETE FROM relay_admission_selector_intents WHERE attempt_id LIKE 'drain-release-%'`
)
await admin().store.reconcileCells([], false)
}
async function deleteHostRows(): Promise<void> {
for (const table of [
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await admin().database.query(`DELETE FROM ${table} WHERE user_id LIKE '${USER_PREFIX}-%'`)
}
}
// `atCap` reports those cells at their connection cap: no headroom left.
async function heartbeatAll(atCap: RelayCellConfig[] = []): Promise<void> {
for (const [index, config] of CELLS.entries()) {
await admin().store.recordCellHeartbeat({
cellId: config.id,
cellUrl: config.url,
cellIncarnation: `2222222${index}-2222-4222-8222-222222222222`,
startedAt: NOW - 1_000,
ready: true,
observedRequests: 0,
region: config.region,
// The cell checks enforced = total + in-flight + reserved.
totalConnections: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
connectionHardCap: CAPPED.connectionHardCap,
connectionUnobservedBound: CAPPED.connectionUnobservedBound
})
}
}
// Holds `cellIds`' relay_cells rows in an open transaction until the returned release runs.
async function holdCellRows(cellIds: string[]): Promise<() => Promise<void>> {
let release!: () => void
const released = new Promise<void>((resolve) => (release = resolve))
let held!: () => void
const acquired = new Promise<void>((resolve) => (held = resolve))
const holder = observer.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT cell_id FROM relay_cells WHERE cell_id IN (${cellIds.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
cellIds
)
held()
await released
})
await acquired
return async () => {
release()
await holder
}
}
// Every host holds a live control lease on the source, then the source is
// isolated for a roll: the state an Asia drain starts from.
async function seed(hosts: Identity[]): Promise<void> {
await deleteHostRows()
await admin().database.query(
`UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id LIKE '${USER_PREFIX}-%'`
)
// Restoring the source to general also clears the previous run's roll stamp.
await applySelector({
[SOURCE.id]: 'general',
...Object.fromEntries(TARGETS.map((target) => [target.id, 'migration-only' as const]))
})
for (const identity of hosts) {
const grant = await admin().store.assign(identity, 'asia-east2', 'asia-east2')
expect(grant.cellId).toBe(SOURCE.id)
await admin().store.activateControl(identity, {
cellId: SOURCE.id,
assignmentEpoch: grant.assignmentEpoch,
generation: 1
})
}
await applySelector(
Object.fromEntries(TARGETS.map((target) => [target.id, 'general' as const]))
)
await applySelector({ [SOURCE.id]: 'migration-only' }, [SOURCE.id])
}
beforeAll(async () => {
for (let index = 0; index < DIRECTOR_INSTANCES; index += 1) {
const database = await openRelayDatabase({
databaseUrl,
dataDir: '',
poolMax: DIRECTOR_POOL_MAX,
applicationName: DIRECTOR_APPLICATION
})
directors.push({
database,
store: new RelayAssignmentStore(database, () => NOW, storeOptions)
})
}
observer = await openRelayDatabase({ databaseUrl, dataDir: '', poolMax: 1 })
await deleteHostRows()
await admin().store.reconcileCells(CELLS, false)
await resetSelectorBoundary()
await heartbeatAll()
})
afterAll(async () => {
if (directors.length > 0) {
await deleteHostRows()
// Cells before the selector rebuild, or its membership names missing rows.
for (const config of CELLS) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await admin().database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [config.id])
}
}
await resetSelectorBoundary()
}
for (const instance of directors) {
await instance.database.close()
}
await observer?.close()
})
it('places a reconnecting host off the isolated source while that source row is held', async () => {
await heartbeatAll()
const host = hostIdentity(9_000)
await seed([host])
// A drain release holds the source row for its whole transaction; target-only placement never asks for it.
const sourceHeld = await holdCellRows([SOURCE.id])
try {
const grant = await Promise.race([
directors[1]!.store.assign(host, 'asia-east2', 'asia-east2'),
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 3_000))
])
expect(grant).not.toBe('blocked')
expect(TARGETS.map(({ id }) => id)).toContain(grant === 'blocked' ? null : grant.cellId)
} finally {
await sourceHeld()
}
// Control: the same hold on the target rows does block a placement, so the hold above was real.
const other = hostIdentity(9_001)
await seed([other])
const targetsHeld = await holdCellRows(TARGETS.map(({ id }) => id))
const placement = directors[2]!.store.assign(other, 'asia-east2', 'asia-east2')
try {
const raced = await Promise.race([
placement.then(() => 'placed' as const),
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 1_000))
])
expect(raced).toBe('blocked')
} finally {
await targetsHeld()
}
expect((await placement).cellId).not.toBe(SOURCE.id)
}, 30_000)
})
@@ -6,7 +6,6 @@ import {
type RelayAssignmentMigration
} from '../assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
@@ -245,39 +244,6 @@ describePostgres('PostgreSQL migration recovery', () => {
)
}
it('reaps an abandoned registered migration onto its healthy target', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advance(
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
expect(
await database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
migration.completed_at, migration.aborted_at
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
AND migration.relay_host_id = assignment.relay_host_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
).toEqual([
{
cell_id: input.cells.failed.id,
assignment_epoch: '2',
completed_at: String(
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
),
aborted_at: null
}
])
})
it('reaps immediately after the retired source has a durable completed fence', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
@@ -290,7 +256,7 @@ describePostgres('PostgreSQL migration recovery', () => {
expect(
await database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
migration.completed_at, migration.aborted_at
(migration.completed_at IS NOT NULL) AS completed, migration.aborted_at
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
@@ -303,177 +269,12 @@ describePostgres('PostgreSQL migration recovery', () => {
{
cell_id: input.cells.failed.id,
assignment_epoch: '2',
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
completed: true,
aborted_at: null
}
])
})
it('reaps immediately after the retired source has an adopted legacy fence', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await input.store.adoptLegacyCellFence(
input.cells.source.id,
input.cellIncarnation
)
await input.store.commitLegacyCellFenceAdoption(
input.cells.source.id,
input.cellIncarnation
)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
expect(
(
await readRegisteredMigrationInventory(
database,
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
)
).abandoned
).toBe(1)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
expect(
await database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
migration.completed_at, migration.aborted_at
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
AND migration.relay_host_id = assignment.relay_host_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
).toEqual([
{
cell_id: input.cells.failed.id,
assignment_epoch: '2',
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
aborted_at: null
}
])
})
it('does not reap after temporary legacy adoption without durable commit', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await input.store.adoptLegacyCellFence(
input.cells.source.id,
input.cellIncarnation
)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
expect(
(
await readRegisteredMigrationInventory(
database,
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
)
).abandoned
).toBe(0)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('invalidates an adopted legacy fence when the source heartbeats', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await input.store.adoptLegacyCellFence(
input.cells.source.id,
input.cellIncarnation
)
await input.store.commitLegacyCellFenceAdoption(
input.cells.source.id,
input.cellIncarnation
)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
await input.heartbeat(input.cells.source)
expect(
await database.query(
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
[input.cells.source.id]
)
).toEqual([])
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('invalidates an adopted legacy fence when the source restarts', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await input.store.adoptLegacyCellFence(
input.cells.source.id,
input.cellIncarnation
)
await input.store.commitLegacyCellFenceAdoption(
input.cells.source.id,
input.cellIncarnation
)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
await input.store.recordCellHeartbeat({
cellId: input.cells.source.id,
cellUrl: input.cells.source.url,
cellIncarnation: '99999999-9999-4999-8999-999999999999',
startedAt: 51,
ready: true,
observedRequests: 0
})
expect(
await database.query(
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
[input.cells.source.id]
)
).toEqual([])
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('serializes legacy adoption with a returning source heartbeat', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await input.store.adoptLegacyCellFence(
input.cells.source.id,
input.cellIncarnation
)
const [commit, heartbeatResult] = await Promise.allSettled([
input.store.commitLegacyCellFenceAdoption(
input.cells.source.id,
input.cellIncarnation
),
input.heartbeat(input.cells.source)
])
expect(heartbeatResult.status).toBe('fulfilled')
if (commit.status === 'rejected') {
expect(commit.reason).toMatchObject({ message: 'legacy_cell_fence_not_active' })
}
expect(
await database.query(
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
[input.cells.source.id]
)
).toEqual([])
input.advance(
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('restores the 24-hour guard when the fenced source heartbeats again', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
@@ -495,83 +296,6 @@ describePostgres('PostgreSQL migration recovery', () => {
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('restores the 24-hour guard when the fenced source restarts', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await completeSourceFence(input)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
await input.store.recordCellHeartbeat({
cellId: input.cells.source.id,
cellUrl: input.cells.source.url,
cellIncarnation: '99999999-9999-4999-8999-999999999999',
startedAt: 51,
ready: true,
observedRequests: 0
})
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(
await database.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('serializes fenced cleanup with a returning source heartbeat', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advancePastHeartbeat()
await completeSourceFence(input)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
const [cleanup, heartbeatResult] = await Promise.allSettled([
input.store.abortExpiredEvacuations(),
input.heartbeat(input.cells.source)
])
expect(cleanup.status).toBe('fulfilled')
expect(heartbeatResult.status).toBe('fulfilled')
if (cleanup.status !== 'fulfilled') throw cleanup.reason
if (heartbeatResult.status !== 'fulfilled') throw heartbeatResult.reason
expect([0, 1]).toContain(cleanup.value)
const migrations = await database.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
if (cleanup.value === 1) {
expect(migrations[0]?.completed_at).not.toBeNull()
} else {
expect(migrations).toEqual([{ completed_at: null, aborted_at: null }])
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
}
})
it('keeps a freshly retired source protected without a durable fence', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(
await database.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
input.advance(STRANDED_MIGRATION_ABANDON_MS)
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
})
it('rolls an abandoned disabled target back to its source', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
@@ -585,7 +309,7 @@ describePostgres('PostgreSQL migration recovery', () => {
expect(
await database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
migration.completed_at, migration.aborted_at
migration.completed_at, (migration.aborted_at IS NOT NULL) AS aborted
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
@@ -599,103 +323,11 @@ describePostgres('PostgreSQL migration recovery', () => {
cell_id: input.cells.source.id,
assignment_epoch: '3',
completed_at: null,
aborted_at: String(
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
)
aborted: true
}
])
})
it('serializes cleanup against supersession without reporting a false target', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await input.store.releaseActivity(input.identity, input.targetControlId)
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advance(
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
)
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
const [cleanup, supersession] = await Promise.allSettled([
input.store.abortExpiredEvacuations(),
input.store.supersedeRegisteredCellEvacuations(
input.cells.source.id,
input.cells.failed.id,
input.cells.replacement.id,
100
)
])
expect(cleanup.status).toBe('fulfilled')
if (cleanup.status !== 'fulfilled') throw cleanup.reason
if (supersession.status === 'fulfilled') {
expect([0, 1]).toContain(supersession.value)
const assignment = await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[input.identity.userId]
)
expect(assignment).toEqual([{
cell_id:
supersession.value === 1
? input.cells.replacement.id
: input.cells.source.id
}])
if (supersession.value === 0) expect(cleanup.value).toBeGreaterThanOrEqual(1)
} else {
expect(cleanup.value).toBeGreaterThanOrEqual(1)
expect(supersession.reason).toMatchObject({ message: 'migration_already_superseded' })
expect(
await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[input.identity.userId]
)
).toEqual([{ cell_id: input.cells.source.id }])
}
})
it('completes a registered migration from a dead source idempotently', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.failed)
await input.store.attestCellFence(
input.cells.source.id,
input.cellIncarnation
)
const completion = {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
targetCellId: input.cells.failed.id
}
await expect(
input.store.cellEvacuationStatus(
input.cells.source.id,
input.cells.failed.id,
true
)
).resolves.toMatchObject({ inProgress: 0, completed: 1 })
await expect(
input.store.completeEvacuationFromDeadSource(input.identity, completion)
).resolves.toMatchObject({ changed: false })
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 1, replacement: 0 })
})
it('requires fence evidence for dead-source completion', async () => {
const input = await fixture()
await input.store.releaseActivity(input.identity, input.sourceControlId)
await expect(
input.store.completeEvacuationFromDeadSource(input.identity, {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
targetCellId: input.cells.failed.id
})
).rejects.toThrow('cell_fence_attestation_missing')
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 2, replacement: 0 })
})
it('supersedes a registered migration with exact accounting and idempotency', async () => {
const input = await fixture()
await database.query(
@@ -760,416 +392,6 @@ describePostgres('PostgreSQL migration recovery', () => {
])
})
it('reconciles durable cell accounting before aggregate supersession', async () => {
const input = await fixture()
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(
input.cells.failed.id,
input.cellIncarnation
)
await database.query(
`UPDATE relay_cells
SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END
WHERE cell_id IN (?, ?)`,
[input.cells.replacement.id, input.cells.source.id, input.cells.replacement.id]
)
await expect(
input.store.supersedeRegisteredCellEvacuations(
input.cells.source.id,
input.cells.failed.id,
input.cells.replacement.id,
100
)
).resolves.toBe(1)
expect(await reservations(input.cells)).toEqual({
source: 1,
failed: 0,
replacement: 2
})
})
it('rebuilds an expired registered migration lease before supersession', async () => {
const input = await fixture()
await pinMigration(input, input.migration.assignmentEpoch)
await clearActivities(input)
await database.query(
`UPDATE relay_assignment_migrations SET expires_at = 0
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
)
await fenceFailedCell(input)
await expect(supersedeAggregate(input)).resolves.toBe(1)
expect(await reservations(input.cells)).toEqual({
source: 0,
failed: 0,
replacement: 1
})
expect(
await database.query(
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
[input.identity.userId]
)
).toEqual([
{
assignment_epoch: '2',
source_request_units: '1',
target_reserved_units: '2',
aborted_at: '45101'
},
{
assignment_epoch: '3',
source_request_units: '0',
target_reserved_units: '1',
aborted_at: null
}
])
})
it('retires an obsolete row without discarding replacement activity', async () => {
const input = await fixture()
await pinMigration(input, input.migration.assignmentEpoch)
await fenceFailedCell(input)
await input.store.supersedeRegisteredEvacuation(input.identity, {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
currentTargetCellId: input.cells.failed.id,
replacementTargetCellId: input.cells.replacement.id
})
await database.query(
`UPDATE relay_assignment_migrations SET aborted_at = NULL
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
)
const activityBefore = await assignmentActivities(input)
await expect(supersedeAggregate(input)).resolves.toBe(1)
expect(await assignmentActivities(input)).toEqual(activityBefore)
expect(await reservations(input.cells)).toEqual({
source: 1,
failed: 0,
replacement: 2
})
expect(
await database.query(
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
WHERE user_id = ? ORDER BY assignment_epoch`,
[input.identity.userId]
)
).toEqual([
{ assignment_epoch: '2', aborted_at: '45101' },
{ assignment_epoch: '3', aborted_at: null }
])
})
it('anchors a newer dormant failed-cell epoch and preserves drain evidence', async () => {
const input = await fixture()
const drainAttemptId = await pinMigration(input, input.migration.assignmentEpoch)
await clearActivities(input)
await database.query(
`UPDATE relay_assignments SET assignment_epoch = 4
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`UPDATE relay_assignment_migrations SET expires_at = 0
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
)
await fenceFailedCell(input)
await expect(supersedeAggregate(input)).resolves.toBe(1)
expect(await reservations(input.cells)).toEqual({
source: 0,
failed: 0,
replacement: 1
})
expect(
await database.query(
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
[input.identity.userId]
)
).toEqual([
{
assignment_epoch: '2',
source_request_units: '1',
target_reserved_units: '2',
aborted_at: '45101'
},
{
assignment_epoch: '4',
source_request_units: '0',
target_reserved_units: '1',
aborted_at: '45101'
},
{
assignment_epoch: '5',
source_request_units: '0',
target_reserved_units: '1',
aborted_at: null
}
])
expect(
await database.query(
`SELECT assignment_epoch, drain_attempt_id, source_request_units,
target_reserved_units
FROM relay_post_drain_migration_pins
WHERE user_id = ? ORDER BY assignment_epoch`,
[input.identity.userId]
)
).toEqual([
{
assignment_epoch: '2',
drain_attempt_id: drainAttemptId,
source_request_units: '1',
target_reserved_units: '2'
},
{
assignment_epoch: '4',
drain_attempt_id: drainAttemptId,
source_request_units: '0',
target_reserved_units: '1'
},
{
assignment_epoch: '5',
drain_attempt_id: drainAttemptId,
source_request_units: '0',
target_reserved_units: '1'
}
])
})
it('uses an existing current-epoch migration once when retiring an older row', async () => {
const input = await fixture()
await pinMigration(input, input.migration.assignmentEpoch)
await clearActivities(input)
await database.query(
`UPDATE relay_assignments SET assignment_epoch = 4
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id,
previous_epoch, assignment_epoch, source_request_units,
target_reserved_units, expires_at, target_registered_at,
completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, 2, 4, 0, 1, 0, 100, NULL, NULL, 100, 100)`,
[
input.identity.userId,
input.identity.relayHostId,
input.cells.source.id,
input.cells.failed.id
]
)
await database.query(
`INSERT INTO relay_assignment_migration_incarnations
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
target_cell_incarnation)
VALUES (?, ?, 4, ?, ?)`,
[
input.identity.userId,
input.identity.relayHostId,
input.cellIncarnation,
input.cellIncarnation
]
)
await pinMigration(input, 4)
await fenceFailedCell(input)
await expect(supersedeAggregate(input)).resolves.toBe(1)
expect(
await database.query(
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
WHERE user_id = ? ORDER BY assignment_epoch`,
[input.identity.userId]
)
).toEqual([
{ assignment_epoch: '2', aborted_at: '45101' },
{ assignment_epoch: '4', aborted_at: '45101' },
{ assignment_epoch: '5', aborted_at: null }
])
expect(await reservations(input.cells)).toEqual({
source: 0,
failed: 0,
replacement: 1
})
})
it('rejects a newer failed-cell epoch with ambiguous activity', async () => {
const input = await fixture()
await database.query(
`UPDATE relay_assignments SET assignment_epoch = 4
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await fenceFailedCell(input)
await expect(supersedeAggregate(input)).rejects.toThrow(
'migration_activity_topology_mismatch'
)
expect(
await database.query(
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
WHERE user_id = ?`,
[input.identity.userId]
)
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
expect(await reservations(input.cells)).toEqual({
source: 1,
failed: 2,
replacement: 0
})
})
it('leaves lease repair retryable when supersession later fails', async () => {
const input = await fixture(1)
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`UPDATE relay_assignments SET migration_leases = 0
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`UPDATE relay_cells SET reserved_requests = reserved_requests - 1
WHERE cell_id = ?`,
[input.cells.failed.id]
)
await database.query(
`UPDATE relay_assignment_migrations SET expires_at = 0
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
)
await fenceFailedCell(input)
await expect(supersedeAggregate(input)).rejects.toThrow(
'relay_capacity_exhausted'
)
expect(
await database.query(
`SELECT activity_id FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_kind = 'migration'`,
[input.identity.userId]
)
).toEqual([{ activity_id: 'migration:2' }])
expect(
await database.query(
`SELECT aborted_at FROM relay_assignment_migrations
WHERE user_id = ? AND assignment_epoch = 2`,
[input.identity.userId]
)
).toEqual([{ aborted_at: null }])
await database.query(
`UPDATE relay_cells SET capacity_requests = 2 WHERE cell_id = ?`,
[input.cells.replacement.id]
)
await expect(supersedeAggregate(input)).resolves.toBe(1)
expect(await reservations(input.cells)).toEqual({
source: 1,
failed: 0,
replacement: 2
})
})
it('serializes concurrent supersession retries without duplicating capacity', async () => {
const input = await fixture()
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(
input.cells.failed.id,
input.cellIncarnation
)
const supersession = {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
currentTargetCellId: input.cells.failed.id,
replacementTargetCellId: input.cells.replacement.id
}
const results = await Promise.all([
input.store.supersedeRegisteredEvacuation(input.identity, supersession),
input.store.supersedeRegisteredEvacuation(input.identity, supersession)
])
expect(results[0]).toEqual(results[1])
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 0, replacement: 2 })
expect(
await database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[input.identity.userId]
)
).toEqual([{ count: '2' }])
})
it('fails a competing aggregate supersession instead of reporting the wrong target', async () => {
const input = await fixture()
const alternate = {
id: `recovery-cell-${sequence}-alternate`,
url: `https://recovery-${sequence}-alternate.example.com`,
capacityRequests: 20
}
await input.store.reconcileCells([...Object.values(input.cells), alternate], false)
await input.heartbeat(alternate)
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.heartbeat(alternate)
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
const results = await Promise.allSettled([
input.store.supersedeRegisteredCellEvacuations(
input.cells.source.id,
input.cells.failed.id,
input.cells.replacement.id,
100
),
input.store.supersedeRegisteredCellEvacuations(
input.cells.source.id,
input.cells.failed.id,
alternate.id,
100
)
])
const fulfilled = results.filter(
(result): result is PromiseFulfilledResult<number> => result.status === 'fulfilled'
)
const rejected = results.filter(
(result): result is PromiseRejectedResult => result.status === 'rejected'
)
expect(fulfilled).toHaveLength(1)
expect(fulfilled[0]!.value).toBe(1)
expect(rejected).toHaveLength(1)
expect(rejected[0]!.reason).toMatchObject({ message: 'migration_already_superseded' })
const successor = await database.query(
`SELECT assignment.cell_id, migration.target_cell_id
FROM relay_assignments assignment
JOIN relay_assignment_migrations migration
ON migration.user_id = assignment.user_id
AND migration.relay_host_id = assignment.relay_host_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE assignment.user_id = ? AND migration.previous_epoch = ?`,
[input.identity.userId, input.migration.assignmentEpoch]
)
expect(successor).toHaveLength(1)
expect(successor[0]!.cell_id).toBe(successor[0]!.target_cell_id)
expect([input.cells.replacement.id, alternate.id]).toContain(
successor[0]!.target_cell_id
)
})
it('rolls back every supersession change when replacement capacity is insufficient', async () => {
const input = await fixture(1)
await input.store.setCellEnabled(input.cells.failed.id, false)
@@ -1205,153 +427,6 @@ describePostgres('PostgreSQL migration recovery', () => {
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
})
it('rolls back supersession when migration request-unit shape drifted', async () => {
const input = await fixture()
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
await database.query(
`UPDATE relay_assignment_activity_leases
SET request_units = request_units + 1
WHERE user_id = ? AND activity_kind = 'migration'`,
[input.identity.userId]
)
await database.query(
`UPDATE relay_cells SET reserved_requests = reserved_requests + 1 WHERE cell_id = ?`,
[input.cells.failed.id]
)
await expect(
input.store.supersedeRegisteredEvacuation(input.identity, {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
currentTargetCellId: input.cells.failed.id,
replacementTargetCellId: input.cells.replacement.id
})
).rejects.toThrow('migration_activity_lease_shape_mismatch')
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 3, replacement: 0 })
expect(
await database.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[input.identity.userId]
)
).toEqual([{ cell_id: input.cells.failed.id, assignment_epoch: '2' }])
})
it('rolls back supersession when locked cell reservation accounting drifted', async () => {
const input = await fixture()
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
await database.query(
`UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`,
[input.cells.replacement.id]
)
await expect(
input.store.supersedeRegisteredEvacuation(input.identity, {
assignmentEpoch: input.migration.assignmentEpoch,
sourceCellId: input.cells.source.id,
currentTargetCellId: input.cells.failed.id,
replacementTargetCellId: input.cells.replacement.id
})
).rejects.toThrow('migration_cell_reservation_accounting_mismatch')
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 2, replacement: 1 })
expect(
await database.query(
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
WHERE user_id = ?`,
[input.identity.userId]
)
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
})
async function fenceFailedCell(input: RecoveryFixture): Promise<void> {
await input.store.setCellEnabled(input.cells.failed.id, false)
input.advancePastHeartbeat()
await input.heartbeat(input.cells.source)
await input.heartbeat(input.cells.replacement)
await input.store.attestCellFence(
input.cells.failed.id,
input.cellIncarnation
)
}
async function supersedeAggregate(input: RecoveryFixture): Promise<number> {
return await input.store.supersedeRegisteredCellEvacuations(
input.cells.source.id,
input.cells.failed.id,
input.cells.replacement.id,
100
)
}
async function clearActivities(input: RecoveryFixture): Promise<void> {
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0,
reserved_invites = 0, pending_installs = 0, pending_confirmations = 0,
migration_leases = 0
WHERE user_id = ? AND relay_host_id = ?`,
[input.identity.userId, input.identity.relayHostId]
)
await database.query(
`UPDATE relay_cells SET reserved_requests = 0
WHERE cell_id IN (?, ?, ?)`,
[input.cells.source.id, input.cells.failed.id, input.cells.replacement.id]
)
}
async function pinMigration(
input: RecoveryFixture,
assignmentEpoch: number
): Promise<string> {
const drainAttemptId = `${input.identity.userId}-drain-${assignmentEpoch}`
await database.query(
`INSERT INTO relay_post_drain_migration_pins
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
source_cell_id, source_cell_incarnation, target_cell_id,
target_cell_incarnation, source_request_units, target_reserved_units,
pinned_at)
SELECT migration.user_id, migration.relay_host_id,
migration.assignment_epoch, ?, migration.source_cell_id,
incarnation.source_cell_incarnation, migration.target_cell_id,
incarnation.target_cell_incarnation, migration.source_request_units,
migration.target_reserved_units, 100
FROM relay_assignment_migrations migration
JOIN relay_assignment_migration_incarnations incarnation
ON incarnation.user_id = migration.user_id
AND incarnation.relay_host_id = migration.relay_host_id
AND incarnation.assignment_epoch = migration.assignment_epoch
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.assignment_epoch = ?`,
[
drainAttemptId,
input.identity.userId,
input.identity.relayHostId,
assignmentEpoch
]
)
return drainAttemptId
}
async function assignmentActivities(input: RecoveryFixture): Promise<unknown[]> {
return await database.query(
`SELECT activity_id, activity_kind, cell_id, request_units
FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`,
[input.identity.userId, input.identity.relayHostId]
)
}
async function reservations(cells: RecoveryFixture['cells']): Promise<Record<string, number>> {
const rows = await database.query(
`SELECT cell_id, reserved_requests FROM relay_cells
@@ -184,7 +184,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
})
now = migration.expiresAt + 1
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
await store.releaseExpiredActivityLeases()
await store.recordCellHeartbeat({
cellId: source.id,
cellUrl: source.url,
@@ -207,7 +207,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
cellId: source.id,
cellIncarnation: sourceIncarnation
})
).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: true })
).resolves.toMatchObject({ shouldSend: true })
await expect(
database.query(
`SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
@@ -1,139 +0,0 @@
import pg from 'pg'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_migration_inventory_test'
describePostgres('PostgreSQL registered migration inventory', () => {
let database: RelayDatabase | undefined
beforeAll(async () => {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
} finally {
await client.end()
}
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
})
afterAll(async () => {
await database?.close()
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await client.end()
}
})
it('counts a disabled-target migration even while its source is active', async () => {
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
await database!.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 1, 0, 0, 1),
('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 1)`
)
await database!.query(
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
VALUES ('cell-a', 'general', 1), ('cell-b', 'existing-only', 1)`
)
await database!.query(
`INSERT INTO relay_assignments
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
pending_installs, pending_confirmations, migration_leases)
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
[now, now]
)
await database!.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES ('user-1', '111111111111', 'control:source:1', 'control', 'cell-a', 1, ?, ?)`,
[now, now]
)
await database!.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
assignment_epoch, source_request_units, target_reserved_units, expires_at,
target_registered_at, created_at, updated_at)
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, 2, 2, 1, 2)`
)
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
open: 1,
inactive: 1,
abandoned: 1,
pairs: [
{
sourceCellId: 'cell-a',
targetCellId: 'cell-b',
open: 1,
inactive: 1,
abandoned: 1
}
]
})
await database!.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES ('cell-c', 'https://c.example.test', 1, 4000, 0, 0, 0, 1)`
)
await database!.query(
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
VALUES ('cell-c', 'migration-only', 1)`
)
await database!.query(
`INSERT INTO relay_assignments
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
pending_installs, pending_confirmations, migration_leases)
VALUES ('user-2', '222222222222', 'cell-c', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
[now, now]
)
await database!.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
assignment_epoch, source_request_units, target_reserved_units, expires_at,
target_registered_at, created_at, updated_at)
VALUES ('user-2', '222222222222', 'cell-a', 'cell-c', 1, 2, 1, 2, ?, NULL, 1, 2)`,
[now + 60_000]
)
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
open: 2,
inactive: 1,
abandoned: 1,
pairs: [
{
sourceCellId: 'cell-a',
targetCellId: 'cell-b',
open: 1,
inactive: 1,
abandoned: 1
},
{
sourceCellId: 'cell-a',
targetCellId: 'cell-c',
open: 1,
inactive: 0,
abandoned: 0
}
]
})
})
})
@@ -1,120 +0,0 @@
import { afterEach, describe, expect, it } from 'vitest'
import { openInMemoryRelayDatabase, type RelayDatabase } from '../../database/database.js'
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
import {
formatRegisteredMigrationInventory,
readRegisteredMigrationInventory
} from './registered-migration-inventory.js'
describe('registered migration inventory', () => {
const databases: RelayDatabase[] = []
afterEach(async () => {
for (const database of databases.splice(0)) {
await database.close()
}
})
it('does not restart abandonment when an old source migration lease is refreshed', async () => {
const database = await openInMemoryRelayDatabase()
databases.push(database)
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
await insertCell(database, 'cell-a', false, 'existing-only')
await insertCell(database, 'cell-b', true, 'migration-only')
await insertMigration(database, now - 1)
const fresh = await readRegisteredMigrationInventory(database, now)
expect(fresh).toEqual({
open: 1,
inactive: 1,
abandoned: 1,
pairs: [
{
sourceCellId: 'cell-a',
targetCellId: 'cell-b',
open: 1,
inactive: 1,
abandoned: 1
}
]
})
expect(formatRegisteredMigrationInventory(fresh)).toEqual([
'[orca-relay] migration inventory open=1 expiredRegisteredInactive=1 abandonedRegistered=1',
'[orca-relay] migration pair sourceCellId=cell-a targetCellId=cell-b open=1 expiredRegisteredInactive=1 abandoned=1'
])
await database.query(`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?`, [
now + 1,
now
])
expect((await readRegisteredMigrationInventory(database, now + 1)).abandoned).toBe(1)
})
it('includes unregistered open migrations without logging a host identity', async () => {
const database = await openInMemoryRelayDatabase()
databases.push(database)
const now = 10_000
await insertCell(database, 'cell-a', false, 'existing-only')
await insertCell(database, 'cell-b', true, 'migration-only')
await insertMigration(database, now + 60_000, null)
const inventory = await readRegisteredMigrationInventory(database, now)
expect(inventory).toEqual({
open: 1,
inactive: 0,
abandoned: 0,
pairs: [
{
sourceCellId: 'cell-a',
targetCellId: 'cell-b',
open: 1,
inactive: 0,
abandoned: 0
}
]
})
expect(formatRegisteredMigrationInventory(inventory).join('\n')).not.toContain('111111111111')
})
})
async function insertCell(
database: RelayDatabase,
cellId: string,
enabled: boolean,
admissionState: string
): Promise<void> {
await database.query(
`INSERT INTO relay_cells
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
observed_requests, last_heartbeat_at, updated_at)
VALUES (?, ?, ?, 4000, 0, 0, 0, 1)`,
[cellId, `https://${cellId}.example.test`, enabled ? 1 : 0]
)
await database.query(
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
VALUES (?, ?, 1)`,
[cellId, admissionState]
)
}
async function insertMigration(
database: RelayDatabase,
expiresAt: number,
targetRegisteredAt: number | null = 2
): Promise<void> {
await database.query(
`INSERT INTO relay_assignments
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
pending_installs, pending_confirmations, migration_leases)
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
[expiresAt, expiresAt]
)
await database.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
assignment_epoch, source_request_units, target_reserved_units, expires_at,
target_registered_at, created_at, updated_at)
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, ?, ?, 1, 2)`,
[expiresAt, targetRegisteredAt]
)
}
@@ -1,23 +1,16 @@
import { describe, expect, it } from 'vitest'
import {
DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS,
RelayDrainReturnAdmission,
type DrainReturnDeferral,
type DrainReturnGrant
} from './drain-return-admission.js'
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
type Timer = { at: number; callback: () => void; cancelled: boolean }
function harness(
overrides: {
maxQueued?: number
maxRetryAfterSeconds?: number
onServiceMs?: (durationMs: number) => void
} = {}
) {
let now = 0
const timers: Timer[] = []
const placement = new RelayPublicAssignmentAdmission({
maxConcurrent: 2,
maxQueued: 8,
@@ -26,32 +19,18 @@ function harness(
maxDrainReturnConcurrent: 1,
maxDrainReturnQueued: overrides.maxQueued ?? 0,
drainReturnWaitMs: 3_000,
drainReturnMinIntervalMs: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS * 1_000,
now: () => now,
schedule: (callback, delayMs) => {
const timer = { at: now + delayMs, callback, cancelled: false }
timers.push(timer)
return () => {
timer.cancelled = true
}
}
drainReturnMinIntervalMs: 2_000,
now: () => 0,
schedule: () => () => undefined
})
const lane = new RelayDrainReturnAdmission(placement, {
maxConcurrent: 1,
maxRetryAfterSeconds: overrides.maxRetryAfterSeconds ?? 300,
now: () => now,
onServiceMs: overrides.onServiceMs
maxRetryAfterSeconds: 300,
now: () => 0
})
return {
lane,
placement,
advance: (ms: number) => {
now += ms
for (const timer of timers.filter((entry) => !entry.cancelled && entry.at <= now)) {
timer.cancelled = true
timer.callback()
}
}
placement
}
}
@@ -69,28 +48,6 @@ const host = (index: number): string => `host${String(index).padStart(12, '0')}`
describe('drain-return admission', () => {
it('shortens the pacing once the measured service time is short', async () => {
const { lane, advance } = harness()
for (let index = 0; index < 30; index++) {
const grant = admitted(await lane.acquire(host(index)))
advance(50)
grant.lease.release()
advance(2_000)
}
admitted(await lane.acquire(host(100)))
const retries = []
for (let index = 101; index <= 120; index++) {
retries.push(deferred(await lane.acquire(host(index))).retryAfterSeconds)
}
// ~50 ms per re-placement: twenty deferrals span about one second, where the
// 860 ms starting estimate would have spread them over seventeen.
expect(Math.max(...retries)).toBe(3)
})
// The unclamped sample: the EWMA's floor and ceiling would hide the real tail.
it('answers a host’s own early retry with its interval, not a place behind the cohort', async () => {
const { lane } = harness()
admitted(await lane.acquire(host(0))).lease.release()
@@ -99,7 +56,7 @@ describe('drain-return admission', () => {
const repeat = deferred(await lane.acquire(host(0)))
expect(repeat).toMatchObject({
reason: 'host-rate-limited',
retryAfterSeconds: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS
retryAfterSeconds: 2
})
// And it booked nothing: the next overflow still gets the first slot.
@@ -107,17 +64,6 @@ describe('drain-return admission', () => {
expect(deferred(await lane.acquire(host(2))).retryAfterSeconds).toBe(2)
})
it('keeps a host’s reserved slot when it comes back early', async () => {
const { lane, advance } = harness()
admitted(await lane.acquire(host(0)))
for (let index = 1; index <= 4; index++) await lane.acquire(host(index))
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(6)
advance(1_000)
// Early and still full: the same instant, not a new slot after everyone else.
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(5)
})
it('lets a queued placement go before a queued drain return', async () => {
const { lane, placement } = harness({ maxQueued: 1 })
const drain = admitted(await lane.acquire(host(0)))
@@ -1,4 +1,4 @@
import { describe, expect, it, vi } from 'vitest'
import { describe, expect, it } from 'vitest'
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
describe('public assignment admission', () => {
@@ -7,7 +7,7 @@ describe('public assignment admission', () => {
const admission = new RelayPublicAssignmentAdmission({
maxConcurrent: 1,
minIntervalMs: 0,
now: vi.fn(() => 0)
now: () => 0
})
const lease = await admission.acquire('host-a')
expect(lease).not.toBeNull()
@@ -46,19 +46,12 @@ describe('public assignment admission', () => {
})
it('gives a bounded reserved waiter the next public slot', async () => {
let expire!: () => void
let cancelled = false
const admission = new RelayPublicAssignmentAdmission({
maxConcurrent: 2,
maxReservedConcurrent: 1,
reservedWaitMs: 1_000,
minIntervalMs: 0,
schedule: (callback) => {
expire = callback
return () => {
cancelled = true
}
}
schedule: () => () => undefined
})
const first = await admission.acquire('host-a')
const second = await admission.acquire('host-b')
@@ -70,12 +63,10 @@ describe('public assignment admission', () => {
const reserved = await reservedPromise
expect(reserved).not.toBeNull()
expect(cancelled).toBe(true)
await expect(admission.acquire('host-d')).resolves.toBeNull()
reserved?.release()
await expect(admission.acquire('host-d')).resolves.not.toBeNull()
second?.release()
expire()
})
it('serializes same-host assignment and reserved recovery without losing priority', async () => {
@@ -100,34 +91,4 @@ describe('public assignment admission', () => {
await expect(admission.acquire('host-b')).resolves.not.toBeNull()
})
it('tells each caller its own rejection reason without crossing requests', async () => {
const admission = new RelayPublicAssignmentAdmission({
maxConcurrent: 1,
maxQueued: 1,
waitMs: 1_000,
maxReservedConcurrent: 1,
reservedWaitMs: 1_000,
minIntervalMs: 5_000,
now: () => 0,
schedule: () => () => undefined
})
const queued: string[] = []
const displaced: string[] = []
const throttled: string[] = []
const active = await admission.acquire('host-a')
// 'host-b' waits, then reserved recovery for the same host displaces it.
const ordinary = admission.acquire('host-b', (reason) => displaced.push(reason))
const reserved = admission.acquireReserved('host-b', (reason) => queued.push(reason))
await expect(ordinary).resolves.toBeNull()
active?.release()
const recovered = await reserved
recovered?.release()
await admission.acquire('host-a', (reason) => throttled.push(reason))
expect(displaced).toEqual(['superseded'])
expect(queued).toEqual([])
expect(throttled).toEqual(['host-rate-limited'])
})
})
@@ -1,55 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import { createRelayApp } from '../../app.js'
import type { RelayConfig } from '../../config.js'
describe('public assignment circuit breaker', () => {
it('rejects assign and resolve without invoking relay state', async () => {
const assign = vi.fn()
const resolveResume = vi.fn()
const app = createRelayApp(config(), {
store: { resolveResume } as never,
assignments: { assign } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
for (const path of ['/v1/assign', '/v1/resolve']) {
const response = await app.request(path, { method: 'POST' })
expect(response.status).toBe(503)
expect(response.headers.get('retry-after')).toBe('5')
expect(await response.json()).toEqual({ error: 'assignments_temporarily_unavailable' })
}
expect(assign).not.toHaveBeenCalled()
expect(resolveResume).not.toHaveBeenCalled()
expect((await app.request('/health')).status).toBe(200)
expect((await app.request('/v1/admin/drain', { method: 'POST' })).status).toBe(401)
})
})
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://relay.example.test',
cellUrl: 'https://relay.example.test',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
role: 'director',
cellId: 'director',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'runtime@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 3,
publicAssignmentsEnabled: false,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data'
}
}
@@ -1,6 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
RelayAssignmentRowBusyError,
type RelayAssignment,
type ResolvedRelayAssignment
} from '../assignment-store.js'
@@ -21,7 +20,6 @@ vi.mock('../../authentication/relay-token-verifier.js', () => ({
import { createRelayApp } from '../../app.js'
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
@@ -36,15 +34,11 @@ describe('drain-return lane', () => {
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
relayHostId === drained ? isolatedHome(relayHostId) : assignment('cell-o', relayHostId)
)
const outcomes: string[] = []
const servedLanes: string[] = []
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
recordAdmissionServiceMs: (lane) => servedLanes.push(lane)
ready: vi.fn(async () => true)
})
// The re-placement holds the drain lane; the sticky lane's only slot is free.
@@ -56,125 +50,12 @@ describe('drain-return lane', () => {
)
expect(reconnect.status).toBe(200)
expect(resolve).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: drained },
{ classifyHomeRollIsolation: true }
)
replacement.resolve(assignment('cell-new', drained))
expect((await moving).status).toBe(200)
expect(outcomes).toEqual(['drain-return', 'sticky'])
// The drained host held a sticky slot for its verification read too.
expect(servedLanes).toEqual(['sticky', 'sticky', 'drain-return'])
})
it('defers an overflowing drain return with a paced Retry-After and says why', async () => {
const holder = 'hhhhhhhhhhhhhhhh'
const replacement = deferred<RelayAssignment>()
const assign = vi.fn(async () => await replacement.promise)
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
isolatedHome(relayHostId)
)
const outcomes: string[] = []
const reasons: string[] = []
const retryAfters: number[] = []
const causes: string[] = []
const app = createRelayApp(config({ drainReturnQueueMax: 1, drainReturnWaitMs: 50 }), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
recordAssignmentRejectionReason: (lane, reason) => reasons.push(`${lane}:${reason}`),
recordDrainReturnRetryAfter: (seconds) => retryAfters.push(seconds),
recordAssignmentUnavailable: (cause) => causes.push(cause)
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const holding = app.request('/v1/assign', assignmentRequest(holder, { reconnect: true }))
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
const queued = app.request(
'/v1/assign',
assignmentRequest('qqqqqqqqqqqqqqqq', { reconnect: true })
)
const overflow = await app.request(
'/v1/assign',
assignmentRequest('ffffffffffffffff', { reconnect: true })
)
expect(overflow.status).toBe(503)
expect(overflow.headers.get('retry-after')).toBe('2')
const timedOut = await queued
expect(timedOut.status).toBe(503)
expect(Number(timedOut.headers.get('retry-after'))).toBeGreaterThanOrEqual(2)
expect(reasons).toEqual(['drain-return:queue-full', 'drain-return:wait-timeout'])
expect(retryAfters).toEqual([2, Number(timedOut.headers.get('retry-after'))])
expect(outcomes.filter((outcome) => outcome === 'drain-return-deferred')).toHaveLength(2)
expect(causes).toEqual(['drain-return-deferred', 'drain-return-deferred'])
expect(warn.mock.calls.flat().join('\n')).toMatch(
/lane=drain-return hinted=true reason=queue-full .*retryAfter=2/
)
replacement.resolve(assignment('cell-new', holder))
expect((await holding).status).toBe(200)
})
it('answers a drained host’s row-busy redial with the short interval', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
vi.spyOn(console, 'warn').mockImplementation(() => {})
const holder = 'hhhhhhhhhhhhhhhh'
const host = 'bbbbbbbbbbbbbbbb'
const holding = deferred<RelayAssignment>()
let busy = true
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
if (relayHostId === holder) return await holding.promise
if (relayHostId === host && busy) {
busy = false
throw new RelayAssignmentRowBusyError()
}
return assignment('cell-new', relayHostId)
})
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
isolatedHome(relayHostId)
)
const app = createRelayApp(config({ drainReturnQueueMax: 1 }), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const request = (relayHostId: string) =>
Promise.resolve(
app.request('/v1/assign', assignmentRequest(relayHostId, { reconnect: true }))
)
const held = request(holder)
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
const queued = request(host)
await vi.advanceTimersByTimeAsync(10)
// A cohort behind it books the next ten service slots.
for (let index = 0; index < 10; index++) {
const overflow = await request(hostId('f', index))
expect(overflow.status).toBe(503)
}
await vi.advanceTimersByTimeAsync(2_490)
holding.resolve(assignment('cell-new', holder))
expect((await held).status).toBe(200)
const refused = await queued
expect(refused.headers.get('retry-after')).toBe('1')
await vi.advanceTimersByTimeAsync(1_000)
const redial = await request(host)
// Its own lane interval, not a slot after the ten hosts that came later.
expect(redial.status).toBe(503)
expect(redial.headers.get('retry-after')).toBe('2')
})
})
function hostId(prefix: string, index: number): string {
return `${prefix}${String(index).padStart(15, '0')}`
}
function assignmentRequest(relayHostId: string, extra: { reconnect?: boolean } = {}): RequestInit {
return {
method: 'POST',
@@ -209,7 +90,7 @@ function deferred<T>() {
return { promise, resolve }
}
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://relay.example.test',
@@ -233,7 +114,6 @@ function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data',
...overrides
dataDir: './data'
}
}
@@ -18,68 +18,10 @@ import { createRelayApp } from '../../app.js'
describe('public assignment overload', () => {
it('fairly drains a bounded incident-scale burst without raising database concurrency', async () => {
const pending = new Map<string, ReturnType<typeof deferred<RelayAssignment>>>()
const admittedHosts: string[] = []
let active = 0
let highWater = 0
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
admittedHosts.push(relayHostId)
active++
highWater = Math.max(highWater, active)
const operation = deferred<RelayAssignment>()
pending.set(relayHostId, operation)
try {
return await operation.promise
} finally {
active--
pending.delete(relayHostId)
}
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const hosts = Array.from(
{ length: 430 },
(_, index) => `host-${String(index).padStart(11, '0')}`
)
const firstWave = hosts.map((host) =>
app.request('/v1/assign', assignmentRequest(host))
)
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
const retryWave = await Promise.all(
hosts.map((host) => app.request('/v1/assign', assignmentRequest(host)))
)
expect(retryWave.every((response) => response.status === 503)).toBe(true)
expect(assign).toHaveBeenCalledTimes(2)
while (assign.mock.calls.length < 130) {
const activeOperations = [...pending]
const expectedCalls = assign.mock.calls.length + activeOperations.length
for (const [host, operation] of activeOperations) {
operation.resolve(assignment(`cell-${host}`, host))
}
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(expectedCalls))
}
for (const [host, operation] of pending) {
operation.resolve(assignment(`cell-${host}`, host))
}
const firstResponses = await Promise.all(firstWave)
expect(firstResponses.filter((response) => response.status === 200)).toHaveLength(130)
expect(firstResponses.filter((response) => response.status === 503)).toHaveLength(300)
expect(admittedHosts).toEqual(hosts.slice(0, 130))
expect(highWater).toBe(2)
})
it('refuses an oversized chunked body before parsing it, sparing only the reserve routes', async () => {
const resolve = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: { resolve } as never,
assignments: {} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
@@ -98,15 +40,11 @@ describe('public assignment overload', () => {
const refused = await app.request('/v1/resolve', chunked(64 * 1024))
expect(refused.status).toBe(413)
expect(await refused.json()).toEqual({ error: 'request_too_large' })
expect(resolve).not.toHaveBeenCalled()
// Reserve bodies may reach 16 KiB, so the 4 KiB app limit must not answer for them. An
// unavailable route refuses before its own limit, including a body past 16 KiB.
// unavailable route refuses before its own limit.
const reserve = await app.request('/internal/reserve', chunked(6 * 1024))
expect(reserve.status).toBe(404)
expect(await reserve.json()).toEqual({ error: 'reserve_unavailable' })
const oversized = await app.request('/internal/reserve', chunked(20 * 1024))
expect(oversized.status).toBe(404)
expect(await oversized.json()).toEqual({ error: 'reserve_unavailable' })
})
it('reserves one bounded resolve lane while assignment work is saturated', async () => {
@@ -133,8 +71,6 @@ describe('public assignment overload', () => {
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
const recovery = app.request('/v1/resolve', resolveRequest('cccccccccccccccc', 1))
await Promise.resolve()
expect(resolveResume).not.toHaveBeenCalled()
const excessResolve = await app.request(
'/v1/resolve',
resolveRequest('dddddddddddddddd', 2)
@@ -143,8 +79,6 @@ describe('public assignment overload', () => {
expect(excessResolve.status).toBe(503)
expect(excessAssign.status).toBe(503)
expect(assign).toHaveBeenCalledTimes(2)
expect(resolveResume).not.toHaveBeenCalled()
pendingAssignments
.get('aaaaaaaaaaaaaaaa')
@@ -153,7 +87,6 @@ describe('public assignment overload', () => {
await vi.waitFor(() => expect(resolveResume).toHaveBeenCalledTimes(1))
pendingResolve.resolve({ userId: 'user-1', relayDeviceId: 'device-1' })
expect((await recovery).status).toBe(200)
expect(resolve).toHaveBeenCalledTimes(1)
for (const [host, operation] of pendingAssignments) {
operation.resolve(assignment(`cell-${host}`, host))
@@ -37,13 +37,11 @@ describe('public assignment reconnect lane', () => {
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
relayHostId === reconnecting ? assignment('cell-r', reconnecting) : null
)
const outcomes: string[] = []
const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
recordAssignmentAdmission: (outcome) => outcomes.push(outcome)
ready: vi.fn(async () => true)
})
// Saturate the placement lane with two in-flight newcomers, zero queue.
@@ -59,18 +57,6 @@ describe('public assignment reconnect lane', () => {
)
expect(reconnected.status).toBe(200)
expect(resolve).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: reconnecting },
{ classifyHomeRollIsolation: true }
)
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: reconnecting },
undefined,
undefined,
undefined,
{ hostRequested: true }
)
expect(outcomes).toContain('sticky')
pending.get(newcomerA)?.resolve(assignment('cell-a', newcomerA))
pending.get(newcomerB)?.resolve(assignment('cell-b', newcomerB))
@@ -78,28 +64,6 @@ describe('public assignment reconnect lane', () => {
expect((await second).status).toBe(200)
})
it('rate-limits repeat fast-lane attempts with the short retry-after', async () => {
const host = 'rrrrrrrrrrrrrrrr'
const assign = vi.fn(async () => assignment('cell-r', host))
const resolve = vi.fn(async () => assignment('cell-r', host))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect(
(await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))).status
).toBe(200)
const repeat = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(repeat.status).toBe(503)
expect(repeat.headers.get('retry-after')).toBe('2')
expect(resolve).toHaveBeenCalledTimes(1)
expect(assign).toHaveBeenCalledTimes(1)
})
it('rejects on the fast lane when the verification probe fails transiently', async () => {
const host = 'tttttttttttttttt'
const assign = vi.fn()
@@ -117,7 +81,6 @@ describe('public assignment reconnect lane', () => {
expect(response.status).toBe(503)
expect(response.headers.get('retry-after')).toBe('2')
expect(assign).not.toHaveBeenCalled()
})
it('refuses a dial whose assignment row is busy at once, and frees the fast lane', async () => {
@@ -143,7 +106,6 @@ describe('public assignment reconnect lane', () => {
expect(refused.status).toBe(503)
expect(refused.headers.get('retry-after')).toBe('1')
expect(await refused.json()).toEqual({ error: 'assignment_row_busy' })
expect(warn.mock.calls.flat().join('\n')).toContain('reason=relay_assignment_row_busy')
// The single fast-lane slot went back with the refusal.
expect(
(await app.request('/v1/assign', assignmentRequest(next, { reconnect: true }))).status
@@ -111,51 +111,15 @@ async function setup() {
}
describe('constrained idle regional assignment transaction', () => {
it.each(['missing', 'disabled', 'future'] as const)(
'does only one read per tick with %s durable control and sees later enablement',
async (state) => {
const { store, database, safety } = await setup()
const control = (await database.query(
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
))[0]!
if (state === 'missing') {
await database.query('DELETE FROM relay_region_rehome_control')
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
)
}
const query = vi.spyOn(database, 'query')
const transaction = vi.spyOn(database, 'transaction')
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
expect(transaction).not.toHaveBeenCalled()
}
if (state === 'missing') {
const columns = Object.keys(control)
await database.query(
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
Object.values(control)
)
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
[safety.observedAt]
)
}
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(1)
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
}
)
it('sees durable control changes between candidate polls', async () => {
const { store, database, safety } = await setup()
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
await database.query("UPDATE relay_region_rehome_control SET enabled = 1 WHERE control_id = 'global'")
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
})
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
const { store, database, safety, request } = await setup()
@@ -184,33 +148,6 @@ describe('constrained idle regional assignment transaction', () => {
})
})
it.each(['next_dispatch_at', 'paused_until'] as const)(
'skips the candidate join while %s holds the durable dispatch budget closed',
async (column) => {
const { store, database, safety } = await setup()
const query = vi.spyOn(database, 'query')
// One assignment only: setup leaves both fields at 0, and naming the other
// one too would assign this column twice, which Postgres rejects.
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt + 1]
)
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(2)
expect(query.mock.calls[1]![0]).toMatch(/FROM relay_region_rehome_worker_state/s)
}
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt]
)
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(2)
}
)
// sourceRequestUnits reads a reserve cell's missing phone leases as zero units; selection must drop it.
it.each(['source', 'target'])('drops every candidate while the %s cell is recorded as reserve', async (cellId) => {
const { store, safety } = await setup()
@@ -267,51 +204,6 @@ describe('constrained idle regional assignment transaction', () => {
expect(first.some((c) => c.relayHostId === next[0]!.relayHostId)).toBe(false)
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const template = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
identity.userId,
identity.relayHostId
])
)[0]!
const columns = Object.keys(template)
for (let index = 0; index < 100; index++) {
const values = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`idle-store-test-${String(index).padStart(3, '0')}`,
`pagehost${String(index).padStart(8, '0')}`,
identity.userId,
identity.relayHostId
]
)
}
}
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first).toHaveLength(100)
expect(next).toHaveLength(1)
expect(next[0]!.relayHostId).toBe('pagehost00000099')
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
regionalRehomeCohortPercent: 100
})
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
})
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
'rechecks generation when replacement wins after the initial authority lookup',
async () => {
@@ -430,20 +322,6 @@ describe('constrained idle regional assignment transaction', () => {
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
const { store, safety, database } = await setup()
const before = await database.query('SELECT * FROM relay_assignments')
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(1)
expect(candidates[0]).toMatchObject({
sourceGeneration: 7,
sourceCellId: 'source',
targetCellId: 'target'
})
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
})
function holdStatement(database: RelayDatabase, fragment: string) {
@@ -151,52 +151,6 @@ describe('idle regional rehome candidate window', () => {
expect(new Set(candidates.map((candidate) => candidate.attemptId)).size).toBe(3)
})
it('drops only the targets without room for the host plus its source activity', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 4 WHERE user_id = ?',
[identity.userId]
)
// Five units needed: four source units plus the assignment the move reserves.
await database.query("UPDATE relay_cells SET capacity_requests = 4 WHERE cell_id = 'asia-idle'")
const short = await store.selectIdleRegionalRehomeCandidates(safety)
expect(short.map((candidate) => candidate.targetCellId)).toEqual(['asia-mid', 'asia-busy'])
// Exactly enough room is enough; it ranks last because the ratio is per capacity.
await database.query("UPDATE relay_cells SET capacity_requests = 5 WHERE cell_id = 'asia-idle'")
const exact = await store.selectIdleRegionalRehomeCandidates(safety)
expect(exact.map((candidate) => candidate.targetCellId)).toEqual([
'asia-mid',
'asia-busy',
'asia-idle'
])
})
it('reads a bounded window of decisions however many hosts are eligible', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, 700)
const query = database.query.bind(database)
const decisionRows: number[] = []
vi.spyOn(database, 'query').mockImplementation(async (sql, params) => {
const rows = await query(sql, params)
if (/^SELECT user_id, relay_host_id FROM relay_region_decisions/.test(sql)) {
decisionRows.push(rows.length)
}
return rows
})
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(decisionRows.length).toBeGreaterThan(0)
for (const count of decisionRows) {
expect(count).toBeLessThanOrEqual(500)
}
expect(first).toHaveLength(102)
expect(next).toHaveLength(102)
const firstKeys = new Set(first.map((c) => `${c.userId}/${c.targetCellId}`))
expect(next.every((c) => !firstKeys.has(`${c.userId}/${c.targetCellId}`))).toBe(true)
})
it("keeps the window's last host when a decision turns eligible between the two reads", async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
@@ -252,7 +206,6 @@ describe('idle regional rehome candidate window', () => {
for (let poll = 0; poll < 20 && !wrapped; poll++) {
const page = await store.selectIdleRegionalRehomeCandidates(safety)
expect(page.length).toBeGreaterThan(0)
expect(page.length).toBeLessThanOrEqual(102)
for (const candidate of page) {
const key = `${candidate.userId}/${candidate.targetCellId}`
if (seen.has(key)) {
@@ -286,23 +239,4 @@ describe('idle regional rehome candidate window', () => {
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
it('excludes a host outside the cohort', async () => {
const { store, database } = await setup()
await seedHost(store, identity)
const now = 100_000_000 + 86_400_000
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
await database.query('UPDATE relay_region_decisions SET cohort_bucket = 40')
const narrow = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 40 })
expect(await narrow.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
const wide = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 41 })
expect(await wide.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
})
@@ -80,7 +80,7 @@ describe('persisted region decisions across director restart', () => {
expect(row.outcome).toBe('inconclusive')
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.generation).toBe(2)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'stale' })
})
@@ -111,7 +111,7 @@ describe('persisted region decisions across director restart', () => {
context.setNow(1_000_000_000 - 60_000)
const restarted = await context.restart()
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.generation).toBe(2)
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
@@ -124,104 +124,7 @@ async function window(context: Awaited<ReturnType<typeof setup>>) {
return result.window!
}
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
const attemptId = '33333333-3333-4333-8333-333333333333'
await context.database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
[
attemptId,
identity.userId,
identity.relayHostId,
cells[0]!.id,
incarnations[0],
cells[1]!.id,
incarnations[1],
migration.previousEpoch,
migration.assignmentEpoch,
context.now(),
context.now()
]
)
return { migration }
}
describe('ordered region decisions and migration outcomes', () => {
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
const context = await setup()
const { migration } = await regionalMigration(context)
context.advance(1_000)
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
const outcomes = await context.store.regionCorrectionOutcomes()
expect(outcomes).toEqual([
expect.objectContaining({
sourceCellId: cells[0]!.id,
targetCellId: cells[1]!.id,
state: 'registering',
count: 1,
oldestOpenMs: 1_000
})
])
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
before
)
await context.store.activateControl(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch
})
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({ state: 'registered' })
])
await context.store.releaseActivity(identity, context.activityId)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({
state: 'completed',
targetReservedUnits: 0,
oldestOpenMs: 0
})
])
})
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
const context = await setup()
const query = context.database.query.bind(context.database)
const transaction = context.database.transaction.bind(context.database)
const queryLocked = context.database.queryLocked.bind(context.database)
context.database.query = async (sql, params) => {
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
return query(sql, params)
}
context.database.transaction = async () => {
throw new Error('preview_must_not_open_mutating_transaction')
}
context.database.queryLocked = async () => {
throw new Error('preview_must_not_lock')
}
try {
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
expect(JSON.stringify(preview)).not.toContain(identity.userId)
} finally {
context.database.query = query
context.database.transaction = transaction
context.database.queryLocked = queryLocked
}
})
it('allocates distinct ordered generations for concurrent window issuers', async () => {
const context = await setup()
const replies = await Promise.all([window(context), window(context), window(context)])
@@ -1,10 +1,6 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import {
openRelayDatabase,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
type RelayDatabase
} from '../../database/database.js'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
@@ -127,25 +123,15 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
await expect(
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
).rejects.toMatchObject({ code: '23514' })
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'
ORDER BY conname`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
// The existing control row keeps its tuning and gains the cooldown.
// Preserve existing control tuning across the upgrade.
const control = await client.query(
`SELECT generation, preference_max_age_ms, host_cooldown_ms
`SELECT generation, preference_max_age_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
)
expect(control.rows).toEqual([
{
generation: '3',
preference_max_age_ms: '86400000',
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
preference_max_age_ms: '86400000'
}
])
})
@@ -182,14 +168,6 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
})
}, 60_000)
})
@@ -1,4 +1,4 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../assignment-store.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
@@ -134,10 +134,6 @@ const moves = (candidates: { userId: string; sourceCellId: string; targetCellId:
to: cells.find(({ id }) => id === targetCellId)!.region
}))
afterEach(() => {
vi.restoreAllMocks()
})
describe('rehome source-region filter', () => {
it('keeps US to Asia moves and drops Asia to US moves', async () => {
const context = await setup('us-central1')
@@ -151,13 +147,5 @@ describe('rehome source-region filter', () => {
await context.database.close()
})
it('reports the filtered hosts in the preview, agreeing with selection', async () => {
const context = await setup('us-central1')
const preview = await context.store.previewRegionalRehomeEligibility(context.safety())
expect(preview.counts).toEqual({
'eligible:us-central1-to-asia-east2': 1,
'source-outside-director-region': 1
})
await context.database.close()
})
})
File diff suppressed because it is too large Load Diff
@@ -1,7 +1,6 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../assignment-store.js'
import {
consumeRelayCellInventoryHold,
openRelayDatabase,
type PostgresDatabase,
type RelayDatabase
@@ -18,21 +17,6 @@ const describePostgres = databaseUrl ? describe : describe.skip
// Roughly the round trip from an Asia cell to the us-central1 database.
const STATEMENT_DELAY_MS = 150
// Probe the formerly fleet-locked tables plus admission/region tables read by commit.
const CELL_TABLES = [
'relay_cells',
'relay_cell_runtime',
'relay_cell_capabilities',
'relay_cell_rehome_safety',
'relay_cell_admission',
'relay_cell_regions'
] as const
// The target-row statement locks exactly these, held from it to COMMIT.
const TARGET_LOCKED = ['target:relay_cells', 'target:relay_cell_admission']
type Trip = { sql: string; lockable: Record<string, boolean>; probeMs: number }
type DelayControl = StatementDelay & { beforeTrip: (sql: string) => Promise<void> }
describePostgres('PostgreSQL regional rehome target-row lock', () => {
@@ -97,27 +81,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
}
}
async function lockable(table: string, cellId: string): Promise<boolean> {
try {
await observer.transaction(
async (transaction) => {
await transaction.queryLocked(
`SELECT cell_id FROM ${table} WHERE cell_id = ?`,
[cellId],
{
failIfUnavailable: true
}
)
},
{ reportRetries: false }
)
return true
} catch (error) {
if (error instanceof Error && error.message === 'database_lock_unavailable') return false
throw error
}
}
async function reservedRequests(cellId: string): Promise<number> {
const row = (
await primary.query(`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`, [cellId])
@@ -152,59 +115,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
)
}
it('holds only the target cell and admission rows until the next COMMIT boundary', async () => {
const context = await fixture()
const request = await context.select()
const trips: Trip[] = []
const probed = {
source: context.source.id,
target: context.target.id,
bystander: context.bystander.id
}
control.beforeTrip = async (sql) => {
const probeStartedAt = performance.now()
const state: Record<string, boolean> = {}
for (const [role, cellId] of Object.entries(probed)) {
for (const table of CELL_TABLES) state[`${role}:${table}`] = await lockable(table, cellId)
}
trips.push({ sql, lockable: state, probeMs: performance.now() - probeStartedAt })
}
consumeRelayCellInventoryHold(delayed)
control.enabled = true
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
control.enabled = false
expect(result).toEqual({ outcome: 'committed' })
expect(trips.length).toBeGreaterThanOrEqual(15)
expect(trips[trips.length - 1]!.sql).toBe('COMMIT')
for (const trip of trips) {
for (const [key, free] of Object.entries(trip.lockable)) {
if (TARGET_LOCKED.includes(key)) continue
expect({ sql: trip.sql, key, free }).toEqual({ sql: trip.sql, key, free: true })
}
}
for (const key of TARGET_LOCKED) {
expect(trips.filter((trip) => !trip.lockable[key]).map((trip) => trip.sql)).toEqual([
'COMMIT'
])
}
const counts = consumeRelayCellInventoryHold(delayed)
const commitProbeMs = trips.at(-1)!.probeMs
console.info(
JSON.stringify({
event: 'rehome_target_row_hold',
trips: trips.length,
commitProbeMs,
...counts
})
)
expect(counts.rehomeTargetRowHolds).toBe(1)
expect(counts.cellInventoryHoldMaxSite).toBe('rehome-target-row')
expect(counts.rehomeTargetRowHoldMsMax).toBeGreaterThanOrEqual(STATEMENT_DELAY_MS)
expect(await reservedRequests(context.target.id)).toBe(context.targetReservedBefore + 2)
}, 45_000)
it('defers when the target leaves general admission after selection', async () => {
const context = await fixture()
const request = await context.select()
@@ -240,23 +150,14 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
})
await lockedPromise
}
consumeRelayCellInventoryHold(delayed)
control.enabled = true
const startedAt = performance.now()
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
const elapsedMs = performance.now() - startedAt
control.enabled = false
release()
await writer
expect(result).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
expect(consumeRelayCellInventoryHold(delayed)).toMatchObject({
cellInventoryLockUnavailable: 1,
rehomeTargetRowHolds: 0
})
// NOWAIT: no 1s lock_timeout wait and no transaction retry behind the writer.
expect(elapsedMs).toBeLessThan(40 * STATEMENT_DELAY_MS)
await expectNothingCommitted(context)
})
@@ -1,202 +0,0 @@
import { describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../assignment-store.js'
import { openInMemoryRelayDatabase } from '../../database/database.js'
import { REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT } from './regional-rehome-safety.js'
const cell = (id: string, region: 'us-central1' | 'asia-east2') => ({
id,
url: `https://${id}.relay.example.test`,
region,
capacityRequests: 100,
connectionHardCap: 1_000 as const,
connectionUnobservedBound: 60
})
const source = cell('us-c1', 'us-central1')
const noHeadroom = cell('asia-a', 'asia-east2')
const unclean = cell('asia-b', 'asia-east2')
const highLoad = cell('asia-c', 'asia-east2')
const lowLoad = cell('asia-d', 'asia-east2')
const incarnation = (n: number) =>
`${String(n).repeat(8)}-${String(n).repeat(4)}-4${String(n).repeat(3)}` +
`-8${String(n).repeat(3)}-${String(n).repeat(12)}`
async function setup() {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
await store.inspectRegionalRehomeControl()
clock += 24 * 60 * 60_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
const beat = async (
config: typeof source,
n: number,
protocol: number,
state: { observedRequests: number; enforcedConnections: number; sqlFailures: number }
) => {
await store.recordCellHeartbeat({
cellId: config.id,
cellUrl: config.url,
region: config.region,
cellIncarnation: incarnation(n),
startedAt: 900_000,
ready: true,
observedRequests: state.observedRequests,
totalConnections: state.enforcedConnections,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: state.enforcedConnections,
connectionInclusionWatermark: clock,
connectionHardCap: 1_000,
connectionUnobservedBound: 60
})
await store.recordCellRegionalRehomeStatus({
cellId: config.id,
cellIncarnation: incarnation(n),
regionalRehomeProtocol: protocol,
safety: {
observedAt: clock,
sqlFailures: state.sqlFailures,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
})
}
const activatePreferredSource = async () => {
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1,
idleRegionalRehome: true,
cellIncarnation: incarnation(1)
})
const { window } = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
expect(window).toBeDefined()
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
const safety = () => ({
observedAt: clock,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
return { database, store, beat, activatePreferredSource, safety }
}
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
describe('regional rehome target selection', () => {
it('never selects a target without connection headroom, even at lowest load', async () => {
const context = await setup()
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
})
await context.activatePreferredSource()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(lowLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
it('falls to the next clean target when the load winner goes unclean', async () => {
const context = await setup()
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.activatePreferredSource()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(highLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
})
@@ -1,189 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from '../assignment-store.js'
import type { RelayConfig } from '../../config.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
afterEach(() => vi.restoreAllMocks())
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
vi.useFakeTimers()
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const worker = startRegionalRehomeWorker(
config(),
{
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => safety(Date.now()),
random: () => 0
}
)!
try {
await vi.advanceTimersByTimeAsync(0)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5_999)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(1)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
worker.stop()
await vi.advanceTimersByTimeAsync(60_000)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
} finally {
worker.stop()
vi.useRealTimers()
}
})
it('stops walking the page when the source names a deferral no candidate can pass', async () => {
const fetchImpl = respondWith([{ outcome: 'deferred', reason: 'concurrency-limit' }])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 3, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(1)
expect(summaries.entries).toEqual([
{
event: 'orca_relay_idle_rehome_dispatch_summary',
candidates: 3,
dispatched: 1,
stoppedBy: 'concurrency-limit',
outcomes: { 'deferred:concurrency-limit': 1 }
}
])
})
it('keeps its whole-page walk when the source sends no reason at all', async () => {
const fetchImpl = respondWith([{ outcome: 'deferred' }])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 3, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(3)
expect(summaries.entries[0]).toMatchObject({
stoppedBy: null,
outcomes: { deferred: 3 }
})
})
it('walks past a deferral that only concerns the one candidate', async () => {
const fetchImpl = respondWith([
{ outcome: 'deferred', reason: 'host-unsupported' },
{ outcome: 'busy' },
{ outcome: 'committed' }
])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 4, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(3)
expect(summaries.entries[0]).toMatchObject({
dispatched: 3,
stoppedBy: 'committed',
outcomes: { 'deferred:host-unsupported': 1, busy: 1, committed: 1 }
})
})
})
function respondWith(bodies: { outcome: string; reason?: string }[]) {
let index = 0
return vi.fn(async () => {
const body = bodies[Math.min(index++, bodies.length - 1)]!
return new Response(JSON.stringify({ v: 1, ...body }), {
headers: { 'content-type': 'application/json' }
})
})
}
function collectSummaries() {
const entries: Record<string, unknown>[] = []
let arrived: (() => void) | undefined
// Construction immediately polls; await first instead of waiting for a timer tick.
const first = new Promise<void>((resolve) => {
arrived = resolve
})
const original = console.warn
console.warn = (line: unknown, ...rest: unknown[]) => {
try {
const parsed = JSON.parse(line as string) as Record<string, unknown>
if (parsed.event === 'orca_relay_idle_rehome_dispatch_summary') {
entries.push(parsed)
arrived?.()
return
}
} catch {
// Not a JSON log line; fall through to the original writer.
}
original(line as string, ...rest)
}
return { entries, first, restore: () => (console.warn = original) }
}
async function runOnePoll(
fetchImpl: typeof fetch,
candidates: number,
summaries: { first: Promise<void> }
): Promise<void> {
const assignments = {
selectIdleRegionalRehomeCandidates: vi.fn(async () =>
Array.from({ length: candidates }, (_, index) => ({
v: 1 as const,
attemptId: `00000000-0000-4000-8000-00000000000${index}`,
userId: `user-${index}`,
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'us-c1',
sourceCellUrl: 'https://us-c1.relay.example.test',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
targetCellId: 'asia-c1'
}))
)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
fetch: fetchImpl,
identityToken: async () => 'token',
safetySnapshot: () => safety(Date.now()),
intervalMs: 60_000
})!
try {
await summaries.first
} finally {
worker.stop()
}
}
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
role: 'director',
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
rehomeDirectorServiceAccount: 'relay-director@example.test',
...overrides
} as RelayConfig
}
function safety(observedAt: number) {
return {
requiredCells: 2,
missingCells: 0,
observedAt,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolTotal: 3,
databasePoolIdle: 3,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolOldestWaitMs: 0,
databasePoolWaitMsMax: 0
}
}
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import { PLACEMENT_INTAKE_SHARE, PlacementLoadBand } from './placement-load-band.js'
import { PlacementLoadBand } from './placement-load-band.js'
import { RelayAssignmentStore } from '../assignment-store.js'
import { openInMemoryRelayDatabase } from '../../database/database.js'
@@ -13,74 +13,8 @@ function seededRandom(seed: number): () => number {
}
}
type SimulatedCell = { cellId: string; reserved: number; capacity: number }
const quiet = () => undefined
// Each pick goes to one of the directors at random, as the load balancer spreads requests.
function simulate(
directors: PlacementLoadBand[],
cells: SimulatedCell[],
picks: number,
at: (index: number) => number,
route: () => number = seededRandom(99)
) {
const counts = new Map(cells.map((cell) => [cell.cellId, 0]))
for (let index = 0; index < picks; index += 1) {
const ranked = cells
.filter((cell) => cell.reserved < cell.capacity)
.map((cell) => ({ cellId: cell.cellId, loadRatio: cell.reserved / cell.capacity, cell }))
.sort(
(left, right) => left.loadRatio - right.loadRatio || left.cellId.localeCompare(right.cellId)
)
const director = directors[Math.floor(route() * directors.length)]
const picked = director?.pick(ranked, at(index))
if (!picked) throw new Error('no candidate')
picked.cell.reserved += 1
counts.set(picked.cellId, (counts.get(picked.cellId) ?? 0) + 1)
}
return counts
}
// c27's drain on 10-08: 2,770 hosts onto one empty Asia cell (2,805 free)
// and four fuller ones (2,311 free between them).
function c27DrainCells(): SimulatedCell[] {
return [
{ cellId: 'cell-empty', reserved: 0, capacity: 4000 },
...['cell-b', 'cell-c', 'cell-d', 'cell-e'].map((cellId, index) => ({
cellId,
reserved: 4000 - 577 - (index === 0 ? 3 : 0),
capacity: 4000
}))
]
}
function drainShare(paceMs: number, seed: number) {
const cells = c27DrainCells()
const directors = Array.from(
{ length: 5 },
(_, index) => new PlacementLoadBand(seededRandom(seed + index), quiet)
)
const hosts = 2770
const counts = simulate(directors, cells, hosts, (index) => (index * paceMs) / hosts)
return { counts, cells, share: (counts.get('cell-empty') ?? 0) / hosts }
}
describe('PlacementLoadBand', () => {
it.each([
['1,200 s', 1_200_000],
['300 s', 300_000]
])('keeps the empty cell near the share across five directors at the %s pace', (_, paceMs) => {
for (const seed of [1, 2, 3]) {
const { counts, cells, share } = drainShare(paceMs, seed * 10)
expect([...counts.values()].reduce((sum, count) => sum + count, 0)).toBe(2770)
expect(share).toBeGreaterThan(0.3)
expect(share).toBeLessThan(0.5)
for (const cell of cells) expect(cell.reserved).toBeLessThanOrEqual(cell.capacity)
}
})
})
describe('leastLoadedCell with the load band', () => {
async function setupRegions() {
const database = await openInMemoryRelayDatabase()
@@ -121,20 +55,6 @@ describe('leastLoadedCell with the load band', () => {
return counts
}
it('spreads within the preferred region and keeps an empty US cell out', async () => {
const { database, store } = await setupRegions()
try {
const counts = await placements(store, 200)
expect(counts.get('us-a')).toBeUndefined()
expect(counts.get('asia-a')).toBeLessThanOrEqual(PLACEMENT_INTAKE_SHARE * 200 + 1)
const used = [...counts.values()].filter((count) => count >= 20)
expect(used.length).toBeGreaterThanOrEqual(3)
} finally {
await database.close()
}
})
it('fills the preferred region to its hard cap before spilling, never past it', async () => {
const { database, store } = await setupRegions()
try {
@@ -111,25 +111,6 @@ function seated(epoch: number, joinedAt = 0) {
describe('reserve assignment on a director', () => {
it('reads what each cell acts on: a tripped dead-man is db, a flip back still leasing is reserve', async () => {
const tripped = setup({
feeds: [feed('c1', { admitModeEffective: 'db' }), feed('c2', { admitModeEffective: 'db' })]
})
expect(await tripped.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
const leasing = setup({ feeds: [feed('c1', { admitMode: 'db', admitModeEffective: 'reserve' }), feed('c2')] })
expect(leasing.directory.admitModeOf('c1')).toBe('reserve')
// Raw db must immediately stop booking/pacing even while effective mode stays reserve during re-registration.
const reregistering = setup({
feeds: [
feed('c1', { admitModeEffective: 'reserve', admitModeRaw: 'db' }),
feed('c2', { admitModeEffective: 'reserve', admitModeRaw: 'db' })
]
})
expect(await reregistering.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
expect(reregistering.reserved).toEqual([])
expect(reregistering.directory.admitModeOf('c1')).toBe('reserve')
})
it('before every live cell has answered, holds back only hosts that may be on an unheard cell', async () => {
const cells = cellList(['c1', 'c2'])
const onUnheard = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
@@ -153,18 +134,6 @@ describe('reserve assignment on a director', () => {
expect((await gated.assignment.plan(onUnheard, { reconnect: true, region: US })).kind).not.toBe('retry')
})
it('answers a reconnect from memory at the same epoch with no booking and no database read', async () => {
const row = vi.fn(async () => null)
const { assignment, reserved } = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')],
row
})
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({ kind: 'answer', lane: 'sticky', assignment: { cellId: 'c1', assignmentEpoch: 7 } })
expect(reserved).toEqual([])
expect(row).not.toHaveBeenCalled()
})
it('re-books a host on a cell that restarted since its seat, and only then answers it', async () => {
const { assignment, directory, reserved, now } = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')]
@@ -198,57 +167,6 @@ describe('reserve assignment on a director', () => {
})
})
it('never sends a drained or demoted host back, and books it elsewhere above every known epoch', async () => {
const { assignment, directory, reserved, now } = setup()
directory.apply(
'c1',
feed('c1', {
full: undefined,
seq: 3,
changes: [
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4503, at: 2 }
]
}),
now.value,
now.value
)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { assignmentEpoch: 8 }
})
expect(reserved.length).toBeGreaterThan(0)
expect(reserved.every((call) => !call.sticky && call.epoch === 8)).toBe(true)
})
it('leaves a cell without a feed (c17/c18) on today’s path without starving the reserve pool', async () => {
const cells = cellList(['c1', 'c2', 'c17'])
let rowOnOld = false
const { assignment, directory, reserved, now } = setup({
cells,
feeds: [feed('c1'), feed('c2', { admitMode: 'db' })],
row: async () => (rowOnOld ? { cellId: 'c17', assignmentEpoch: 5 } : null)
})
directory.markNoFeed('c17', now.value)
expect(directory.isComplete()).toBe(true)
// An old image reports no ceiling; counting it as a database cell would zero the share.
expect(await assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement',
assignment: { cellId: 'c1' }
})
expect(reserved.map((call) => call.cellId)).not.toContain('c17')
rowOnOld = true
expect(
await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: true, region: US })
).toMatchObject({ kind: 'database', epochFloor: 5 })
directory.markNoFeed('c1', now.value)
expect(directory.cellState('c1')?.ceiling).toBeUndefined()
expect(assignment.placementCells().find((cell) => cell.cellId === 'c1')?.general).toBe(false)
})
it('reads the row once for a host the map never saw, and refuses it while the database is down', async () => {
const down = setup({ row: async () => Promise.reject(new Error('timeout')) })
expect(await down.assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
@@ -318,95 +236,6 @@ describe('reserve assignment on a director', () => {
}
})
describe('for a share-draw loser', () => {
const ASIA = 'asia-east2' as const
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) =>
setup({
...(reserve ? { reserve } : {}),
cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])],
feeds: [
reserveCell,
// Database room far above the reserve cell's, so the 0.25 draw loses.
feed('c2', {
admitMode: 'db',
counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 }
}),
feed('c3')
]
})
const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => {
const plan = await assignment.plan(identity, { reconnect: false, region: ASIA })
expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 })
return plan.kind === 'database' ? plan.regionReserve : undefined
}
it('offers the reserve cell in its region only, only while it has a seat', async () => {
const { assignment, reserved, directory, now } = setupAsia()
const regionReserve = await regionReserveOf(assignment)
expect(reserved).toEqual([])
await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA })
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } })
directory.apply('c1', full, now.value, now.value)
expect(await regionReserveOf(assignment, OTHER)).toBeUndefined()
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
})
it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => {
const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } }))
const regionReserve = await regionReserveOf(assignment)
await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError)
expect(reserved).toEqual([])
})
it('gives the region up when the reserve cell refuses with a seat still on the map', async () => {
for (const outcome of ['full', 'unreachable']) {
const { assignment, reserved } = setupAsia(feed('c1'), (cellId) =>
cellId === 'c1' ? { outcome } : { outcome: 'ok' }
)
const regionReserve = await regionReserveOf(assignment)
// Null sends the host to the database lane's cross-region fallback, not a 1 s retry.
await expect(regionReserve?.()).resolves.toBeNull()
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
}
})
})
describe('for a region-correction request', () => {
const request = { reconnect: true, region: US, regionCorrection: true }
it('leaves the answer to the database lane, offering only the in-region reserve fallback and the spill', async () => {
const row = vi.fn(async () => null)
// A memory answer would skip the window exchange; the share draw is not offered either.
const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row })
const plan = await assignment.plan(HOST, request)
expect(plan).toEqual({
kind: 'database',
regionReserve: expect.any(Function),
reserveSpill: expect.any(Function)
})
expect(reserved).toEqual([])
expect(row).not.toHaveBeenCalled()
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({
assignmentEpoch: 8
})
})
it('keeps today’s path for a host the map never saw, without reading its row, but may spill', async () => {
const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 }))
const { assignment, reserved } = setup({ row })
const plan = await assignment.plan(HOST, request)
expect(plan).toEqual({ kind: 'database', reserveSpill: expect.any(Function) })
expect(row).not.toHaveBeenCalled()
expect(reserved).toEqual([])
await expect(
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 3 }) : null
).resolves.toMatchObject({ assignmentEpoch: 4 })
})
})
describe('after the database lane found no cell anywhere', () => {
const ASIA = 'asia-east2' as const
const paced = { intake: { perSec: 1, burst: 20, tokens: 0 } }
@@ -428,31 +257,6 @@ describe('reserve assignment on a director', () => {
return plan.kind === 'database' ? plan.reserveSpill : undefined
}
it('books its own region first, minting above the row the database lane read', async () => {
const harness = setupSpill({})
const spill = await spillOf(harness)
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 9 })).resolves.toMatchObject({
cellId: 'asia-r',
region: ASIA,
assignmentEpoch: 10
})
expect(harness.reserved).toEqual([{ cellId: 'asia-r', epoch: 10, sticky: false, dryRun: false }])
})
it('books the other region when its own is full, or already tried', async () => {
const full = setupSpill(fullCounts)
await expect((await spillOf(full))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
cellId: 'us-r',
region: US,
assignmentEpoch: 5
})
const tried = setupSpill({})
await expect((await spillOf(tried))?.({ skipOwnRegion: true, rowEpoch: 0 })).resolves.toMatchObject({
cellId: 'us-r'
})
expect(tried.reserved.map((call) => call.cellId)).toEqual(['us-r'])
})
it('tries the other region after a pace, and answers the pace only if nothing places', async () => {
const abroad = setupSpill(paced)
await expect((await spillOf(abroad))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
@@ -464,39 +268,10 @@ describe('reserve assignment on a director', () => {
)
})
it('returns null with no reserve seat anywhere, so the refusal stands', async () => {
const harness = setupSpill(fullCounts, fullCounts)
await expect((await spillOf(harness))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toBeNull()
expect(harness.reserved).toEqual([])
})
it('is not offered while the map is incomplete', async () => {
const gated = setup({
cells: [...cellList(['asia-r', 'asia-db'], ASIA), ...cellList(['us-r'])],
feeds: [feed('asia-r'), feed('asia-db', { admitMode: 'db', full: [seated(4)] })],
startedAt: 100_500
})
const plan = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA })
expect(plan).toMatchObject({ kind: 'database' })
expect(plan.kind === 'database' && plan.reserveSpill).toBeUndefined()
const correction = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA, regionCorrection: true })
expect(correction).toEqual({ kind: 'database' })
})
it('answers the shortest pace when every region paces', async () => {
const harness = setupSpill(paced, { intake: { perSec: 2, burst: 20, tokens: 0 } })
const spill = await spillOf(harness)
// Asia refills its token in 4 s, US in 2 s.
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 0 })).rejects.toSatisfy(
(error) => error instanceof ReservePaceError && error.retryAfterSeconds === 2
)
})
})
describe('rehome on reconnect', () => {
const ASIA = 'asia-east2' as const
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
const ask = { reconnect: true, region: ASIA, preferred: true }
// 2,000 controls on the US cell refill its move budget once a minute.
const usCounts = { counts: { seats: 1, controls: 2_000, bookings: 0, units: 1, ceiling: 4_000 } }
@@ -546,107 +321,6 @@ describe('reserve assignment on a director', () => {
expect(demoted).toEqual([{ cellId: 'us-r', epoch: 5, joinedAt: 1_000 }])
})
it('answers a region-correction request on the rehome lane', async () => {
const { assignment, warm } = setupRehome()
await warm()
expect(await assignment.plan(HOST, { ...ask, regionCorrection: true })).toMatchObject({
kind: 'answer',
lane: 'rehome'
})
})
it('moves no one into Asia at 98% full, or anywhere under 10% free', async () => {
for (const share of [0.98, 0.91]) {
const { assignment, reserved, warm } = setupRehome({ asia: asiaFull(share) })
await warm()
expect(await assignment.plan(HOST, ask), String(share)).toMatchObject({
kind: 'answer',
lane: 'sticky',
assignment: { cellId: 'us-r' }
})
expect(reserved.filter((call) => call.cellId === 'asia-r')).toEqual([])
}
const { assignment, warm } = setupRehome({ asia: asiaFull(0.9) })
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
})
// 10% rehome headroom above the 0% spill line prevents spilled hosts bouncing back.
it('keeps a gap above the spill line: 5% free takes placements, not rehomes', async () => {
const { assignment, warm } = setupRehome({ asia: asiaFull(0.95) })
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky', assignment: { cellId: 'us-r' } })
expect(await assignment.plan(OTHER, { reconnect: false, region: ASIA })).toMatchObject({
lane: 'placement',
assignment: { cellId: 'asia-r' }
})
})
it('keeps today’s answer when any rule fails', async () => {
const sticky = { kind: 'answer', lane: 'sticky', assignment: { cellId: 'us-r' } }
const unhinted = setupRehome()
await unhinted.warm()
expect(await unhinted.assignment.plan(HOST, { ...ask, preferred: false })).toMatchObject(sticky)
const off = setupRehome({ percent: 0 })
await off.warm()
expect(await off.assignment.plan(HOST, ask)).toMatchObject(sticky)
const closed = setupRehome({ switchOpen: false })
await closed.warm()
expect(await closed.assignment.plan(HOST, ask)).toMatchObject(sticky)
const cold = setupRehome()
expect(await cold.assignment.plan(HOST, ask)).toMatchObject(sticky)
const sameRegion = setupRehome()
await sameRegion.warm()
expect(await sameRegion.assignment.plan(HOST, { ...ask, region: US })).toMatchObject(sticky)
})
it('never runs for a fresh ask, an unknown host, a database-mode source, or an incomplete map', async () => {
const fresh = setupRehome()
await fresh.warm()
expect(await fresh.assignment.plan(HOST, { ...ask, reconnect: false })).toMatchObject({ lane: 'placement' })
expect(await fresh.assignment.plan(OTHER, ask)).toMatchObject({ kind: 'answer', lane: 'placement' })
const onDatabase = setup({
cells: [...cellList(['us-db']), ...cellList(['asia-r'], ASIA)],
feeds: [feed('us-db', { admitMode: 'db', ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
})
expect(await onDatabase.assignment.plan(HOST, ask)).toMatchObject({ kind: 'database' })
const gated = setup({
cells: [...cellList(['us-r']), ...cellList(['asia-r', 'asia-unheard'], ASIA)],
feeds: [feed('us-r', { ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
startedAt: 100_500,
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
})
expect(await gated.assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
expect(gated.reserved).toEqual([])
})
it('falls through to the sticky answer when the rehome booking is paced', async () => {
const { assignment, reserved, warm } = setupRehome({
reserve: (cellId) => (cellId === 'asia-r' ? { outcome: 'intake' } : { outcome: 'ok' })
})
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ kind: 'answer', lane: 'sticky' })
expect(reserved.map((call) => call.cellId)).toEqual(['asia-r'])
})
it('leaves a host another install keeps dropping', async () => {
const { assignment, directory, now, warm } = setupRehome()
await warm()
directory.apply(
'us-r',
feed('us-r', {
...usCounts,
full: undefined,
seq: 3,
changes: [{ seq: 3, kind: 'leave', ...HOST, epoch: 4, generation: 1, closeCode: 4408, at: now.value - 1 }]
}),
now.value,
now.value
)
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
})
it('falls through to the sticky answer when the booking is refused, and gives the move back', async () => {
let refuse = true
const { assignment, warm } = setupRehome({
@@ -657,52 +331,6 @@ describe('reserve assignment on a director', () => {
refuse = false
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
})
it('moves one host per source budget a minute, and logs moves and skips per region pair', async () => {
const lines: Record<string, unknown>[] = []
const { assignment, now, warm } = setupRehome({
usSeats: [seated(5, 1_000), { ...OTHER, epoch: 5, generation: 1, state: 'active', joinedAt: 1_000 }],
log: (line) => lines.push(JSON.parse(line))
})
await warm()
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
expect(await assignment.plan(OTHER, ask)).toMatchObject({ lane: 'sticky' })
now.value += 60_000
await assignment.plan({ ...HOST, relayHostId: 'flushflushflush1' }, ask)
expect(lines.filter((line) => line.event === 'orca_relay_reserve_rehome')).toEqual([
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 0, skippedByRule: { 'source-budget': 1 } },
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 1, skippedByRule: { 'source-budget': 1 } }
])
})
})
it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => {
const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) })
await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toMatchObject({
message: 'reserve_cells_paced',
retryAfterSeconds: expect.any(Number)
})
})
it('tells the old reserve seat to go when it books the host elsewhere, naming that seat exactly', async () => {
const { assignment, directory, demoted, now } = setup({
feeds: [feed('c1', { full: [seated(5, 1_000)] }), feed('c2')],
reserve: (cellId) => (cellId === 'c1' ? { outcome: 'full' } : { outcome: 'ok' })
})
// The host left c1 for a reason that moves it: its next placement is fresh, above 5.
directory.apply(
'c1',
feed('c1', {
full: undefined,
seq: 2,
changes: [{ seq: 2, kind: 'drain-only', ...HOST, epoch: 5, generation: 1, at: 2 }]
}),
now.value,
now.value
)
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
expect(plan).toMatchObject({ kind: 'answer', lane: 'placement', assignment: { assignmentEpoch: 6 } })
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
})
// A half-open seat missed by the booking map must be demoted by the feed, before its 75s silence close.
@@ -714,91 +342,4 @@ describe('reserve assignment on a director', () => {
directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
})
it('answers a reconnect at the newest epoch even when an older epoch joined later', async () => {
const { assignment, reserved } = setup({
feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
})
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
kind: 'answer',
lane: 'sticky',
assignment: { cellId: 'c1', assignmentEpoch: 6 }
})
expect(reserved).toEqual([])
})
it('in dry run only asks a cell to check a booking, and never changes the answer', async () => {
const { assignment, reserved } = setup({ mode: 'dry-run' })
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
expect(plan.kind).toBe('database')
expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
expect(reserved).toMatchObject([{ dryRun: true }])
})
describe('when a cell answers a demotion', () => {
const outcomes = (lines: string[]) =>
lines
.map((line) => JSON.parse(line))
.filter((entry) => entry.event === 'orca_relay_reserve_demote_outcome')
it('logs the answer by host digest and leaves the seat to the feed', async () => {
const lines: string[] = []
const logged = Promise.withResolvers<void>()
const { assignment, directory } = setup({
feeds: [feed('c1', { full: [seated(10, 1_000)] }), feed('c2')],
demote: () => 'not-seated',
log: (line) => {
lines.push(line)
if (JSON.parse(line).event === 'orca_relay_reserve_demote_outcome') {
logged.resolve()
}
}
})
assignment.demoteSeat({ ...HOST, cellId: 'c1', epoch: 10, joinedAt: 1_000 }, 'reconcile')
await logged.promise
expect(outcomes(lines)).toMatchObject([
{ via: 'reconcile', cellId: 'c1', epoch: 10, outcome: 'not-seated' }
])
expect(outcomes(lines)[0].relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(lines.join('\n')).not.toContain(HOST.relayHostId)
expect(directory.seatsOnCell('c1')).toHaveLength(1)
})
})
describe('for a host re-asking off a draining cell', () => {
const events = (lines: string[], event: string) =>
lines.map((line) => JSON.parse(line)).filter((entry) => entry.event === event)
it('marks its pace as the drain’s, on the reserve lane, in the region fallback and in the spill', async () => {
const { assignment } = setup({
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
reserve: () => ({ outcome: 'intake' })
})
await expect(assignment.plan(HOST, { reconnect: true, region: US })).rejects.toMatchObject({ drained: true })
const plan = await assignment.plan(HOST, { reconnect: true, region: US, regionCorrection: true })
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).rejects.toMatchObject({ drained: true })
await expect(
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 0 }) : null
).rejects.toMatchObject({ drained: true })
})
it('leaves every other host as today: no drain count, and a plain pace', async () => {
const lines: string[] = []
const { assignment } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], log: (line) => lines.push(line) })
const paced = setup({
feeds: [feed('c1', { full: [seated(7)] }), feed('c2', { draining: true, full: [seated(6)] })],
reserve: () => ({ outcome: 'intake' })
})
expect(await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: false, region: US })).toMatchObject({
kind: 'answer',
lane: 'placement'
})
await expect(
paced.assignment.plan(HOST, { reconnect: false, region: US })
).rejects.toMatchObject({ drained: false })
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([])
})
})
})
@@ -57,30 +57,15 @@ describe('reserve ledger writer', () => {
expect(await row(db, 'bbbbbbbbbbbbbbbb')).toMatchObject({ cell_id: 'c1', reserved_controls: 2 })
})
it('keeps its queue through a database stall and drops the oldest past the bound', async () => {
it('persists a booked join after a database stall', async () => {
let now = 1_000
const { database: db, writer } = await setup(() => now)
const query = vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
writer.enqueue({ ...HOST, cellId: 'c1', epoch: 5 })
await writer.flush()
expect(writer.pending()).toBe(1)
await writer.flush()
expect(query).toHaveBeenCalledTimes(1)
now += 1_000
await writer.flush()
expect(writer.pending()).toBe(0)
expect(await row(db)).toMatchObject({ cell_id: 'c1', assignment_epoch: 5 })
const bounded = new ReserveLedgerWriter(db, { queueMax: 2, log: () => undefined })
const hosts = ['oldest0000000000', 'middle0000000000', 'newest0000000000']
for (const relayHostId of hosts) {
bounded.enqueue({ ...HOST, relayHostId, cellId: 'c1', epoch: 1 })
}
expect(bounded.pending()).toBe(2)
await bounded.flush()
expect(await row(db, hosts[0])).toBeUndefined()
expect(await row(db, hosts[1])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
expect(await row(db, hosts[2])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
})
it('reconciles the map’s seats and never lowers an epoch', async () => {
@@ -1,179 +0,0 @@
import { describe, expect, it } from 'vitest'
import {
runReservePlacementSimulation,
type SimulationConfig,
type SimulationReport
} from './reserve-placement-simulation.js'
const SEEDS = [11, 23, 47]
const rtt = (region: string) => (region === 'asia-east2' ? 170 : 10)
function fleet(): SimulationConfig['cells'] {
const cells: SimulationConfig['cells'] = []
for (let index = 0; index < 24; index += 1) {
cells.push({
cellId: `us${String(index).padStart(2, '0')}`,
region: 'us-central1',
hardCap: 600,
ceiling: 480,
intakePerSec: 5,
// A third of the fleet stays in database mode, one cell on an image without step 5.
mode: index % 3 === 0 ? 'db' : 'reserve',
old: index === 0,
preseated: 150
})
}
for (let index = 0; index < 6; index += 1) {
cells.push({
cellId: `asia${index}`,
region: 'asia-east2',
hardCap: 600,
ceiling: 480,
intakePerSec: 2,
mode: index === 0 ? 'db' : 'reserve',
preseated: 200
})
}
return cells
}
function expectInvariants(report: SimulationReport): void {
expect(report.violations).toEqual([])
}
describe('step 5 deterministic simulation', () => {
// Invariant 9 (dead-man) never trips here; its own tests below cover it.
it.each(SEEDS)(
'holds invariants 1-8 across mixed versions, lagging polls, restarts and database stalls (seed %i)',
async (seed) => {
const report = await runReservePlacementSimulation({
seed,
durationMs: 25 * 60_000,
directors: 5,
// One director still on the database-only revision.
oldDirectors: 1,
cells: fleet(),
hosts: 6_600,
hostRegion: (index) => (index % 5 === 0 ? 'asia-east2' : 'us-central1'),
arrivalPerSec: 20,
meanSessionMs: 8 * 60_000,
reconnectDelayMs: [500, 5_000],
duplicateAssignShare: 0.05,
halfOpenShare: 0.05,
pollLagMs: (director) => [0, 200, 1_000, 4_000, 10_000][director]!,
rttMs: rtt,
cellRestarts: [{ at: 6 * 60_000, cellId: 'us04' }, { at: 14 * 60_000, cellId: 'asia2' }],
directorRestarts: [{ at: 8 * 60_000, director: 2 }, { at: 16 * 60_000, director: 4 }],
// Drained hosts land elsewhere while their old seat lingers: placement supersedes it.
drains: [{ at: 12 * 60_000, cellId: 'us05', paceMs: 60_000 }],
databaseStalls: Array.from({ length: 3 }, (_, index) => ({
at: (index * 610 + 120) * 1_000,
durationMs: 7_000
}))
})
expectInvariants(report)
expect(report.reservePlacements).toBeGreaterThan(500)
expect(report.stickyAnswers).toBeGreaterThan(500)
expect(report.seatedAtEnd).toBeGreaterThan(4_000)
// The run reached the paths the invariants guard, so a pass is not vacuous.
expect(report.supersedes).toBeGreaterThan(0)
expect(report.rowAheadDemotions).toBeGreaterThan(0)
expect(report.directorRestarts).toBe(2)
expect(report.stickyReserves).toBeGreaterThan(0)
expect(report.databasePlacements).toBeGreaterThan(0)
},
120_000
)
it.each(SEEDS)(
'near full: extra reserve calls stay within 10% and exhausted requests within 1% (seed %i)',
async (seed) => {
const cells = Array.from({ length: 17 }, (_, index) => ({
cellId: `us${index}`,
region: 'us-central1' as const,
hardCap: 700,
ceiling: 600,
intakePerSec: 50,
mode: 'reserve' as const,
preseated: 582
}))
const report = await runReservePlacementSimulation({
seed,
durationMs: 3 * 60_000,
directors: 5,
cells,
hosts: 17 * 582 + 300,
hostRegion: () => 'us-central1',
arrivalPerSec: 10,
meanSessionMs: 0,
reconnectDelayMs: [1_000, 3_000],
duplicateAssignShare: 0,
halfOpenShare: 0,
pollLagMs: () => 0,
rttMs: rtt
})
expectInvariants(report)
expect(report.reservePlacements).toBeGreaterThan(250)
expect(report.reserveCalls / report.reservePlacements - 1).toBeLessThanOrEqual(0.1)
expect(report.exhausted / report.requests).toBeLessThanOrEqual(0.01)
for (const share of Object.values(report.maxOccupancyShare)) expect(share).toBeLessThanOrEqual(1)
},
120_000
)
it.each(SEEDS.flatMap((seed) => [[1_200_000, seed], [300_000, seed]] as const))(
'a drain onto an empty cell at the %i ms pace spreads within each cell intake budget (seed %i)',
async (paceMs, seed) => {
// Production c27 drain: 1,200s pacing toward cells with 2,805/841/777/667/26 free seats.
const free = [2_805, 841, 777, 667, 26]
const cells: SimulationConfig['cells'] = free.map((seats, index) => ({
cellId: `asia${index + 1}`,
region: 'asia-east2',
hardCap: 3_000,
ceiling: 2_830,
intakePerSec: 2,
mode: 'reserve',
preseated: 2_830 - seats
}))
cells.unshift({
cellId: 'asia0',
region: 'asia-east2',
hardCap: 3_000,
ceiling: 2_830,
intakePerSec: 2,
mode: 'reserve',
preseated: 2_770
})
const preseated = cells.reduce((sum, cell) => sum + (cell.preseated ?? 0), 0)
const report = await runReservePlacementSimulation({
seed,
durationMs: 30 * 60_000,
directors: 5,
cells,
hosts: preseated,
hostRegion: () => 'asia-east2',
arrivalPerSec: 1,
meanSessionMs: 0,
reconnectDelayMs: [1_000, 3_000],
duplicateAssignShare: 0.02,
halfOpenShare: 0,
pollLagMs: () => 0,
rttMs: rtt,
drains: [{ at: 10_000, cellId: 'asia0', paceMs }]
})
expectInvariants(report)
// Below every cell's budget nothing waits; above it the cells' buckets pace the drain.
if (paceMs === 300_000) expect(report.intakeRefusals + report.paced).toBeGreaterThan(0)
const moved = Object.entries(report.arrivalsByCell)
const total = moved.reduce((sum, [, count]) => sum + count, 0)
expect(total).toBeGreaterThan(2_000)
// Spread: the emptiest cell takes a share, not the drain.
expect((report.arrivalsByCell.asia1 ?? 0) / total).toBeLessThan(0.6)
expect(moved.filter(([, count]) => count / total > 0.1).length).toBeGreaterThanOrEqual(3)
},
120_000
)
})
@@ -2,7 +2,7 @@ import { exportJWK, generateKeyPair, SignJWT, type CryptoKey, type JWK } from 'j
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { RelayConfig } from '../config.js'
import { createRelayTokenVerifier } from './relay-token-verifier.js'
import { createStaleIfErrorJWKSet, RELAY_JWKS_MAX_STALE_MS } from './stale-if-error-jwks.js'
import { createStaleIfErrorJWKSet } from './stale-if-error-jwks.js'
const config = {
jwksUrl: 'https://auth.example.test/.well-known/jwks.json',
@@ -43,37 +43,7 @@ beforeEach(() => vi.useFakeTimers({ now: Date.parse('2026-10-11T13:00:00Z') }))
afterEach(() => vi.useRealTimers())
describe('relay token keys while auth is down', () => {
it('accepts a valid token from the last served set once the cache is past its 10 min age', async () => {
const key = await signingKey('auth-1')
const authServer = auth(key)
const jwks = createStaleIfErrorJWKSet(new URL(config.jwksUrl), { fetch: authServer.fetch })
const verify = createRelayTokenVerifier(config as RelayConfig, jwks)
expect(await verify(await relayToken(key))).not.toBeNull()
vi.setSystemTime(Date.now() + 20 * MINUTE)
authServer.state.up = false
const fetches = authServer.fetch.mock.calls.length
expect(await verify(await relayToken(key))).toMatchObject({ sub: 'user-1' })
expect(authServer.fetch.mock.calls.length).toBe(fetches + 1)
// Inside the retry window the stale set answers without waiting on another fetch.
expect(await verify(await relayToken(key))).not.toBeNull()
expect(authServer.fetch.mock.calls.length).toBe(fetches + 1)
})
it('rejects a token signed by a key the last served set never had', async () => {
const key = await signingKey('auth-1')
const authServer = auth(key)
const jwks = createStaleIfErrorJWKSet(new URL(config.jwksUrl), { fetch: authServer.fetch })
const verify = createRelayTokenVerifier(config as RelayConfig, jwks)
expect(await verify(await relayToken(key))).not.toBeNull()
vi.setSystemTime(Date.now() + MINUTE)
authServer.state.up = false
expect(await verify(await relayToken(await signingKey('unknown')))).toBeNull()
})
it('bounds the stale set at 30 minutes since the last served set', async () => {
expect(RELAY_JWKS_MAX_STALE_MS).toBe(30 * MINUTE)
const key = await signingKey('auth-1')
const authServer = auth(key)
const verify = createRelayTokenVerifier(
@@ -1,83 +0,0 @@
import pg from 'pg'
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
import { reconcileCellAdmissionAtStartup } from './cell-admission-startup.js'
import type { RelayCellConfig } from '../../config.js'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_startup_retry_test'
const cell: RelayCellConfig = {
id: 'startup-retry-cell',
url: 'https://startup-retry.example.test',
capacityRequests: 4_000
}
afterEach(() => vi.restoreAllMocks())
describePostgres('PostgreSQL director startup reconciliation', () => {
const databases: RelayDatabase[] = []
let scopedDatabaseUrl = ''
beforeAll(async () => {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
} finally {
await client.end()
}
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
scopedDatabaseUrl = url.toString()
databases.push(
await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' }),
await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' })
)
})
afterAll(async () => {
for (const database of databases) await database.close()
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await client.end()
}
})
it('recovers after the cell inventory lock outlasts transaction retries', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells([cell], false)
let releaseLock: () => void = () => undefined
let reportLocked: () => void = () => undefined
const lockReleased = new Promise<void>((resolve) => (releaseLock = resolve))
const lockAcquired = new Promise<void>((resolve) => (reportLocked = resolve))
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`)
reportLocked()
await lockReleased
})
await lockAcquired
const releaseTimer = setTimeout(releaseLock, 3_500)
try {
await reconcileCellAdmissionAtStartup({ role: 'director', cells: [cell] }, store)
} finally {
clearTimeout(releaseTimer)
releaseLock()
await holder
}
expect(await databases[0]!.query(`SELECT cell_id FROM relay_cells`)).toEqual([
{ cell_id: cell.id }
])
const warnings = warn.mock.calls.flat().join('\n')
expect(warnings).toContain('orca_relay_startup_reconcile_recovered')
expect(warnings).not.toContain('orca_relay_postgres_transaction_exhausted')
}, 10_000)
})
@@ -1,44 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
reconcileCellAdmissionAtStartup
} from './cell-admission-startup.js'
import type { RelayCellConfig } from '../../config.js'
const cells: RelayCellConfig[] = [
{
id: 'candidate',
url: 'https://candidate.relay.example.com',
capacityRequests: 4_000,
initiallyEnabled: false
}
]
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
describe('cell admission startup authority', () => {
it('bounds retry wall time when each reconciliation is slow', async () => {
vi.useFakeTimers()
vi.setSystemTime(0)
vi.spyOn(Math, 'random').mockReturnValue(0)
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const lockUnavailable = Object.assign(new Error('lock unavailable'), { code: '55P03' })
const reconcileCellsAtStartup = vi.fn().mockImplementation(async () => {
vi.setSystemTime(Date.now() + 10_000)
throw lockUnavailable
})
const startup = reconcileCellAdmissionAtStartup(
{ role: 'director', cells },
{ reconcileCellsAtStartup }
)
const rejection = expect(startup).rejects.toBe(lockUnavailable)
await vi.runAllTimersAsync()
await rejection
expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(5)
})
})
@@ -1,119 +0,0 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
import type { RelayCellConfig } from '../../config.js'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const CELL_PREFIX = 'connection-cap-min-postgres'
const FLEET_SIZE = 16
function cellConfig(id: string, connectionHardCap: 1_000 | 3_000): RelayCellConfig {
return {
id,
url: `https://${id}.example.com`,
capacityRequests: 1_000,
connectionHardCap,
connectionUnobservedBound: 60
}
}
async function removeCells(database: RelayDatabase): Promise<void> {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_capabilities',
'relay_cell_rehome_safety',
'relay_cell_runtime',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id LIKE ?`, [`${CELL_PREFIX}-%`])
}
}
function storeOn(database: RelayDatabase): RelayAssignmentStore {
return new RelayAssignmentStore(database, () => 100, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
}
async function heartbeat(
store: RelayAssignmentStore,
cell: RelayCellConfig,
connectionHardCap: number,
units: number
): Promise<void> {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: units,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: units,
connectionHardCap,
connectionUnobservedBound: 60
})
}
describePostgres('PostgreSQL effective connection cap', () => {
let database: RelayDatabase
let racer: RelayDatabase
beforeAll(async () => {
database = await openRelayDatabase({ databaseUrl, dataDir: '' })
racer = await openRelayDatabase({ databaseUrl, dataDir: '' })
await removeCells(database)
})
afterAll(async () => {
await removeCells(database)
await racer.close()
await database.close()
})
it('keeps 16 raised cells placeable while a heartbeat races the raise', async () => {
const ids = Array.from({ length: FLEET_SIZE }, (_, index) => `${CELL_PREFIX}-fleet-${index}`)
const lower = ids.map((id) => cellConfig(id, 1_000))
const director = storeOn(database)
const otherInstance = storeOn(racer)
await director.reconcileCells(lower, true)
for (const cell of lower) await heartbeat(director, cell, 1_000, 10)
// The heartbeat lands before or after the raise; either order must leave the same state.
await Promise.all([
director.reconcileCells(
ids.map((id) => cellConfig(id, 3_000)),
true
),
heartbeat(otherInstance, lower[0]!, 1_000, 11)
])
for (const id of ids) {
await expect(director.cellDeploymentStatus(id)).resolves.toMatchObject({
connectionCapacity: {
hardCap: 3_000,
reportedHardCap: 1_000,
effectiveHardCap: 1_000,
heartbeatFresh: true
}
})
const rows = await database.query(
`SELECT cell_id FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[id]
)
expect(rows).toHaveLength(1)
}
// The old processes keep heartbeating their 1,000 under the raised config.
for (const cell of lower) {
await expect(heartbeat(otherInstance, cell, 1_000, 12)).resolves.toBeUndefined()
}
})
})
@@ -1,55 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayConfig } from '../config.js'
import { startCellHeartbeat } from './cell-heartbeat-client.js'
const CONFIG = {
role: 'cell',
cellId: 'cell-a',
cellUrl: 'https://relay-a.example.com',
directorUrl: 'https://relay.example.com',
heartbeatAudience: 'https://relay.example.com/v1/admin/cell-heartbeat'
} as RelayConfig
describe('cell heartbeat client', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an immediate authenticated heartbeat without putting credentials in the URL', async () => {
const requests: Array<{ url: string; init?: RequestInit }> = []
const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => {
requests.push({ url: String(url), init })
return new Response('{}', { status: 200 })
}) as typeof fetch
const client = startCellHeartbeat(CONFIG, {
ready: async () => true,
observedRequests: () => 7,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
}),
fetch: fetchImpl,
identityToken: async () => 'secret-token',
now: () => 123,
incarnation: '11111111-1111-4111-8111-111111111111',
intervalMs: 60_000
})!
await vi.waitFor(() => expect(requests).toHaveLength(1))
client.stop()
expect(requests[0]!.url).toBe('https://relay.example.com/v1/admin/cell-heartbeat')
expect(requests[0]!.url).not.toContain('secret-token')
expect(requests[0]!.init?.headers).toMatchObject({ authorization: 'Bearer secret-token' })
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
v: 1,
cellId: 'cell-a',
cellUrl: 'https://relay-a.example.com',
region: 'us-central1',
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 123,
ready: true,
observedRequests: 7
})
})
})
@@ -5,7 +5,7 @@ import {
cellFlagObjectName,
cellFlagParser
} from './cell-flags.js'
import { CONTROL_FLAG_POLL_MS, startControlFlagChannel } from './relay-control-flag-channel.js'
import { startControlFlagChannel } from './relay-control-flag-channel.js'
const CELL_ID = 'production-gce-c7'
const ACCESS_TOKEN = 'ya29.secret-access-token'
@@ -46,7 +46,7 @@ function cellObject(generation: number, flags: Record<string, unknown>, cellId =
}
}
function cellChannel(fetchImpl: typeof fetch, autoStart = false) {
function cellChannel(fetchImpl: typeof fetch) {
return startControlFlagChannel({
objectName: cellFlagObjectName(CELL_ID),
defaults: CELL_FLAG_DEFAULTS,
@@ -54,7 +54,7 @@ function cellChannel(fetchImpl: typeof fetch, autoStart = false) {
appliedEvent: CELL_FLAGS_APPLIED_EVENT,
fetch: fetchImpl,
random: () => 0.5,
autoStart
autoStart: false
})
}
@@ -230,19 +230,4 @@ describe('control flag channel', () => {
expect(channel.applied()).toMatchObject({ generation: 5, flags: { readinessLocal: true } })
})
it('reads one at a time and on a jittered 5 s cadence', async () => {
vi.useFakeTimers()
const google = fakeGoogle(cellObject(5, { readinessLocal: true }))
const channel = cellChannel(google.fetchImpl)
await Promise.all([channel.poll(), channel.poll()])
expect(google.state.storageRequests).toHaveLength(1)
const running = cellChannel(google.fetchImpl, true)
await vi.advanceTimersByTimeAsync(0)
expect(google.state.storageRequests).toHaveLength(2)
await vi.advanceTimersByTimeAsync(CONTROL_FLAG_POLL_MS)
expect(google.state.storageRequests).toHaveLength(3)
running.stop()
await vi.advanceTimersByTimeAsync(CONTROL_FLAG_POLL_MS * 3)
expect(google.state.storageRequests).toHaveLength(3)
})
})
@@ -4,69 +4,34 @@ import {
CellAdmitEffectiveWriter,
DEAD_MAN_SLOT_CLAIM_RETRY_MS
} from './cell-admit-effective-writer.js'
import { DEAD_MAN_SLOT_HOLD_MS } from './cell-dead-man-trip-slot.js'
function setup(
options: { fail?: boolean; busy?: boolean; busyWhen?: () => boolean; slotFree?: () => boolean } = {}
options: { slotFree?: () => boolean } = {}
) {
const clock = { value: 1_000 }
const writes: ('db' | 'reserve' | 'claim')[] = []
const pending: Array<() => void> = []
const logs: string[] = []
const writer = new CellAdmitEffectiveWriter({
write: async (mode) => {
writes.push(mode)
await new Promise<void>((resolve) => pending.push(resolve))
if (options.fail) throw new Error('connection refused')
},
claimTrip: async (slots) => {
writes.push('claim')
await new Promise<void>((resolve) => pending.push(resolve))
return slots > 0 && (options.slotFree?.() ?? true)
},
databaseBusy: () => options.busy === true || options.busyWhen?.() === true,
now: () => clock.value,
log: (line) => logs.push(line)
databaseBusy: () => false,
now: () => clock.value
})
const settle = async () => {
while (pending.length > 0) pending.shift()!()
await new Promise((resolve) => setTimeout(resolve, 0))
}
return { clock, writes, logs, writer, settle }
return { clock, writes, writer, settle }
}
describe('CellAdmitEffectiveWriter', () => {
it('writes on the first tick, on every change, and every 15 s, one write at a time', async () => {
const { clock, writes, writer, settle } = setup()
writer.tick('reserve', true)
writer.tick('db', true)
expect(writes).toEqual(['reserve'])
await settle()
// Tripped: the switch still says reserve, the raw mode db.
writer.tick('db', true)
await settle()
writer.tick('db', true)
expect(writes).toEqual(['reserve', 'db'])
clock.value += CELL_ADMIT_EFFECTIVE_REFRESH_MS
writer.tick('db', true)
expect(writes).toEqual(['reserve', 'db', 'db'])
})
// Step 5 off: a db switch never writes, so the dark roll adds no database write.
// Best-effort: a failed write never throws into the cell, logs at most once a minute, and retries.
it('retries a failed write on the next tick and rate-limits its log line', async () => {
const { clock, writes, logs, writer, settle } = setup({ fail: true })
writer.tick('db', true)
await settle()
clock.value += 1_000
writer.tick('db', true)
await settle()
expect(writes).toEqual(['db', 'db'])
expect(logs).toHaveLength(1)
expect(JSON.parse(logs[0]!)).toMatchObject({ event: 'orca_relay_cell_admit_effective_write_failed', mode: 'db' })
})
it('claims a held trip slot in place of its refresh, and permits the trip only once it lands', async () => {
let free = false
const { clock, writes, writer, settle } = setup({ slotFree: () => free })
@@ -97,20 +62,4 @@ describe('CellAdmitEffectiveWriter', () => {
expect(writes).toEqual(['reserve', 'claim', 'reserve', 'claim'])
})
it('refreshes a tripped row past a saturated pool before its slot hold runs out', async () => {
let busy = false
const { clock, writes, writer, settle } = setup({ busyWhen: () => busy })
writer.tick('db', true)
await settle()
busy = true
clock.value += 4 * 60_000
writer.tick('db', true)
expect(writes).toEqual(['db'])
clock.value += 60_000
writer.tick('db', true)
await settle()
expect(writes).toEqual(['db', 'db'])
expect(DEAD_MAN_SLOT_HOLD_MS).toBeGreaterThan(5 * 60_000)
})
})
@@ -8,8 +8,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
import WebSocket from 'ws'
import type { RawData } from 'ws'
import { CELL_FLAG_DEFAULTS, type CellFlags } from '../control-flags/cell-flags.js'
import { CELL_SEAT_DRIFT_SWEEP_MS } from '../seats/cell-seat-drift.js'
import type { CellSeatLog } from '../seats/cell-seat-log.js'
import type { RelayConfig } from '../../config.js'
import { PostgresDatabase } from '../../database/database.js'
import type { AppliedControlFlags } from '../control-flags/relay-control-flag-channel.js'
@@ -73,20 +71,6 @@ function cellConfig(port: number, issuer: string): RelayConfig {
}
}
// The registry's private feed log, to plant the drift the cell must repair.
function seatLogOf(sessions: object): CellSeatLog {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: HostSessionRegistry holds its feed log as `seatLog`.
return (sessions as { seatLog: CellSeatLog }).seatLog
}
// The JSON lines a test's console.warn spy saw for one event.
function warned(event: string): Array<Record<string, unknown>> {
return vi
.mocked(console.warn)
.mock.calls.map(([line]) => (typeof line === 'string' && line.startsWith('{') ? JSON.parse(line) : null))
.filter((entry) => entry?.event === event)
}
function nextMessage(socket: WebSocket): Promise<Record<string, unknown>> {
return new Promise((resolve, reject) => {
socket.once('message', (data: RawData) => {
@@ -265,9 +249,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
book,
setFlags: (next: Partial<CellFlags>) => {
applied = { generation: applied.generation + 1, flags: { ...CELL_FLAG_DEFAULTS, ...next } }
},
advance: (ms: number) => {
elapsedMs += ms
}
}
}
@@ -338,97 +319,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
return (page as { full: Array<{ relayHostId: string; epoch: number }> }).full.map((entry) => entry.epoch)
}
it('repairs at once the seat a director names when no session backs it', async () => {
const cell = await startCell({ admitMode: 'reserve' })
const identity = await cell.host()
cell.book(identity, 9)
await cell.connect(identity, 9)
// A stale seat at a generation above the live session's: its changes could never replace it.
const log = seatLogOf(cell.relay.sessions)
log.append({ kind: 'join', userId: 'user-1', relayHostId: identity.hostId, epoch: 8, generation: 5, at: 4_242 })
const before = cell.relay.sessions.seatFeed(null)
const seq = 'full' in before ? before.seq : 0
const seat = { v: 1 as const, userId: 'user-1', relayHostId: identity.hostId, epoch: 8, joinedAt: 4_242 }
expect(cell.relay.sessions.demote(seat)).toBe('not-seated')
expect(cell.relay.sessions.seatFeed(seq)).toMatchObject({
changes: [
{ kind: 'leave', epoch: 8, generation: 5, closeCode: 4409 },
{ kind: 'join', epoch: 9, generation: 1 }
]
})
expect(seatEpochs(cell)).toEqual([9])
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([
{ kind: 'unbacked-seat', via: 'demote', reason: 'epoch', seatEpoch: 8, sessionEpoch: 9, relisted: true }
])
expect(JSON.stringify(warned('orca_relay_cell_seat_log_drift'))).not.toContain(identity.hostId)
// Asked again, there is nothing left to repair.
expect(cell.relay.sessions.demote(seat)).toBe('not-seated')
expect(cell.relay.sessions.seatFeed(seq + 2)).toMatchObject({ changes: [] })
})
it('leaves a seat no session backs once two sweeps in a row find it, with no director asking', async () => {
const cell = await startCell({ admitMode: 'db' })
const ghost = await cell.host()
seatLogOf(cell.relay.sessions).append({
kind: 'join',
userId: 'user-1',
relayHostId: ghost.hostId,
epoch: 10,
generation: 3,
at: 1_000
})
expect(seatEpochs(cell)).toEqual([10])
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
expect(seatEpochs(cell)).toEqual([])
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([
{ kind: 'unbacked-seat', via: 'sweep', reason: 'no-session', seatEpoch: 10, seatGeneration: 3, relisted: false }
])
// Left as moved: the cell will not seat it again at 10 from memory.
expect(cell.relay.sessions.seatFeed(0)).toMatchObject({ changes: [{}, { kind: 'leave', closeCode: 4409 }] })
})
it('lists again a live session the feed lost, once two sweeps in a row find it', async () => {
const cell = await startCell({ admitMode: 'reserve' })
const identity = await cell.host()
cell.book(identity, 9)
await cell.connect(identity, 9)
seatLogOf(cell.relay.sessions).append({
kind: 'leave',
userId: 'user-1',
relayHostId: identity.hostId,
epoch: 9,
generation: 1,
at: 5_000
})
expect(seatEpochs(cell)).toEqual([])
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
expect(seatEpochs(cell)).toEqual([9])
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([{ kind: 'unlisted-session', sessionEpoch: 9 }])
})
// A drained session's seat stays listed, with no session, until its socket's close lands.
it('repairs nothing for a seat that one sweep finds unbacked and is gone by the next', async () => {
const cell = await startCell({ admitMode: 'reserve' })
const identity = await cell.host()
cell.book(identity, 9)
const live = await cell.connect(identity, 9)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: ws keeps its net.Socket as `_socket`.
;(live.socket as unknown as { _socket: { pause(): void } })._socket.pause()
cell.relay.sessions.drainHost({
attemptId: 'attempt-1',
userId: 'user-1',
relayHostId: identity.hostId,
sourceAssignmentEpoch: 9,
graceMs: 0
})
await new Promise((resolve) => setTimeout(resolve, 20))
expect(seatEpochs(cell)).toEqual([9])
await new Promise((resolve) => setTimeout(resolve, RELAY_WEBSOCKET_FORCE_CLOSE_MS + 300))
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
expect(seatEpochs(cell)).toEqual([])
expect(warned('orca_relay_cell_seat_log_drift')).toEqual([])
}, 10_000)
// Drained sockets can close seconds after map removal; reusing their generation let late leaves erase reconnected live seats.
it('keeps a session that reconnected while its drained predecessor was still closing', async () => {
const cell = await startCell({ admitMode: 'reserve' })
@@ -456,19 +346,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
expect(seatEpochs(cell)).toEqual([10])
}, 10_000)
// Repeated fresh joins for one identity replace session and seat together, leaving no drift to repair.
it('stays on the live session when the same identity connects afresh each turn', async () => {
const cell = await startCell({ admitMode: 'reserve' })
const identity = await cell.host()
for (const epoch of [9, 10, 11]) {
cell.book(identity, epoch)
expect((await cell.connect(identity, epoch)).ack).toMatchObject({ type: 'host-hello-ack' })
expect(seatEpochs(cell)).toEqual([epoch])
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
}
expect(seatEpochs(cell)).toEqual([11])
expect(warned('orca_relay_cell_seat_log_drift')).toEqual([])
})
})
it('gives every booking unit back when the cell starts to drain', async () => {
@@ -542,44 +419,4 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
}).toEqual(before)
}, 30_000)
it('never crosses the hard cap under seeded bookings, hellos, rebooks and closes', async () => {
const hardCap = 6
const cell = await startCell({ admitMode: 'reserve' }, { hardCap, controlReserve: 1 })
const hosts = await Promise.all(Array.from({ length: 12 }, () => cell.host()))
const epochs = new Map<string, number>()
const open: WebSocket[] = []
let seed = 7
const random = () => {
seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648
return seed / 2_147_483_648
}
let peak = 0
for (let step = 0; step < 240; step += 1) {
const identity = hosts[Math.floor(random() * hosts.length)]!
const roll = random()
if (roll < 0.4) {
// A director's fresh booking, or a re-book at a newer epoch.
const epoch = (epochs.get(identity.hostId) ?? 1) + (random() < 0.5 ? 0 : 1)
if (cell.book(identity, epoch)[0]?.outcome === 'ok') epochs.set(identity.hostId, epoch)
} else if (roll < 0.8) {
const reply = await cell.connect(identity, epochs.get(identity.hostId) ?? 1)
if (reply.ack?.type === 'host-hello-ack') open.push(reply.socket!)
} else if (open.length > 0) {
const socket = open.splice(Math.floor(random() * open.length), 1)[0]!
// A newer control for the same host may already have replaced it.
if (socket.readyState !== WebSocket.CLOSED) {
const closed = new Promise((resolve) => socket.once('close', resolve))
socket.close()
await closed
}
}
const counts = cell.relay.connectionSnapshot()!
peak = Math.max(peak, counts.physicalConnections)
expect(counts.physicalConnections).toBeLessThanOrEqual(hardCap)
expect(counts.enforcedConnectionUnits).toBeLessThanOrEqual(hardCap)
}
// The run did reach the cap, so the bound was exercised.
expect(peak).toBeGreaterThanOrEqual(hardCap - 1)
}, 60_000)
})
@@ -1,41 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import { CellReserveClient } from './cell-reserve-client.js'
const target = { cellId: 'c1', cellUrl: 'https://c1.example' }
const item = (userId: string, epoch = 3) => ({
userId,
relayHostId: 'abcdefghijklmnop',
epoch,
ttlMs: 30_000
})
describe('cell reserve client', () => {
it('batches a cell’s bookings into one request and maps each answer back', async () => {
const fetchImpl = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
const body = JSON.parse(String(init?.body)) as { items: Array<{ userId: string }> }
return Response.json({
v: 1,
results: body.items.map((entry) =>
entry.userId === 'b' ? { outcome: 'seated-newer', epoch: 9 } : { outcome: 'ok' }
)
})
})
const client = new CellReserveClient({
directorId: 'director-a',
identityToken: async () => 'token',
fetch: fetchImpl
})
const answers = await Promise.all([
client.reserve(target, item('a')),
client.reserve(target, item('b')),
client.reserve(target, item('c'), true)
])
expect(answers).toEqual([{ outcome: 'ok' }, { outcome: 'seated-newer', epoch: 9 }, { outcome: 'ok' }])
// One booking batch and one dry-run batch.
expect(fetchImpl).toHaveBeenCalledTimes(2)
const first = JSON.parse(String(fetchImpl.mock.calls[0]![1]?.body)) as Record<string, unknown>
expect(first).toMatchObject({ v: 1, directorId: 'director-a', items: [{ userId: 'a' }, { userId: 'b' }] })
expect(String(fetchImpl.mock.calls[0]![0])).toBe('https://c1.example/internal/reserve')
})
})
@@ -1,114 +0,0 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import WebSocket from 'ws'
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
import { openDelayedPostgresDatabase, type StatementDelay } from '../../test-fixtures/delayed-postgres-database.js'
import { RESERVE_MODE_CELL_ID, startReserveModeCell } from '../../test-fixtures/reserve-mode-cell.js'
// PostgreSQL flip-back keeps controls connected, reserves most pool capacity for DB work and folds cell deltas per tick even across cross-region round trips (E-mix).
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
async function until(check: () => Promise<boolean> | boolean, timeoutMs: number): Promise<void> {
const deadline = Date.now() + timeoutMs
while (!(await check())) {
if (Date.now() > deadline) throw new Error('condition not reached')
await new Promise((resolve) => setTimeout(resolve, 50))
}
}
describePostgres('flipping a reserve-mode cell back on PostgreSQL', () => {
const cleanup: Array<() => Promise<void> | void> = []
afterEach(async () => {
for (const close of cleanup.splice(0).reverse()) await close()
vi.restoreAllMocks()
// The database is shared with the other PostgreSQL suites, some of which read every cell.
const database = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
try {
const tables = await database.query(
`SELECT table_name FROM information_schema.columns
WHERE table_schema = current_schema() AND column_name = 'cell_id' AND table_name LIKE 'relay_%'`
)
for (const row of tables) {
await database.query(`DELETE FROM ${String(row.table_name)} WHERE cell_id = ?`, [RESERVE_MODE_CELL_ID])
}
} finally {
await database.close()
}
})
async function cellOn(database: RelayDatabase, databasePoolMax = 10) {
vi.spyOn(console, 'log').mockImplementation(() => undefined)
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-flip-back-pg-'))
cleanup.push(() => rmSync(dataDir, { recursive: true, force: true }))
return await startReserveModeCell({ database, dataDir, cleanup, databasePoolMax })
}
it('re-registers every memory-admitted control, ten flips in a row', async () => {
const database = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
cleanup.push(() => database.close())
for (let flip = 0; flip < 10; flip += 1) {
const cell = await cellOn(database)
const hosts = []
for (let index = 0; index < 3; index += 1) hosts.push(await cell.bookedHost(5 + index))
const ids = new Set(hosts.map((entry) => entry.identity.hostId))
const leased = async () =>
(await cell.controlLeases()).filter((row) => ids.has(String(row.relay_host_id)))
expect(await leased()).toEqual([])
cell.setAdmitMode('db')
await until(async () => (await leased()).length === 3, 10_000)
await until(() => cell.relay.sessions.reregistrationPending() === 0, 10_000)
expect(hosts.every((entry) => entry.socket.readyState === WebSocket.OPEN)).toBe(true)
for (const entry of hosts) entry.socket.close()
}
}, 120_000)
it('keeps re-registration to a third of the pool and one cell-row write per tick, at 170 ms a trip', async () => {
const delay: StatementDelay = { enabled: false, delayMs: 170 }
let cellRowWrites = 0
let leaseWritesInFlight = 0
let peakInFlight = 0
delay.beforeTrip = async (sql) => {
if (/UPDATE relay_cells\b/.test(sql)) cellRowWrites += 1
}
const database = openDelayedPostgresDatabase(databaseUrl!, delay, 10)
cleanup.push(() => database.close())
// The schema and the cell row, without the delay.
const setup = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
await setup.close()
const cell = await cellOn(database, 10)
const deferred = cell.relay.assignments.activateControlDeferringCell.bind(cell.relay.assignments)
vi.spyOn(cell.relay.assignments, 'activateControlDeferringCell').mockImplementation(async (...args) => {
leaseWritesInFlight += 1
peakInFlight = Math.max(peakInFlight, leaseWritesInFlight)
try {
return await deferred(...args)
} finally {
leaseWritesInFlight -= 1
}
})
const hosts = []
for (let index = 0; index < 9; index += 1) hosts.push(await cell.bookedHost(5 + index))
const ids = new Set(hosts.map((entry) => entry.identity.hostId))
delay.enabled = true
const startedAt = Date.now()
cell.setAdmitMode('db')
await until(() => cell.relay.sessions.reregistrationPending() > 0, 5_000)
await until(() => cell.relay.sessions.reregistrationPending() === 0, 45_000)
const elapsedMs = Date.now() - startedAt
delay.enabled = false
const leased = (await cell.controlLeases()).filter((row) => ids.has(String(row.relay_host_id)))
expect(leased).toHaveLength(9)
expect(hosts.every((entry) => entry.socket.readyState === WebSocket.OPEN)).toBe(true)
// Pool of 10: at most 3 activations at once, leaving 7 for hellos and renewals.
expect(peakInFlight).toBeLessThanOrEqual(3)
// Far fewer cell-row writes than activations: deltas fold into one write per tick.
expect(cellRowWrites).toBeLessThan(9)
process.stderr.write(`flip_back_latency ${JSON.stringify({ elapsedMs, peakInFlight, cellRowWrites })}\n`)
}, 60_000)
})
@@ -3,7 +3,6 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import WebSocket from 'ws'
import { CELL_ADMIT_EFFECTIVE_REFRESH_MS } from './cell-admit-effective-writer.js'
import { RESERVE_DEAD_MAN_MS, reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
import { openRelayDatabase } from '../../database/database.js'
import {
@@ -34,7 +33,7 @@ describe('flipping a reserve-mode cell back to the database', () => {
vi.restoreAllMocks()
})
async function cell(now?: () => number, options: { flagsUnread?: boolean } = {}) {
async function cell(now?: () => number) {
vi.spyOn(console, 'log').mockImplementation(() => undefined)
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
vi.spyOn(console, 'error').mockImplementation(() => undefined)
@@ -42,7 +41,7 @@ describe('flipping a reserve-mode cell back to the database', () => {
cleanup.push(() => rmSync(dataDir, { recursive: true, force: true }))
const database = await openRelayDatabase({ dataDir })
cleanup.push(() => database.close())
return await startReserveModeCell({ database, dataDir, cleanup, ...options, ...(now ? { now } : {}) })
return await startReserveModeCell({ database, dataDir, cleanup, ...(now ? { now } : {}) })
}
it('registers every memory-admitted control without disconnecting any, matching the seat report', async () => {
@@ -159,47 +158,6 @@ describe('flipping a reserve-mode cell back to the database', () => {
expect(host.socket.readyState).toBe(WebSocket.OPEN)
}, 60_000)
it('takes its flip-back pace from the switch file', async () => {
const reserve = await cell()
for (let index = 0; index < 4; index += 1) await reserve.bookedHost(5 + index)
const real = reserve.relay.assignments.activateControlDeferringCell.bind(reserve.relay.assignments)
let inFlight = 0
let peak = 0
vi.spyOn(reserve.relay.assignments, 'activateControlDeferringCell').mockImplementation(async (...args) => {
inFlight += 1
peak = Math.max(peak, inFlight)
await new Promise((resolve) => setTimeout(resolve, 50))
try {
return await real(...args)
} finally {
inFlight -= 1
}
})
reserve.setAdmitMode('db', { reregisterInFlight: 1 })
await until(async () => (await reserve.controlLeases()).length === 4)
expect(peak).toBe(1)
}, 30_000)
// Step 5 off adds no writes before/after flag reads; leaving reserve owes exactly one db write.
it('writes an admit-effective row only while its switch says reserve, plus one db write on the way out', async () => {
let now = Date.now()
const booting = await cell(() => now, { flagsUnread: true })
await new Promise((resolve) => setTimeout(resolve, 2_500))
expect(await booting.admitEffectiveRow()).toBeUndefined()
booting.setAdmitMode('db')
now += 3 * CELL_ADMIT_EFFECTIVE_REFRESH_MS
await new Promise((resolve) => setTimeout(resolve, 2_500))
expect(await booting.admitEffectiveRow()).toBeUndefined()
booting.setAdmitMode('reserve')
await until(async () => (await booting.admitEffectiveRow())?.mode === 'reserve')
booting.setAdmitMode('db')
await until(async () => (await booting.admitEffectiveRow())?.mode === 'db')
const last = (await booting.admitEffectiveRow())!.updated_at
now += 3 * CELL_ADMIT_EFFECTIVE_REFRESH_MS
await new Promise((resolve) => setTimeout(resolve, 2_500))
expect((await booting.admitEffectiveRow())!.updated_at).toEqual(last)
}, 30_000)
it('flips itself back to db when no placing director has been in touch for the dead-man window', async () => {
let now = Date.now()
const reserve = await cell(() => now)
@@ -1,336 +0,0 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { z } from 'zod'
import type { RelayAssignmentStore } from '../../assignment/assignment-store.js'
import type { CellFlags } from '../control-flags/cell-flags.js'
import type { RelayConfig } from '../../config.js'
import type { RelayCredentialStore } from '../../credentials/credential-store.js'
import { HostSessionRegistry, type HostSession } from '../../connections/host-session-registry.js'
import type { AppliedControlFlags } from '../control-flags/relay-control-flag-channel.js'
import type { RelayRuntimeObserver } from '../../observability/relay-observability.js'
import type { RelayTokenClaims } from '../../authentication/relay-token-verifier.js'
import { ProcessQueuedByteBudget } from '../../connections/splice-forwarder.js'
const adminTokenChecks = vi.hoisted(() => ({ count: 0 }))
vi.mock('../../authentication/admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string) => {
adminTokenChecks.count += 1
return token === 'deploy-token'
},
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
createRegionalRehomeTokenVerifier: () => async (token: string) => token === 'rehome-token',
createRuntimeTokenVerifier: () => async () => false
}))
import { createRelayApp } from '../../app.js'
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number, reason?: string) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(reason ?? ''))
})
}
const incarnation = '11111111-1111-4111-8111-111111111111'
// The published v1 shape, so a field rename fails here before it reaches a director.
const SeatChangeSchema = z.object({
seq: z.number(),
kind: z.enum(['join', 'leave', 'drain-only', 'active']),
userId: z.string(),
relayHostId: z.string(),
epoch: z.number(),
generation: z.number(),
state: z.enum(['active', 'drain-only']).optional(),
closeCode: z.number().optional(),
at: z.number()
})
const SeatFeedReplySchema = z.object({
v: z.literal(1),
cellId: z.string(),
incarnation: z.string(),
at: z.number(),
draining: z.boolean(),
counts: z.object({
controls: z.number(),
seats: z.number()
}),
flagsApplied: z
.object({
generation: z.number(),
flags: z.object({
readinessLocal: z.boolean(),
ticketCheck: z.enum(['off', 'shadow', 'enforce']),
rejectionFence: z.boolean(),
admitMode: z.enum(['db', 'reserve']),
reserveDryRun: z.boolean(),
phonePath: z.enum(['db', 'memory'])
}),
ignoredKeys: z.array(z.string()).optional()
})
.optional(),
seq: z.number(),
changes: z.array(SeatChangeSchema).optional(),
more: z.boolean().optional(),
full: z
.array(
z.object({
userId: z.string(),
relayHostId: z.string(),
epoch: z.number(),
generation: z.number(),
state: z.enum(['active', 'drain-only']),
joinedAt: z.number()
})
)
.optional()
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
publicUrl: 'https://c7.relay.example.test',
cellUrl: 'https://c7.relay.example.test',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c7',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
rehomeDirectorServiceAccount: 'relay-director@example.test',
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data',
...overrides
}
}
function hostIdentity(index: number, exp = 4_102_444_800): RelayTokenClaims {
return {
sub: `user-${index}`,
prof: 'profile-1',
relayHostId: `host${String(index).padStart(12, '0')}`,
purpose: 'host-control',
exp
}
}
// What a host's auth-refresh token verifies as.
const verifyRelayToken = vi.fn<(token: string) => Promise<RelayTokenClaims | null>>()
function createCell(relayConfig = config(), cellFlags?: () => AppliedControlFlags<CellFlags>) {
const assignments = {
activateControl: vi.fn().mockResolvedValue('control:production-gce-c7:1'),
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
releaseActivity: vi.fn().mockResolvedValue(true),
renewControlActivities: vi.fn().mockResolvedValue([])
} as unknown as RelayAssignmentStore
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
relayConfig,
verifyRelayToken,
{} as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer,
Date.now,
Math.random,
incarnation
)
const activate = (
registry as unknown as {
activate: (
socket: WebSocket,
identity: RelayTokenClaims,
existing: HostSession | null,
generation: number,
rebind: boolean,
assignmentEpoch: number,
appVersion: string
) => Promise<void>
}
).activate.bind(registry)
const connect = async (
index: number,
options: { generation?: number; epoch?: number; rebind?: boolean; exp?: number } = {}
): Promise<FakeSocket> => {
const socket = new FakeSocket()
const identity = hostIdentity(index, options.exp)
await activate(
socket as unknown as WebSocket,
identity,
registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }),
options.generation ?? 1,
options.rebind ?? false,
options.epoch ?? 1,
'1.4.200'
)
return socket
}
const app = createRelayApp(relayConfig, {
store: {} as never,
assignments: {} as never,
drain: (graceMs, options) => registry.drain(graceMs, options ?? {}),
cellIncarnation: incarnation,
cellSeatFeed: (sinceSeq) => registry.seatFeed(sinceSeq),
cellFlags,
cellPhonePathEffective: () => registry.phonePathEffective(),
isDraining: () => registry.isDraining(),
runtimeCounts: () => ({
totalConnections: 0,
preAuthConnections: 0,
queuedBytes: 0,
...registry.runtimeCounts()
}),
ready: vi.fn(async () => true)
})
const poll = async (since?: string, token = 'rehome-token') =>
await app.request(
`/v1/admin/cell-seats${since === undefined ? '' : `?since=${encodeURIComponent(since)}`}`,
{ headers: { authorization: `Bearer ${token}` } }
)
const read = async (since?: string) => SeatFeedReplySchema.parse(await (await poll(since)).json())
return { registry, connect, app, poll, read }
}
describe('cell seat feed', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('keeps a closing socket seated until its close lands, as the change log does', async () => {
const { connect, read } = createCell()
const socket = await connect(1)
// The ws close handshake: no longer OPEN, close event not yet emitted.
socket.readyState = 2
const closing = await read()
expect(closing.counts).toMatchObject({ controls: 0, seats: 1 })
expect(closing.full).toHaveLength(1)
socket.readyState = socket.CLOSED
socket.emit('close', 1000, Buffer.from(''))
expect((await read()).counts).toMatchObject({ controls: 0, seats: 0 })
})
it('reports join, drain-only and leave as changes after the cursor', async () => {
const { registry, connect, read } = createCell()
const socket = await connect(1)
const { seq } = await read()
registry.drain(60_000)
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
await connect(2)
const body = await read(`${incarnation}:${seq}`)
expect(body.full).toBeUndefined()
expect(body.draining).toBe(true)
expect(body.changes?.map((change) => change.kind)).toEqual(['drain-only', 'leave'])
expect(body.changes?.[1]).toMatchObject({
userId: 'user-1',
closeCode: RELAY_CLOSE_CODE.DRAINING,
generation: 1
})
// The second connect landed after the drain, so the cell refused it: no join.
expect(body.seq).toBe(seq + 2)
})
it('reports an auth expiry as drain-only and the refresh that clears it as active', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const { connect, read } = createCell()
const exp = Math.floor(Date.now() / 1000) + 5
const socket = await connect(1, { exp })
const { seq } = await read()
// The 15 s heartbeat finds the token expired, inside its grace.
await vi.advanceTimersByTimeAsync(15_000)
verifyRelayToken.mockResolvedValueOnce(hostIdentity(1, exp + 3_600))
socket.emit(
'message',
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'refreshed' })),
false
)
await vi.waitFor(() => expect(verifyRelayToken).toHaveBeenCalled())
const body = await read(`${incarnation}:${seq}`)
expect(body.changes?.map((change) => change.kind)).toEqual(['drain-only', 'active'])
expect(body.counts.seats).toBe(1)
expect((await read()).full).toEqual([expect.objectContaining({ state: 'active' })])
})
it('does not report a rebind as a leave, and joins the rebound socket', async () => {
const { connect, read } = createCell()
await connect(1)
const { seq } = await read()
await connect(1, { rebind: true, epoch: 2 })
const body = await read(`${incarnation}:${seq}`)
expect(body.changes).toEqual([
expect.objectContaining({ kind: 'join', userId: 'user-1', epoch: 2, state: 'active' })
])
})
it('reports a superseded generation leaving after its successor joined', async () => {
const { connect, read } = createCell()
await connect(1)
const { seq } = await read()
await connect(1, { generation: 2 })
const body = await read(`${incarnation}:${seq}`)
// Readers drop the leave: its generation is older than the seat's.
expect(body.changes?.map((change) => [change.kind, change.generation])).toEqual([
['leave', 1],
['join', 2]
])
})
it('resyncs a cursor from another incarnation with a full snapshot', async () => {
const { connect, poll, read } = createCell()
await connect(1)
const body = await read('22222222-2222-4222-8222-222222222222:1')
expect(body.full).toHaveLength(1)
expect((await poll('not a cursor')).status).toBe(400)
})
it('accepts only the directors rehome identity, verified once per poll', async () => {
const { poll } = createCell()
adminTokenChecks.count = 0
expect((await poll(undefined, 'deploy-token')).status).toBe(401)
expect((await poll(undefined, 'wrong')).status).toBe(401)
expect((await poll()).status).toBe(200)
// The admin middleware leaves this route to its own rehome check.
expect(adminTokenChecks.count).toBe(0)
})
it('is unavailable on a cell without the rehome pair, and on directors', async () => {
const withoutPair = createCell(
config({ rehomeAudience: undefined, rehomeDirectorServiceAccount: undefined })
)
const response = await withoutPair.poll()
expect(response.status).toBe(404)
expect(await response.json()).toEqual({ error: 'seat_feed_unavailable' })
const director = createCell(config({ role: 'director', cellId: 'director' }))
expect((await director.poll()).status).toBe(404)
})
})
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import { CELL_SEAT_FEED_PAGE_MAX, CellSeatLog } from './cell-seat-log.js'
import { CellSeatLog } from './cell-seat-log.js'
function append(log: CellSeatLog, count: number): void {
for (let index = 0; index < count; index += 1) {
@@ -24,16 +24,6 @@ describe('cell seat log', () => {
expect(log.read(5)).toMatchObject({ seq: 10, full: expect.any(Array) })
})
it('pages a long backlog and resumes from the returned cursor', () => {
const log = new CellSeatLog(CELL_SEAT_FEED_PAGE_MAX * 2)
append(log, CELL_SEAT_FEED_PAGE_MAX + 5)
const first = log.read(0)
expect(first).toMatchObject({ seq: CELL_SEAT_FEED_PAGE_MAX, more: true })
expect('changes' in first && first.changes).toHaveLength(CELL_SEAT_FEED_PAGE_MAX)
const second = log.read(first.seq)
expect(second).toMatchObject({ seq: CELL_SEAT_FEED_PAGE_MAX + 5, more: false })
expect('changes' in second && second.changes).toHaveLength(5)
})
})
describe('cell seat log snapshot', () => {
@@ -85,20 +75,5 @@ describe('cell seat cursor', () => {
expect(log.recentlyLeftOf(host.userId, host.relayHostId)).toBeUndefined()
})
it('sends only the newest page of leavers in a full snapshot', () => {
const log = new CellSeatLog(CELL_SEAT_FEED_PAGE_MAX * 4, () => 0)
const leavers = CELL_SEAT_FEED_PAGE_MAX + 5
for (let index = 0; index < leavers; index += 1) {
const host = { userId: `user-${index}`, relayHostId: 'abcdefghijklmnop', epoch: 1, generation: 1 }
log.append({ kind: 'join', ...host, at: 0 })
log.append({ kind: 'leave', ...host, closeCode: 1006, at: 0 })
}
const page = log.read(null)
const left = 'full' in page ? page.recentlyLeft : []
expect(left).toHaveLength(CELL_SEAT_FEED_PAGE_MAX)
expect(left.at(-1)?.userId).toBe(`user-${leavers - 1}`)
// The cell itself still remembers the oldest.
expect(log.recentlyLeftOf('user-0', 'abcdefghijklmnop')).toBeDefined()
})
})
@@ -1,166 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import type { RelayAssignment } from '../../assignment/assignment-store.js'
import type { RelayConfig } from '../../config.js'
import {
ShadowDirectoryCompare
} from './shadow-directory-compare.js'
import { ShadowSeatDirectory, type SeatFeedResponse } from './shadow-seat-directory.js'
const fakes = vi.hoisted(() => ({
verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-a', relayHostId: token }))
}))
vi.mock('../../authentication/relay-token-verifier.js', () => ({
createRelayTokenVerifier: () => fakes.verifyRelayToken,
readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
}))
import { createRelayApp } from '../../app.js'
const HOST = 'hhhhhhhhhhhhhhhh'
const IDENTITY = { userId: 'user-a', relayHostId: HOST }
function feed(cellId: string, overrides: Partial<SeatFeedResponse> = {}): SeatFeedResponse {
return { v: 1, cellId, incarnation: `inc-${cellId}`, seq: 1, at: 1, full: [], ...overrides }
}
function seat(epoch: number, state = 'active', relayHostId = HOST, joinedAt = 1) {
return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt }
}
function directoryWith(
seats: Record<string, ReturnType<typeof seat>[]>,
noFeed: string[] = [],
unlive: string[] = []
) {
const directory = new ShadowSeatDirectory()
const cellIds = [...Object.keys(seats), ...noFeed]
directory.setCells(cellIds, cellIds, {
readAt: 0,
expiresAt: new Map(
cellIds.filter((cellId) => !unlive.includes(cellId)).map((cellId) => [cellId, 1e12])
)
})
for (const [cellId, full] of Object.entries(seats)) {
directory.apply(cellId, feed(cellId, { full }), 10)
}
for (const cellId of noFeed) directory.markNoFeed(cellId, 10)
return directory
}
describe('unexplained samples', () => {
it('log only the cell and epoch of the real database record, never raw ids', () => {
const lines: Record<string, unknown>[] = []
const compare = new ShadowDirectoryCompare(
directoryWith({ 'cell-a': [seat(3)] }),
() => 20,
(line) => lines.push(JSON.parse(line))
)
compare.compare('sticky-verify', IDENTITY, { ...assignment('cell-a'), assignmentEpoch: 2 })
compare.flush(20)
const sample = lines.find((line) => line.event === 'orca_relay_shadow_compare_unexplained')
expect(sample).toMatchObject({
class: 'epoch-mismatch',
db: { cellId: 'cell-a', assignmentEpoch: 2 }
})
expect(JSON.stringify(lines)).not.toContain(HOST)
expect(JSON.stringify(lines)).not.toContain('user-a')
})
})
describe('compare hooks on the director routes', () => {
async function respond(compareShadowSeats?: () => void) {
const resolved = assignment('cell-a')
const app = createRelayApp(config(), {
// SAFETY: /v1/resolve reads only resolveResume.
store: { resolveResume: vi.fn(async () => ({ userId: 'user-a' })) } as never,
// SAFETY: both routes read only resolve and assign.
assignments: {
resolve: vi.fn(async () => resolved),
assign: vi.fn(async () => resolved)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true),
compareShadowSeats
})
const sticky = await app.request('/v1/assign', {
method: 'POST',
headers: { authorization: `Bearer ${HOST}`, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, relayHostId: HOST, reconnect: true })
})
const resolve = await app.request('/v1/resolve', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
v: 1,
relayHostId: HOST,
resumeToken: Buffer.alloc(32, 1).toString('base64url')
})
})
return [
{ status: sticky.status, body: await sticky.text() },
{ status: resolve.status, body: await resolve.text() }
]
}
it('compares after each database answer without changing a response byte', async () => {
vi.useFakeTimers({ now: 1_000_000 })
try {
const baseline = await respond()
const routes: unknown[] = []
const compared = await respond((...args: unknown[]) => {
routes.push(args[0])
})
const throwing = await respond(() => {
throw new Error('compare broke')
})
expect(baseline[0]?.status).toBe(200)
expect(baseline[1]?.status).toBe(200)
expect(compared).toEqual(baseline)
expect(throwing).toEqual(baseline)
expect(routes).toEqual(['sticky-verify', 'resolve'])
} finally {
vi.useRealTimers()
}
})
})
function assignment(cellId: string): RelayAssignment {
return {
userId: 'user-a',
relayHostId: HOST,
cellId,
cellUrl: `https://${cellId}.relay.example.test`,
assignmentEpoch: 3,
leaseExpiresAt: 1_300_000
}
}
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://relay.example.test',
cellUrl: 'https://relay.example.test',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
role: 'director',
cellId: 'director',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'runtime@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 3,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data'
}
}
@@ -3,7 +3,6 @@ import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
import { openInMemoryRelayDatabase } from '../../database/database.js'
import {
SHADOW_SEAT_OUTRANK_SKEW_MS,
SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS,
ShadowSeatDirectory,
startShadowSeatPoller,
type SeatFeedCell,
@@ -141,7 +140,6 @@ describe('startShadowSeatPoller', () => {
{ cellId: 'cell-a', epoch: 3, generation: 1, state: 'active' }
])
expect(cell.directory.cellState('cell-a')).toMatchObject({ status: 'live', seq: 5 })
expect(cell.identityToken).toHaveBeenCalledTimes(1)
})
it('resyncs from a full snapshot when the cell restarts with a new incarnation', async () => {
@@ -284,7 +282,6 @@ describe('startShadowSeatPoller', () => {
await cell.tick()
await cell.tick()
cell.stop()
expect(call).toBe(3)
expect(cell.requests[2]?.searchParams.get('since')).toBe(`${INCARNATION}:1`)
expect(cell.directory.seatsOf('user-a', 'host-2')).toHaveLength(1)
})
@@ -336,31 +333,6 @@ describe('startShadowSeatPoller', () => {
expect(cell.directory.isComplete()).toBe(true)
})
it('forgets the switches a cell reported once its image no longer has the feed', async () => {
let rolledBack = false
const cell = poller({
'cell-a': () => json(feed({ seq: 1, full: [] })),
'cell-b': () =>
rolledBack
? new Response('Not Found', { status: 404 })
: json(
feed({
cellId: 'cell-b',
seq: 1,
full: [],
flagsApplied: { generation: 3, flags: { readinessLocal: true } }
})
)
})
await cell.tick()
expect(cell.directory.cellState('cell-b')?.flagsApplied?.generation).toBe(3)
rolledBack = true
await cell.tick()
cell.stop()
expect(cell.directory.cellState('cell-b')).toMatchObject({ status: 'no-feed' })
expect(cell.directory.cellState('cell-b')?.flagsApplied).toBeUndefined()
})
it('keeps a reserve switch through a 404 until a feed says db or the process is replaced', () => {
const directory = new ShadowSeatDirectory()
directory.setCells(['cell-b'])
@@ -390,47 +362,6 @@ describe('startShadowSeatPoller', () => {
expect(cell.directory.isComplete()).toBe(true)
})
it('backs off cell-list reads while the database fails', async () => {
const listCells = vi.fn(async (): Promise<SeatFeedCell[]> => {
throw new Error('pool timeout')
})
const cell = poller({}, 'all', listCells)
for (let second = 0; second < 40; second += 1) {
await cell.tick()
cell.advance(1_000)
}
cell.stop()
// Retries at +1, +2, +4, +8, +16 s, then at most every 30 s.
expect(listCells.mock.calls.length).toBeLessThanOrEqual(7)
expect(listCells.mock.calls.length).toBeGreaterThanOrEqual(5)
})
it('keeps polling the last cell list while a cell-list read hangs', async () => {
const readEntered = Promise.withResolvers<void>()
const readDone = Promise.withResolvers<SeatFeedCell[]>()
const listCells = vi
.fn<() => Promise<SeatFeedCell[]>>()
.mockResolvedValueOnce(CELLS)
.mockImplementation(() => {
readEntered.resolve()
return readDone.promise
})
const cell = poller({ 'cell-a': () => json(feed({ seq: 1, full: [] })) }, ['cell-a'], listCells)
await cell.tick()
cell.advance(31_000)
const round = cell.tick()
await readEntered.promise
await Promise.resolve()
expect(listCells).toHaveBeenCalledTimes(2)
expect(cell.requests.map((url) => url.hostname)).toEqual([
'cell-a.example.test',
'cell-a.example.test'
])
readDone.resolve(CELLS)
await round
cell.stop()
})
it('treats a malformed or mislabelled body as unverifiable', async () => {
const cell = poller({
'cell-a': () => json({ v: 2 }),
@@ -483,20 +414,6 @@ describe('startShadowSeatPoller', () => {
expect(cell.directory.seatsOf('user-a', 'host-1')).toEqual([])
})
it('bounds the recently-left memory', () => {
const directory = new ShadowSeatDirectory()
directory.setCells(['cell-a'])
const hosts = SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS + 10
directory.apply(
'cell-a',
feed({ seq: 1, full: Array.from({ length: hosts }, (_, index) => seat(`host-${index}`)) }),
10
)
directory.apply('cell-a', feed({ incarnation: 'restarted', seq: 1, full: [] }), 20)
expect(directory.summary(20).recentlyLeftHosts).toBe(SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS)
expect(directory.recentlyLeftOf('user-a', 'host-0', 20)).toEqual([])
expect(directory.recentlyLeftOf('user-a', `host-${hosts - 1}`, 20)).toHaveLength(1)
})
})
describe('RelayAssignmentStore.seatFeedCells', () => {
@@ -22,11 +22,6 @@ const OTHER_CELL = {
capacityRequests: 900
}
const RESUME_DEVICE_READ = expect.stringMatching(/^SELECT .* FROM relay_devices\b/)
const HOST_ASSIGNMENT_READ = expect.stringMatching(
/^SELECT assignment\.\*.* FROM relay_assignments\b/
)
async function until(check: () => Promise<boolean>, timeoutMs = 10_000): Promise<void> {
const deadline = Date.now() + timeoutMs
while (!(await check())) {
@@ -58,11 +53,7 @@ describe('the memory phone path', () => {
cleanup,
...(now ? { now } : {})
})
// The admit-effective writer ticks on its own; every other statement is the phone's.
const phoneStatements = () =>
database.statements.filter((sql) => !sql.includes('relay_cell_admit_effective'))
// The test's own reads and writes go around the recorder.
return { ...reserve, database, raw: inner, phoneStatements }
return { ...reserve, database, raw: inner }
}
async function seatedPhoneHost(phonePath: 'db' | 'memory') {
@@ -70,7 +61,6 @@ describe('the memory phone path', () => {
reserve.setAdmitMode('reserve', { phonePath })
const seated = await reserve.bookedHost(3)
await reserve.installDevice(seated.identity.hostId, TOKEN)
reserve.database.statements.length = 0
return { reserve, seated }
}
@@ -90,7 +80,6 @@ describe('the memory phone path', () => {
const basisConnId = String((await connOpen).connId)
await reserve.attachData(await connOpen, seated.ack.generation)
expect(await phone.hello).toMatchObject({ ok: true })
reserve.database.statements.length = 0
// The desktop's confirm, sent in reply to the phone over the end-to-end channel.
const confirm = async (reqId: string, basis = basisConnId) => {
const answer = Promise.race([
@@ -108,26 +97,17 @@ describe('the memory phone path', () => {
const deviceRow = async (reserve: Awaited<ReturnType<typeof cell>>) =>
(await reserve.raw.query(`SELECT current_expires_at FROM relay_devices`))[0]
const warnings = (event: string) =>
vi
.mocked(console.warn)
.mock.calls.map(([line]) => String(line))
.filter((line) => line.includes(event))
.map((line) => JSON.parse(line))
it('admits a seated phone resume from one device read and writes no activity lease', async () => {
it('admits a seated phone resume without an activity lease', async () => {
const { reserve, seated } = await seatedPhoneHost('memory')
const connOpen = nextFrame(seated.socket, 'conn-open')
const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
expect(await connOpen).toMatchObject({ kind: 'resume', relayDeviceId: 'device-1' })
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
await reserve.attachData(await connOpen, seated.ack.generation)
expect(await phone.hello).toMatchObject({
type: 'relay-hello',
ok: true,
credentialKind: 'resume'
})
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
expect(await reserve.activityLeases()).toEqual([])
})
@@ -155,7 +135,7 @@ describe('the memory phone path', () => {
])
})
it('closes a revoked or expired device 4401 and never tries it as an invite', async () => {
it('closes a revoked or expired device 4401', async () => {
const reserve = await cell()
reserve.setAdmitMode('reserve', { phonePath: 'memory' })
const seated = await reserve.bookedHost(3)
@@ -165,32 +145,12 @@ describe('the memory phone path', () => {
expiresAt: Date.now() - 1_000
})
for (const refused of [token(2), token(3)]) {
reserve.database.statements.length = 0
const phone = await reserve.dialPhone(seated.identity.hostId, refused)
expect(await phone.hello).toEqual({ type: 'relay-hello', ok: false, code: 4401 })
expect((await phone.closed).code).toBe(4401)
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
}
})
it('closes 4503 when the device read fails, without trying the invite path', async () => {
const { reserve, seated } = await seatedPhoneHost('memory')
vi.spyOn(reserve.relay.store, 'lookupResume').mockRejectedValue(
new Error('connection terminated')
)
const invite = vi.spyOn(reserve.relay.store, 'resolveInviteForMove')
for (let dial = 0; dial < 2; dial += 1) {
const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
expect((await phone.closed).code).toBe(4503)
}
expect(invite).not.toHaveBeenCalled()
expect(reserve.phoneStatements()).toEqual([])
// One line opens the window; the repeat waits for its closing line.
expect(warnings('orca_relay_client_accept_db_error')).toEqual([
{ event: 'orca_relay_client_accept_db_error', cause: 'error', cellId: reserve.config.cellId }
])
})
it('falls through to the invite path only when no device row matches', async () => {
const { reserve, seated } = await seatedPhoneHost('memory')
const created = nextFrame(seated.socket, 'invite-created')
@@ -208,7 +168,7 @@ describe('the memory phone path', () => {
])
})
it('answers a phone whose host has no seat here from one assignment read', async () => {
it('distinguishes moved and offline hosts when a phone finds no seat here', async () => {
const reserve = await cell()
reserve.setAdmitMode('reserve', { phonePath: 'memory' })
await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL])
@@ -222,31 +182,19 @@ describe('the memory phone path', () => {
)
}
await reserve.installDevice(host.hostId, TOKEN)
reserve.database.statements.length = 0
const phone = await reserve.dialPhone(host.hostId, TOKEN)
const closed = await phone.closed
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ, HOST_ASSIGNMENT_READ])
return closed.code
}
expect(await dial('elsewhere')).toBe(4409)
expect(await dial('none')).toBe(4404)
expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([])
expect(await dial('here')).toBe(4404)
expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([
expect.objectContaining({
cellId: reserve.config.cellId,
relayHostIdDigest: expect.any(String)
})
])
vi.spyOn(reserve.relay.assignments, 'resolve').mockRejectedValueOnce(
new Error('connection terminated')
)
const host = await reserve.host()
await reserve.installDevice(host.hostId, TOKEN)
expect((await (await reserve.dialPhone(host.hostId, TOKEN)).closed).code).toBe(4409)
expect(warnings('orca_relay_client_miss_read_error')).toEqual([
{ event: 'orca_relay_client_miss_read_error', cause: 'error', cellId: reserve.config.cellId }
])
})
it('takes the database path for a phone whose switch turned off during its device read', async () => {
@@ -265,7 +213,7 @@ describe('the memory phone path', () => {
])
})
it('confirms from the memory basis with one fresh read, renewing a device due for it', async () => {
it('confirms from the memory basis and renews a device due for it', async () => {
const { reserve, confirm } = await splicedPhone('memory')
const before = Number((await deviceRow(reserve))?.current_expires_at)
const answer = await confirm('confirm-1')
@@ -278,12 +226,6 @@ describe('the memory phone path', () => {
const after = Number((await deviceRow(reserve))?.current_expires_at)
expect(after).toBeGreaterThan(before)
expect(answer.resumeExpiresAt).toBe(after)
expect(reserve.phoneStatements()).toEqual([
RESUME_DEVICE_READ,
expect.stringMatching(
/^UPDATE relay_devices SET current_expires_at = \?.*RETURNING current_expires_at$/
)
])
expect(await reserve.activityLeases()).toEqual([])
// Its audit row leaves with the next batch.
await reserve.relay.store.resumeAudit.flush()
@@ -293,10 +235,8 @@ describe('the memory phone path', () => {
expect(audits.map((row) => JSON.parse(String(row.detail_json)))).toEqual([
{ reqId: 'confirm-1', basisConnId: expect.any(String), renewed: true }
])
// A retry of the same request answers from memory; another basis cannot reuse its reqId.
reserve.database.statements.length = 0
// A retry keeps its answer; another basis cannot reuse its reqId.
expect(await confirm('confirm-1')).toEqual(answer)
expect(reserve.phoneStatements()).toEqual([])
expect(await confirm('confirm-1', 'another-basis')).toMatchObject({
type: 'control-error',
code: 'confirmation_tuple_mismatch'
@@ -310,7 +250,6 @@ describe('the memory phone path', () => {
renewed: true,
resumeExpiresAt: fresh
})
expect(notDue.reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
const due = Date.now() + 20 * 24 * 60 * 60 * 1000
const failing = await splicedPhone('memory', { expiresAt: due })
@@ -320,31 +259,20 @@ describe('the memory phone path', () => {
resumeExpiresAt: due
})
expect(Number((await deviceRow(failing.reserve))?.current_expires_at)).toBe(due)
// Never silent: a renewal left failing ages phones toward re-pairing.
expect(warnings('orca_relay_resume_renewal_failed')).toEqual([
{
event: 'orca_relay_resume_renewal_failed',
cause: 'error',
cellId: failing.reserve.config.cellId
}
])
})
it('replays a stored answer after its splice closed, and drops the basis itself', async () => {
const { reserve, phone, confirm } = await splicedPhone('memory')
const { phone, confirm } = await splicedPhone('memory')
const answer = await confirm('confirm-1')
phone.socket.close()
await phone.closed
await new Promise((resolve) => setTimeout(resolve, 50))
reserve.database.statements.length = 0
expect(await confirm('confirm-1')).toEqual(answer)
expect(await confirm('confirm-1', 'another-basis')).toMatchObject({
code: 'confirmation_tuple_mismatch'
})
expect(reserve.phoneStatements()).toEqual([])
// A new request on the closed basis finds none in memory, so it goes to the database.
expect(await confirm('confirm-2')).toMatchObject({ code: 'confirmation_not_active' })
expect(reserve.phoneStatements()[0]).toBe('BEGIN')
})
it('answers a grace-version confirm renewed:false with its expiry unchanged, as today', async () => {
@@ -363,31 +291,6 @@ describe('the memory phone path', () => {
renewed: false,
resumeExpiresAt: stored
})
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
})
it('answers every device change since the accept read as the database confirm does', async () => {
const changes: Record<string, [string, unknown[]]> = {
revoked: ['UPDATE relay_devices SET revoked_at = ?', [Date.now() - 1]],
expired: ['UPDATE relay_devices SET current_expires_at = ?', [Date.now() - 1]],
retired: [
"UPDATE relay_devices SET current_hash = 'rotated-twice', current_version = 3, grace_version = 2",
[]
],
unchanged: ['UPDATE relay_devices SET updated_at = updated_at', []]
}
const outcome = async (phonePath: 'db' | 'memory', [sql, params]: [string, unknown[]]) => {
const { reserve, confirm } = await splicedPhone(phonePath)
await reserve.raw.query(sql, params)
const answer = await confirm('confirm-1')
return answer.type === 'control-error'
? answer.code
: `${answer.type}:${answer.acceptedAs}:${answer.renewed}`
}
for (const [name, change] of Object.entries(changes)) {
expect(await outcome('memory', change), name).toBe(await outcome('db', change))
}
expect(await outcome('memory', changes.revoked!)).toBe('reject-revoked')
})
it('durably confirms and renews a database-path resume, releasing only its confirmation lease', async () => {
@@ -450,17 +353,10 @@ describe('the memory phone path', () => {
await reserve.relay.assignments.setCellAdmitMode(reserve.config.cellId, 'reserve')
rollback.roll(reserve, advance)
expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
reserve.database.statements.length = 0
expect(await spliced.confirm('confirm-1')).toMatchObject({
type: 'device-resume-confirmed',
renewed: true
})
expect(
reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))
).toEqual([
RESUME_DEVICE_READ,
expect.stringMatching(/^UPDATE relay_devices SET current_expires_at/)
])
// Only the control lease a flip back re-registers; nothing for the phone.
const leases = async () =>
(await reserve.activityLeases()).map((lease) => lease.activity_kind)
@@ -474,14 +370,9 @@ describe('the memory phone path', () => {
).rejects.toThrow('cell_in_reserve_mode')
// The splice still runs, and closing it releases nothing it never took.
expect(spliced.phone.socket.readyState).toBe(spliced.phone.socket.OPEN)
// Let a flip back finish leasing its control (and fold its relay_cells write) first.
await until(async () => reserve.relay.sessions.reregistrationPending() === 0)
await new Promise((resolve) => setTimeout(resolve, 250))
reserve.database.statements.length = 0
spliced.phone.socket.close()
await spliced.phone.closed
await new Promise((resolve) => setTimeout(resolve, 50))
expect(reserve.phoneStatements()).toEqual([])
expect((await leases()).filter((kind) => kind !== 'control')).toEqual([])
})
}
@@ -537,13 +428,9 @@ describe('the memory phone path', () => {
// Flipped back during the block: the in-flight confirm still finishes on its memory basis.
reserve.setAdmitMode('db', { phonePath: 'memory' })
reserve.database.failing = null
reserve.database.statements.length = 0
expect(await confirm('confirm-c', inFlight.basisConnId)).toMatchObject({
type: 'device-resume-confirmed'
})
expect(reserve.phoneStatements().find((sql) => !sql.includes('relay_assignment'))).toEqual(
RESUME_DEVICE_READ
)
// And new resumes take the database path again.
const connOpen = nextFrame(seated.socket, 'conn-open')
await reserve.dialPhone(seated.identity.hostId, token(1))
@@ -1,37 +0,0 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
import { RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract/host-close-reason'
import { describe, expect, it } from 'vitest'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
function memoryAt(clock: { now: number }): HostCloseReasonMemory {
return new HostCloseReasonMemory(() => clock.now)
}
describe('HostCloseReasonMemory', () => {
it('expires an entry once its host may have been rebalanced away', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += 1
expect(memory.read('a')).toBeNull()
expect(memory.size()).toBe(0)
})
it('drops the oldest survivors rather than growing without bound', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
for (let index = 0; index < 50_050; index++) {
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
}
expect(memory.size()).toBe(50_000)
expect(memory.read('host-0')).toBeNull()
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
})
@@ -1,4 +1,3 @@
import nacl from 'tweetnacl'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
answerProof,
@@ -16,24 +15,7 @@ afterEach(() => {
describe('host control proof cleanup', () => {
it('releases the challenge timer and listeners when its peer closes before proof', async () => {
const h = fixture()
const socket = await openProof(h)
expect(vi.getTimerCount()).toBe(1)
expect(socket.listenerCount('message')).toBe(1)
socket.peerClose()
expect({
timers: vi.getTimerCount(),
message: socket.listenerCount('message'),
close: socket.listenerCount('close')
}).toEqual({ timers: 0, message: 0, close: 0 })
await vi.advanceTimersByTimeAsync(10_000)
expect(socket.close).not.toHaveBeenCalled()
expect(h.activate).not.toHaveBeenCalled()
expect(h.database.query).not.toHaveBeenCalled()
})
it('does no challenge crypto, send, timer or registration after a closed peer finishes verification', async () => {
it('does not challenge or register a closed peer after verification finishes', async () => {
const h = fixture()
let finish!: (valid: boolean) => void
h.verify.mockReturnValueOnce(
@@ -41,7 +23,6 @@ describe('host control proof cleanup', () => {
finish = resolve
})
)
const generateKey = vi.spyOn(nacl.box, 'keyPair')
const socket = new ProofSocket()
h.registry.acceptControl(socket.registrySocket(), h.identity)
socket.emit('message', Buffer.from(h.hello), false)
@@ -50,11 +31,6 @@ describe('host control proof cleanup', () => {
finish(true)
await vi.advanceTimersByTimeAsync(0)
expect(socket.send).not.toHaveBeenCalled()
expect(generateKey).not.toHaveBeenCalled()
expect(vi.getTimerCount()).toBe(0)
expect(socket.listenerCount('message')).toBe(0)
expect(h.activate).not.toHaveBeenCalled()
expect(h.database.query).not.toHaveBeenCalled()
expect(
h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })
).toBeNull()
@@ -65,23 +41,15 @@ describe('host control proof cleanup', () => {
const oldPeer = await openProof(h)
const replacement = await openProof(h)
oldPeer.peerClose()
expect(vi.getTimerCount()).toBe(1)
answerProof(replacement, h.keyPair)
await vi.advanceTimersByTimeAsync(0)
expect(h.activate).toHaveBeenCalledOnce()
expect(h.recordAuth).toHaveBeenCalledExactlyOnceWith(true)
expect(
h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })?.socket
).toBe(replacement)
expect(replacement.send).toHaveBeenCalledTimes(2)
expect(replacement.listenerCount('message')).toBe(1)
expect(replacement.listenerCount('close')).toBe(2)
expect(vi.getTimerCount()).toBe(1)
await vi.advanceTimersByTimeAsync(10_000)
expect(oldPeer.close).not.toHaveBeenCalled()
expect(replacement.close).not.toHaveBeenCalled()
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
expect(vi.getTimerCount()).toBe(0)
})
})
@@ -1,4 +1,3 @@
import { CONTROL_CONTINUITY_LIMITS } from '@orca-cloud/relay-contract/control-continuity'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
@@ -46,7 +45,6 @@ async function lapsedHost() {
)
const session = h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })
expect(session?.state).toBe('orphaned')
expect(CONTROL_CONTINUITY_LIMITS.orphanGraceMs).toBeGreaterThan(5_000)
await vi.advanceTimersByTimeAsync(5_000)
return { h, ack, session: session!, resumeHello: resumeHelloFor(h.hello, ack) }
}
@@ -229,7 +229,6 @@ describe('client accept abandoned mid-DB-phase', () => {
capacity
)
await vi.advanceTimersByTimeAsync(0)
expect(h.acquireActivity).toHaveBeenCalledOnce()
// The phone's 12s bound fires while the cell still waits on Postgres.
client.close(1000, 'client bound')
capacity.release()
@@ -247,54 +246,12 @@ describe('client accept abandoned mid-DB-phase', () => {
{ userId: identity.sub, relayHostId: identity.relayHostId },
expect.stringMatching(/^confirmation:/)
)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'activity',
expect.any(Number)
)
const line = warn.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('orca_relay_client_accept_abandoned'))
expect(line).toBeDefined()
expect(JSON.parse(line!)).toMatchObject({ stage: 'activity' })
expect(line).not.toContain(identity.relayHostId)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('stops after a slow credential reservation without acquiring an activity lease', async () => {
const h = harness()
await activeHost(h)
const slowReserve = deferred<CredentialReservation>()
h.store.reserveCredential.mockReturnValueOnce(slowReserve.promise)
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowReserve.resolve(reservation)
await accepting
expect(h.acquireActivity).not.toHaveBeenCalled()
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'credential',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
// The attach resolves its owning session once and hands it to the unfenced leg;
@@ -360,7 +317,6 @@ describe('host data attach ownership', () => {
1
)
await vi.advanceTimersByTimeAsync(0)
expect(h.store.recordConnectionBasis).toHaveBeenCalledOnce()
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('relay-hello'))
basis.resolve()
expect(await attaching).toBe(true)
@@ -420,17 +376,4 @@ describe('paced drain and the phones of a host not yet told', () => {
expect(served.close).not.toHaveBeenCalled()
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
})
it("refuses that host's phones as soon as its own drain is sent", async () => {
const { h, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(40_000)
// Told by its zero-grace close, a timer nested 1 ms after the send.
await vi.advanceTimersByTimeAsync(1)
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
const refused = await dial(h, laterHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
})
})

Some files were not shown because too many files have changed in this diff Show More