mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
Merge main into brennanb2025/main-terminal-replay
Resolve conflicts onto main's reorganized modules: provider imports move to providers/pty and providers/working-directory; the terminal CLI recovery hints move to src/cli/terminals beside main's cli-error layout. Accept main's deletion of two low-value tests whose only changes here adapted them to this branch's APIs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -4,7 +4,7 @@
|
||||
/skill-stubs/*.md text eol=lf
|
||||
/skill-stubs/_shared/*.md text eol=lf
|
||||
/skills/*/SKILL.md text eol=lf
|
||||
/src/cli/bundled-skill-guides.ts text eol=lf
|
||||
/src/cli/skills/bundled-skill-guides.ts text eol=lf
|
||||
# SVG bytes feed embedded data URLs and asset hashes on every build host.
|
||||
*.svg text eol=lf
|
||||
# Compiler input line breaks must not depend on the checkout OS.
|
||||
|
||||
@@ -12,24 +12,57 @@ body:
|
||||
|
||||
Please write your issue in English.
|
||||
|
||||
- type: dropdown
|
||||
id: where
|
||||
attributes:
|
||||
label: Where were you using Orca?
|
||||
description: Where you saw the bug.
|
||||
options:
|
||||
- Desktop only
|
||||
- Desktop + SSH host
|
||||
- Desktop + Orca server
|
||||
- Desktop + WSL
|
||||
- Mobile app
|
||||
- Web browser
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
description: 'The device you were using. WSL: pick Windows. Phone: pick iOS or Android.'
|
||||
options:
|
||||
- macOS
|
||||
- Windows
|
||||
- Linux
|
||||
- iOS
|
||||
- Android
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: remote_os
|
||||
attributes:
|
||||
label: Remote host operating system
|
||||
description: Only for SSH, Orca server or web.
|
||||
options:
|
||||
- Linux
|
||||
- macOS
|
||||
- Windows
|
||||
- Other
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: orca_version
|
||||
attributes:
|
||||
label: Orca version
|
||||
description: Include the exact Orca version if you can. You can find it in `Orca > About Orca`.
|
||||
placeholder: v0.14.2
|
||||
description: 'macOS: Orca > About Orca. Windows/Linux: Help > About Orca. Mobile: Settings > About.'
|
||||
placeholder: v1.4.2
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
@@ -1,28 +1,16 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
CloudSqlRolloutLease,
|
||||
LEASE_TTL_MS,
|
||||
LeaseConflict,
|
||||
LeaseUnreadable
|
||||
} from './storage-lease.mjs'
|
||||
import { CloudSqlRolloutLease, LeaseConflict, LeaseUnreadable } from './storage-lease.mjs'
|
||||
|
||||
const BUCKET = 'onorca-cloud-terraform-state'
|
||||
const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock'
|
||||
const NOW = 1_756_000_000_000
|
||||
|
||||
/** Model real GCS generation increments, ifGenerationMatch and 412 conflicts; faults inject failures. */
|
||||
function fakeStorage({ object = null, faults = [] } = {}) {
|
||||
const state = { object, requests: [] }
|
||||
function fakeStorage({ object = null } = {}) {
|
||||
const state = { object }
|
||||
const fetcher = async (rawUrl, init = {}) => {
|
||||
const url = new URL(rawUrl)
|
||||
const method = init.method ?? 'GET'
|
||||
const record = { method, url, path: url.pathname, search: url.searchParams }
|
||||
state.requests.push(record)
|
||||
const fault = faults.find((candidate) => candidate.when(record))
|
||||
if (fault) {
|
||||
return json(fault.status, fault.body ?? {})
|
||||
}
|
||||
|
||||
if (url.pathname.startsWith('/upload/')) {
|
||||
const want = url.searchParams.get('ifGenerationMatch')
|
||||
@@ -63,15 +51,10 @@ function json(status, body) {
|
||||
}
|
||||
}
|
||||
|
||||
function storedRecord({
|
||||
holderKey,
|
||||
expiresAt,
|
||||
repository = 'stablyai/orca',
|
||||
workflow = 'Deploy Relay Production'
|
||||
}) {
|
||||
function storedRecord({ holderKey, expiresAt }) {
|
||||
return {
|
||||
repository,
|
||||
workflow,
|
||||
repository: 'stablyai/orca',
|
||||
workflow: 'Deploy Relay Production',
|
||||
run_id: '9001',
|
||||
run_url: 'https://github.com/stablyai/orca/actions/runs/9001',
|
||||
run_attempt: '1',
|
||||
@@ -81,14 +64,14 @@ function storedRecord({
|
||||
}
|
||||
}
|
||||
|
||||
function leaseFor(storage, { warn = () => {} } = {}) {
|
||||
function leaseFor(storage) {
|
||||
return new CloudSqlRolloutLease({
|
||||
bucket: BUCKET,
|
||||
objectName: OBJECT,
|
||||
accessToken: 'test-token',
|
||||
fetcher: storage.fetcher,
|
||||
now: () => NOW,
|
||||
warn
|
||||
warn: () => {}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -114,77 +97,24 @@ test('refuses a live lease held by another run and never writes', async () => {
|
||||
.catch((thrown) => thrown)
|
||||
|
||||
assert.ok(error instanceof LeaseConflict)
|
||||
assert.equal(error.holder.repository, 'stablyai/orca')
|
||||
assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001')
|
||||
assert.equal(
|
||||
storage.state.requests.filter((request) => request.method !== 'GET').length,
|
||||
0,
|
||||
'a foreign live lease must not be written'
|
||||
)
|
||||
assert.equal(storage.state.object.generation, '1500')
|
||||
})
|
||||
|
||||
test('re-enters a live lease this run already holds and extends it', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
generation: '1500',
|
||||
body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 })
|
||||
}
|
||||
})
|
||||
const warnings = []
|
||||
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
|
||||
HOLDER
|
||||
)
|
||||
|
||||
assert.equal(claim.state, 'reentrant')
|
||||
assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover')
|
||||
assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved')
|
||||
assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS)
|
||||
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
|
||||
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
|
||||
assert.equal(storage.state.object.generation, '1501')
|
||||
})
|
||||
|
||||
test('takes over an expired lease and warns naming the stale holder', async () => {
|
||||
test('takes over an expired lease', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
generation: '1500',
|
||||
body: storedRecord({
|
||||
holderKey: 'stablyai/orca/9001',
|
||||
expiresAt: NOW - 1,
|
||||
repository: 'stablyai/orca',
|
||||
workflow: 'Deploy Relay Production Capacity'
|
||||
expiresAt: NOW - 1
|
||||
})
|
||||
}
|
||||
})
|
||||
const warnings = []
|
||||
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
|
||||
HOLDER
|
||||
)
|
||||
await leaseFor(storage).acquire(HOLDER)
|
||||
|
||||
assert.equal(claim.state, 'takeover')
|
||||
assert.equal(warnings.length, 1)
|
||||
assert.match(warnings[0], /stablyai\/orca/)
|
||||
assert.match(warnings[0], /Deploy Relay Production Capacity/)
|
||||
assert.match(warnings[0], /actions\/runs\/9001/)
|
||||
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
|
||||
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
|
||||
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
|
||||
})
|
||||
|
||||
test('releases with a generation match and leaves the object gone', async () => {
|
||||
const storage = fakeStorage()
|
||||
const lease = leaseFor(storage)
|
||||
const claim = await lease.acquire(HOLDER)
|
||||
|
||||
const released = await lease.release(HOLDER.holderKey)
|
||||
|
||||
assert.deepEqual(released, { released: true, generation: claim.generation })
|
||||
const remove = storage.state.requests.find((request) => request.method === 'DELETE')
|
||||
assert.equal(remove.search.get('ifGenerationMatch'), claim.generation)
|
||||
assert.equal(storage.state.object, null)
|
||||
})
|
||||
|
||||
test('refuses to release a lease another run now holds', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
@@ -212,17 +142,3 @@ test('treats an unreadable record as held, not free', async () => {
|
||||
assert.ok(error instanceof LeaseUnreadable)
|
||||
assert.equal(storage.state.object.generation, '1500')
|
||||
})
|
||||
|
||||
test('renewal rewrites only expires_at on the observed generation', async () => {
|
||||
const storage = fakeStorage()
|
||||
const lease = leaseFor(storage)
|
||||
await lease.acquire(HOLDER)
|
||||
storage.state.object.body.expires_at = NOW - 1
|
||||
|
||||
const renewed = await lease.renew(HOLDER.holderKey)
|
||||
|
||||
assert.equal(renewed.renewed, true)
|
||||
assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS)
|
||||
assert.equal(storage.state.object.body.acquired_at, NOW)
|
||||
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
|
||||
})
|
||||
|
||||
@@ -62,11 +62,24 @@ jobs:
|
||||
current.closed_at === pr.closed_at
|
||||
}
|
||||
if (!Number.isSafeInteger(pr.number) || pr.number <= 0 ||
|
||||
(superseded ? !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? '') :
|
||||
!Number.isFinite(Date.parse(pr.closed_at)))) {
|
||||
(superseded && !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? ''))) {
|
||||
throw new Error('Missing PR identity')
|
||||
}
|
||||
if (!await stillCurrent()) return
|
||||
if (!superseded && !Number.isFinite(Date.parse(pr.closed_at))) {
|
||||
// A close quickly followed by a reopen can deliver a payload rendered after the reopen.
|
||||
const { data: current } = await github.rest.pulls.get({
|
||||
...context.repo, pull_number: pr.number
|
||||
})
|
||||
if (current.state !== 'closed' || current.merged_at !== null) {
|
||||
core.info(`PR #${pr.number} changed after this close event; skipping cancellation`)
|
||||
return
|
||||
}
|
||||
throw new Error('Missing PR identity')
|
||||
}
|
||||
if (!await stillCurrent()) {
|
||||
core.info(`PR #${pr.number} changed after this event; skipping cancellation`)
|
||||
return
|
||||
}
|
||||
for (const [prefix, workflow, suffix = ''] of targets) {
|
||||
const group = prefix === 'cloud-verify-refs/pull'
|
||||
? `${prefix}/${pr.number}${suffix}` : `${prefix}-${pr.number}${suffix}`
|
||||
|
||||
@@ -38,8 +38,8 @@ on:
|
||||
- '!src/main/daemon/executable-host/windows/testing/daemon-host-relocation.test-fixture.ts'
|
||||
- '!src/main/daemon/endpoint/publication/testing/daemon-endpoint-reachability-test-harness.ts'
|
||||
- 'src/main/providers/local-pty/local-pty-provider.ts'
|
||||
- 'src/main/providers/windows-shell-args.ts'
|
||||
- 'src/main/providers/windows-shell-preflight-runtime.windows.test.ts'
|
||||
- 'src/main/providers/shell-launch/windows/shell-args.ts'
|
||||
- 'src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts'
|
||||
- 'native/computer-use-macos/**'
|
||||
- 'native/computer-use-linux/**'
|
||||
- 'native/computer-use-windows/**'
|
||||
@@ -53,7 +53,8 @@ on:
|
||||
- 'src/shared/computer-use-*.ts'
|
||||
- 'tests/e2e/computer-linux.e2e.ts'
|
||||
- 'tests/e2e/computer-windows.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-store.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-background.e2e.ts'
|
||||
- 'tests/e2e/helpers/windows-semantic-controls-fixture.*'
|
||||
- 'tests/e2e/helpers/computer-cli-driver.ts'
|
||||
- 'tests/e2e/helpers/computer-driver.ts'
|
||||
- 'tests/e2e/vitest.config.ts'
|
||||
@@ -95,11 +96,10 @@ jobs:
|
||||
config/scripts/performance/macos-computer-owner/macos-computer-helper-owner-loss-processes.test.mjs
|
||||
src/main/computer/sidecar-transport/sidecar-client.test.ts
|
||||
src/main/computer/macos-native/macos-native-provider-client.test.ts
|
||||
src/main/providers/windows-shell-preflight-runtime.windows.test.ts
|
||||
src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts
|
||||
src/main/computer/desktop-script/actions/desktop-script-provider-actions.test.ts
|
||||
src/cli/format.test.ts
|
||||
src/cli/handlers/computer-action-validation.test.ts
|
||||
src/cli/handlers/computer-state-formatting.test.ts
|
||||
src/cli/output/command-result-output.test.ts
|
||||
src/cli/computer/computer-state-formatting.test.ts
|
||||
src/cli/index.test.ts
|
||||
src/cli/runtime/envelope-schema.test.ts
|
||||
src/shared/remote-runtime/remote-runtime-client.test.ts
|
||||
@@ -152,4 +152,4 @@ jobs:
|
||||
- run: pnpm verify:computer-native
|
||||
- run: pnpm build:cli
|
||||
- run: pnpm run build:electron-vite:parallel
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-background.e2e.ts
|
||||
|
||||
@@ -38,6 +38,8 @@ jobs:
|
||||
mac-native-owner-smoke:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: macos-15
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
timeout-minutes: 180
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -14,48 +14,248 @@ jobs:
|
||||
apply-os-label:
|
||||
runs-on: ubuntu-slim
|
||||
timeout-minutes: 5
|
||||
# Serialize runs per issue; each run reconciles from current state, so order does not matter.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-issue-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Apply OS label from issue form
|
||||
- name: Reconcile OS and mobile labels from issue form
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const issue = context.payload.issue;
|
||||
const body = issue.body || "";
|
||||
const osMatch = body.match(/^### Operating system\s+(.+)$/m);
|
||||
// The only labels this workflow may add or remove.
|
||||
const MANAGED_LABELS = ["os:macos", "os:Windows", "os:linux", "os:mobile", "mobile"];
|
||||
const MAX_PASSES = 3;
|
||||
|
||||
if (!osMatch) {
|
||||
core.info("No operating system field found. Skipping.");
|
||||
return;
|
||||
const NOTE_PREFIX =
|
||||
"> \u{1F310} **Translation note (machine-translated by an assistant on ";
|
||||
const NOTE_SUFFIX =
|
||||
"):** This issue's title was originally written in Chinese and has been translated to English (the body is unchanged). The original, untranslated text is preserved intact in the collapsible section below.";
|
||||
const NOTE_DATE = /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/;
|
||||
|
||||
// Metadata headings in accepted order; equal rank = current/legacy names of one field.
|
||||
const HEADING_RANKS = new Map([
|
||||
["Summary", 1],
|
||||
["Where were you using Orca?", 2],
|
||||
["Client", 2],
|
||||
["Operating system", 3],
|
||||
["Host operating system", 3],
|
||||
["Remote host operating system", 4],
|
||||
["Orca version", 5],
|
||||
["Coding harness and version", 6],
|
||||
["Details", 7],
|
||||
["Reproduction steps", 7],
|
||||
]);
|
||||
const FREE_TEXT_RANK = 7;
|
||||
|
||||
const OS_LABELS = new Map([
|
||||
["macos", "os:macos"],
|
||||
["windows", "os:Windows"],
|
||||
["linux", "os:linux"],
|
||||
["ios", "os:mobile"],
|
||||
["android", "os:mobile"],
|
||||
["other", null],
|
||||
]);
|
||||
// Value: whether the choice means the mobile app. Retired option strings stay for open issues.
|
||||
const WHERE_IS_MOBILE = new Map([
|
||||
["desktop only", false],
|
||||
["desktop + ssh host", false],
|
||||
["desktop + orca server", false],
|
||||
["desktop + wsl", false],
|
||||
["mobile app", true],
|
||||
["web browser", false],
|
||||
["other", false],
|
||||
["desktop (this computer)", false],
|
||||
["desktop + ssh", false],
|
||||
["desktop + orca remote server", false],
|
||||
["mobile (ios)", true],
|
||||
["mobile (android)", true],
|
||||
["web / browser", false],
|
||||
]);
|
||||
const CLIENT_IS_MOBILE = new Map([
|
||||
...WHERE_IS_MOBILE,
|
||||
["desktop", false],
|
||||
["mobile", true],
|
||||
["web", false],
|
||||
]);
|
||||
const REMOTE_OS_VALUES = new Set(["linux", "macos", "windows", "other", "not sure"]);
|
||||
|
||||
const isBlank = (line) => /^[ \t]*$/.test(line);
|
||||
const isHeading = (line) => /^###(?:[ \t]|$)/.test(line);
|
||||
const isUnselected = (value) => value === "" || value === "_no response_";
|
||||
const isTranslationNote = (line) =>
|
||||
line.length === NOTE_PREFIX.length + 10 + NOTE_SUFFIX.length &&
|
||||
line.startsWith(NOTE_PREFIX) &&
|
||||
line.endsWith(NOTE_SUFFIX) &&
|
||||
NOTE_DATE.test(line.slice(NOTE_PREFIX.length, NOTE_PREFIX.length + 10));
|
||||
|
||||
// Reads only the leading metadata prefix; null means "not a recognizable bug form".
|
||||
function readFormFields(body) {
|
||||
if (typeof body !== "string") {
|
||||
return null;
|
||||
}
|
||||
const lines = body.replace(/\r\n/g, "\n").split("\n");
|
||||
let index = 0;
|
||||
const skipBlankLines = () => {
|
||||
while (index < lines.length && isBlank(lines[index])) {
|
||||
index += 1;
|
||||
}
|
||||
};
|
||||
|
||||
skipBlankLines();
|
||||
if (index < lines.length && isTranslationNote(lines[index])) {
|
||||
index += 1;
|
||||
if (index >= lines.length || !isBlank(lines[index])) {
|
||||
return null;
|
||||
}
|
||||
skipBlankLines();
|
||||
}
|
||||
|
||||
const fields = new Map();
|
||||
let lastRank = 0;
|
||||
while (index < lines.length) {
|
||||
const line = lines[index];
|
||||
if (!isHeading(line)) {
|
||||
return null;
|
||||
}
|
||||
// Exact "### <heading>" only; any other heading shape is unknown.
|
||||
const name = line.startsWith("### ") ? line.slice(4) : null;
|
||||
const rank = HEADING_RANKS.get(name);
|
||||
if (rank === undefined || rank < lastRank || fields.has(name)) {
|
||||
return null;
|
||||
}
|
||||
if (rank === FREE_TEXT_RANK) {
|
||||
// Free text starts here and is never inspected.
|
||||
return fields;
|
||||
}
|
||||
lastRank = rank;
|
||||
index += 1;
|
||||
|
||||
let value = "";
|
||||
let valueLines = 0;
|
||||
while (index < lines.length && !isHeading(lines[index])) {
|
||||
if (!isBlank(lines[index])) {
|
||||
value = lines[index];
|
||||
valueLines += 1;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
if (valueLines > 1) {
|
||||
return null;
|
||||
}
|
||||
fields.set(name, value.trim().toLowerCase());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const selectedOs = osMatch[1].trim().toLowerCase();
|
||||
const osLabelMap = {
|
||||
macos: "os:macos",
|
||||
windows: "os:Windows",
|
||||
linux: "os:linux",
|
||||
};
|
||||
// Returns the managed labels the body asks for, or null to leave the issue untouched.
|
||||
function desiredLabelsFor(body) {
|
||||
const fields = readFormFields(body);
|
||||
if (!fields || !fields.has("Orca version")) {
|
||||
return null;
|
||||
}
|
||||
const osHeading = ["Operating system", "Host operating system"].find((name) =>
|
||||
fields.has(name),
|
||||
);
|
||||
if (!osHeading) {
|
||||
return null;
|
||||
}
|
||||
const desired = new Set();
|
||||
|
||||
const desiredLabel = osLabelMap[selectedOs];
|
||||
if (!desiredLabel) {
|
||||
core.info(`No OS label configured for "${selectedOs}". Skipping.`);
|
||||
return;
|
||||
const os = fields.get(osHeading);
|
||||
if (!isUnselected(os)) {
|
||||
if (!OS_LABELS.has(os)) {
|
||||
return null;
|
||||
}
|
||||
const osLabel = OS_LABELS.get(os);
|
||||
if (osLabel) {
|
||||
desired.add(osLabel);
|
||||
}
|
||||
}
|
||||
|
||||
// The current heading wins by presence, even when blank.
|
||||
const whereChoices = fields.has("Where were you using Orca?")
|
||||
? [fields.get("Where were you using Orca?"), WHERE_IS_MOBILE]
|
||||
: fields.has("Client")
|
||||
? [fields.get("Client"), CLIENT_IS_MOBILE]
|
||||
: null;
|
||||
if (whereChoices && !isUnselected(whereChoices[0])) {
|
||||
const [where, isMobileByChoice] = whereChoices;
|
||||
if (!isMobileByChoice.has(where)) {
|
||||
return null;
|
||||
}
|
||||
if (isMobileByChoice.get(where)) {
|
||||
desired.add("mobile");
|
||||
}
|
||||
}
|
||||
|
||||
// Validated only; the remote host never contributes a label.
|
||||
const remoteOs = fields.get("Remote host operating system") ?? "";
|
||||
if (!isUnselected(remoteOs) && !REMOTE_OS_VALUES.has(remoteOs)) {
|
||||
return null;
|
||||
}
|
||||
return desired;
|
||||
}
|
||||
|
||||
const managedLabels = Object.values(osLabelMap);
|
||||
const existingLabels = issue.labels.map((label) =>
|
||||
typeof label === "string" ? label : label.name,
|
||||
);
|
||||
const labelsToKeep = existingLabels.filter(
|
||||
(label) => !managedLabels.includes(label),
|
||||
);
|
||||
const nextLabels = [...new Set([...labelsToKeep, desiredLabel])];
|
||||
|
||||
await github.rest.issues.setLabels({
|
||||
const issueRef = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: nextLabels,
|
||||
});
|
||||
issue_number: context.payload.issue.number,
|
||||
};
|
||||
|
||||
core.info(`Applied label ${desiredLabel} to issue #${issue.number}.`);
|
||||
// The event is only a signal: decisions come from the issue as it is now.
|
||||
async function readCurrentIssue() {
|
||||
const { data } = await github.rest.issues.get(issueRef);
|
||||
const labelNames = (data.labels ?? []).map((label) =>
|
||||
typeof label === "string" ? label : label.name,
|
||||
);
|
||||
const desired = desiredLabelsFor(data.body);
|
||||
const present = MANAGED_LABELS.filter((label) => labelNames.includes(label));
|
||||
return {
|
||||
desired,
|
||||
toAdd: desired ? MANAGED_LABELS.filter((l) => desired.has(l) && !present.includes(l)) : [],
|
||||
toRemove: desired ? present.filter((l) => !desired.has(l)) : [],
|
||||
};
|
||||
}
|
||||
|
||||
let state = await readCurrentIssue();
|
||||
let hasWritten = false;
|
||||
for (let pass = 1; ; pass += 1) {
|
||||
if (!state.desired) {
|
||||
if (hasWritten) {
|
||||
throw new Error(
|
||||
`Issue #${issueRef.issue_number} stopped matching the bug form while labels were being updated. Managed labels may be incomplete; rerun this job once the issue body is settled.`,
|
||||
);
|
||||
}
|
||||
core.info("No recognizable bug form at the start of the issue body. Labels left untouched.");
|
||||
return;
|
||||
}
|
||||
if (state.toAdd.length === 0 && state.toRemove.length === 0) {
|
||||
core.info(
|
||||
`Issue #${issueRef.issue_number} managed labels match the form: [${[...state.desired].join(", ")}].`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (pass > MAX_PASSES) {
|
||||
throw new Error(
|
||||
`Issue #${issueRef.issue_number} labels still differ from the form after ${MAX_PASSES} passes (to add: [${state.toAdd.join(", ")}], to remove: [${state.toRemove.join(", ")}]). The issue is probably being edited; rerun this job.`,
|
||||
);
|
||||
}
|
||||
|
||||
hasWritten = true;
|
||||
if (state.toAdd.length > 0) {
|
||||
await github.rest.issues.addLabels({ ...issueRef, labels: state.toAdd });
|
||||
}
|
||||
for (const name of state.toRemove) {
|
||||
try {
|
||||
await github.rest.issues.removeLabel({ ...issueRef, name });
|
||||
} catch (error) {
|
||||
// Already gone is the desired outcome; the re-read below verifies real state.
|
||||
if (error?.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
state = await readCurrentIssue();
|
||||
}
|
||||
|
||||
@@ -11,12 +11,12 @@ on:
|
||||
- 'native/macos-shipit-scheduling/**'
|
||||
- 'config/scripts/macos-shipit-scheduling*'
|
||||
- 'config/electron-builder.config.cjs'
|
||||
- 'src/main/startup/main-process-quit*'
|
||||
- 'src/main/window/main-window-state-lifecycle*'
|
||||
- 'src/main/startup/quit/main-process-quit*'
|
||||
- 'src/main/window/main-window/main-window-state-lifecycle*'
|
||||
- 'src/main/window/dashboard-popout-window*'
|
||||
- 'src/packages/process-host/**'
|
||||
- 'src/main/runtime/agent-session/lifecycle/structured-agent-cli-writers*'
|
||||
- 'src/main/providers/macos-tcc-login-shell*'
|
||||
- 'src/main/providers/shell-launch/macos/tcc-login-shell*'
|
||||
- 'src/shared/process-table-snapshot*'
|
||||
- 'src/shared/darwin-process-table-capture*'
|
||||
- 'src/shared/update-status-types.ts'
|
||||
@@ -55,8 +55,8 @@ jobs:
|
||||
src/main/updater/mac-install/updater.mac-install.test.ts
|
||||
src/main/updater/updater.headless-serve-install.test.ts
|
||||
src/main/updater/mac-install/updater-mac-quit-guard.test.ts
|
||||
src/main/startup/main-process-quit-update-veto.test.ts
|
||||
src/main/window/main-window-state-lifecycle.test.ts
|
||||
src/main/startup/quit/main-process-quit-update-veto.test.ts
|
||||
src/main/window/main-window/main-window-state-lifecycle.test.ts
|
||||
- name: Prove login-backed terminal writer census on the disposable runner
|
||||
env:
|
||||
ORCA_MACOS_LOGIN_CENSUS_CI: '1'
|
||||
|
||||
@@ -34,13 +34,19 @@ on:
|
||||
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
|
||||
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
|
||||
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
|
||||
- 'src/main/github/issue-field-options.ts'
|
||||
- 'src/main/github/issues/issue-field-options.ts'
|
||||
- 'config/tsconfig.test.mobile-host-declarations.json'
|
||||
- 'config/tsconfig.node.json'
|
||||
- 'src/types/build-constants.d.ts'
|
||||
- 'src/main/native-chat/agent-session-wire/**'
|
||||
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
|
||||
- 'src/main/native-chat/*transcript*.ts'
|
||||
- 'src/main/native-chat/transcript-discovery/**'
|
||||
- 'src/main/native-chat/transcript-filesystem/**'
|
||||
- 'src/main/native-chat/transcript-decoding/**'
|
||||
- 'src/main/native-chat/transcript-streaming/**'
|
||||
- 'src/main/native-chat/transcript-watching/**'
|
||||
- 'src/main/native-chat/transcript-history/**'
|
||||
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
|
||||
@@ -58,13 +64,11 @@ on:
|
||||
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-json-file.ts'
|
||||
- 'src/main/ai-vault/discovery/session-path-absence.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-root-overrides.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
|
||||
- 'src/main/ai-vault/parsing/session-json-object.ts'
|
||||
@@ -73,7 +77,6 @@ on:
|
||||
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-conversation-name-requests.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
|
||||
@@ -81,24 +84,17 @@ on:
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parser-stream-cleanup.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-timeline.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-document-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-every-agent-fixture.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-log-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-vault-roots.ts'
|
||||
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
|
||||
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
|
||||
@@ -180,13 +176,19 @@ on:
|
||||
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
|
||||
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
|
||||
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
|
||||
- 'src/main/github/issue-field-options.ts'
|
||||
- 'src/main/github/issues/issue-field-options.ts'
|
||||
- 'config/tsconfig.test.mobile-host-declarations.json'
|
||||
- 'config/tsconfig.node.json'
|
||||
- 'src/types/build-constants.d.ts'
|
||||
- 'src/main/native-chat/agent-session-wire/**'
|
||||
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
|
||||
- 'src/main/native-chat/*transcript*.ts'
|
||||
- 'src/main/native-chat/transcript-discovery/**'
|
||||
- 'src/main/native-chat/transcript-filesystem/**'
|
||||
- 'src/main/native-chat/transcript-decoding/**'
|
||||
- 'src/main/native-chat/transcript-streaming/**'
|
||||
- 'src/main/native-chat/transcript-watching/**'
|
||||
- 'src/main/native-chat/transcript-history/**'
|
||||
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
|
||||
@@ -204,13 +206,11 @@ on:
|
||||
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-json-file.ts'
|
||||
- 'src/main/ai-vault/discovery/session-path-absence.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-root-overrides.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
|
||||
- 'src/main/ai-vault/parsing/session-json-object.ts'
|
||||
@@ -219,7 +219,6 @@ on:
|
||||
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-conversation-name-requests.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
|
||||
@@ -227,24 +226,17 @@ on:
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parser-stream-cleanup.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-timeline.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-document-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-every-agent-fixture.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-log-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-vault-roots.ts'
|
||||
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
|
||||
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
|
||||
|
||||
@@ -247,7 +247,7 @@ jobs:
|
||||
run: node "$(if [ -f config/scripts/orcad-distribution/template/build-orcad-template.mjs ]; then printf %s config/scripts/orcad-distribution/template/build-orcad-template.mjs; else printf %s config/scripts/build-orcad-template.mjs; fi)" --targets linux-${{ runner.arch == 'ARM64' && 'arm64' || 'x64' }}-glibc
|
||||
- name: Verify a structured chat from the desktop server template
|
||||
if: runner.os == 'Linux'
|
||||
run: pnpm test:node-server --artifact src/main/orcad/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
|
||||
run: pnpm test:node-server --artifact src/main/orcad/testing/structured-chat/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
|
||||
# Only a Windows runner compiles it; arm64 cross-compiles here.
|
||||
# Before the Node 18 check below: the build script imports TypeScript, which Node 18 cannot load.
|
||||
# orcad's instance lock and stop proof read a PID's creation time through it; the x64 binary this
|
||||
|
||||
@@ -853,8 +853,8 @@ jobs:
|
||||
src/main/pty/shell-startup/zsh-startup-hook-user-config-equivalence.live-shell.test.ts \
|
||||
src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts \
|
||||
src/main/runtime/agent-session/environment/structured-session-cli-login-shell.live-shell.test.ts \
|
||||
src/main/startup/shell-probe-terminal-detach.live-shell.test.ts \
|
||||
src/main/startup/shell-probe-lifetime.live-shell.test.ts \
|
||||
src/main/startup/shell-environment/shell-probe-terminal-detach.live-shell.test.ts \
|
||||
src/main/startup/shell-environment/shell-probe-lifetime.live-shell.test.ts \
|
||||
src/renderer/src/components/terminal-pane/pty-connection/emulation/fish-color-scheme-child-stdin.node-pty.test.ts \
|
||||
src/shared/fish-wsl-cwd-report.node-pty.test.ts \
|
||||
src/main/wsl/filesystem/wsl-sh-cwd-report.node-pty.test.ts \
|
||||
@@ -1117,7 +1117,6 @@ jobs:
|
||||
src/main/browser/cookies/browser-google-cookie-clear.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/presentation/popup-origin-chrome.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/navigation/browser-link-materialization.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-compatibility.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-native-compatibility.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-background-compatibility.electron.test.ts
|
||||
src/main/browser/extensions/api-runtime/browser-extension-api.electron.test.ts
|
||||
|
||||
@@ -47,8 +47,8 @@ jobs:
|
||||
src/main/ipc/preflight-runnable-local-cli.test.ts
|
||||
src/main/ipc/preflight-wsl-agent-detection.test.ts
|
||||
src/main/ipc/preflight-windows-path-refresh.repro.test.ts
|
||||
src/main/startup/hydrate-shell-path.windows.test.ts
|
||||
src/main/startup/login-shell-environment.test.ts
|
||||
src/main/startup/shell-environment/hydrate-shell-path.windows.test.ts
|
||||
src/main/startup/shell-environment/login-shell-environment.test.ts
|
||||
src/packages/process-host/src/process-tree-termination.test.ts
|
||||
src/packages/process-host/src/run-process.test.ts
|
||||
src/packages/process-host/src/run-process-termination-failure.test.ts
|
||||
|
||||
@@ -15,8 +15,8 @@ on:
|
||||
- 'src/main/managed-server/**'
|
||||
- 'src/main/ssh/migration-source/**'
|
||||
- 'src/main/persistence/migrating-orcad-catalog/**'
|
||||
- 'src/main/orcad/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/orcad-host-unavailable.ts'
|
||||
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
|
||||
- 'src/main/node-runtime/**'
|
||||
- 'src/main/ssh/remote-exec/**'
|
||||
- 'src/main/ssh/remote-host/**'
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- 'src/main/managed-server/**'
|
||||
- 'src/main/ssh/migration-source/**'
|
||||
- 'src/main/persistence/migrating-orcad-catalog/**'
|
||||
- 'src/main/orcad/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/orcad-host-unavailable.ts'
|
||||
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
|
||||
- 'src/main/node-runtime/**'
|
||||
- 'src/main/ssh/remote-exec/**'
|
||||
- 'src/main/ssh/remote-host/**'
|
||||
@@ -38,7 +38,7 @@ on:
|
||||
- 'config/scripts/native-build/windows/build-windows-cli-launcher.mjs'
|
||||
- 'config/scripts/native-build/windows/windows-rust-executable-build.mjs'
|
||||
- 'native/windows-cli-launcher/**'
|
||||
- 'src/main/orcad/orcad-cli-launcher.ts'
|
||||
- 'src/main/orcad/serve-startup/orcad-cli-launcher.ts'
|
||||
- '.github/actions/prepare-windows-cli-launcher/**'
|
||||
- 'config/patches/node-pty*'
|
||||
- 'config/patches/@vscode__windows-process-tree*'
|
||||
|
||||
@@ -221,8 +221,8 @@ jobs:
|
||||
pnpm-lock.yaml
|
||||
cloud/pnpm-lock.yaml
|
||||
|
||||
# These two tests import the cloud relay workspace directly. Keeping them in one job
|
||||
# avoids installing and building the same workspace once per unit-test shard.
|
||||
# This test imports the cloud relay workspace directly. Its dedicated job avoids
|
||||
# installing and building the same workspace once per unit-test shard.
|
||||
- name: Install relay integration dependencies
|
||||
id: relay_dependencies
|
||||
if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }}
|
||||
@@ -237,5 +237,4 @@ jobs:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
run: >-
|
||||
pnpm exec vitest run --config config/vitest.config.ts
|
||||
tests/e2e/relay-region-compatibility.unit.test.ts
|
||||
tests/e2e/relay-region-correction.unit.test.ts
|
||||
|
||||
@@ -7,8 +7,7 @@ on:
|
||||
- 'src/main/ai-vault/scanners/opencode/**'
|
||||
- 'src/main/ai-vault/scanners/opencode2/**'
|
||||
- 'src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime*'
|
||||
- 'src/main/native-chat/transcript-opencode*'
|
||||
- 'src/main/native-chat/transcript-zcode*'
|
||||
- 'src/main/native-chat/transcript-history/opencode/**'
|
||||
- 'src/shared/native-chat/native-chat-types.ts'
|
||||
- 'src/shared/native-chat/transcript/native-chat-transcript-projection*'
|
||||
- '.github/workflows/zcode-transcript-contract.yml'
|
||||
@@ -41,4 +40,4 @@ jobs:
|
||||
bun-version-file: config/.bun-version
|
||||
no-cache: true
|
||||
- name: Verify SQLite ordering, legacy readers, and host routing
|
||||
run: pnpm test src/main/native-chat/transcript-zcode.test.ts src/main/native-chat/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
|
||||
run: pnpm test src/main/native-chat/transcript-history/opencode/transcript-zcode.test.ts src/main/native-chat/transcript-history/opencode/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
|
||||
|
||||
+1
-1
@@ -308,7 +308,7 @@
|
||||
"ignorePatterns": [
|
||||
"src/main/acp/generated/acp-protocol.generated.ts",
|
||||
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
|
||||
"src/cli/bundled-skill-guides.ts",
|
||||
"src/cli/skills/bundled-skill-guides.ts",
|
||||
"resources/plugins/declarative/*.mjs",
|
||||
"**/node_modules",
|
||||
"**/dist",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
|
||||
import { DELIVERY_LEASE_MS, DurablePushStore } from './durable-push-store.js'
|
||||
import { DurablePushStore } from './durable-push-store.js'
|
||||
|
||||
import {
|
||||
cleanupDurablePushFixtures,
|
||||
@@ -63,20 +63,6 @@ it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
}
|
||||
)
|
||||
|
||||
it('leases exactly one delivery per device across concurrent claimers', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
for (let seq = 1; seq <= 5; seq++) {
|
||||
await store.accept('host', 'phone', notification(seq))
|
||||
advance(1)
|
||||
}
|
||||
const claims = await Promise.all(
|
||||
Array.from({ length: 8 }, () => new DurablePushStore(db, clock).claim())
|
||||
)
|
||||
const leased = claims.filter((claim) => claim !== null)
|
||||
expect(leased).toHaveLength(1)
|
||||
expect(leased[0]?.notification.notificationSeq).toBe(1)
|
||||
})
|
||||
|
||||
it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
'does not lease the row behind a head another claimer holds',
|
||||
async () => {
|
||||
@@ -138,21 +124,3 @@ it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
expect((await first)?.notification.notificationSeq).toBe(1)
|
||||
}
|
||||
)
|
||||
|
||||
// Only the TTL term keeps the candidate scan off an unpruned backlog; expires_at is not indexed first.
|
||||
|
||||
it('frees a device held by a crashed worker once its lease lapses', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
advance(1)
|
||||
await store.accept('host', 'phone', notification(2))
|
||||
const crashed = (await new DurablePushStore(db, clock).claim())!
|
||||
advance(DELIVERY_LEASE_MS - 1)
|
||||
expect(await store.claim()).toBeNull()
|
||||
advance(1)
|
||||
const reclaimed = (await store.claim())!
|
||||
expect(reclaimed.id).toBe(crashed.id)
|
||||
expect(reclaimed.attempts).toBe(2)
|
||||
await store.finish(reclaimed)
|
||||
expect((await store.claim())?.notification.notificationSeq).toBe(2)
|
||||
})
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openPushDatabase } from '../database/push-database.js'
|
||||
import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js'
|
||||
import {
|
||||
cleanupDurablePushFixtures,
|
||||
durablePushTestDatabaseUrl,
|
||||
fixture,
|
||||
notification
|
||||
} from './durable-push-store.test-fixture.js'
|
||||
@@ -11,20 +9,6 @@ import {
|
||||
afterEach(cleanupDurablePushFixtures)
|
||||
|
||||
describe('durable push acceptance', () => {
|
||||
it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
for (let i = 0; i < 300; i++) {
|
||||
expect(await store.accept('host', 'phone1', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone2', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued')
|
||||
}
|
||||
expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited')
|
||||
expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited')
|
||||
expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued')
|
||||
advance(15 * 60_000)
|
||||
expect(await store.accept('host', 'phone1', notification(301))).toBe('queued')
|
||||
})
|
||||
|
||||
it('queues one delivery per event and recovers work across service instances', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
@@ -32,16 +16,6 @@ describe('durable push acceptance', () => {
|
||||
await restarted.accept('host', 'phone', notification(1))
|
||||
advance(1)
|
||||
await restarted.accept('host', 'phone', notification(2))
|
||||
const rows = await db.query(
|
||||
"SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id",
|
||||
['phone']
|
||||
)
|
||||
expect(rows).toHaveLength(2)
|
||||
expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([
|
||||
notification(1),
|
||||
notification(2)
|
||||
])
|
||||
expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true)
|
||||
const delivery = await restarted.claim()
|
||||
expect(delivery?.notification.notificationSeq).toBe(1)
|
||||
expect(await store.claim()).toBeNull()
|
||||
@@ -93,18 +67,9 @@ describe('durable push acceptance', () => {
|
||||
const { db, store } = await fixture()
|
||||
await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable')
|
||||
try {
|
||||
const databaseUrl = durablePushTestDatabaseUrl
|
||||
if (databaseUrl) {
|
||||
const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' })
|
||||
try {
|
||||
await expect(
|
||||
concurrent.query('SELECT COUNT(*) FROM push_delivery_batches')
|
||||
).resolves.toHaveLength(1)
|
||||
} finally {
|
||||
await concurrent.close()
|
||||
}
|
||||
}
|
||||
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow()
|
||||
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow(
|
||||
/push_delivery_batches/
|
||||
)
|
||||
expect(await db.query('SELECT * FROM push_events')).toEqual([])
|
||||
expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([])
|
||||
} finally {
|
||||
@@ -160,38 +125,3 @@ it('does not resurrect an in-flight alert after a dismissal and transient provid
|
||||
expect(await store.claim()).toBeNull()
|
||||
expect(await store.pendingCount('phone')).toBe(0)
|
||||
})
|
||||
|
||||
it('fences late renew and finish after an expired claim is dismissed', async () => {
|
||||
const { db, store, advance } = await fixture()
|
||||
const alert = notification(1)
|
||||
await store.accept('host', 'phone', alert)
|
||||
const stale = (await store.claim())!
|
||||
advance(DELIVERY_LEASE_MS)
|
||||
await store.accept('host', 'phone', {
|
||||
...notification(2, 'dismiss'),
|
||||
notificationId: alert.notificationId
|
||||
})
|
||||
const read = async () =>
|
||||
(
|
||||
await db.query(
|
||||
'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?',
|
||||
[stale.id]
|
||||
)
|
||||
)[0]
|
||||
expect(await read()).toBeUndefined()
|
||||
await store.renew(stale)
|
||||
await store.finish(stale, 1000)
|
||||
await store.finish(stale)
|
||||
expect(await read()).toBeUndefined()
|
||||
const dismissal = (await store.claim())!
|
||||
expect(dismissal.notification.kind).toBe('dismiss')
|
||||
await store.finish(dismissal)
|
||||
await store.accept('host', 'phone', notification(3))
|
||||
const fresh = (await store.claim())!
|
||||
await store.finish(fresh, 1000)
|
||||
advance(1000)
|
||||
const retry = (await store.claim())!
|
||||
expect(retry.id).toBe(fresh.id)
|
||||
await store.finish(retry)
|
||||
expect(await store.claim()).toBeNull()
|
||||
})
|
||||
|
||||
@@ -13,7 +13,6 @@ import { pauseAfter, within } from './durable-push-store.test-fixture.js'
|
||||
const cleanups: (() => Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0)) await cleanup()
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
const note = (seq: number, overrides: Partial<PushNotification> = {}): PushNotification => ({
|
||||
@@ -42,9 +41,8 @@ async function fixture() {
|
||||
const send = vi.fn(async (_delivery: PushDelivery): Promise<PushProviderOutcome> => ({
|
||||
status: 'sent'
|
||||
}))
|
||||
const onRetry = vi.fn()
|
||||
const dispatcher = new PushDispatcher({ devices, fcm: { send } as never })
|
||||
const worker = new DurablePushWorker(store, dispatcher, { now, onRetry })
|
||||
const worker = new DurablePushWorker(store, dispatcher, { now })
|
||||
cleanups.push(async () => {
|
||||
await worker.stop()
|
||||
await db.close()
|
||||
@@ -57,7 +55,6 @@ async function fixture() {
|
||||
worker,
|
||||
dispatcher,
|
||||
send,
|
||||
onRetry,
|
||||
now,
|
||||
registrationId: device.registrationId,
|
||||
accept: (notification: PushNotification) =>
|
||||
@@ -102,7 +99,6 @@ it('stops queued claim admissions without leasing the remaining device', async (
|
||||
paused.release()
|
||||
await within(Promise.all([pending, stopping]), 2_000)
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
expect(h.send.mock.calls[0]?.[0].orca.notificationSeq).toBe(1)
|
||||
const [remaining] = await h.db.query(
|
||||
'SELECT attempts, lease_until FROM push_delivery_batches WHERE registration_id = ?',
|
||||
[other.registrationId]
|
||||
|
||||
@@ -26,18 +26,10 @@ describe('push host challenge store', () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => {
|
||||
it('rejects a proof issued for another challenge', async () => {
|
||||
const owner = createPushHostKeypair(6)
|
||||
const intruder = createPushHostKeypair(7)
|
||||
const ownerChallenge = await store.issue(hostPublicKeyB64(owner))
|
||||
expect(
|
||||
answerPushHostChallenge(ownerChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: intruder,
|
||||
now: () => clock
|
||||
})
|
||||
).toBeNull()
|
||||
|
||||
const intruderChallenge = await store.issue(hostPublicKeyB64(intruder))
|
||||
const intruderProof = answerPushHostChallenge(intruderChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
|
||||
@@ -19,11 +19,6 @@ async function concurrentSessions(db: PushDatabase) {
|
||||
sessions.map((session) => store.resolve(session.sessionToken))
|
||||
)
|
||||
expect(decisions.filter((decision) => decision.ok)).toHaveLength(1)
|
||||
const [row] = await db.query(
|
||||
'SELECT COUNT(*) AS count FROM push_sessions WHERE host_fingerprint = ?',
|
||||
[host]
|
||||
)
|
||||
expect(Number(row?.count)).toBe(1)
|
||||
} finally {
|
||||
await db.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [host])
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
const harnesses: Awaited<ReturnType<typeof createPushServerHarness>>[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(harnesses.splice(0).map((h) => h.close()))
|
||||
})
|
||||
|
||||
it('returns queued for concurrent retries without double quota or delivery', async () => {
|
||||
const h = await createPushServerHarness()
|
||||
harnesses.push(h)
|
||||
const token = await h.signIn(createPushHostKeypair(2))
|
||||
const registrationId = await h.registerAndroid(token)
|
||||
const body = { v: 1, registrationIds: [registrationId], notification: notification() }
|
||||
const responses = await Promise.all(
|
||||
Array.from({ length: 10 }, () => h.post('/v1/send', body, token))
|
||||
)
|
||||
for (const response of responses)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1)
|
||||
await h.flushDeliveries()
|
||||
await h.post('/v1/send', body, token)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined()
|
||||
expect(
|
||||
Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count)
|
||||
).toBe(1)
|
||||
await h.post(
|
||||
'/v1/send',
|
||||
{ ...body, notification: notification({ notificationEpoch: 'new-epoch' }) },
|
||||
token
|
||||
)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(2)
|
||||
})
|
||||
@@ -2,7 +2,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
import {
|
||||
createPushServerHarness,
|
||||
FCM_TOKEN,
|
||||
notification
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
@@ -34,12 +33,6 @@ describe('push gateway send route', () => {
|
||||
})
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
|
||||
message: {
|
||||
token: FCM_TOKEN,
|
||||
data: { title: 'Agent needs input', tag: expect.stringMatching(/^[a-f0-9]{64}$/) }
|
||||
}
|
||||
})
|
||||
|
||||
const afterDeath = await harness.post(
|
||||
'/v1/send',
|
||||
@@ -47,11 +40,6 @@ describe('push gateway send route', () => {
|
||||
sessionToken
|
||||
)
|
||||
expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] })
|
||||
|
||||
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(await listed.json()).toEqual({
|
||||
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }]
|
||||
})
|
||||
})
|
||||
|
||||
it('reports an error for a registration the host does not own', async () => {
|
||||
|
||||
@@ -36,7 +36,6 @@ it('delivers again after a topic error without re-registering the phone', async
|
||||
expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
await h.flushDeliveries()
|
||||
if (seq === 1) {
|
||||
expect(h.server.observability.consume().delivery_error).toBe(1)
|
||||
h.setApnsResponse({ status: 200, body: '' })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,11 +67,10 @@ describe('fcm client', () => {
|
||||
error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' }
|
||||
})
|
||||
})
|
||||
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toMatchObject({
|
||||
status: 'error',
|
||||
reason: 'INVALID_ARGUMENT',
|
||||
retryable: false,
|
||||
retryAfterMs: 10000
|
||||
retryable: false
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,7 +5,7 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs
|
||||
import { SignJWT, exportJWK, generateKeyPair, importJWK, type JWK } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// client wrapper, the echo service, phone sockets and workerd memory sampling.
|
||||
import { execSync } from 'node:child_process'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const PORT = 8787
|
||||
|
||||
@@ -39,7 +39,7 @@ export async function runLifecycleChecks(ctx: Ctx, main: Control, mainAck: Ack):
|
||||
mute.on('close', (c) => (muteClose = c))
|
||||
await new Promise((res) => mute.on('open', res))
|
||||
// Complete the handshake by hand with the shipped proof answerer, then never pong.
|
||||
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/relay-host-proof')
|
||||
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-host-proof')
|
||||
mute.send(JSON.stringify({ type: 'host-hello', v: 1, relayHostId: ctx.relayHostId, assignmentEpoch: 7,
|
||||
hostPublicKeyB64: ctx.keypair.publicKeyB64, appVersion: 'spike' }))
|
||||
mute.on('message', (raw) => {
|
||||
|
||||
@@ -12,7 +12,7 @@ const ERASER = 'https://erasure.test.invalid'
|
||||
const BODY = JSON.stringify({ relayHostId: 'AAAAAAAAAAAAAAAA' })
|
||||
const JTI = 'jti-0123456789abcdef'
|
||||
|
||||
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string; iat?: number; exp?: number }
|
||||
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string }
|
||||
|
||||
async function sign(opts: Sign = {}): Promise<string> {
|
||||
const claims: Record<string, unknown> = {
|
||||
@@ -26,8 +26,8 @@ async function sign(opts: Sign = {}): Promise<string> {
|
||||
.setProtectedHeader({ alg: 'HS256' })
|
||||
.setIssuer(opts.issuer ?? ISSUER)
|
||||
.setAudience('orca-relay-edge-admin')
|
||||
.setIssuedAt(opts.iat ?? now)
|
||||
.setExpirationTime(opts.exp ?? now + 60)
|
||||
.setIssuedAt(now)
|
||||
.setExpirationTime(now + 60)
|
||||
.sign(new TextEncoder().encode(opts.key ?? KEY))
|
||||
}
|
||||
|
||||
@@ -45,24 +45,19 @@ const baseEnv = { ADMIN_SIGNING_KEY: KEY, ADMIN_ISSUER: ISSUER } as Env
|
||||
|
||||
describe('admin token verify (jti)', () => {
|
||||
|
||||
it('still binds op, body, key and age', async () => {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
it('binds the operation and exact body', async () => {
|
||||
expect(await verifyWith(await sign({ jti: JTI, op: 'erase' }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, body: '{}' }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, key: NEXT }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, iat: now - 180, exp: now + 60 }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('admin request verify (keys)', () => {
|
||||
|
||||
it('accepts the primary or the NEXT key, nothing else', async () => {
|
||||
it('accepts the primary key and only a configured NEXT key', async () => {
|
||||
const env = { ...baseEnv, ADMIN_SIGNING_KEY_NEXT: NEXT } as Env
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI })), 'drain')).not.toBeNull()
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: NEXT })), 'drain')).not.toBeNull()
|
||||
expect(await verifyAdminRequest(baseEnv, request(await sign({ jti: JTI, key: NEXT })), 'drain')).toBeNull()
|
||||
const wrong = randomBytes(36).toString('base64url')
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: wrong })), 'drain')).toBeNull()
|
||||
})
|
||||
|
||||
it('the erasure keys (primary or NEXT) verify only erase, under their own issuer', async () => {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { readFileSync } from 'node:fs'
|
||||
import { SignJWT, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const PORT = 8789
|
||||
|
||||
@@ -8,7 +8,7 @@ import { join } from 'node:path'
|
||||
import { SignJWT, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ const mint = (key: CryptoKey = privateKey, kid = 'k1') =>
|
||||
.sign(key)
|
||||
|
||||
// A Cache API stand-in shared by every "isolate" (module instance) in the test.
|
||||
function stubCache(): Map<string, Response> {
|
||||
function stubCache(): void {
|
||||
const store = new Map<string, Response>()
|
||||
vi.stubGlobal('caches', {
|
||||
default: {
|
||||
@@ -38,7 +38,6 @@ function stubCache(): Map<string, Response> {
|
||||
match: async (request: Request) => store.get(request.url)?.clone()
|
||||
}
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
// Each call is a fresh isolate: a new module instance with an empty in-memory key cache.
|
||||
@@ -64,24 +63,21 @@ afterEach(() => {
|
||||
|
||||
describe('relay JWT key availability', () => {
|
||||
|
||||
it('still says invalid for a bad token when keys are available', async () => {
|
||||
it('rejects an unknown key when the key set is available', async () => {
|
||||
stubCache()
|
||||
authUp(() => true)
|
||||
const verify = await coldVerifier()
|
||||
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
|
||||
expect(await verify(env, await mint(otherKey))).toEqual({ ok: false, reason: 'invalid' })
|
||||
expect(await verify(env, 'not-a-jwt')).toEqual({ ok: false, reason: 'invalid' })
|
||||
expect(await verify(env, await mint(privateKey, 'unknown-kid'))).toEqual({ ok: false, reason: 'invalid' })
|
||||
})
|
||||
|
||||
it('a cold isolate verifies from the persisted set while auth is down', async () => {
|
||||
const store = stubCache()
|
||||
stubCache()
|
||||
let up = true
|
||||
authUp(() => up)
|
||||
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
|
||||
const warm = await coldVerifier()
|
||||
expect((await warm(env, await mint())).ok).toBe(true)
|
||||
expect(store.size).toBe(1)
|
||||
up = false
|
||||
const cold = await coldVerifier()
|
||||
const result = await cold(env, await mint())
|
||||
|
||||
@@ -15,7 +15,7 @@ import { SignJWT, exportJWK, generateKeyPair, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import pg from 'pg'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
|
||||
|
||||
|
||||
@@ -10,4 +10,3 @@ vi.mock('../../relay/src/database/database.js', async (importOriginal) => ({
|
||||
}))
|
||||
|
||||
await import('../../relay/src/credentials/credential-store.test.ts')
|
||||
await import('../../relay/src/credentials/credential-store-cleanup.test.ts')
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const cell = {
|
||||
id: 'control-supersession-postgres',
|
||||
url: 'https://control-supersession-postgres.example.com',
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
const identity = {
|
||||
userId: 'control-supersession-postgres-user',
|
||||
relayHostId: 'supersedehost001'
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL control supersession', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
databases.push(
|
||||
await openRelayDatabase({ databaseUrl, dataDir: '' }),
|
||||
await openRelayDatabase({ databaseUrl, dataDir: '' })
|
||||
)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
const database = databases[0]
|
||||
if (database) {
|
||||
await database.query(
|
||||
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
|
||||
// A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
|
||||
await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
|
||||
cell.id
|
||||
])
|
||||
await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id])
|
||||
}
|
||||
for (const connection of databases) await connection.close()
|
||||
})
|
||||
|
||||
it('serializes parallel generations into one durable control', async () => {
|
||||
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
|
||||
await stores[0]!.reconcileCells([cell])
|
||||
await stores[0]!.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionInclusionWatermark: 1,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 50
|
||||
})
|
||||
const assignment = await stores[0]!.assign(identity)
|
||||
|
||||
await Promise.all([
|
||||
stores[0]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1,
|
||||
connectionInclusionWatermark: 10
|
||||
}),
|
||||
stores[1]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 2,
|
||||
connectionInclusionWatermark: 11
|
||||
})
|
||||
])
|
||||
await databases[0]!.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
|
||||
request_units, expires_at, updated_at)
|
||||
VALUES (?, ?, ?, 'control', ?, 1, 90100, 100),
|
||||
(?, ?, ?, 'control', ?, 1, 90100, 100)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`control:${cell.id}:100`,
|
||||
cell.id,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`control:${cell.id}:101`,
|
||||
cell.id
|
||||
]
|
||||
)
|
||||
// The store reserves a unit per control lease, so a hand-written pair has to
|
||||
// carry its own reservation or the fixture starts out of balance.
|
||||
await databases[0]!.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`,
|
||||
[cell.id]
|
||||
)
|
||||
await stores[0]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 102,
|
||||
connectionInclusionWatermark: 12
|
||||
})
|
||||
|
||||
const controls = await databases[0]!.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND activity_kind = 'control'`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(Number(controls[0]!.count)).toBe(1)
|
||||
const assignments = await databases[0]!.query(
|
||||
`SELECT reserved_controls FROM relay_assignments WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(Number(assignments[0]!.reserved_controls)).toBe(1)
|
||||
const cells = await databases[0]!.query(
|
||||
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
|
||||
[cell.id]
|
||||
)
|
||||
expect(Number(cells[0]!.reserved_requests)).toBe(1)
|
||||
const claims = await databases[0]!.query(
|
||||
`SELECT claim_activity_id FROM relay_control_connection_reservations
|
||||
WHERE user_id = ? AND state = 'claimed'`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(claims).toEqual([{ claim_activity_id: `control:${cell.id}:102` }])
|
||||
}, 15_000)
|
||||
})
|
||||
@@ -169,7 +169,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
|
||||
const ledger = new ReserveLedgerWriter(database, { now: () => now, log: () => undefined })
|
||||
for (const epoch of [6, 7]) ledger.enqueue({ userId: USER, relayHostId: host, cellId: dead.id, epoch })
|
||||
await ledger.flush()
|
||||
expect(ledger.pending()).toBe(0)
|
||||
expect(await rowOf(host)).toEqual({ cell_id: live.id, assignment_epoch: 7 })
|
||||
})
|
||||
|
||||
@@ -181,63 +180,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
|
||||
await expect(hostAssign(host, true, null)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
})
|
||||
|
||||
it('a withheld verdict (fleet or region guard) keeps the fence', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
await killDeadCell()
|
||||
store = new RelayAssignmentStore(database, () => now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: HEARTBEAT_TTL_MS,
|
||||
deadCellVerdict: () => ({
|
||||
dead: false,
|
||||
withheld: 'region',
|
||||
fleetFailing: 6,
|
||||
fleetPolled: 34,
|
||||
regionFailing: 6,
|
||||
regionPolled: 6
|
||||
})
|
||||
})
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
})
|
||||
|
||||
// Only a heartbeat that stopped (cause 'unheard') is death; polls failing for 45 s add nothing else.
|
||||
it('a cell heartbeating but not ready keeps its hosts, however long its polls fail', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
now += HEARTBEAT_TTL_MS + 15_000
|
||||
await heartbeat(dead, false)
|
||||
await heartbeat(live)
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
})
|
||||
|
||||
it('drain then death keeps the fence: a disabled cell is draining, not unheard', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
await database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [dead.id])
|
||||
try {
|
||||
await killDeadCell()
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
} finally {
|
||||
await database.query(`UPDATE relay_cells SET enabled = 1 WHERE cell_id = ?`, [dead.id])
|
||||
}
|
||||
})
|
||||
|
||||
it('LB drain: heartbeat fresh while polls fail, so the host is pinned to its cell', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
now += 10_000
|
||||
await heartbeat(dead)
|
||||
await heartbeat(live)
|
||||
expect(await hostAssign(host)).toMatchObject({ cellId: dead.id, assignmentEpoch: 3 })
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
})
|
||||
|
||||
it('DB outage or director freeze: heartbeat stale but polls answered (no verdict), so 503', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
|
||||
@@ -84,27 +84,4 @@ describePostgres('PostgreSQL deployment status', () => {
|
||||
restartBlockingReservedRequests: -1
|
||||
})
|
||||
})
|
||||
|
||||
it('reads the same counts through the reserve-aware status, and none for a reserve-mode cell', async () => {
|
||||
const store = new RelayAssignmentStore(database!, () => 100)
|
||||
const reserveCell = { ...cell, id: `${cell.id}-reserve`, url: 'https://deployment-status-reserve.example.com' }
|
||||
await store.reconcileCells([cell, reserveCell])
|
||||
await store.assign({ userId: 'aware-user', relayHostId: 'd1e2f3a4b5c6' })
|
||||
expect(await store.reserveAwareCellDeploymentStatus(cell.id)).toEqual({
|
||||
admitMode: 'db',
|
||||
status: await store.cellDeploymentStatus(cell.id)
|
||||
})
|
||||
|
||||
await store.setCellAdmitMode(reserveCell.id, 'reserve')
|
||||
await expect(store.cellDeploymentStatus(reserveCell.id)).rejects.toThrow('cell_in_reserve_mode')
|
||||
const reserve = await store.reserveAwareCellDeploymentStatus(reserveCell.id)
|
||||
expect(reserve).toMatchObject({
|
||||
admitMode: 'reserve',
|
||||
status: { cellId: reserveCell.id, cellUrl: reserveCell.url, enabled: true, capacityRequests: 20 }
|
||||
})
|
||||
for (const field of ['assignments', 'activityLeases', 'reservedRequests', 'restartBlockingReservedRequests']) {
|
||||
expect(reserve.status).not.toHaveProperty(field)
|
||||
}
|
||||
await expect(store.reserveAwareCellDeploymentStatus('missing-cell')).rejects.toThrow('cell_not_found')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,42 +3,27 @@ import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const NOW = 100_000
|
||||
const CELL_COUNT = 32
|
||||
const identity = { userId: 'headroom-user', relayHostId: 'headroomhost0001' }
|
||||
|
||||
function observeHeadroom(database: RelayDatabase, rowCounts: number[]): RelayDatabase {
|
||||
return {
|
||||
dialect: database.dialect,
|
||||
async query(sql, params) {
|
||||
const rows = await database.query(sql, params)
|
||||
if (sql.includes('FROM relay_cell_connection_limits limits')) rowCounts.push(rows.length)
|
||||
return rows
|
||||
},
|
||||
queryLocked: (sql, params, options) => database.queryLocked(sql, params, options),
|
||||
transaction: (operation, options) =>
|
||||
database.transaction((transaction) => operation(observeHeadroom(transaction, rowCounts)), options),
|
||||
close: () => database.close()
|
||||
}
|
||||
}
|
||||
|
||||
describe('assignment headroom query scope', () => {
|
||||
describe('assignment headroom admission', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
async function setup() {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
const rowCounts: number[] = []
|
||||
const store = new RelayAssignmentStore(observeHeadroom(database, rowCounts), () => NOW, {
|
||||
const store = new RelayAssignmentStore(database, () => NOW, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
const cells = Array.from({ length: CELL_COUNT }, (_, index) => ({
|
||||
id: `cell-${String(index).padStart(2, '0')}`,
|
||||
url: `https://cell-${index}.example.com`,
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}))
|
||||
const cells = [
|
||||
{
|
||||
id: 'cell-00',
|
||||
url: 'https://cell-0.example.com',
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
]
|
||||
await store.reconcileCells(cells)
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
@@ -69,29 +54,9 @@ describe('assignment headroom query scope', () => {
|
||||
kind: 'splice',
|
||||
cellId: 'cell-00'
|
||||
})
|
||||
rowCounts.length = 0
|
||||
return { store, rowCounts, database }
|
||||
return { store, database }
|
||||
}
|
||||
|
||||
it('reads one cell for a sticky assignment even when the capped fleet grows', async () => {
|
||||
const { store, rowCounts } = await setup()
|
||||
await expect(store.assign(identity)).resolves.toMatchObject({
|
||||
...identity,
|
||||
cellId: 'cell-00',
|
||||
assignmentEpoch: 1
|
||||
})
|
||||
console.info('sticky headroom rows', { fleetCells: CELL_COUNT, rowCounts })
|
||||
expect(rowCounts).toEqual([1])
|
||||
})
|
||||
|
||||
it('still considers the full fleet for a new placement', async () => {
|
||||
const { store, rowCounts } = await setup()
|
||||
await expect(
|
||||
store.assign({ userId: 'new-user', relayHostId: 'newheadroomhost1' })
|
||||
).resolves.toMatchObject({ cellId: 'cell-01', assignmentEpoch: 1 })
|
||||
expect(rowCounts).toEqual([CELL_COUNT])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['missing snapshot', `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = 'cell-00'`],
|
||||
['expired snapshot', `UPDATE relay_cell_connection_snapshots SET snapshot_at = ${NOW - 45_000} WHERE cell_id = 'cell-00'`],
|
||||
@@ -104,10 +69,9 @@ describe('assignment headroom query scope', () => {
|
||||
})
|
||||
|
||||
it('preserves admission for cells without a connection limit', async () => {
|
||||
const { store, rowCounts, database } = await setup()
|
||||
const { store, database } = await setup()
|
||||
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = 'cell-00'`)
|
||||
await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'cell-00' })
|
||||
expect(rowCounts).toEqual([0])
|
||||
})
|
||||
|
||||
it('counts outstanding reservations at the admission boundary', async () => {
|
||||
|
||||
@@ -106,15 +106,6 @@ describe('home cell unavailable', () => {
|
||||
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
|
||||
})
|
||||
|
||||
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
|
||||
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
|
||||
await store.assign(IDENTITY)
|
||||
|
||||
await expect(
|
||||
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
|
||||
).rejects.toThrow('relay_capacity_exhausted')
|
||||
})
|
||||
|
||||
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
|
||||
const home: RelayCellConfig = {
|
||||
id: 'home',
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
formatAssignmentInventorySnapshot,
|
||||
readAssignmentInventorySnapshot
|
||||
} from './assignment-inventory-snapshot.js'
|
||||
import { openInMemoryRelayDatabase } from '../database/database.js'
|
||||
|
||||
describe('assignment inventory snapshot', () => {
|
||||
it('reports per-cell counters, lease backlog, and reservation debt', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const now = 1_000_000
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 3999, 5, ?, ?)`,
|
||||
[now, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-a', 'general', ?)`,
|
||||
[now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_runtime
|
||||
(cell_id, cell_url, cell_incarnation, started_at, ready, observed_requests,
|
||||
last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 'inc-1', ?, 1, 5, ?, ?)`,
|
||||
[now - 60_000, now - 10_000, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units,
|
||||
expires_at, updated_at)
|
||||
VALUES
|
||||
('user-1', 'host-1', 'control:1', 'control', 'cell-a', 1, ?, ?),
|
||||
('user-1', 'host-2', 'control:1', 'control', 'cell-a', 3, ?, ?)`,
|
||||
[now - 1, now, now + 90_000, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_control_connection_reservations
|
||||
(reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch,
|
||||
cell_id, state, created_at, timeout_at, updated_at)
|
||||
VALUES
|
||||
('r1', 'k1', 'user-1', 'host-1', 1, 'cell-a', 'late-arrival-debt', ?, ?, ?),
|
||||
('r2', 'k2', 'user-1', 'host-2', 1, 'cell-a', 'reserved', ?, ?, ?),
|
||||
('r3', 'k3', 'user-1', 'host-3', 1, 'cell-a', 'claimed', ?, ?, ?),
|
||||
('r4', 'k4', 'user-1', 'host-4', 1, 'cell-a', 'released', ?, ?, ?)`,
|
||||
[now, now, now, now, now, now, now, now, now, now, now, now]
|
||||
)
|
||||
|
||||
const snapshot = await readAssignmentInventorySnapshot(database, now)
|
||||
|
||||
expect(snapshot.cells).toEqual([
|
||||
{
|
||||
cellId: 'cell-a',
|
||||
region: 'us-central1',
|
||||
admissionState: 'general',
|
||||
enabled: true,
|
||||
capacityRequests: 4000,
|
||||
reservedRequests: 3999,
|
||||
runtimeReady: true,
|
||||
heartbeatAgeMs: 10_000
|
||||
}
|
||||
])
|
||||
expect(snapshot.activityLeases).toEqual({ total: 2, expired: 1, requestUnits: 4 })
|
||||
expect(snapshot.connectionReservations).toEqual({ outstanding: 3, lateArrivalDebt: 1 })
|
||||
expect(snapshot.regionalRehomes).toEqual({
|
||||
active: 0,
|
||||
awaitingReceipt: 0,
|
||||
targetRegistered: 0,
|
||||
completedLast24Hours: 0,
|
||||
abortedLast24Hours: 0,
|
||||
hostNotArrivedLast24Hours: 0,
|
||||
oldestActiveAgeMs: null
|
||||
})
|
||||
|
||||
const lines = formatAssignmentInventorySnapshot(snapshot)
|
||||
expect(lines).toHaveLength(3)
|
||||
expect(lines[0]).toContain('cellId=cell-a')
|
||||
expect(lines[0]).toContain('reserved=3999')
|
||||
expect(lines[1]).toContain('expiredLeases=1')
|
||||
expect(lines[1]).toContain('lateArrivalDebt=1')
|
||||
expect(lines[2]).toContain('active=0')
|
||||
})
|
||||
|
||||
it('reports cells missing runtime and admission rows without failing', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 0)`
|
||||
)
|
||||
|
||||
const snapshot = await readAssignmentInventorySnapshot(database, 5_000)
|
||||
|
||||
expect(snapshot.cells[0]).toMatchObject({
|
||||
cellId: 'cell-b',
|
||||
admissionState: 'unset',
|
||||
enabled: false,
|
||||
runtimeReady: null,
|
||||
heartbeatAgeMs: null
|
||||
})
|
||||
expect(snapshot.activityLeases).toEqual({ total: 0, expired: 0, requestUnits: 0 })
|
||||
expect(snapshot.regionalRehomes.active).toBe(0)
|
||||
})
|
||||
})
|
||||
+1
-4
@@ -302,7 +302,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
|
||||
).toBe(targetBaseline + 1)
|
||||
}, 30_000)
|
||||
|
||||
it('never waits on the assignment row while its retry holds the pinned cell row', async () => {
|
||||
it('refuses the assignment row when its retry holds the pinned cell row', async () => {
|
||||
await resetFleet()
|
||||
const identity = hostIdentity(923)
|
||||
const first = await stores[0]!.assign(identity, 'us-central1')
|
||||
@@ -324,10 +324,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
await releaseCell()
|
||||
const cellFreedAt = performance.now()
|
||||
expect(await regrant).toBeInstanceOf(RelayAssignmentRowBusyError)
|
||||
// NOWAIT once the cell row is held: nowhere near the 1s bounded wait.
|
||||
expect(performance.now() - cellFreedAt).toBeLessThan(300)
|
||||
} finally {
|
||||
await releaseAssignment?.()
|
||||
await releaseCell()
|
||||
|
||||
@@ -1,22 +1,10 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import { SignJWT } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
ASSIGNMENT_LEASE_AUDIENCE,
|
||||
assignmentLeaseKeyId,
|
||||
classifyAssignmentLease
|
||||
} from './assignment-lease.js'
|
||||
import { AssignmentLeaseShadow } from './assignment-lease-shadow.js'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import type { RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import { HostSessionRegistry } from '../connections/host-session-registry.js'
|
||||
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from '../connections/splice-forwarder.js'
|
||||
|
||||
const KEY = new TextEncoder().encode('0123456789abcdef0123456789abcdef')
|
||||
const OTHER_KEY = new TextEncoder().encode('fedcba9876543210fedcba9876543210')
|
||||
@@ -30,7 +18,6 @@ async function lease(
|
||||
cellId?: string
|
||||
assignmentEpoch?: number
|
||||
sub?: string
|
||||
relayHostId?: string
|
||||
expiresAt?: string | number
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
@@ -40,7 +27,7 @@ async function lease(
|
||||
cellId: overrides.cellId ?? CELL_ID,
|
||||
cellUrl: 'https://c7.relay.example.test',
|
||||
assignmentEpoch: overrides.assignmentEpoch ?? 4,
|
||||
relayHostId: overrides.relayHostId ?? HOST.relayHostId
|
||||
relayHostId: HOST.relayHostId
|
||||
})
|
||||
.setProtectedHeader(kid === null ? { alg: 'HS256' } : { alg: 'HS256', kid })
|
||||
.setIssuer('https://relay.example.test')
|
||||
@@ -73,7 +60,6 @@ describe('assignment lease classification', () => {
|
||||
expect(await classify(await lease({ kid: assignmentLeaseKeyId(OTHER_KEY) }))).toBe(
|
||||
'bad-signature'
|
||||
)
|
||||
expect(await classify('not.a.jwt')).toBe('bad-signature')
|
||||
expect(await classify('x'.repeat(5_000))).toBe('bad-signature')
|
||||
expect(await classify(await lease({ sub: 'user-2' }))).toBe('wrong-host')
|
||||
expect(await classify(await lease({ cellId: 'production-gce-c8' }))).toBe('wrong-cell')
|
||||
@@ -81,127 +67,3 @@ describe('assignment lease classification', () => {
|
||||
expect(await classify(await lease({ assignmentEpoch: 5 }))).toBe('epoch-ahead')
|
||||
})
|
||||
})
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(''))
|
||||
})
|
||||
}
|
||||
|
||||
describe('host hello with the lease shadow on', () => {
|
||||
async function hello(input: { withLease: boolean; databaseSays: boolean }) {
|
||||
const keyPair = nacl.box.keyPair()
|
||||
const relayHostId = createHash('sha256')
|
||||
.update(keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
const identity: RelayTokenClaims = {
|
||||
sub: HOST.userId,
|
||||
prof: 'profile-1',
|
||||
relayHostId,
|
||||
purpose: 'host-control',
|
||||
exp: 4_102_444_800
|
||||
}
|
||||
const recorded: string[] = []
|
||||
const shadow = new AssignmentLeaseShadow({ enabled: () => true, key: KEY, cellId: CELL_ID })
|
||||
const record = shadow.record.bind(shadow)
|
||||
vi.spyOn(shadow, 'record').mockImplementation((check, dbValid, dbReadMs) => {
|
||||
void check.then((leaseClass) => recorded.push(`${leaseClass}:${dbValid}`))
|
||||
record(check, dbValid, dbReadMs)
|
||||
})
|
||||
const verifyCellAssignment = vi.fn(async () => input.databaseSays)
|
||||
const registry = new HostSessionRegistry(
|
||||
config(),
|
||||
vi.fn(),
|
||||
{} as RelayCredentialStore,
|
||||
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
|
||||
new ProcessQueuedByteBudget(),
|
||||
{
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn()
|
||||
},
|
||||
Date.now,
|
||||
Math.random,
|
||||
'incarnation-1',
|
||||
shadow
|
||||
)
|
||||
const socket = new FakeSocket()
|
||||
const hostLease = input.withLease ? await lease({ relayHostId }) : undefined
|
||||
registry.acceptControl(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
undefined,
|
||||
undefined,
|
||||
hostLease
|
||||
)
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId,
|
||||
assignmentEpoch: 4,
|
||||
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
|
||||
appVersion: '1.4.200'
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
await vi.waitFor(() => expect(recorded).toHaveLength(1))
|
||||
return { socket, recorded, verifyCellAssignment }
|
||||
}
|
||||
|
||||
it('still refuses a host the database refuses, whatever its lease says', async () => {
|
||||
const { socket, recorded } = await hello({ withLease: true, databaseSays: false })
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
|
||||
expect(recorded).toEqual(['valid:false'])
|
||||
})
|
||||
|
||||
it('still admits a host the database admits, with no lease at all', async () => {
|
||||
const { socket, recorded, verifyCellAssignment } = await hello({
|
||||
withLease: false,
|
||||
databaseSays: true
|
||||
})
|
||||
expect(verifyCellAssignment).toHaveBeenCalledTimes(1)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
expect(String(socket.send.mock.calls[0]?.[0])).toContain('"type":"host-challenge"')
|
||||
expect(recorded).toEqual(['absent:true'])
|
||||
})
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://c7.relay.example.test',
|
||||
cellUrl: 'https://c7.relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: KEY,
|
||||
role: 'cell',
|
||||
cellId: CELL_ID,
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const CELL = { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
|
||||
const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
|
||||
describe('assign marks only the answers it minted as placed', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
async function setup(now: () => number): Promise<RelayAssignmentStore> {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayAssignmentStore(database, now)
|
||||
await store.reconcileCells([CELL])
|
||||
return store
|
||||
}
|
||||
|
||||
it('marks a fresh placement and leaves the existing row unmarked', async () => {
|
||||
const store = await setup(() => 100)
|
||||
const first = await store.assign(IDENTITY)
|
||||
const again = await store.assign(IDENTITY)
|
||||
|
||||
expect(first.placed).toBe(true)
|
||||
expect(again).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
|
||||
expect(again.placed).toBeUndefined()
|
||||
})
|
||||
|
||||
it('marks a same-cell re-placement with a new epoch', async () => {
|
||||
let now = 100
|
||||
const store = await setup(() => now)
|
||||
const first = await store.assign(IDENTITY)
|
||||
await store.changeActivity(IDENTITY, 'control', -1)
|
||||
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.releaseExpiredActivity()
|
||||
now += ASSIGNMENT_LIMITS.dormantTtlMs
|
||||
const replaced = await store.assign(IDENTITY)
|
||||
|
||||
expect(replaced).toMatchObject({
|
||||
cellId: first.cellId,
|
||||
assignmentEpoch: first.assignmentEpoch + 1
|
||||
})
|
||||
expect(replaced.placed).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -36,28 +36,8 @@ describe('assignment rejection log window', () => {
|
||||
|
||||
expect(window.admit('assign:placement:host-rate-limited', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:sticky:host-rate-limited', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(false)
|
||||
})
|
||||
|
||||
it('bounds the tracked keys and reports what an evicted window suppressed', () => {
|
||||
const timers = manualTimers()
|
||||
const closed: { key: string; suppressed: number }[] = []
|
||||
const window = new AssignmentRejectionLogWindow<string>({
|
||||
windowMs: 10_000,
|
||||
schedule: timers.schedule,
|
||||
onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed })
|
||||
})
|
||||
|
||||
window.admit('key-0', 'host-a')
|
||||
window.admit('key-0', 'host-b')
|
||||
for (let index = 1; index < 200; index++) window.admit(`key-${index}`, 'host-a')
|
||||
|
||||
expect(closed[0]).toEqual({ key: 'key-0', suppressed: 1 })
|
||||
// Evicted keys emit again rather than staying silently suppressed.
|
||||
expect(window.admit('key-0', 'host-a')).toBe(true)
|
||||
expect(window.admit('key-199', 'host-a')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
function manualTimers(): {
|
||||
|
||||
@@ -15,11 +15,9 @@ vi.mock('../authentication/relay-token-verifier.js', () => ({
|
||||
}))
|
||||
|
||||
import { createRelayApp } from '../app.js'
|
||||
import { relayHostLogDigest } from '../observability/relay-host-log-digest.js'
|
||||
|
||||
describe('assignment rejection logging', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
@@ -49,44 +47,9 @@ describe('assignment rejection logging', () => {
|
||||
expect(line).toContain('lane=sticky')
|
||||
expect(line).toContain('hinted=true')
|
||||
expect(line).toContain('reason=relay_capacity_exhausted')
|
||||
expect(line).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
expect(line).toContain('host=af342ad00ee4')
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('names the throttled host once per window and closes it with the suppressed count', async () => {
|
||||
vi.useFakeTimers()
|
||||
const host = 'mmmmmmmmmmmmmmmm'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
assign: vi.fn(async () => assignment('cell-r', host)),
|
||||
resolve: vi.fn(async () => null)
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
|
||||
|
||||
const throttled = (): string[] =>
|
||||
warn.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.filter((entry) => entry.includes('reason=host-rate-limited'))
|
||||
expect(throttled()).toHaveLength(1)
|
||||
expect(throttled()[0]).toContain('route=assign')
|
||||
expect(throttled()[0]).toContain('lane=placement')
|
||||
expect(throttled()[0]).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
expect(throttled()[0]).not.toContain('suppressed=')
|
||||
expect(throttled()[0]).not.toContain(host)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(throttled()).toHaveLength(2)
|
||||
expect(throttled()[1]).toContain('suppressed=1')
|
||||
expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
})
|
||||
})
|
||||
|
||||
function assignmentRequest(
|
||||
|
||||
@@ -202,49 +202,5 @@ describePostgres('reserve-mode census on PostgreSQL', () => {
|
||||
await holder.close()
|
||||
}
|
||||
}, 20_000)
|
||||
|
||||
// A stall must not double connection demand: concurrent assigns share one reserve-set read,
|
||||
// and after it fails the last answer stands for the backoff without another query.
|
||||
it('reads the reserve set once for concurrent assigns during a stall, and not again in the backoff', async () => {
|
||||
const timed = await openRelayDatabase({ databaseUrl, dataDir: '/tmp', statementTimeoutMs: 300 })
|
||||
const holder = await openRelayDatabase({ databaseUrl, dataDir: '/tmp' })
|
||||
let release!: () => void
|
||||
const released = new Promise<void>((resolve) => (release = resolve))
|
||||
let held!: () => void
|
||||
const holding = new Promise<void>((resolve) => (held = resolve))
|
||||
const lock = holder.transaction(async (transaction) => {
|
||||
await transaction.query(`LOCK TABLE relay_cell_admit_modes IN ACCESS EXCLUSIVE MODE`)
|
||||
held()
|
||||
await released
|
||||
})
|
||||
try {
|
||||
await holding
|
||||
const store = new RelayAssignmentStore(timed, () => 500_000)
|
||||
await store.reconcileCells([cellA, cellB], false)
|
||||
await heartbeat(store, cellA)
|
||||
await heartbeat(store, cellB)
|
||||
const query = timed.query.bind(timed)
|
||||
let reads = 0
|
||||
timed.query = async (sql, params) => {
|
||||
if (sql.includes('FROM relay_cell_admit_modes m')) reads += 1
|
||||
return await query(sql, params)
|
||||
}
|
||||
const placed = await Promise.all(
|
||||
Array.from({ length: 6 }, async (_, index) =>
|
||||
await store.assign({ userId: USER, relayHostId: `hoststall0000${String(index).padStart(3, '0')}` })
|
||||
)
|
||||
)
|
||||
expect(placed).toHaveLength(6)
|
||||
expect(reads).toBe(1)
|
||||
await store.assign({ userId: USER, relayHostId: 'hoststall0000100' })
|
||||
expect(await store.reserveModeCells()).toBeNull()
|
||||
expect(reads).toBe(1)
|
||||
} finally {
|
||||
release()
|
||||
await lock
|
||||
await timed.close()
|
||||
await holder.close()
|
||||
}
|
||||
}, 20_000)
|
||||
})
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ describe('assignment reserve-mode behavior', () => {
|
||||
const { unknown, restore } = await unknownSet(() => now)
|
||||
expect(await unknown.reserveModeCells()).toBeNull()
|
||||
expect(await unknown.releaseExpiredActivityLeases()).toBe(0)
|
||||
expect(await unknown.evacuateDeadCells()).toBe(0)
|
||||
await restore()
|
||||
await setReserve(store, [])
|
||||
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
|
||||
@@ -125,7 +124,7 @@ describe('assignment reserve-mode behavior', () => {
|
||||
}
|
||||
return await query(sql, params)
|
||||
}
|
||||
return { counter, release, restore: () => (database!.query = query) }
|
||||
return { release, restore: () => (database!.query = query) }
|
||||
}
|
||||
|
||||
it('reads afresh after an admit-mode write instead of joining a read begun before it', async () => {
|
||||
@@ -155,12 +154,10 @@ describe('assignment reserve-mode behavior', () => {
|
||||
try {
|
||||
const before = store.reserveModeCells()
|
||||
await store.setCellAdmitMode('cell-a', 'reserve')
|
||||
const reads = held.counter.reads
|
||||
held.release()
|
||||
await before
|
||||
// A write cleared the cache: the next caller reads now, not after a 5 s backoff.
|
||||
expect([...((await store.reserveModeCells()) ?? [])]).toEqual(['cell-a'])
|
||||
expect(held.counter.reads).toBeGreaterThan(reads)
|
||||
} finally {
|
||||
held.release()
|
||||
held.restore()
|
||||
@@ -197,13 +194,8 @@ describe('assignment reserve-mode behavior', () => {
|
||||
|
||||
it('never places a fresh host on a reserve-mode cell through the database', async () => {
|
||||
const store = await setup(() => 1_000, ['cell-a'])
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
const placed = await store.assign({
|
||||
userId: `user-${index}`,
|
||||
relayHostId: `host${String(index).padStart(12, '0')}`
|
||||
})
|
||||
expect(placed.cellId).toBe('cell-b')
|
||||
}
|
||||
const placed = await store.assign({ userId: 'user-0', relayHostId: 'host000000000000' })
|
||||
expect(placed.cellId).toBe('cell-b')
|
||||
})
|
||||
|
||||
it('refuses drains, evacuations and status reads that would treat a reserve-mode cell as empty', async () => {
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
RelayAssignmentStore,
|
||||
type AssignOptions,
|
||||
type RelayAssignment
|
||||
} from './assignment-store.js'
|
||||
import type { RelayCellConfig } from '../config.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
type RelayDatabase,
|
||||
type RelayLockOptions,
|
||||
type RelayTransactionOptions,
|
||||
type SqlRow
|
||||
} from '../database/database.js'
|
||||
import { ReservePaceError } from './reserve-placement/reserve-assignment.js'
|
||||
|
||||
const CELLS: RelayCellConfig[] = [
|
||||
{ id: 'us-c1', url: 'https://us-c1.relay.example.com', region: 'us-central1', capacityRequests: 100 },
|
||||
{ id: 'asia-c1', url: 'https://asia-c1.relay.example.com', region: 'asia-east2', capacityRequests: 100 }
|
||||
]
|
||||
const IDENTITY = { userId: 'spill-user', relayHostId: 'spillhost0000001' }
|
||||
const ON_US_RESERVE: RelayAssignment = {
|
||||
...IDENTITY,
|
||||
cellId: 'us-reserve',
|
||||
cellUrl: 'https://us-reserve.relay.example.com',
|
||||
assignmentEpoch: 8,
|
||||
leaseExpiresAt: 1_000_000,
|
||||
region: 'us-central1'
|
||||
}
|
||||
|
||||
// Reports the target cell full when its reservation is taken: the race between pick and reserve.
|
||||
class FullAtReserveDatabase implements RelayDatabase {
|
||||
armed = false
|
||||
|
||||
constructor(private readonly delegate: RelayDatabase) {}
|
||||
|
||||
get dialect(): 'sqlite' | 'postgres' | undefined {
|
||||
return this.delegate.dialect
|
||||
}
|
||||
|
||||
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
||||
return await this.delegate.query(sql, params)
|
||||
}
|
||||
|
||||
async queryLocked(sql: string, params?: unknown[], options?: RelayLockOptions): Promise<SqlRow[]> {
|
||||
const rows = await this.delegate.queryLocked(sql, params, options)
|
||||
if (!this.armed || sql.trim() !== 'SELECT * FROM relay_cells WHERE cell_id = ?') {
|
||||
return rows
|
||||
}
|
||||
return rows.map((row) => ({ ...row, reserved_requests: row.capacity_requests }))
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>, options?: RelayTransactionOptions): Promise<T> {
|
||||
return await this.delegate.transaction(async (inner) => {
|
||||
const wrapped = new FullAtReserveDatabase(inner)
|
||||
wrapped.armed = this.armed
|
||||
return await operation(wrapped)
|
||||
}, options)
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.delegate.close()
|
||||
}
|
||||
}
|
||||
|
||||
describe('reserve spill before relay_capacity_exhausted', () => {
|
||||
let database: FullAtReserveDatabase
|
||||
let store: RelayAssignmentStore
|
||||
let now: number
|
||||
|
||||
beforeEach(async () => {
|
||||
now = 100
|
||||
database = new FullAtReserveDatabase(await openInMemoryRelayDatabase())
|
||||
store = new RelayAssignmentStore(database, () => now)
|
||||
await store.reconcileCells(CELLS)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await database.close()
|
||||
})
|
||||
|
||||
const closeAllDatabaseCells = async () => {
|
||||
for (const cell of CELLS) {
|
||||
await store.configureCell(cell, 'migration-only')
|
||||
}
|
||||
}
|
||||
const assignAsia = async (step5: AssignOptions) =>
|
||||
await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, step5)
|
||||
|
||||
it('books a reserve seat when no database cell has room anywhere', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ reserveSpill })).resolves.toEqual({ ...ON_US_RESERVE, placed: true })
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: 0 })
|
||||
expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
|
||||
})
|
||||
|
||||
it('skips the own region after the in-region reserve fallback already tried it', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const regionReserve = vi.fn(async () => null)
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ regionReserve, reserveSpill })).resolves.toMatchObject({ cellId: 'us-reserve' })
|
||||
expect(regionReserve).toHaveBeenCalledOnce()
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: true, rowEpoch: 0 })
|
||||
})
|
||||
|
||||
it('hands the spill the epoch of the host row it read', async () => {
|
||||
const first = await assignAsia({})
|
||||
// Dormant: its next assign is a placement, which reads the row and finds no cell.
|
||||
await store.changeActivity(IDENTITY, 'control', -1)
|
||||
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.releaseExpiredActivity()
|
||||
now += ASSIGNMENT_LIMITS.dormantTtlMs
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await assignAsia({ reserveSpill })
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: first.assignmentEpoch })
|
||||
})
|
||||
|
||||
it('refuses with relay_capacity_exhausted when no reserve seat exists either', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
await expect(assignAsia({ reserveSpill: async () => null })).rejects.toThrow('relay_capacity_exhausted')
|
||||
})
|
||||
|
||||
it('answers a paced spill with its pace, not a refusal', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => {
|
||||
throw new ReservePaceError(4)
|
||||
})
|
||||
await expect(assignAsia({ reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
|
||||
})
|
||||
|
||||
it('answers an in-region reserve pace before any spill', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
const regionReserve = async () => {
|
||||
throw new ReservePaceError(2)
|
||||
}
|
||||
await expect(assignAsia({ regionReserve, reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
|
||||
expect(reserveSpill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not spill a reservation race: the host retries and re-picks', async () => {
|
||||
database.armed = true
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ reserveSpill })).rejects.toThrow('relay_capacity_exhausted')
|
||||
expect(reserveSpill).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -143,18 +143,12 @@ async function setup(cells: RelayCellConfig[] = [SOURCE, TARGET]) {
|
||||
// The old process stopped heartbeating at START_MS, so it is unheard from here.
|
||||
const OLD_UNHEARD_AT = START_MS + HEARTBEAT_TTL_MS + 1_000
|
||||
|
||||
// Both routes that refused in production: a hinted assign (placement lane after its
|
||||
// verification read finds the cell not live) and resolve, which then assigns.
|
||||
async function hintedAssign(store: RelayAssignmentStore) {
|
||||
return await store.assign(IDENTITY, 'us-central1', 'us-central1', undefined, {
|
||||
hostRequested: true
|
||||
})
|
||||
}
|
||||
|
||||
async function resolveRoute(store: RelayAssignmentStore) {
|
||||
return (await store.resolve(IDENTITY)) ?? (await store.assign(IDENTITY))
|
||||
}
|
||||
|
||||
async function expectRefused(
|
||||
attempt: Promise<unknown>,
|
||||
cause: 'not_ready' | 'unheard'
|
||||
@@ -164,10 +158,7 @@ async function expectRefused(
|
||||
expect(error).toMatchObject({ unavailableCause: cause })
|
||||
}
|
||||
|
||||
describe.each([
|
||||
['hinted assign', hintedAssign],
|
||||
['resolve', resolveRoute]
|
||||
] as const)('a host offline through a roll redials via %s', (_route, redial) => {
|
||||
describe('a host offline through a roll redials via hinted assign', () => {
|
||||
describe('while the replacement process boots (not ready)', () => {
|
||||
it('is re-placed when that process started after the roll isolated the cell', async () => {
|
||||
const { store, first, heartbeat, at, isolateForRoll } = await setup()
|
||||
@@ -175,7 +166,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
const moved = await redial(store)
|
||||
const moved = await hintedAssign(store)
|
||||
expect(moved).toMatchObject({ cellId: TARGET.id })
|
||||
expect(moved.assignmentEpoch).toBeGreaterThan(first.assignmentEpoch)
|
||||
})
|
||||
@@ -186,7 +177,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused while the new process reports holding connections', async () => {
|
||||
@@ -200,7 +191,7 @@ describe.each([
|
||||
totalConnections: 1
|
||||
})
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused once the new process itself goes unheard', async () => {
|
||||
@@ -210,7 +201,7 @@ describe.each([
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
await at(20_000 + HEARTBEAT_TTL_MS + 1_000)
|
||||
|
||||
await expectRefused(redial(store), 'unheard')
|
||||
await expectRefused(hintedAssign(store), 'unheard')
|
||||
})
|
||||
|
||||
it('is refused on an ordinary readiness dip with no roll stamp', async () => {
|
||||
@@ -218,7 +209,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused on a migration-only cell parked without a roll stamp', async () => {
|
||||
@@ -227,7 +218,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -238,7 +229,7 @@ describe.each([
|
||||
await isolateForRoll()
|
||||
await at(OLD_UNHEARD_AT)
|
||||
|
||||
await expectRefused(redial(store), 'unheard')
|
||||
await expectRefused(hintedAssign(store), 'unheard')
|
||||
})
|
||||
|
||||
it('is refused when the roll stamp is stale, even with a new process', async () => {
|
||||
@@ -248,7 +239,7 @@ describe.each([
|
||||
await at(stale)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: stale - 1, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
|
||||
const cells = [
|
||||
{
|
||||
id: 'row-lock-order-a',
|
||||
url: 'https://row-lock-order-a.example.com',
|
||||
capacityRequests: 200,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
},
|
||||
{
|
||||
id: 'row-lock-order-b',
|
||||
url: 'https://row-lock-order-b.example.com',
|
||||
capacityRequests: 200,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
]
|
||||
const userId = 'row-lock-order-user'
|
||||
const hosts = ['rowlockhost00001', 'rowlockhost00002', 'rowlockhost00003', 'rowlockhost00004'].map(
|
||||
(relayHostId) => ({ userId, relayHostId })
|
||||
)
|
||||
|
||||
// pg_stat_database exposes cycles hidden by three 40P01 retries; eventual success alone misses deadlocks that exhaust retries under load.
|
||||
describePostgres('PostgreSQL row lock order', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
for (let index = 0; index < 4; index++) {
|
||||
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
|
||||
}
|
||||
})
|
||||
|
||||
async function removeTestRows(database: RelayDatabase): Promise<void> {
|
||||
await database.query(
|
||||
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
|
||||
[userId]
|
||||
)
|
||||
for (const table of [
|
||||
'relay_control_capabilities',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_post_drain_migration_pins',
|
||||
'relay_assignment_migration_incarnations',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [userId])
|
||||
}
|
||||
for (const cell of cells) {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_runtime',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterAll(async () => {
|
||||
if (databases[0]) await removeTestRows(databases[0])
|
||||
for (const connection of databases) await connection.close()
|
||||
})
|
||||
|
||||
async function deadlockCount(): Promise<number> {
|
||||
const rows = await databases[0]!.query(
|
||||
`SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()`
|
||||
)
|
||||
return Number(rows[0]!.deadlocks)
|
||||
}
|
||||
|
||||
it('runs the cell accept and the placement retry concurrently without a cycle', async () => {
|
||||
await removeTestRows(databases[0]!)
|
||||
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
|
||||
await stores[0]!.reconcileCells(cells)
|
||||
for (const cell of cells) {
|
||||
await stores[0]!.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionInclusionWatermark: 1,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 50
|
||||
})
|
||||
}
|
||||
const placements = new Map<string, { cellId: string; assignmentEpoch: number }>()
|
||||
for (const identity of hosts) {
|
||||
const assignment = await stores[0]!.assign(identity)
|
||||
placements.set(identity.relayHostId, assignment)
|
||||
}
|
||||
|
||||
const before = await deadlockCount()
|
||||
// The accept path (host rows, then the cell row last) against the paths
|
||||
// that must read the inventory first: placement and evacuation.
|
||||
for (let round = 0; round < 12; round++) {
|
||||
await Promise.allSettled(
|
||||
hosts.flatMap((identity, index) => {
|
||||
const placement = placements.get(identity.relayHostId)!
|
||||
const store = stores[index % stores.length]!
|
||||
const other = stores[(index + 1) % stores.length]!
|
||||
return [
|
||||
store.activateControl(identity, {
|
||||
cellId: placement.cellId,
|
||||
assignmentEpoch: placement.assignmentEpoch,
|
||||
generation: round + 2
|
||||
}),
|
||||
other.assign(identity),
|
||||
other.startEvacuation(
|
||||
identity,
|
||||
placement.cellId === cells[0]!.id ? cells[1]!.id : cells[0]!.id
|
||||
)
|
||||
]
|
||||
})
|
||||
)
|
||||
}
|
||||
const after = await deadlockCount()
|
||||
|
||||
expect(after - before).toBe(0)
|
||||
}, 120_000)
|
||||
})
|
||||
@@ -5,46 +5,16 @@ import type { RelayCellConfig } from '../../config.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
openRelayDatabase,
|
||||
type RelayDatabase,
|
||||
type SqlRow
|
||||
type RelayDatabase
|
||||
} from '../../database/database.js'
|
||||
|
||||
// Both dialects must skip host scans with no actionable cells; stale existing-only rows previously scanned every assignment for nothing.
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const schema = 'relay_dead_cell_precheck_test'
|
||||
const HOST_QUERY = 'FROM relay_assignments assignment'
|
||||
const CELLS: RelayCellConfig[] = [
|
||||
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
|
||||
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
|
||||
]
|
||||
|
||||
class RecordingDatabase implements RelayDatabase {
|
||||
readonly statements: string[] = []
|
||||
|
||||
constructor(private readonly delegate: RelayDatabase) {}
|
||||
|
||||
get dialect() {
|
||||
return this.delegate.dialect
|
||||
}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
this.statements.push(sql)
|
||||
return await this.delegate.query(sql, params)
|
||||
}
|
||||
|
||||
async queryLocked(...args: Parameters<RelayDatabase['queryLocked']>): Promise<SqlRow[]> {
|
||||
return await this.delegate.queryLocked(...args)
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
|
||||
return await this.delegate.transaction(operation)
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.delegate.close()
|
||||
}
|
||||
}
|
||||
|
||||
function scopedUrl(): string {
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
@@ -91,13 +61,12 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
|
||||
async function setup(now: () => number) {
|
||||
database = await open()
|
||||
const recording = new RecordingDatabase(database)
|
||||
const store = new RelayAssignmentStore(recording, now, {
|
||||
const store = new RelayAssignmentStore(database, now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
await store.reconcileCells(CELLS)
|
||||
return { store, recording }
|
||||
return { store }
|
||||
}
|
||||
|
||||
async function heartbeat(store: RelayAssignmentStore, cell: RelayCellConfig): Promise<void> {
|
||||
@@ -111,31 +80,9 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
})
|
||||
}
|
||||
|
||||
it('skips the host query when the only dead cell is unfenced and existing-only', async () => {
|
||||
let now = 100
|
||||
const { store, recording } = await setup(() => now)
|
||||
for (const cell of CELLS) await heartbeat(store, cell)
|
||||
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
expect(await store.assign(identity)).toMatchObject({ cellId: 'cell-a' })
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
await store.setCellEnabled('cell-a', false)
|
||||
now += 45_001
|
||||
await heartbeat(store, CELLS[1]!)
|
||||
recording.statements.length = 0
|
||||
|
||||
expect(await store.evacuateDeadCells()).toBe(0)
|
||||
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(false)
|
||||
expect(
|
||||
await database!.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
|
||||
identity.userId
|
||||
])
|
||||
).toEqual([{ cell_id: 'cell-a' }])
|
||||
})
|
||||
|
||||
it('still evacuates hosts from a dead uncapped cell that admits', async () => {
|
||||
let now = 100
|
||||
const { store, recording } = await setup(() => now)
|
||||
const { store } = await setup(() => now)
|
||||
for (const cell of CELLS) await heartbeat(store, cell)
|
||||
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
@@ -143,10 +90,8 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
now += 45_001
|
||||
await heartbeat(store, CELLS[1]!)
|
||||
recording.statements.length = 0
|
||||
|
||||
expect(await store.evacuateDeadCells()).toBe(1)
|
||||
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(true)
|
||||
expect((await store.resolve(identity))?.cellId).toBe('cell-b')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
|
||||
import type { SqlRow } from '../../database/database.js'
|
||||
|
||||
function text(row: SqlRow, field: string): string {
|
||||
const value = row[field]
|
||||
if (typeof value !== 'string') {
|
||||
throw new Error(`invalid_${field}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
it('keeps first assignment matches, lease order, row identity, and separate identity components', () => {
|
||||
const rows = [
|
||||
{ user_id: 'a:b', relay_host_id: 'c', value: 1 },
|
||||
{ user_id: 'a', relay_host_id: 'b:c', value: 2 },
|
||||
{ user_id: 'a:b', relay_host_id: 'c', value: 3 },
|
||||
{ user_id: '', relay_host_id: '', value: 4 }
|
||||
]
|
||||
const lookup = createDrainMigrationRowLookup(rows, text)
|
||||
for (const identity of [
|
||||
{ userId: 'a:b', relayHostId: 'c' },
|
||||
{ userId: 'a', relayHostId: 'b:c' },
|
||||
{ userId: '', relayHostId: '' },
|
||||
{ userId: 'missing', relayHostId: 'c' }
|
||||
]) {
|
||||
const expected = rows.filter(
|
||||
(row) =>
|
||||
row.user_id === identity.userId && row.relay_host_id === identity.relayHostId
|
||||
)
|
||||
expect(lookup.first(identity)).toBe(expected[0])
|
||||
expect(lookup.all(identity)).toEqual(expected)
|
||||
lookup.all(identity).forEach((row, index) => expect(row).toBe(expected[index]))
|
||||
}
|
||||
})
|
||||
|
||||
it('retains lazy validation and short circuiting when an inventory is malformed', () => {
|
||||
const first = { user_id: 'user', relay_host_id: 'host' }
|
||||
const identity = { userId: 'user', relayHostId: 'host' }
|
||||
const lookup = createDrainMigrationRowLookup(
|
||||
[first, { user_id: null, relay_host_id: 'bad' }],
|
||||
text
|
||||
)
|
||||
expect(lookup.first(identity)).toBe(first)
|
||||
expect(() => lookup.all(identity)).toThrow('invalid_user_id')
|
||||
const unrelated = createDrainMigrationRowLookup(
|
||||
[first, { user_id: 'other', relay_host_id: null }],
|
||||
text
|
||||
)
|
||||
expect(unrelated.all(identity)).toEqual([first])
|
||||
expect(() => unrelated.first({ userId: 'other', relayHostId: 'host' })).toThrow(
|
||||
'invalid_relay_host_id'
|
||||
)
|
||||
})
|
||||
@@ -1,122 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from '../../database/database.js'
|
||||
|
||||
let database: RelayDatabase | undefined
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
it.each([false, true])(
|
||||
'looks up a whole drain inventory with linear identity reads (expired: %s)',
|
||||
async (expired) => {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
let measuring = false
|
||||
let identityReads = 0
|
||||
let indexedRows = 0
|
||||
const instrument = (delegate: RelayDatabase): RelayDatabase => ({
|
||||
dialect: delegate.dialect,
|
||||
query: (sql, params) => delegate.query(sql, params),
|
||||
queryLocked: async (sql, params, options) => {
|
||||
const rows = await delegate.queryLocked(sql, params, options)
|
||||
if (
|
||||
!measuring ||
|
||||
!sql.includes('WHERE EXISTS') ||
|
||||
!(sql.includes('SELECT assignment.*') || sql.includes('SELECT lease.*'))
|
||||
) {
|
||||
return rows
|
||||
}
|
||||
indexedRows += rows.length
|
||||
return rows.map(
|
||||
(row): SqlRow =>
|
||||
new Proxy(row, {
|
||||
get(target, key) {
|
||||
if (key === 'user_id' || key === 'relay_host_id') {
|
||||
identityReads++
|
||||
}
|
||||
return Reflect.get(target, key)
|
||||
}
|
||||
})
|
||||
)
|
||||
},
|
||||
transaction: (operation, options) =>
|
||||
delegate.transaction((tx) => operation(instrument(tx)), options),
|
||||
close: () => delegate.close()
|
||||
})
|
||||
let now = 100
|
||||
const store = new RelayAssignmentStore(instrument(database), () => now, {
|
||||
requireLiveCells: true
|
||||
})
|
||||
const cells = ['a', 'b'].map((id) => ({
|
||||
id: `cell-${id}`,
|
||||
url: `https://relay-${id}.example.com`,
|
||||
capacityRequests: 500
|
||||
}))
|
||||
await store.reconcileCells(cells)
|
||||
const incarnation = '11111111-1111-4111-8111-111111111111'
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: incarnation,
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
}
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
const identities = Array.from({ length: 50 }, (_, index) => ({
|
||||
userId: `user-${index % 5}`,
|
||||
relayHostId: `host${String(index).padStart(12, '0')}`
|
||||
}))
|
||||
for (const identity of identities) {
|
||||
await store.assign(identity)
|
||||
}
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
await store.setCellEnabled('cell-a', false)
|
||||
for (const identity of identities) {
|
||||
const migration = await store.startEvacuation(identity, 'cell-b')
|
||||
await store.markMigrationTargetRegistered(identity, {
|
||||
cellId: 'cell-b',
|
||||
assignmentEpoch: migration.assignmentEpoch
|
||||
})
|
||||
}
|
||||
const attempt = {
|
||||
attemptId: '55555555-5555-4555-8555-555555555555',
|
||||
cellId: 'cell-a',
|
||||
cellIncarnation: incarnation,
|
||||
traceValue: '66666666-6666-4666-8666-666666666666',
|
||||
plannedGraceMs: 120_000
|
||||
}
|
||||
await store.prepareCellDrainAttempt(attempt)
|
||||
if (expired) {
|
||||
now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: incarnation,
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
}
|
||||
}
|
||||
measuring = true
|
||||
await expect(store.beginCellDrainSend(attempt)).resolves.toMatchObject({
|
||||
state: 'send-may-have-started',
|
||||
shouldSend: true
|
||||
})
|
||||
expect(indexedRows).toBeGreaterThanOrEqual(100)
|
||||
expect(identityReads).toBeLessThanOrEqual(indexedRows * 2)
|
||||
const migrations = await database.query(
|
||||
'SELECT expires_at FROM relay_assignment_migrations'
|
||||
)
|
||||
expect(migrations).toHaveLength(50)
|
||||
expect(
|
||||
migrations.every(
|
||||
(row) => row.expires_at === now + ASSIGNMENT_LIMITS.migrationLeaseMs
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
)
|
||||
-282
@@ -1,282 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import {
|
||||
encodeMembership,
|
||||
type CellAdmissionMembership,
|
||||
type CellAdmissionState
|
||||
} from '../../cell/admission/cell-admission-selector.js'
|
||||
import type { RelayCellConfig } from '../../config.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const DIRECTOR_INSTANCES = 5
|
||||
const DIRECTOR_POOL_MAX = 3
|
||||
const DIRECTOR_APPLICATION = 'drain-release-contention-director'
|
||||
|
||||
const USER_PREFIX = 'drain-release-contention'
|
||||
const NOW = 1_000_000
|
||||
const CAPPED = {
|
||||
capacityRequests: 6_000,
|
||||
connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
} as const
|
||||
const SOURCE: RelayCellConfig = {
|
||||
id: 'drain-release-contention-a-source',
|
||||
url: 'https://drain-release-contention-source.example.test',
|
||||
region: 'asia-east2',
|
||||
...CAPPED
|
||||
}
|
||||
const TARGETS: RelayCellConfig[] = ['b', 'c', 'd'].map((suffix) => ({
|
||||
id: `drain-release-contention-${suffix}-target`,
|
||||
url: `https://drain-release-contention-${suffix}-target.example.test`,
|
||||
region: 'asia-east2',
|
||||
...CAPPED
|
||||
}))
|
||||
// Next to the database. Taken only once every Asia neighbour is at its cap.
|
||||
const US_CELL: RelayCellConfig = {
|
||||
id: 'drain-release-contention-e-us',
|
||||
url: 'https://drain-release-contention-us.example.test',
|
||||
region: 'us-central1',
|
||||
...CAPPED
|
||||
}
|
||||
const CELLS = [SOURCE, ...TARGETS, US_CELL]
|
||||
|
||||
type Identity = { userId: string; relayHostId: string }
|
||||
type Instance = { database: RelayDatabase; store: RelayAssignmentStore }
|
||||
|
||||
function hostIdentity(index: number): Identity {
|
||||
return {
|
||||
userId: `${USER_PREFIX}-${index}`,
|
||||
// Relay host ids are fixed-width opaque ids.
|
||||
relayHostId: `drainrel${String(index).padStart(8, '0')}`
|
||||
}
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL drain releases against director placement', () => {
|
||||
const storeOptions = { requireLiveCells: true, heartbeatTtlMs: 45_000 }
|
||||
const directors: Instance[] = []
|
||||
let observer: RelayDatabase
|
||||
|
||||
function admin(): Instance {
|
||||
return directors[0]!
|
||||
}
|
||||
|
||||
async function applySelector(
|
||||
states: Record<string, CellAdmissionState>,
|
||||
rollIsolatedCells?: string[]
|
||||
): Promise<void> {
|
||||
const current = await admin().store.inspectCellAdmissionSelector()
|
||||
// Built from relay_cells: the apply requires exact coverage of a shared fleet.
|
||||
const fleet = await admin().database.query(
|
||||
`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`
|
||||
)
|
||||
const membership: CellAdmissionMembership = {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
for (const row of fleet) {
|
||||
const cellId = String(row['cell_id'])
|
||||
const state =
|
||||
states[cellId] ??
|
||||
(current.selector.membership.existingOnly.includes(cellId)
|
||||
? 'existing-only'
|
||||
: current.selector.membership.migrationOnly.includes(cellId)
|
||||
? 'migration-only'
|
||||
: 'general')
|
||||
if (state === 'existing-only') membership.existingOnly.push(cellId)
|
||||
else if (state === 'migration-only') membership.migrationOnly.push(cellId)
|
||||
else membership.general.push(cellId)
|
||||
}
|
||||
await admin().store.applyCellAdmissionSelector({
|
||||
attemptId: `drain-release-${current.selector.generation}`,
|
||||
expectedGeneration: current.selector.generation,
|
||||
...(current.selector.generation === 0
|
||||
? {
|
||||
expectedMembershipSha256: createHash('sha256')
|
||||
.update(encodeMembership(current.selector.membership))
|
||||
.digest('hex')
|
||||
}
|
||||
: {}),
|
||||
membership,
|
||||
...(rollIsolatedCells ? { rollIsolatedCells } : {})
|
||||
})
|
||||
}
|
||||
|
||||
// Other Postgres files write admission through the generation-0 helpers, so
|
||||
// the selector goes back to generation 0 exactly as found.
|
||||
async function resetSelectorBoundary(): Promise<void> {
|
||||
await admin().database.query(
|
||||
`UPDATE relay_admission_selectors SET generation = 0, attempt_id = NULL
|
||||
WHERE selector_id = 'general'`
|
||||
)
|
||||
await admin().database.query(
|
||||
`DELETE FROM relay_admission_selector_intents WHERE attempt_id LIKE 'drain-release-%'`
|
||||
)
|
||||
await admin().store.reconcileCells([], false)
|
||||
}
|
||||
|
||||
async function deleteHostRows(): Promise<void> {
|
||||
for (const table of [
|
||||
'relay_control_capabilities',
|
||||
'relay_control_connection_reservations',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await admin().database.query(`DELETE FROM ${table} WHERE user_id LIKE '${USER_PREFIX}-%'`)
|
||||
}
|
||||
}
|
||||
|
||||
// `atCap` reports those cells at their connection cap: no headroom left.
|
||||
async function heartbeatAll(atCap: RelayCellConfig[] = []): Promise<void> {
|
||||
for (const [index, config] of CELLS.entries()) {
|
||||
await admin().store.recordCellHeartbeat({
|
||||
cellId: config.id,
|
||||
cellUrl: config.url,
|
||||
cellIncarnation: `2222222${index}-2222-4222-8222-222222222222`,
|
||||
startedAt: NOW - 1_000,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
region: config.region,
|
||||
// The cell checks enforced = total + in-flight + reserved.
|
||||
totalConnections: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
|
||||
connectionHardCap: CAPPED.connectionHardCap,
|
||||
connectionUnobservedBound: CAPPED.connectionUnobservedBound
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Holds `cellIds`' relay_cells rows in an open transaction until the returned release runs.
|
||||
async function holdCellRows(cellIds: string[]): Promise<() => Promise<void>> {
|
||||
let release!: () => void
|
||||
const released = new Promise<void>((resolve) => (release = resolve))
|
||||
let held!: () => void
|
||||
const acquired = new Promise<void>((resolve) => (held = resolve))
|
||||
const holder = observer.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT cell_id FROM relay_cells WHERE cell_id IN (${cellIds.map(() => '?').join(', ')})
|
||||
ORDER BY cell_id ASC`,
|
||||
cellIds
|
||||
)
|
||||
held()
|
||||
await released
|
||||
})
|
||||
await acquired
|
||||
return async () => {
|
||||
release()
|
||||
await holder
|
||||
}
|
||||
}
|
||||
|
||||
// Every host holds a live control lease on the source, then the source is
|
||||
// isolated for a roll: the state an Asia drain starts from.
|
||||
async function seed(hosts: Identity[]): Promise<void> {
|
||||
await deleteHostRows()
|
||||
await admin().database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id LIKE '${USER_PREFIX}-%'`
|
||||
)
|
||||
// Restoring the source to general also clears the previous run's roll stamp.
|
||||
await applySelector({
|
||||
[SOURCE.id]: 'general',
|
||||
...Object.fromEntries(TARGETS.map((target) => [target.id, 'migration-only' as const]))
|
||||
})
|
||||
for (const identity of hosts) {
|
||||
const grant = await admin().store.assign(identity, 'asia-east2', 'asia-east2')
|
||||
expect(grant.cellId).toBe(SOURCE.id)
|
||||
await admin().store.activateControl(identity, {
|
||||
cellId: SOURCE.id,
|
||||
assignmentEpoch: grant.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
}
|
||||
await applySelector(
|
||||
Object.fromEntries(TARGETS.map((target) => [target.id, 'general' as const]))
|
||||
)
|
||||
await applySelector({ [SOURCE.id]: 'migration-only' }, [SOURCE.id])
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
for (let index = 0; index < DIRECTOR_INSTANCES; index += 1) {
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl,
|
||||
dataDir: '',
|
||||
poolMax: DIRECTOR_POOL_MAX,
|
||||
applicationName: DIRECTOR_APPLICATION
|
||||
})
|
||||
directors.push({
|
||||
database,
|
||||
store: new RelayAssignmentStore(database, () => NOW, storeOptions)
|
||||
})
|
||||
}
|
||||
observer = await openRelayDatabase({ databaseUrl, dataDir: '', poolMax: 1 })
|
||||
await deleteHostRows()
|
||||
await admin().store.reconcileCells(CELLS, false)
|
||||
await resetSelectorBoundary()
|
||||
await heartbeatAll()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (directors.length > 0) {
|
||||
await deleteHostRows()
|
||||
// Cells before the selector rebuild, or its membership names missing rows.
|
||||
for (const config of CELLS) {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await admin().database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [config.id])
|
||||
}
|
||||
}
|
||||
await resetSelectorBoundary()
|
||||
}
|
||||
for (const instance of directors) {
|
||||
await instance.database.close()
|
||||
}
|
||||
await observer?.close()
|
||||
})
|
||||
|
||||
it('places a reconnecting host off the isolated source while that source row is held', async () => {
|
||||
await heartbeatAll()
|
||||
const host = hostIdentity(9_000)
|
||||
await seed([host])
|
||||
// A drain release holds the source row for its whole transaction; target-only placement never asks for it.
|
||||
const sourceHeld = await holdCellRows([SOURCE.id])
|
||||
try {
|
||||
const grant = await Promise.race([
|
||||
directors[1]!.store.assign(host, 'asia-east2', 'asia-east2'),
|
||||
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 3_000))
|
||||
])
|
||||
expect(grant).not.toBe('blocked')
|
||||
expect(TARGETS.map(({ id }) => id)).toContain(grant === 'blocked' ? null : grant.cellId)
|
||||
} finally {
|
||||
await sourceHeld()
|
||||
}
|
||||
// Control: the same hold on the target rows does block a placement, so the hold above was real.
|
||||
const other = hostIdentity(9_001)
|
||||
await seed([other])
|
||||
const targetsHeld = await holdCellRows(TARGETS.map(({ id }) => id))
|
||||
const placement = directors[2]!.store.assign(other, 'asia-east2', 'asia-east2')
|
||||
try {
|
||||
const raced = await Promise.race([
|
||||
placement.then(() => 'placed' as const),
|
||||
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 1_000))
|
||||
])
|
||||
expect(raced).toBe('blocked')
|
||||
} finally {
|
||||
await targetsHeld()
|
||||
}
|
||||
expect((await placement).cellId).not.toBe(SOURCE.id)
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
type RelayAssignmentMigration
|
||||
} from '../assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -245,39 +244,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
)
|
||||
}
|
||||
|
||||
it('reaps an abandoned registered migration onto its healthy target', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(
|
||||
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
),
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('reaps immediately after the retired source has a durable completed fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -290,7 +256,7 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
(migration.completed_at IS NOT NULL) AS completed, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
@@ -303,177 +269,12 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
|
||||
completed: true,
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('reaps immediately after the retired source has an adopted legacy fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
expect(
|
||||
(
|
||||
await readRegisteredMigrationInventory(
|
||||
database,
|
||||
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
)
|
||||
).abandoned
|
||||
).toBe(1)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('does not reap after temporary legacy adoption without durable commit', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
expect(
|
||||
(
|
||||
await readRegisteredMigrationInventory(
|
||||
database,
|
||||
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
)
|
||||
).abandoned
|
||||
).toBe(0)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('invalidates an adopted legacy fence when the source heartbeats', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.heartbeat(input.cells.source)
|
||||
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('invalidates an adopted legacy fence when the source restarts', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.store.recordCellHeartbeat({
|
||||
cellId: input.cells.source.id,
|
||||
cellUrl: input.cells.source.url,
|
||||
cellIncarnation: '99999999-9999-4999-8999-999999999999',
|
||||
startedAt: 51,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('serializes legacy adoption with a returning source heartbeat', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
|
||||
const [commit, heartbeatResult] = await Promise.allSettled([
|
||||
input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
),
|
||||
input.heartbeat(input.cells.source)
|
||||
])
|
||||
expect(heartbeatResult.status).toBe('fulfilled')
|
||||
if (commit.status === 'rejected') {
|
||||
expect(commit.reason).toMatchObject({ message: 'legacy_cell_fence_not_active' })
|
||||
}
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('restores the 24-hour guard when the fenced source heartbeats again', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -495,83 +296,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('restores the 24-hour guard when the fenced source restarts', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await completeSourceFence(input)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.store.recordCellHeartbeat({
|
||||
cellId: input.cells.source.id,
|
||||
cellUrl: input.cells.source.url,
|
||||
cellIncarnation: '99999999-9999-4999-8999-999999999999',
|
||||
startedAt: 51,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('serializes fenced cleanup with a returning source heartbeat', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await completeSourceFence(input)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
const [cleanup, heartbeatResult] = await Promise.allSettled([
|
||||
input.store.abortExpiredEvacuations(),
|
||||
input.heartbeat(input.cells.source)
|
||||
])
|
||||
expect(cleanup.status).toBe('fulfilled')
|
||||
expect(heartbeatResult.status).toBe('fulfilled')
|
||||
if (cleanup.status !== 'fulfilled') throw cleanup.reason
|
||||
if (heartbeatResult.status !== 'fulfilled') throw heartbeatResult.reason
|
||||
expect([0, 1]).toContain(cleanup.value)
|
||||
const migrations = await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
if (cleanup.value === 1) {
|
||||
expect(migrations[0]?.completed_at).not.toBeNull()
|
||||
} else {
|
||||
expect(migrations).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a freshly retired source protected without a durable fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('rolls an abandoned disabled target back to its source', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -585,7 +309,7 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
migration.completed_at, (migration.aborted_at IS NOT NULL) AS aborted
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
@@ -599,103 +323,11 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
cell_id: input.cells.source.id,
|
||||
assignment_epoch: '3',
|
||||
completed_at: null,
|
||||
aborted_at: String(
|
||||
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
aborted: true
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('serializes cleanup against supersession without reporting a false target', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
|
||||
const [cleanup, supersession] = await Promise.allSettled([
|
||||
input.store.abortExpiredEvacuations(),
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
])
|
||||
expect(cleanup.status).toBe('fulfilled')
|
||||
if (cleanup.status !== 'fulfilled') throw cleanup.reason
|
||||
if (supersession.status === 'fulfilled') {
|
||||
expect([0, 1]).toContain(supersession.value)
|
||||
const assignment = await database.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
expect(assignment).toEqual([{
|
||||
cell_id:
|
||||
supersession.value === 1
|
||||
? input.cells.replacement.id
|
||||
: input.cells.source.id
|
||||
}])
|
||||
if (supersession.value === 0) expect(cleanup.value).toBeGreaterThanOrEqual(1)
|
||||
} else {
|
||||
expect(cleanup.value).toBeGreaterThanOrEqual(1)
|
||||
expect(supersession.reason).toMatchObject({ message: 'migration_already_superseded' })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ cell_id: input.cells.source.id }])
|
||||
}
|
||||
})
|
||||
|
||||
it('completes a registered migration from a dead source idempotently', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.failed)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
const completion = {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
targetCellId: input.cells.failed.id
|
||||
}
|
||||
|
||||
await expect(
|
||||
input.store.cellEvacuationStatus(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
true
|
||||
)
|
||||
).resolves.toMatchObject({ inProgress: 0, completed: 1 })
|
||||
await expect(
|
||||
input.store.completeEvacuationFromDeadSource(input.identity, completion)
|
||||
).resolves.toMatchObject({ changed: false })
|
||||
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 1, replacement: 0 })
|
||||
})
|
||||
|
||||
it('requires fence evidence for dead-source completion', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
|
||||
await expect(
|
||||
input.store.completeEvacuationFromDeadSource(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
targetCellId: input.cells.failed.id
|
||||
})
|
||||
).rejects.toThrow('cell_fence_attestation_missing')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 2, replacement: 0 })
|
||||
})
|
||||
|
||||
it('supersedes a registered migration with exact accounting and idempotency', async () => {
|
||||
const input = await fixture()
|
||||
await database.query(
|
||||
@@ -760,416 +392,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('reconciles durable cell accounting before aggregate supersession', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells
|
||||
SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END
|
||||
WHERE cell_id IN (?, ?)`,
|
||||
[input.cells.replacement.id, input.cells.source.id, input.cells.replacement.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
})
|
||||
|
||||
it('rebuilds an expired registered migration lease before supersession', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
|
||||
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '3',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('retires an obsolete row without discarding replacement activity', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await fenceFailedCell(input)
|
||||
await input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET aborted_at = NULL
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
const activityBefore = await assignmentActivities(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await assignmentActivities(input)).toEqual(activityBefore)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{ assignment_epoch: '2', aborted_at: '45101' },
|
||||
{ assignment_epoch: '3', aborted_at: null }
|
||||
])
|
||||
})
|
||||
|
||||
it('anchors a newer dormant failed-cell epoch and preserves drain evidence', async () => {
|
||||
const input = await fixture()
|
||||
const drainAttemptId = await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
|
||||
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '4',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '5',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, drain_attempt_id, source_request_units,
|
||||
target_reserved_units
|
||||
FROM relay_post_drain_migration_pins
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '4',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '5',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1'
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('uses an existing current-epoch migration once when retiring an older row', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id,
|
||||
previous_epoch, assignment_epoch, source_request_units,
|
||||
target_reserved_units, expires_at, target_registered_at,
|
||||
completed_at, aborted_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 2, 4, 0, 1, 0, 100, NULL, NULL, 100, 100)`,
|
||||
[
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id
|
||||
]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migration_incarnations
|
||||
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
|
||||
target_cell_incarnation)
|
||||
VALUES (?, ?, 4, ?, ?)`,
|
||||
[
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
input.cellIncarnation,
|
||||
input.cellIncarnation
|
||||
]
|
||||
)
|
||||
await pinMigration(input, 4)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{ assignment_epoch: '2', aborted_at: '45101' },
|
||||
{ assignment_epoch: '4', aborted_at: '45101' },
|
||||
{ assignment_epoch: '5', aborted_at: null }
|
||||
])
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a newer failed-cell epoch with ambiguous activity', async () => {
|
||||
const input = await fixture()
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).rejects.toThrow(
|
||||
'migration_activity_topology_mismatch'
|
||||
)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 2,
|
||||
replacement: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves lease repair retryable when supersession later fails', async () => {
|
||||
const input = await fixture(1)
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET migration_leases = 0
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests - 1
|
||||
WHERE cell_id = ?`,
|
||||
[input.cells.failed.id]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).rejects.toThrow(
|
||||
'relay_capacity_exhausted'
|
||||
)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT activity_id FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ activity_id: 'migration:2' }])
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND assignment_epoch = 2`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ aborted_at: null }])
|
||||
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET capacity_requests = 2 WHERE cell_id = ?`,
|
||||
[input.cells.replacement.id]
|
||||
)
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
})
|
||||
|
||||
it('serializes concurrent supersession retries without duplicating capacity', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
const supersession = {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
}
|
||||
|
||||
const results = await Promise.all([
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, supersession),
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, supersession)
|
||||
])
|
||||
expect(results[0]).toEqual(results[1])
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 0, replacement: 2 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '2' }])
|
||||
})
|
||||
|
||||
it('fails a competing aggregate supersession instead of reporting the wrong target', async () => {
|
||||
const input = await fixture()
|
||||
const alternate = {
|
||||
id: `recovery-cell-${sequence}-alternate`,
|
||||
url: `https://recovery-${sequence}-alternate.example.com`,
|
||||
capacityRequests: 20
|
||||
}
|
||||
await input.store.reconcileCells([...Object.values(input.cells), alternate], false)
|
||||
await input.heartbeat(alternate)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.heartbeat(alternate)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
),
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
alternate.id,
|
||||
100
|
||||
)
|
||||
])
|
||||
const fulfilled = results.filter(
|
||||
(result): result is PromiseFulfilledResult<number> => result.status === 'fulfilled'
|
||||
)
|
||||
const rejected = results.filter(
|
||||
(result): result is PromiseRejectedResult => result.status === 'rejected'
|
||||
)
|
||||
expect(fulfilled).toHaveLength(1)
|
||||
expect(fulfilled[0]!.value).toBe(1)
|
||||
expect(rejected).toHaveLength(1)
|
||||
expect(rejected[0]!.reason).toMatchObject({ message: 'migration_already_superseded' })
|
||||
const successor = await database.query(
|
||||
`SELECT assignment.cell_id, migration.target_cell_id
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND migration.previous_epoch = ?`,
|
||||
[input.identity.userId, input.migration.assignmentEpoch]
|
||||
)
|
||||
expect(successor).toHaveLength(1)
|
||||
expect(successor[0]!.cell_id).toBe(successor[0]!.target_cell_id)
|
||||
expect([input.cells.replacement.id, alternate.id]).toContain(
|
||||
successor[0]!.target_cell_id
|
||||
)
|
||||
})
|
||||
|
||||
it('rolls back every supersession change when replacement capacity is insufficient', async () => {
|
||||
const input = await fixture(1)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
@@ -1205,153 +427,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
})
|
||||
|
||||
it('rolls back supersession when migration request-unit shape drifted', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_activity_leases
|
||||
SET request_units = request_units + 1
|
||||
WHERE user_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests + 1 WHERE cell_id = ?`,
|
||||
[input.cells.failed.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
).rejects.toThrow('migration_activity_lease_shape_mismatch')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 3, replacement: 0 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ cell_id: input.cells.failed.id, assignment_epoch: '2' }])
|
||||
})
|
||||
|
||||
it('rolls back supersession when locked cell reservation accounting drifted', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`,
|
||||
[input.cells.replacement.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
).rejects.toThrow('migration_cell_reservation_accounting_mismatch')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 2, replacement: 1 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
})
|
||||
|
||||
async function fenceFailedCell(input: RecoveryFixture): Promise<void> {
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
}
|
||||
|
||||
async function supersedeAggregate(input: RecoveryFixture): Promise<number> {
|
||||
return await input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
}
|
||||
|
||||
async function clearActivities(input: RecoveryFixture): Promise<void> {
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0,
|
||||
reserved_invites = 0, pending_installs = 0, pending_confirmations = 0,
|
||||
migration_leases = 0
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 0
|
||||
WHERE cell_id IN (?, ?, ?)`,
|
||||
[input.cells.source.id, input.cells.failed.id, input.cells.replacement.id]
|
||||
)
|
||||
}
|
||||
|
||||
async function pinMigration(
|
||||
input: RecoveryFixture,
|
||||
assignmentEpoch: number
|
||||
): Promise<string> {
|
||||
const drainAttemptId = `${input.identity.userId}-drain-${assignmentEpoch}`
|
||||
await database.query(
|
||||
`INSERT INTO relay_post_drain_migration_pins
|
||||
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
|
||||
source_cell_id, source_cell_incarnation, target_cell_id,
|
||||
target_cell_incarnation, source_request_units, target_reserved_units,
|
||||
pinned_at)
|
||||
SELECT migration.user_id, migration.relay_host_id,
|
||||
migration.assignment_epoch, ?, migration.source_cell_id,
|
||||
incarnation.source_cell_incarnation, migration.target_cell_id,
|
||||
incarnation.target_cell_incarnation, migration.source_request_units,
|
||||
migration.target_reserved_units, 100
|
||||
FROM relay_assignment_migrations migration
|
||||
JOIN relay_assignment_migration_incarnations incarnation
|
||||
ON incarnation.user_id = migration.user_id
|
||||
AND incarnation.relay_host_id = migration.relay_host_id
|
||||
AND incarnation.assignment_epoch = migration.assignment_epoch
|
||||
WHERE migration.user_id = ? AND migration.relay_host_id = ?
|
||||
AND migration.assignment_epoch = ?`,
|
||||
[
|
||||
drainAttemptId,
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
assignmentEpoch
|
||||
]
|
||||
)
|
||||
return drainAttemptId
|
||||
}
|
||||
|
||||
async function assignmentActivities(input: RecoveryFixture): Promise<unknown[]> {
|
||||
return await database.query(
|
||||
`SELECT activity_id, activity_kind, cell_id, request_units
|
||||
FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
}
|
||||
|
||||
async function reservations(cells: RecoveryFixture['cells']): Promise<Record<string, number>> {
|
||||
const rows = await database.query(
|
||||
`SELECT cell_id, reserved_requests FROM relay_cells
|
||||
|
||||
@@ -184,7 +184,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
|
||||
})
|
||||
|
||||
now = migration.expiresAt + 1
|
||||
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: source.id,
|
||||
cellUrl: source.url,
|
||||
@@ -207,7 +207,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
|
||||
cellId: source.id,
|
||||
cellIncarnation: sourceIncarnation
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: true })
|
||||
).resolves.toMatchObject({ shouldSend: true })
|
||||
await expect(
|
||||
database.query(
|
||||
`SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
|
||||
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
|
||||
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_migration_inventory_test'
|
||||
|
||||
describePostgres('PostgreSQL registered migration inventory', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
|
||||
beforeAll(async () => {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await database?.close()
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
})
|
||||
|
||||
it('counts a disabled-target migration even while its source is active', async () => {
|
||||
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 1, 0, 0, 1),
|
||||
('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-a', 'general', 1), ('cell-b', 'existing-only', 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
|
||||
request_units, expires_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'control:source:1', 'control', 'cell-a', 1, ?, ?)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, 2, 2, 1, 2)`
|
||||
)
|
||||
|
||||
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-c', 'https://c.example.test', 1, 4000, 0, 0, 0, 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-c', 'migration-only', 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-2', '222222222222', 'cell-c', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-2', '222222222222', 'cell-a', 'cell-c', 1, 2, 1, 2, ?, NULL, 1, 2)`,
|
||||
[now + 60_000]
|
||||
)
|
||||
|
||||
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
|
||||
open: 2,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
},
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-c',
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,120 +0,0 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
|
||||
import {
|
||||
formatRegisteredMigrationInventory,
|
||||
readRegisteredMigrationInventory
|
||||
} from './registered-migration-inventory.js'
|
||||
|
||||
describe('registered migration inventory', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
for (const database of databases.splice(0)) {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
it('does not restart abandonment when an old source migration lease is refreshed', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
|
||||
await insertCell(database, 'cell-a', false, 'existing-only')
|
||||
await insertCell(database, 'cell-b', true, 'migration-only')
|
||||
await insertMigration(database, now - 1)
|
||||
|
||||
const fresh = await readRegisteredMigrationInventory(database, now)
|
||||
|
||||
expect(fresh).toEqual({
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
expect(formatRegisteredMigrationInventory(fresh)).toEqual([
|
||||
'[orca-relay] migration inventory open=1 expiredRegisteredInactive=1 abandonedRegistered=1',
|
||||
'[orca-relay] migration pair sourceCellId=cell-a targetCellId=cell-b open=1 expiredRegisteredInactive=1 abandoned=1'
|
||||
])
|
||||
await database.query(`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?`, [
|
||||
now + 1,
|
||||
now
|
||||
])
|
||||
expect((await readRegisteredMigrationInventory(database, now + 1)).abandoned).toBe(1)
|
||||
})
|
||||
|
||||
it('includes unregistered open migrations without logging a host identity', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const now = 10_000
|
||||
await insertCell(database, 'cell-a', false, 'existing-only')
|
||||
await insertCell(database, 'cell-b', true, 'migration-only')
|
||||
await insertMigration(database, now + 60_000, null)
|
||||
|
||||
const inventory = await readRegisteredMigrationInventory(database, now)
|
||||
|
||||
expect(inventory).toEqual({
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0
|
||||
}
|
||||
]
|
||||
})
|
||||
expect(formatRegisteredMigrationInventory(inventory).join('\n')).not.toContain('111111111111')
|
||||
})
|
||||
})
|
||||
|
||||
async function insertCell(
|
||||
database: RelayDatabase,
|
||||
cellId: string,
|
||||
enabled: boolean,
|
||||
admissionState: string
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES (?, ?, ?, 4000, 0, 0, 0, 1)`,
|
||||
[cellId, `https://${cellId}.example.test`, enabled ? 1 : 0]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES (?, ?, 1)`,
|
||||
[cellId, admissionState]
|
||||
)
|
||||
}
|
||||
|
||||
async function insertMigration(
|
||||
database: RelayDatabase,
|
||||
expiresAt: number,
|
||||
targetRegisteredAt: number | null = 2
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[expiresAt, expiresAt]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, ?, ?, 1, 2)`,
|
||||
[expiresAt, targetRegisteredAt]
|
||||
)
|
||||
}
|
||||
@@ -1,23 +1,16 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS,
|
||||
RelayDrainReturnAdmission,
|
||||
type DrainReturnDeferral,
|
||||
type DrainReturnGrant
|
||||
} from './drain-return-admission.js'
|
||||
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
|
||||
|
||||
type Timer = { at: number; callback: () => void; cancelled: boolean }
|
||||
|
||||
function harness(
|
||||
overrides: {
|
||||
maxQueued?: number
|
||||
maxRetryAfterSeconds?: number
|
||||
onServiceMs?: (durationMs: number) => void
|
||||
} = {}
|
||||
) {
|
||||
let now = 0
|
||||
const timers: Timer[] = []
|
||||
const placement = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 2,
|
||||
maxQueued: 8,
|
||||
@@ -26,32 +19,18 @@ function harness(
|
||||
maxDrainReturnConcurrent: 1,
|
||||
maxDrainReturnQueued: overrides.maxQueued ?? 0,
|
||||
drainReturnWaitMs: 3_000,
|
||||
drainReturnMinIntervalMs: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS * 1_000,
|
||||
now: () => now,
|
||||
schedule: (callback, delayMs) => {
|
||||
const timer = { at: now + delayMs, callback, cancelled: false }
|
||||
timers.push(timer)
|
||||
return () => {
|
||||
timer.cancelled = true
|
||||
}
|
||||
}
|
||||
drainReturnMinIntervalMs: 2_000,
|
||||
now: () => 0,
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const lane = new RelayDrainReturnAdmission(placement, {
|
||||
maxConcurrent: 1,
|
||||
maxRetryAfterSeconds: overrides.maxRetryAfterSeconds ?? 300,
|
||||
now: () => now,
|
||||
onServiceMs: overrides.onServiceMs
|
||||
maxRetryAfterSeconds: 300,
|
||||
now: () => 0
|
||||
})
|
||||
return {
|
||||
lane,
|
||||
placement,
|
||||
advance: (ms: number) => {
|
||||
now += ms
|
||||
for (const timer of timers.filter((entry) => !entry.cancelled && entry.at <= now)) {
|
||||
timer.cancelled = true
|
||||
timer.callback()
|
||||
}
|
||||
}
|
||||
placement
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,28 +48,6 @@ const host = (index: number): string => `host${String(index).padStart(12, '0')}`
|
||||
|
||||
describe('drain-return admission', () => {
|
||||
|
||||
it('shortens the pacing once the measured service time is short', async () => {
|
||||
const { lane, advance } = harness()
|
||||
for (let index = 0; index < 30; index++) {
|
||||
const grant = admitted(await lane.acquire(host(index)))
|
||||
advance(50)
|
||||
grant.lease.release()
|
||||
advance(2_000)
|
||||
}
|
||||
admitted(await lane.acquire(host(100)))
|
||||
|
||||
const retries = []
|
||||
for (let index = 101; index <= 120; index++) {
|
||||
retries.push(deferred(await lane.acquire(host(index))).retryAfterSeconds)
|
||||
}
|
||||
|
||||
// ~50 ms per re-placement: twenty deferrals span about one second, where the
|
||||
// 860 ms starting estimate would have spread them over seventeen.
|
||||
expect(Math.max(...retries)).toBe(3)
|
||||
})
|
||||
|
||||
// The unclamped sample: the EWMA's floor and ceiling would hide the real tail.
|
||||
|
||||
it('answers a host’s own early retry with its interval, not a place behind the cohort', async () => {
|
||||
const { lane } = harness()
|
||||
admitted(await lane.acquire(host(0))).lease.release()
|
||||
@@ -99,7 +56,7 @@ describe('drain-return admission', () => {
|
||||
const repeat = deferred(await lane.acquire(host(0)))
|
||||
expect(repeat).toMatchObject({
|
||||
reason: 'host-rate-limited',
|
||||
retryAfterSeconds: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS
|
||||
retryAfterSeconds: 2
|
||||
})
|
||||
|
||||
// And it booked nothing: the next overflow still gets the first slot.
|
||||
@@ -107,17 +64,6 @@ describe('drain-return admission', () => {
|
||||
expect(deferred(await lane.acquire(host(2))).retryAfterSeconds).toBe(2)
|
||||
})
|
||||
|
||||
it('keeps a host’s reserved slot when it comes back early', async () => {
|
||||
const { lane, advance } = harness()
|
||||
admitted(await lane.acquire(host(0)))
|
||||
for (let index = 1; index <= 4; index++) await lane.acquire(host(index))
|
||||
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(6)
|
||||
|
||||
advance(1_000)
|
||||
// Early and still full: the same instant, not a new slot after everyone else.
|
||||
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(5)
|
||||
})
|
||||
|
||||
it('lets a queued placement go before a queued drain return', async () => {
|
||||
const { lane, placement } = harness({ maxQueued: 1 })
|
||||
const drain = admitted(await lane.acquire(host(0)))
|
||||
|
||||
+3
-42
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
|
||||
|
||||
describe('public assignment admission', () => {
|
||||
@@ -7,7 +7,7 @@ describe('public assignment admission', () => {
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 1,
|
||||
minIntervalMs: 0,
|
||||
now: vi.fn(() => 0)
|
||||
now: () => 0
|
||||
})
|
||||
const lease = await admission.acquire('host-a')
|
||||
expect(lease).not.toBeNull()
|
||||
@@ -46,19 +46,12 @@ describe('public assignment admission', () => {
|
||||
})
|
||||
|
||||
it('gives a bounded reserved waiter the next public slot', async () => {
|
||||
let expire!: () => void
|
||||
let cancelled = false
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 2,
|
||||
maxReservedConcurrent: 1,
|
||||
reservedWaitMs: 1_000,
|
||||
minIntervalMs: 0,
|
||||
schedule: (callback) => {
|
||||
expire = callback
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const first = await admission.acquire('host-a')
|
||||
const second = await admission.acquire('host-b')
|
||||
@@ -70,12 +63,10 @@ describe('public assignment admission', () => {
|
||||
const reserved = await reservedPromise
|
||||
|
||||
expect(reserved).not.toBeNull()
|
||||
expect(cancelled).toBe(true)
|
||||
await expect(admission.acquire('host-d')).resolves.toBeNull()
|
||||
reserved?.release()
|
||||
await expect(admission.acquire('host-d')).resolves.not.toBeNull()
|
||||
second?.release()
|
||||
expire()
|
||||
})
|
||||
|
||||
it('serializes same-host assignment and reserved recovery without losing priority', async () => {
|
||||
@@ -100,34 +91,4 @@ describe('public assignment admission', () => {
|
||||
await expect(admission.acquire('host-b')).resolves.not.toBeNull()
|
||||
})
|
||||
|
||||
it('tells each caller its own rejection reason without crossing requests', async () => {
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 1,
|
||||
maxQueued: 1,
|
||||
waitMs: 1_000,
|
||||
maxReservedConcurrent: 1,
|
||||
reservedWaitMs: 1_000,
|
||||
minIntervalMs: 5_000,
|
||||
now: () => 0,
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const queued: string[] = []
|
||||
const displaced: string[] = []
|
||||
const throttled: string[] = []
|
||||
|
||||
const active = await admission.acquire('host-a')
|
||||
// 'host-b' waits, then reserved recovery for the same host displaces it.
|
||||
const ordinary = admission.acquire('host-b', (reason) => displaced.push(reason))
|
||||
const reserved = admission.acquireReserved('host-b', (reason) => queued.push(reason))
|
||||
await expect(ordinary).resolves.toBeNull()
|
||||
active?.release()
|
||||
const recovered = await reserved
|
||||
recovered?.release()
|
||||
await admission.acquire('host-a', (reason) => throttled.push(reason))
|
||||
|
||||
expect(displaced).toEqual(['superseded'])
|
||||
expect(queued).toEqual([])
|
||||
expect(throttled).toEqual(['host-rate-limited'])
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createRelayApp } from '../../app.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
|
||||
describe('public assignment circuit breaker', () => {
|
||||
it('rejects assign and resolve without invoking relay state', async () => {
|
||||
const assign = vi.fn()
|
||||
const resolveResume = vi.fn()
|
||||
const app = createRelayApp(config(), {
|
||||
store: { resolveResume } as never,
|
||||
assignments: { assign } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
for (const path of ['/v1/assign', '/v1/resolve']) {
|
||||
const response = await app.request(path, { method: 'POST' })
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('retry-after')).toBe('5')
|
||||
expect(await response.json()).toEqual({ error: 'assignments_temporarily_unavailable' })
|
||||
}
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
expect((await app.request('/health')).status).toBe(200)
|
||||
expect((await app.request('/v1/admin/drain', { method: 'POST' })).status).toBe(401)
|
||||
})
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
cellUrl: 'https://relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
|
||||
role: 'director',
|
||||
cellId: 'director',
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'runtime@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 3,
|
||||
publicAssignmentsEnabled: false,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
+3
-123
@@ -1,6 +1,5 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
RelayAssignmentRowBusyError,
|
||||
type RelayAssignment,
|
||||
type ResolvedRelayAssignment
|
||||
} from '../assignment-store.js'
|
||||
@@ -21,7 +20,6 @@ vi.mock('../../authentication/relay-token-verifier.js', () => ({
|
||||
import { createRelayApp } from '../../app.js'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
@@ -36,15 +34,11 @@ describe('drain-return lane', () => {
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
relayHostId === drained ? isolatedHome(relayHostId) : assignment('cell-o', relayHostId)
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const servedLanes: string[] = []
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
|
||||
recordAdmissionServiceMs: (lane) => servedLanes.push(lane)
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
// The re-placement holds the drain lane; the sticky lane's only slot is free.
|
||||
@@ -56,125 +50,12 @@ describe('drain-return lane', () => {
|
||||
)
|
||||
|
||||
expect(reconnect.status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: drained },
|
||||
{ classifyHomeRollIsolation: true }
|
||||
)
|
||||
replacement.resolve(assignment('cell-new', drained))
|
||||
expect((await moving).status).toBe(200)
|
||||
expect(outcomes).toEqual(['drain-return', 'sticky'])
|
||||
// The drained host held a sticky slot for its verification read too.
|
||||
expect(servedLanes).toEqual(['sticky', 'sticky', 'drain-return'])
|
||||
})
|
||||
|
||||
it('defers an overflowing drain return with a paced Retry-After and says why', async () => {
|
||||
const holder = 'hhhhhhhhhhhhhhhh'
|
||||
const replacement = deferred<RelayAssignment>()
|
||||
const assign = vi.fn(async () => await replacement.promise)
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
isolatedHome(relayHostId)
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const reasons: string[] = []
|
||||
const retryAfters: number[] = []
|
||||
const causes: string[] = []
|
||||
const app = createRelayApp(config({ drainReturnQueueMax: 1, drainReturnWaitMs: 50 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
|
||||
recordAssignmentRejectionReason: (lane, reason) => reasons.push(`${lane}:${reason}`),
|
||||
recordDrainReturnRetryAfter: (seconds) => retryAfters.push(seconds),
|
||||
recordAssignmentUnavailable: (cause) => causes.push(cause)
|
||||
})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
const holding = app.request('/v1/assign', assignmentRequest(holder, { reconnect: true }))
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
|
||||
const queued = app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('qqqqqqqqqqqqqqqq', { reconnect: true })
|
||||
)
|
||||
const overflow = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('ffffffffffffffff', { reconnect: true })
|
||||
)
|
||||
|
||||
expect(overflow.status).toBe(503)
|
||||
expect(overflow.headers.get('retry-after')).toBe('2')
|
||||
const timedOut = await queued
|
||||
expect(timedOut.status).toBe(503)
|
||||
expect(Number(timedOut.headers.get('retry-after'))).toBeGreaterThanOrEqual(2)
|
||||
expect(reasons).toEqual(['drain-return:queue-full', 'drain-return:wait-timeout'])
|
||||
expect(retryAfters).toEqual([2, Number(timedOut.headers.get('retry-after'))])
|
||||
expect(outcomes.filter((outcome) => outcome === 'drain-return-deferred')).toHaveLength(2)
|
||||
expect(causes).toEqual(['drain-return-deferred', 'drain-return-deferred'])
|
||||
expect(warn.mock.calls.flat().join('\n')).toMatch(
|
||||
/lane=drain-return hinted=true reason=queue-full .*retryAfter=2/
|
||||
)
|
||||
replacement.resolve(assignment('cell-new', holder))
|
||||
expect((await holding).status).toBe(200)
|
||||
})
|
||||
|
||||
it('answers a drained host’s row-busy redial with the short interval', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const holder = 'hhhhhhhhhhhhhhhh'
|
||||
const host = 'bbbbbbbbbbbbbbbb'
|
||||
const holding = deferred<RelayAssignment>()
|
||||
let busy = true
|
||||
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
if (relayHostId === holder) return await holding.promise
|
||||
if (relayHostId === host && busy) {
|
||||
busy = false
|
||||
throw new RelayAssignmentRowBusyError()
|
||||
}
|
||||
return assignment('cell-new', relayHostId)
|
||||
})
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
isolatedHome(relayHostId)
|
||||
)
|
||||
const app = createRelayApp(config({ drainReturnQueueMax: 1 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const request = (relayHostId: string) =>
|
||||
Promise.resolve(
|
||||
app.request('/v1/assign', assignmentRequest(relayHostId, { reconnect: true }))
|
||||
)
|
||||
|
||||
const held = request(holder)
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
|
||||
const queued = request(host)
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
// A cohort behind it books the next ten service slots.
|
||||
for (let index = 0; index < 10; index++) {
|
||||
const overflow = await request(hostId('f', index))
|
||||
expect(overflow.status).toBe(503)
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(2_490)
|
||||
holding.resolve(assignment('cell-new', holder))
|
||||
expect((await held).status).toBe(200)
|
||||
const refused = await queued
|
||||
expect(refused.headers.get('retry-after')).toBe('1')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
const redial = await request(host)
|
||||
|
||||
// Its own lane interval, not a slot after the ten hosts that came later.
|
||||
expect(redial.status).toBe(503)
|
||||
expect(redial.headers.get('retry-after')).toBe('2')
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
function hostId(prefix: string, index: number): string {
|
||||
return `${prefix}${String(index).padStart(15, '0')}`
|
||||
}
|
||||
|
||||
function assignmentRequest(relayHostId: string, extra: { reconnect?: boolean } = {}): RequestInit {
|
||||
return {
|
||||
method: 'POST',
|
||||
@@ -209,7 +90,7 @@ function deferred<T>() {
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
@@ -233,7 +114,6 @@ function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data',
|
||||
...overrides
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
|
||||
+2
-69
@@ -18,68 +18,10 @@ import { createRelayApp } from '../../app.js'
|
||||
|
||||
describe('public assignment overload', () => {
|
||||
|
||||
it('fairly drains a bounded incident-scale burst without raising database concurrency', async () => {
|
||||
const pending = new Map<string, ReturnType<typeof deferred<RelayAssignment>>>()
|
||||
const admittedHosts: string[] = []
|
||||
let active = 0
|
||||
let highWater = 0
|
||||
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
admittedHosts.push(relayHostId)
|
||||
active++
|
||||
highWater = Math.max(highWater, active)
|
||||
const operation = deferred<RelayAssignment>()
|
||||
pending.set(relayHostId, operation)
|
||||
try {
|
||||
return await operation.promise
|
||||
} finally {
|
||||
active--
|
||||
pending.delete(relayHostId)
|
||||
}
|
||||
})
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const hosts = Array.from(
|
||||
{ length: 430 },
|
||||
(_, index) => `host-${String(index).padStart(11, '0')}`
|
||||
)
|
||||
|
||||
const firstWave = hosts.map((host) =>
|
||||
app.request('/v1/assign', assignmentRequest(host))
|
||||
)
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
|
||||
const retryWave = await Promise.all(
|
||||
hosts.map((host) => app.request('/v1/assign', assignmentRequest(host)))
|
||||
)
|
||||
|
||||
expect(retryWave.every((response) => response.status === 503)).toBe(true)
|
||||
expect(assign).toHaveBeenCalledTimes(2)
|
||||
while (assign.mock.calls.length < 130) {
|
||||
const activeOperations = [...pending]
|
||||
const expectedCalls = assign.mock.calls.length + activeOperations.length
|
||||
for (const [host, operation] of activeOperations) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
}
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(expectedCalls))
|
||||
}
|
||||
for (const [host, operation] of pending) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
}
|
||||
const firstResponses = await Promise.all(firstWave)
|
||||
expect(firstResponses.filter((response) => response.status === 200)).toHaveLength(130)
|
||||
expect(firstResponses.filter((response) => response.status === 503)).toHaveLength(300)
|
||||
expect(admittedHosts).toEqual(hosts.slice(0, 130))
|
||||
expect(highWater).toBe(2)
|
||||
})
|
||||
|
||||
it('refuses an oversized chunked body before parsing it, sparing only the reserve routes', async () => {
|
||||
const resolve = vi.fn()
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { resolve } as never,
|
||||
assignments: {} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
@@ -98,15 +40,11 @@ describe('public assignment overload', () => {
|
||||
const refused = await app.request('/v1/resolve', chunked(64 * 1024))
|
||||
expect(refused.status).toBe(413)
|
||||
expect(await refused.json()).toEqual({ error: 'request_too_large' })
|
||||
expect(resolve).not.toHaveBeenCalled()
|
||||
// Reserve bodies may reach 16 KiB, so the 4 KiB app limit must not answer for them. An
|
||||
// unavailable route refuses before its own limit, including a body past 16 KiB.
|
||||
// unavailable route refuses before its own limit.
|
||||
const reserve = await app.request('/internal/reserve', chunked(6 * 1024))
|
||||
expect(reserve.status).toBe(404)
|
||||
expect(await reserve.json()).toEqual({ error: 'reserve_unavailable' })
|
||||
const oversized = await app.request('/internal/reserve', chunked(20 * 1024))
|
||||
expect(oversized.status).toBe(404)
|
||||
expect(await oversized.json()).toEqual({ error: 'reserve_unavailable' })
|
||||
})
|
||||
|
||||
it('reserves one bounded resolve lane while assignment work is saturated', async () => {
|
||||
@@ -133,8 +71,6 @@ describe('public assignment overload', () => {
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
|
||||
|
||||
const recovery = app.request('/v1/resolve', resolveRequest('cccccccccccccccc', 1))
|
||||
await Promise.resolve()
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
const excessResolve = await app.request(
|
||||
'/v1/resolve',
|
||||
resolveRequest('dddddddddddddddd', 2)
|
||||
@@ -143,8 +79,6 @@ describe('public assignment overload', () => {
|
||||
|
||||
expect(excessResolve.status).toBe(503)
|
||||
expect(excessAssign.status).toBe(503)
|
||||
expect(assign).toHaveBeenCalledTimes(2)
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
|
||||
pendingAssignments
|
||||
.get('aaaaaaaaaaaaaaaa')
|
||||
@@ -153,7 +87,6 @@ describe('public assignment overload', () => {
|
||||
await vi.waitFor(() => expect(resolveResume).toHaveBeenCalledTimes(1))
|
||||
pendingResolve.resolve({ userId: 'user-1', relayDeviceId: 'device-1' })
|
||||
expect((await recovery).status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledTimes(1)
|
||||
|
||||
for (const [host, operation] of pendingAssignments) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
|
||||
+1
-39
@@ -37,13 +37,11 @@ describe('public assignment reconnect lane', () => {
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
relayHostId === reconnecting ? assignment('cell-r', reconnecting) : null
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome)
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
// Saturate the placement lane with two in-flight newcomers, zero queue.
|
||||
@@ -59,18 +57,6 @@ describe('public assignment reconnect lane', () => {
|
||||
)
|
||||
|
||||
expect(reconnected.status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: reconnecting },
|
||||
{ classifyHomeRollIsolation: true }
|
||||
)
|
||||
expect(assign).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: reconnecting },
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{ hostRequested: true }
|
||||
)
|
||||
expect(outcomes).toContain('sticky')
|
||||
|
||||
pending.get(newcomerA)?.resolve(assignment('cell-a', newcomerA))
|
||||
pending.get(newcomerB)?.resolve(assignment('cell-b', newcomerB))
|
||||
@@ -78,28 +64,6 @@ describe('public assignment reconnect lane', () => {
|
||||
expect((await second).status).toBe(200)
|
||||
})
|
||||
|
||||
it('rate-limits repeat fast-lane attempts with the short retry-after', async () => {
|
||||
const host = 'rrrrrrrrrrrrrrrr'
|
||||
const assign = vi.fn(async () => assignment('cell-r', host))
|
||||
const resolve = vi.fn(async () => assignment('cell-r', host))
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
expect(
|
||||
(await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))).status
|
||||
).toBe(200)
|
||||
const repeat = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
|
||||
|
||||
expect(repeat.status).toBe(503)
|
||||
expect(repeat.headers.get('retry-after')).toBe('2')
|
||||
expect(resolve).toHaveBeenCalledTimes(1)
|
||||
expect(assign).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rejects on the fast lane when the verification probe fails transiently', async () => {
|
||||
const host = 'tttttttttttttttt'
|
||||
const assign = vi.fn()
|
||||
@@ -117,7 +81,6 @@ describe('public assignment reconnect lane', () => {
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('retry-after')).toBe('2')
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses a dial whose assignment row is busy at once, and frees the fast lane', async () => {
|
||||
@@ -143,7 +106,6 @@ describe('public assignment reconnect lane', () => {
|
||||
expect(refused.status).toBe(503)
|
||||
expect(refused.headers.get('retry-after')).toBe('1')
|
||||
expect(await refused.json()).toEqual({ error: 'assignment_row_busy' })
|
||||
expect(warn.mock.calls.flat().join('\n')).toContain('reason=relay_assignment_row_busy')
|
||||
// The single fast-lane slot went back with the refusal.
|
||||
expect(
|
||||
(await app.request('/v1/assign', assignmentRequest(next, { reconnect: true }))).status
|
||||
|
||||
+9
-131
@@ -111,51 +111,15 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('constrained idle regional assignment transaction', () => {
|
||||
it.each(['missing', 'disabled', 'future'] as const)(
|
||||
'does only one read per tick with %s durable control and sees later enablement',
|
||||
async (state) => {
|
||||
const { store, database, safety } = await setup()
|
||||
const control = (await database.query(
|
||||
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
|
||||
))[0]!
|
||||
if (state === 'missing') {
|
||||
await database.query('DELETE FROM relay_region_rehome_control')
|
||||
} else {
|
||||
await database.query(
|
||||
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
|
||||
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
|
||||
)
|
||||
}
|
||||
const query = vi.spyOn(database, 'query')
|
||||
const transaction = vi.spyOn(database, 'transaction')
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
|
||||
expect(transaction).not.toHaveBeenCalled()
|
||||
}
|
||||
if (state === 'missing') {
|
||||
const columns = Object.keys(control)
|
||||
await database.query(
|
||||
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
|
||||
Object.values(control)
|
||||
)
|
||||
} else {
|
||||
await database.query(
|
||||
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
|
||||
[safety.observedAt]
|
||||
)
|
||||
}
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
expect(query.mock.calls.length).toBeGreaterThan(1)
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
}
|
||||
)
|
||||
it('sees durable control changes between candidate polls', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 1 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
})
|
||||
|
||||
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
|
||||
const { store, database, safety, request } = await setup()
|
||||
@@ -184,33 +148,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['next_dispatch_at', 'paused_until'] as const)(
|
||||
'skips the candidate join while %s holds the durable dispatch budget closed',
|
||||
async (column) => {
|
||||
const { store, database, safety } = await setup()
|
||||
const query = vi.spyOn(database, 'query')
|
||||
// One assignment only: setup leaves both fields at 0, and naming the other
|
||||
// one too would assign this column twice, which Postgres rejects.
|
||||
await database.query(
|
||||
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
|
||||
[safety.observedAt + 1]
|
||||
)
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
expect(query.mock.calls[1]![0]).toMatch(/FROM relay_region_rehome_worker_state/s)
|
||||
}
|
||||
await database.query(
|
||||
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
|
||||
[safety.observedAt]
|
||||
)
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
expect(query.mock.calls.length).toBeGreaterThan(2)
|
||||
}
|
||||
)
|
||||
|
||||
// sourceRequestUnits reads a reserve cell's missing phone leases as zero units; selection must drop it.
|
||||
it.each(['source', 'target'])('drops every candidate while the %s cell is recorded as reserve', async (cellId) => {
|
||||
const { store, safety } = await setup()
|
||||
@@ -267,51 +204,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
expect(first.some((c) => c.relayHostId === next[0]!.relayHostId)).toBe(false)
|
||||
})
|
||||
|
||||
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
for (const table of [
|
||||
'relay_assignments',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_control_capabilities',
|
||||
'relay_region_decisions'
|
||||
]) {
|
||||
const template = (
|
||||
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
])
|
||||
)[0]!
|
||||
const columns = Object.keys(template)
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const values = columns.map((column) =>
|
||||
column === 'user_id' || column === 'relay_host_id' ? '?' : column
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
`idle-store-test-${String(index).padStart(3, '0')}`,
|
||||
`pagehost${String(index).padStart(8, '0')}`,
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
]
|
||||
)
|
||||
}
|
||||
}
|
||||
const first = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
const next = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(first).toHaveLength(100)
|
||||
expect(next).toHaveLength(1)
|
||||
expect(next[0]!.relayHostId).toBe('pagehost00000099')
|
||||
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
|
||||
regionalRehomeCohortPercent: 100
|
||||
})
|
||||
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
|
||||
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
|
||||
})
|
||||
|
||||
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
|
||||
'rechecks generation when replacement wins after the initial authority lookup',
|
||||
async () => {
|
||||
@@ -430,20 +322,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
|
||||
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
|
||||
const { store, safety, database } = await setup()
|
||||
const before = await database.query('SELECT * FROM relay_assignments')
|
||||
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(candidates).toHaveLength(1)
|
||||
expect(candidates[0]).toMatchObject({
|
||||
sourceGeneration: 7,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target'
|
||||
})
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
|
||||
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
function holdStatement(database: RelayDatabase, fragment: string) {
|
||||
|
||||
@@ -151,52 +151,6 @@ describe('idle regional rehome candidate window', () => {
|
||||
expect(new Set(candidates.map((candidate) => candidate.attemptId)).size).toBe(3)
|
||||
})
|
||||
|
||||
it('drops only the targets without room for the host plus its source activity', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await seedHost(store, identity)
|
||||
await database.query(
|
||||
'UPDATE relay_assignment_activity_leases SET request_units = 4 WHERE user_id = ?',
|
||||
[identity.userId]
|
||||
)
|
||||
// Five units needed: four source units plus the assignment the move reserves.
|
||||
await database.query("UPDATE relay_cells SET capacity_requests = 4 WHERE cell_id = 'asia-idle'")
|
||||
const short = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(short.map((candidate) => candidate.targetCellId)).toEqual(['asia-mid', 'asia-busy'])
|
||||
// Exactly enough room is enough; it ranks last because the ratio is per capacity.
|
||||
await database.query("UPDATE relay_cells SET capacity_requests = 5 WHERE cell_id = 'asia-idle'")
|
||||
const exact = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(exact.map((candidate) => candidate.targetCellId)).toEqual([
|
||||
'asia-mid',
|
||||
'asia-busy',
|
||||
'asia-idle'
|
||||
])
|
||||
})
|
||||
|
||||
it('reads a bounded window of decisions however many hosts are eligible', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await seedHost(store, identity)
|
||||
await cloneHosts(database, identity, 700)
|
||||
const query = database.query.bind(database)
|
||||
const decisionRows: number[] = []
|
||||
vi.spyOn(database, 'query').mockImplementation(async (sql, params) => {
|
||||
const rows = await query(sql, params)
|
||||
if (/^SELECT user_id, relay_host_id FROM relay_region_decisions/.test(sql)) {
|
||||
decisionRows.push(rows.length)
|
||||
}
|
||||
return rows
|
||||
})
|
||||
const first = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
const next = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(decisionRows.length).toBeGreaterThan(0)
|
||||
for (const count of decisionRows) {
|
||||
expect(count).toBeLessThanOrEqual(500)
|
||||
}
|
||||
expect(first).toHaveLength(102)
|
||||
expect(next).toHaveLength(102)
|
||||
const firstKeys = new Set(first.map((c) => `${c.userId}/${c.targetCellId}`))
|
||||
expect(next.every((c) => !firstKeys.has(`${c.userId}/${c.targetCellId}`))).toBe(true)
|
||||
})
|
||||
|
||||
it("keeps the window's last host when a decision turns eligible between the two reads", async () => {
|
||||
const { store, database, safety, now } = await setup()
|
||||
await seedHost(store, identity)
|
||||
@@ -252,7 +206,6 @@ describe('idle regional rehome candidate window', () => {
|
||||
for (let poll = 0; poll < 20 && !wrapped; poll++) {
|
||||
const page = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(page.length).toBeGreaterThan(0)
|
||||
expect(page.length).toBeLessThanOrEqual(102)
|
||||
for (const candidate of page) {
|
||||
const key = `${candidate.userId}/${candidate.targetCellId}`
|
||||
if (seen.has(key)) {
|
||||
@@ -286,23 +239,4 @@ describe('idle regional rehome candidate window', () => {
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('excludes a host outside the cohort', async () => {
|
||||
const { store, database } = await setup()
|
||||
await seedHost(store, identity)
|
||||
const now = 100_000_000 + 86_400_000
|
||||
const safety = {
|
||||
observedAt: now,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
await database.query('UPDATE relay_region_decisions SET cohort_bucket = 40')
|
||||
const narrow = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 40 })
|
||||
expect(await narrow.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
const wide = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 41 })
|
||||
expect(await wide.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -80,7 +80,7 @@ describe('persisted region decisions across director restart', () => {
|
||||
expect(row.outcome).toBe('inconclusive')
|
||||
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(successor.generation).toBe(2)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
})
|
||||
@@ -111,7 +111,7 @@ describe('persisted region decisions across director restart', () => {
|
||||
context.setNow(1_000_000_000 - 60_000)
|
||||
const restarted = await context.restart()
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(successor.generation).toBe(2)
|
||||
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
|
||||
@@ -124,104 +124,7 @@ async function window(context: Awaited<ReturnType<typeof setup>>) {
|
||||
return result.window!
|
||||
}
|
||||
|
||||
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
|
||||
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
|
||||
const attemptId = '33333333-3333-4333-8333-333333333333'
|
||||
await context.database.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
|
||||
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
|
||||
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
|
||||
[
|
||||
attemptId,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
cells[0]!.id,
|
||||
incarnations[0],
|
||||
cells[1]!.id,
|
||||
incarnations[1],
|
||||
migration.previousEpoch,
|
||||
migration.assignmentEpoch,
|
||||
context.now(),
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
return { migration }
|
||||
}
|
||||
|
||||
describe('ordered region decisions and migration outcomes', () => {
|
||||
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
|
||||
const context = await setup()
|
||||
const { migration } = await regionalMigration(context)
|
||||
context.advance(1_000)
|
||||
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
|
||||
const outcomes = await context.store.regionCorrectionOutcomes()
|
||||
expect(outcomes).toEqual([
|
||||
expect.objectContaining({
|
||||
sourceCellId: cells[0]!.id,
|
||||
targetCellId: cells[1]!.id,
|
||||
state: 'registering',
|
||||
count: 1,
|
||||
oldestOpenMs: 1_000
|
||||
})
|
||||
])
|
||||
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
|
||||
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
|
||||
before
|
||||
)
|
||||
await context.store.activateControl(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.markMigrationTargetRegistered(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch
|
||||
})
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({ state: 'registered' })
|
||||
])
|
||||
await context.store.releaseActivity(identity, context.activityId)
|
||||
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({
|
||||
state: 'completed',
|
||||
targetReservedUnits: 0,
|
||||
oldestOpenMs: 0
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
|
||||
const context = await setup()
|
||||
const query = context.database.query.bind(context.database)
|
||||
const transaction = context.database.transaction.bind(context.database)
|
||||
const queryLocked = context.database.queryLocked.bind(context.database)
|
||||
context.database.query = async (sql, params) => {
|
||||
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
|
||||
return query(sql, params)
|
||||
}
|
||||
context.database.transaction = async () => {
|
||||
throw new Error('preview_must_not_open_mutating_transaction')
|
||||
}
|
||||
context.database.queryLocked = async () => {
|
||||
throw new Error('preview_must_not_lock')
|
||||
}
|
||||
try {
|
||||
const preview = await context.store.previewRegionalRehomeEligibility()
|
||||
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
|
||||
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.userId)
|
||||
} finally {
|
||||
context.database.query = query
|
||||
context.database.transaction = transaction
|
||||
context.database.queryLocked = queryLocked
|
||||
}
|
||||
})
|
||||
|
||||
it('allocates distinct ordered generations for concurrent window issuers', async () => {
|
||||
const context = await setup()
|
||||
const replies = await Promise.all([window(context), window(context), window(context)])
|
||||
|
||||
+4
-26
@@ -1,10 +1,6 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
type RelayDatabase
|
||||
} from '../../database/database.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -127,25 +123,15 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
await expect(
|
||||
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
|
||||
).rejects.toMatchObject({ code: '23514' })
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'
|
||||
ORDER BY conname`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
// The existing control row keeps its tuning and gains the cooldown.
|
||||
// Preserve existing control tuning across the upgrade.
|
||||
const control = await client.query(
|
||||
`SELECT generation, preference_max_age_ms, host_cooldown_ms
|
||||
`SELECT generation, preference_max_age_ms
|
||||
FROM relay_region_rehome_control WHERE control_id = 'global'`
|
||||
)
|
||||
expect(control.rows).toEqual([
|
||||
{
|
||||
generation: '3',
|
||||
preference_max_age_ms: '86400000',
|
||||
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
|
||||
preference_max_age_ms: '86400000'
|
||||
}
|
||||
])
|
||||
})
|
||||
@@ -182,14 +168,6 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
})
|
||||
}, 60_000)
|
||||
})
|
||||
|
||||
+2
-14
@@ -1,4 +1,4 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
|
||||
|
||||
@@ -134,10 +134,6 @@ const moves = (candidates: { userId: string; sourceCellId: string; targetCellId:
|
||||
to: cells.find(({ id }) => id === targetCellId)!.region
|
||||
}))
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('rehome source-region filter', () => {
|
||||
it('keeps US to Asia moves and drops Asia to US moves', async () => {
|
||||
const context = await setup('us-central1')
|
||||
@@ -151,13 +147,5 @@ describe('rehome source-region filter', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('reports the filtered hosts in the preview, agreeing with selection', async () => {
|
||||
const context = await setup('us-central1')
|
||||
const preview = await context.store.previewRegionalRehomeEligibility(context.safety())
|
||||
expect(preview.counts).toEqual({
|
||||
'eligible:us-central1-to-asia-east2': 1,
|
||||
'source-outside-director-region': 1
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-99
@@ -1,7 +1,6 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import {
|
||||
consumeRelayCellInventoryHold,
|
||||
openRelayDatabase,
|
||||
type PostgresDatabase,
|
||||
type RelayDatabase
|
||||
@@ -18,21 +17,6 @@ const describePostgres = databaseUrl ? describe : describe.skip
|
||||
// Roughly the round trip from an Asia cell to the us-central1 database.
|
||||
const STATEMENT_DELAY_MS = 150
|
||||
|
||||
// Probe the formerly fleet-locked tables plus admission/region tables read by commit.
|
||||
const CELL_TABLES = [
|
||||
'relay_cells',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_capabilities',
|
||||
'relay_cell_rehome_safety',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions'
|
||||
] as const
|
||||
|
||||
// The target-row statement locks exactly these, held from it to COMMIT.
|
||||
const TARGET_LOCKED = ['target:relay_cells', 'target:relay_cell_admission']
|
||||
|
||||
type Trip = { sql: string; lockable: Record<string, boolean>; probeMs: number }
|
||||
|
||||
type DelayControl = StatementDelay & { beforeTrip: (sql: string) => Promise<void> }
|
||||
|
||||
describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
@@ -97,27 +81,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function lockable(table: string, cellId: string): Promise<boolean> {
|
||||
try {
|
||||
await observer.transaction(
|
||||
async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT cell_id FROM ${table} WHERE cell_id = ?`,
|
||||
[cellId],
|
||||
{
|
||||
failIfUnavailable: true
|
||||
}
|
||||
)
|
||||
},
|
||||
{ reportRetries: false }
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'database_lock_unavailable') return false
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function reservedRequests(cellId: string): Promise<number> {
|
||||
const row = (
|
||||
await primary.query(`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`, [cellId])
|
||||
@@ -152,59 +115,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
)
|
||||
}
|
||||
|
||||
it('holds only the target cell and admission rows until the next COMMIT boundary', async () => {
|
||||
const context = await fixture()
|
||||
const request = await context.select()
|
||||
const trips: Trip[] = []
|
||||
const probed = {
|
||||
source: context.source.id,
|
||||
target: context.target.id,
|
||||
bystander: context.bystander.id
|
||||
}
|
||||
control.beforeTrip = async (sql) => {
|
||||
const probeStartedAt = performance.now()
|
||||
const state: Record<string, boolean> = {}
|
||||
for (const [role, cellId] of Object.entries(probed)) {
|
||||
for (const table of CELL_TABLES) state[`${role}:${table}`] = await lockable(table, cellId)
|
||||
}
|
||||
trips.push({ sql, lockable: state, probeMs: performance.now() - probeStartedAt })
|
||||
}
|
||||
consumeRelayCellInventoryHold(delayed)
|
||||
control.enabled = true
|
||||
|
||||
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
|
||||
control.enabled = false
|
||||
|
||||
expect(result).toEqual({ outcome: 'committed' })
|
||||
expect(trips.length).toBeGreaterThanOrEqual(15)
|
||||
expect(trips[trips.length - 1]!.sql).toBe('COMMIT')
|
||||
for (const trip of trips) {
|
||||
for (const [key, free] of Object.entries(trip.lockable)) {
|
||||
if (TARGET_LOCKED.includes(key)) continue
|
||||
expect({ sql: trip.sql, key, free }).toEqual({ sql: trip.sql, key, free: true })
|
||||
}
|
||||
}
|
||||
for (const key of TARGET_LOCKED) {
|
||||
expect(trips.filter((trip) => !trip.lockable[key]).map((trip) => trip.sql)).toEqual([
|
||||
'COMMIT'
|
||||
])
|
||||
}
|
||||
const counts = consumeRelayCellInventoryHold(delayed)
|
||||
const commitProbeMs = trips.at(-1)!.probeMs
|
||||
console.info(
|
||||
JSON.stringify({
|
||||
event: 'rehome_target_row_hold',
|
||||
trips: trips.length,
|
||||
commitProbeMs,
|
||||
...counts
|
||||
})
|
||||
)
|
||||
expect(counts.rehomeTargetRowHolds).toBe(1)
|
||||
expect(counts.cellInventoryHoldMaxSite).toBe('rehome-target-row')
|
||||
expect(counts.rehomeTargetRowHoldMsMax).toBeGreaterThanOrEqual(STATEMENT_DELAY_MS)
|
||||
expect(await reservedRequests(context.target.id)).toBe(context.targetReservedBefore + 2)
|
||||
}, 45_000)
|
||||
|
||||
it('defers when the target leaves general admission after selection', async () => {
|
||||
const context = await fixture()
|
||||
const request = await context.select()
|
||||
@@ -240,23 +150,14 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
})
|
||||
await lockedPromise
|
||||
}
|
||||
consumeRelayCellInventoryHold(delayed)
|
||||
control.enabled = true
|
||||
|
||||
const startedAt = performance.now()
|
||||
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
|
||||
const elapsedMs = performance.now() - startedAt
|
||||
control.enabled = false
|
||||
release()
|
||||
await writer
|
||||
|
||||
expect(result).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
|
||||
expect(consumeRelayCellInventoryHold(delayed)).toMatchObject({
|
||||
cellInventoryLockUnavailable: 1,
|
||||
rehomeTargetRowHolds: 0
|
||||
})
|
||||
// NOWAIT: no 1s lock_timeout wait and no transaction retry behind the writer.
|
||||
expect(elapsedMs).toBeLessThan(40 * STATEMENT_DELAY_MS)
|
||||
await expectNothingCommitted(context)
|
||||
})
|
||||
|
||||
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from '../../database/database.js'
|
||||
import { REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT } from './regional-rehome-safety.js'
|
||||
|
||||
const cell = (id: string, region: 'us-central1' | 'asia-east2') => ({
|
||||
id,
|
||||
url: `https://${id}.relay.example.test`,
|
||||
region,
|
||||
capacityRequests: 100,
|
||||
connectionHardCap: 1_000 as const,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
const source = cell('us-c1', 'us-central1')
|
||||
const noHeadroom = cell('asia-a', 'asia-east2')
|
||||
const unclean = cell('asia-b', 'asia-east2')
|
||||
const highLoad = cell('asia-c', 'asia-east2')
|
||||
const lowLoad = cell('asia-d', 'asia-east2')
|
||||
|
||||
const incarnation = (n: number) =>
|
||||
`${String(n).repeat(8)}-${String(n).repeat(4)}-4${String(n).repeat(3)}` +
|
||||
`-8${String(n).repeat(3)}-${String(n).repeat(12)}`
|
||||
|
||||
async function setup() {
|
||||
let clock = 1_000_000
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayAssignmentStore(database, () => clock, {
|
||||
regionalRehomeCohortPercent: 100,
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
await store.inspectRegionalRehomeControl()
|
||||
clock += 24 * 60 * 60_000
|
||||
await store.applyRegionalRehomeControl({
|
||||
expectedGeneration: 0,
|
||||
enabled: true,
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
|
||||
const beat = async (
|
||||
config: typeof source,
|
||||
n: number,
|
||||
protocol: number,
|
||||
state: { observedRequests: number; enforcedConnections: number; sqlFailures: number }
|
||||
) => {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: config.id,
|
||||
cellUrl: config.url,
|
||||
region: config.region,
|
||||
cellIncarnation: incarnation(n),
|
||||
startedAt: 900_000,
|
||||
ready: true,
|
||||
observedRequests: state.observedRequests,
|
||||
totalConnections: state.enforcedConnections,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: state.enforcedConnections,
|
||||
connectionInclusionWatermark: clock,
|
||||
connectionHardCap: 1_000,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
await store.recordCellRegionalRehomeStatus({
|
||||
cellId: config.id,
|
||||
cellIncarnation: incarnation(n),
|
||||
regionalRehomeProtocol: protocol,
|
||||
safety: {
|
||||
observedAt: clock,
|
||||
sqlFailures: state.sqlFailures,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
})
|
||||
}
|
||||
const activatePreferredSource = async () => {
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1,
|
||||
idleRegionalRehome: true,
|
||||
cellIncarnation: incarnation(1)
|
||||
})
|
||||
const { window } = await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
expect(window).toBeDefined()
|
||||
await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: window!.generation,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 180, 'asia-east2': 40 }
|
||||
},
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
}
|
||||
const safety = () => ({
|
||||
observedAt: clock,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
return { database, store, beat, activatePreferredSource, safety }
|
||||
}
|
||||
|
||||
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
|
||||
|
||||
describe('regional rehome target selection', () => {
|
||||
it('never selects a target without connection headroom, even at lowest load', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
// Lowest load but the connection hard cap is exhausted.
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(lowLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('falls to the next clean target when the load winner goes unclean', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(highLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
})
|
||||
@@ -1,189 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
|
||||
|
||||
describe('regional rehome worker', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
|
||||
vi.useFakeTimers()
|
||||
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
|
||||
const worker = startRegionalRehomeWorker(
|
||||
config(),
|
||||
{
|
||||
selectIdleRegionalRehomeCandidates
|
||||
} as unknown as RelayAssignmentStore,
|
||||
{
|
||||
safetySnapshot: () => safety(Date.now()),
|
||||
random: () => 0
|
||||
}
|
||||
)!
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(5_999)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
|
||||
worker.stop()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
|
||||
} finally {
|
||||
worker.stop()
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('stops walking the page when the source names a deferral no candidate can pass', async () => {
|
||||
const fetchImpl = respondWith([{ outcome: 'deferred', reason: 'concurrency-limit' }])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 3, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1)
|
||||
expect(summaries.entries).toEqual([
|
||||
{
|
||||
event: 'orca_relay_idle_rehome_dispatch_summary',
|
||||
candidates: 3,
|
||||
dispatched: 1,
|
||||
stoppedBy: 'concurrency-limit',
|
||||
outcomes: { 'deferred:concurrency-limit': 1 }
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps its whole-page walk when the source sends no reason at all', async () => {
|
||||
const fetchImpl = respondWith([{ outcome: 'deferred' }])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 3, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3)
|
||||
expect(summaries.entries[0]).toMatchObject({
|
||||
stoppedBy: null,
|
||||
outcomes: { deferred: 3 }
|
||||
})
|
||||
})
|
||||
|
||||
it('walks past a deferral that only concerns the one candidate', async () => {
|
||||
const fetchImpl = respondWith([
|
||||
{ outcome: 'deferred', reason: 'host-unsupported' },
|
||||
{ outcome: 'busy' },
|
||||
{ outcome: 'committed' }
|
||||
])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 4, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3)
|
||||
expect(summaries.entries[0]).toMatchObject({
|
||||
dispatched: 3,
|
||||
stoppedBy: 'committed',
|
||||
outcomes: { 'deferred:host-unsupported': 1, busy: 1, committed: 1 }
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function respondWith(bodies: { outcome: string; reason?: string }[]) {
|
||||
let index = 0
|
||||
return vi.fn(async () => {
|
||||
const body = bodies[Math.min(index++, bodies.length - 1)]!
|
||||
return new Response(JSON.stringify({ v: 1, ...body }), {
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function collectSummaries() {
|
||||
const entries: Record<string, unknown>[] = []
|
||||
let arrived: (() => void) | undefined
|
||||
// Construction immediately polls; await first instead of waiting for a timer tick.
|
||||
const first = new Promise<void>((resolve) => {
|
||||
arrived = resolve
|
||||
})
|
||||
const original = console.warn
|
||||
console.warn = (line: unknown, ...rest: unknown[]) => {
|
||||
try {
|
||||
const parsed = JSON.parse(line as string) as Record<string, unknown>
|
||||
if (parsed.event === 'orca_relay_idle_rehome_dispatch_summary') {
|
||||
entries.push(parsed)
|
||||
arrived?.()
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// Not a JSON log line; fall through to the original writer.
|
||||
}
|
||||
original(line as string, ...rest)
|
||||
}
|
||||
return { entries, first, restore: () => (console.warn = original) }
|
||||
}
|
||||
|
||||
async function runOnePoll(
|
||||
fetchImpl: typeof fetch,
|
||||
candidates: number,
|
||||
summaries: { first: Promise<void> }
|
||||
): Promise<void> {
|
||||
const assignments = {
|
||||
selectIdleRegionalRehomeCandidates: vi.fn(async () =>
|
||||
Array.from({ length: candidates }, (_, index) => ({
|
||||
v: 1 as const,
|
||||
attemptId: `00000000-0000-4000-8000-00000000000${index}`,
|
||||
userId: `user-${index}`,
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'us-c1',
|
||||
sourceCellUrl: 'https://us-c1.relay.example.test',
|
||||
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
sourceAssignmentEpoch: 1,
|
||||
sourceGeneration: 1,
|
||||
targetCellId: 'asia-c1'
|
||||
}))
|
||||
)
|
||||
} as unknown as RelayAssignmentStore
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
fetch: fetchImpl,
|
||||
identityToken: async () => 'token',
|
||||
safetySnapshot: () => safety(Date.now()),
|
||||
intervalMs: 60_000
|
||||
})!
|
||||
try {
|
||||
await summaries.first
|
||||
} finally {
|
||||
worker.stop()
|
||||
}
|
||||
}
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
return {
|
||||
role: 'director',
|
||||
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
|
||||
rehomeDirectorServiceAccount: 'relay-director@example.test',
|
||||
...overrides
|
||||
} as RelayConfig
|
||||
}
|
||||
|
||||
function safety(observedAt: number) {
|
||||
return {
|
||||
requiredCells: 2,
|
||||
missingCells: 0,
|
||||
observedAt,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolTotal: 3,
|
||||
databasePoolIdle: 3,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolOldestWaitMs: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { PLACEMENT_INTAKE_SHARE, PlacementLoadBand } from './placement-load-band.js'
|
||||
import { PlacementLoadBand } from './placement-load-band.js'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from '../../database/database.js'
|
||||
|
||||
@@ -13,74 +13,8 @@ function seededRandom(seed: number): () => number {
|
||||
}
|
||||
}
|
||||
|
||||
type SimulatedCell = { cellId: string; reserved: number; capacity: number }
|
||||
|
||||
const quiet = () => undefined
|
||||
|
||||
// Each pick goes to one of the directors at random, as the load balancer spreads requests.
|
||||
function simulate(
|
||||
directors: PlacementLoadBand[],
|
||||
cells: SimulatedCell[],
|
||||
picks: number,
|
||||
at: (index: number) => number,
|
||||
route: () => number = seededRandom(99)
|
||||
) {
|
||||
const counts = new Map(cells.map((cell) => [cell.cellId, 0]))
|
||||
for (let index = 0; index < picks; index += 1) {
|
||||
const ranked = cells
|
||||
.filter((cell) => cell.reserved < cell.capacity)
|
||||
.map((cell) => ({ cellId: cell.cellId, loadRatio: cell.reserved / cell.capacity, cell }))
|
||||
.sort(
|
||||
(left, right) => left.loadRatio - right.loadRatio || left.cellId.localeCompare(right.cellId)
|
||||
)
|
||||
const director = directors[Math.floor(route() * directors.length)]
|
||||
const picked = director?.pick(ranked, at(index))
|
||||
if (!picked) throw new Error('no candidate')
|
||||
picked.cell.reserved += 1
|
||||
counts.set(picked.cellId, (counts.get(picked.cellId) ?? 0) + 1)
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// c27's drain on 10-08: 2,770 hosts onto one empty Asia cell (2,805 free)
|
||||
// and four fuller ones (2,311 free between them).
|
||||
function c27DrainCells(): SimulatedCell[] {
|
||||
return [
|
||||
{ cellId: 'cell-empty', reserved: 0, capacity: 4000 },
|
||||
...['cell-b', 'cell-c', 'cell-d', 'cell-e'].map((cellId, index) => ({
|
||||
cellId,
|
||||
reserved: 4000 - 577 - (index === 0 ? 3 : 0),
|
||||
capacity: 4000
|
||||
}))
|
||||
]
|
||||
}
|
||||
|
||||
function drainShare(paceMs: number, seed: number) {
|
||||
const cells = c27DrainCells()
|
||||
const directors = Array.from(
|
||||
{ length: 5 },
|
||||
(_, index) => new PlacementLoadBand(seededRandom(seed + index), quiet)
|
||||
)
|
||||
const hosts = 2770
|
||||
const counts = simulate(directors, cells, hosts, (index) => (index * paceMs) / hosts)
|
||||
return { counts, cells, share: (counts.get('cell-empty') ?? 0) / hosts }
|
||||
}
|
||||
|
||||
describe('PlacementLoadBand', () => {
|
||||
it.each([
|
||||
['1,200 s', 1_200_000],
|
||||
['300 s', 300_000]
|
||||
])('keeps the empty cell near the share across five directors at the %s pace', (_, paceMs) => {
|
||||
for (const seed of [1, 2, 3]) {
|
||||
const { counts, cells, share } = drainShare(paceMs, seed * 10)
|
||||
expect([...counts.values()].reduce((sum, count) => sum + count, 0)).toBe(2770)
|
||||
expect(share).toBeGreaterThan(0.3)
|
||||
expect(share).toBeLessThan(0.5)
|
||||
for (const cell of cells) expect(cell.reserved).toBeLessThanOrEqual(cell.capacity)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('leastLoadedCell with the load band', () => {
|
||||
async function setupRegions() {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
@@ -121,20 +55,6 @@ describe('leastLoadedCell with the load band', () => {
|
||||
return counts
|
||||
}
|
||||
|
||||
it('spreads within the preferred region and keeps an empty US cell out', async () => {
|
||||
const { database, store } = await setupRegions()
|
||||
try {
|
||||
const counts = await placements(store, 200)
|
||||
|
||||
expect(counts.get('us-a')).toBeUndefined()
|
||||
expect(counts.get('asia-a')).toBeLessThanOrEqual(PLACEMENT_INTAKE_SHARE * 200 + 1)
|
||||
const used = [...counts.values()].filter((count) => count >= 20)
|
||||
expect(used.length).toBeGreaterThanOrEqual(3)
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('fills the preferred region to its hard cap before spilling, never past it', async () => {
|
||||
const { database, store } = await setupRegions()
|
||||
try {
|
||||
|
||||
@@ -111,25 +111,6 @@ function seated(epoch: number, joinedAt = 0) {
|
||||
|
||||
describe('reserve assignment on a director', () => {
|
||||
|
||||
it('reads what each cell acts on: a tripped dead-man is db, a flip back still leasing is reserve', async () => {
|
||||
const tripped = setup({
|
||||
feeds: [feed('c1', { admitModeEffective: 'db' }), feed('c2', { admitModeEffective: 'db' })]
|
||||
})
|
||||
expect(await tripped.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
|
||||
const leasing = setup({ feeds: [feed('c1', { admitMode: 'db', admitModeEffective: 'reserve' }), feed('c2')] })
|
||||
expect(leasing.directory.admitModeOf('c1')).toBe('reserve')
|
||||
// Raw db must immediately stop booking/pacing even while effective mode stays reserve during re-registration.
|
||||
const reregistering = setup({
|
||||
feeds: [
|
||||
feed('c1', { admitModeEffective: 'reserve', admitModeRaw: 'db' }),
|
||||
feed('c2', { admitModeEffective: 'reserve', admitModeRaw: 'db' })
|
||||
]
|
||||
})
|
||||
expect(await reregistering.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
|
||||
expect(reregistering.reserved).toEqual([])
|
||||
expect(reregistering.directory.admitModeOf('c1')).toBe('reserve')
|
||||
})
|
||||
|
||||
it('before every live cell has answered, holds back only hosts that may be on an unheard cell', async () => {
|
||||
const cells = cellList(['c1', 'c2'])
|
||||
const onUnheard = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
@@ -153,18 +134,6 @@ describe('reserve assignment on a director', () => {
|
||||
expect((await gated.assignment.plan(onUnheard, { reconnect: true, region: US })).kind).not.toBe('retry')
|
||||
})
|
||||
|
||||
it('answers a reconnect from memory at the same epoch with no booking and no database read', async () => {
|
||||
const row = vi.fn(async () => null)
|
||||
const { assignment, reserved } = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')],
|
||||
row
|
||||
})
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({ kind: 'answer', lane: 'sticky', assignment: { cellId: 'c1', assignmentEpoch: 7 } })
|
||||
expect(reserved).toEqual([])
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('re-books a host on a cell that restarted since its seat, and only then answers it', async () => {
|
||||
const { assignment, directory, reserved, now } = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')]
|
||||
@@ -198,57 +167,6 @@ describe('reserve assignment on a director', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('never sends a drained or demoted host back, and books it elsewhere above every known epoch', async () => {
|
||||
const { assignment, directory, reserved, now } = setup()
|
||||
directory.apply(
|
||||
'c1',
|
||||
feed('c1', {
|
||||
full: undefined,
|
||||
seq: 3,
|
||||
changes: [
|
||||
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
|
||||
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4503, at: 2 }
|
||||
]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement',
|
||||
assignment: { assignmentEpoch: 8 }
|
||||
})
|
||||
expect(reserved.length).toBeGreaterThan(0)
|
||||
expect(reserved.every((call) => !call.sticky && call.epoch === 8)).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves a cell without a feed (c17/c18) on today’s path without starving the reserve pool', async () => {
|
||||
const cells = cellList(['c1', 'c2', 'c17'])
|
||||
let rowOnOld = false
|
||||
const { assignment, directory, reserved, now } = setup({
|
||||
cells,
|
||||
feeds: [feed('c1'), feed('c2', { admitMode: 'db' })],
|
||||
row: async () => (rowOnOld ? { cellId: 'c17', assignmentEpoch: 5 } : null)
|
||||
})
|
||||
directory.markNoFeed('c17', now.value)
|
||||
expect(directory.isComplete()).toBe(true)
|
||||
// An old image reports no ceiling; counting it as a database cell would zero the share.
|
||||
expect(await assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement',
|
||||
assignment: { cellId: 'c1' }
|
||||
})
|
||||
expect(reserved.map((call) => call.cellId)).not.toContain('c17')
|
||||
rowOnOld = true
|
||||
expect(
|
||||
await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: true, region: US })
|
||||
).toMatchObject({ kind: 'database', epochFloor: 5 })
|
||||
directory.markNoFeed('c1', now.value)
|
||||
expect(directory.cellState('c1')?.ceiling).toBeUndefined()
|
||||
expect(assignment.placementCells().find((cell) => cell.cellId === 'c1')?.general).toBe(false)
|
||||
})
|
||||
|
||||
it('reads the row once for a host the map never saw, and refuses it while the database is down', async () => {
|
||||
const down = setup({ row: async () => Promise.reject(new Error('timeout')) })
|
||||
expect(await down.assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
|
||||
@@ -318,95 +236,6 @@ describe('reserve assignment on a director', () => {
|
||||
}
|
||||
})
|
||||
|
||||
describe('for a share-draw loser', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) =>
|
||||
setup({
|
||||
...(reserve ? { reserve } : {}),
|
||||
cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])],
|
||||
feeds: [
|
||||
reserveCell,
|
||||
// Database room far above the reserve cell's, so the 0.25 draw loses.
|
||||
feed('c2', {
|
||||
admitMode: 'db',
|
||||
counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 }
|
||||
}),
|
||||
feed('c3')
|
||||
]
|
||||
})
|
||||
const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => {
|
||||
const plan = await assignment.plan(identity, { reconnect: false, region: ASIA })
|
||||
expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 })
|
||||
return plan.kind === 'database' ? plan.regionReserve : undefined
|
||||
}
|
||||
|
||||
it('offers the reserve cell in its region only, only while it has a seat', async () => {
|
||||
const { assignment, reserved, directory, now } = setupAsia()
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
expect(reserved).toEqual([])
|
||||
await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA })
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } })
|
||||
directory.apply('c1', full, now.value, now.value)
|
||||
expect(await regionReserveOf(assignment, OTHER)).toBeUndefined()
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
})
|
||||
|
||||
it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => {
|
||||
const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } }))
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError)
|
||||
expect(reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('gives the region up when the reserve cell refuses with a seat still on the map', async () => {
|
||||
for (const outcome of ['full', 'unreachable']) {
|
||||
const { assignment, reserved } = setupAsia(feed('c1'), (cellId) =>
|
||||
cellId === 'c1' ? { outcome } : { outcome: 'ok' }
|
||||
)
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
// Null sends the host to the database lane's cross-region fallback, not a 1 s retry.
|
||||
await expect(regionReserve?.()).resolves.toBeNull()
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('for a region-correction request', () => {
|
||||
const request = { reconnect: true, region: US, regionCorrection: true }
|
||||
|
||||
it('leaves the answer to the database lane, offering only the in-region reserve fallback and the spill', async () => {
|
||||
const row = vi.fn(async () => null)
|
||||
// A memory answer would skip the window exchange; the share draw is not offered either.
|
||||
const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row })
|
||||
const plan = await assignment.plan(HOST, request)
|
||||
expect(plan).toEqual({
|
||||
kind: 'database',
|
||||
regionReserve: expect.any(Function),
|
||||
reserveSpill: expect.any(Function)
|
||||
})
|
||||
expect(reserved).toEqual([])
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({
|
||||
assignmentEpoch: 8
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps today’s path for a host the map never saw, without reading its row, but may spill', async () => {
|
||||
const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 }))
|
||||
const { assignment, reserved } = setup({ row })
|
||||
const plan = await assignment.plan(HOST, request)
|
||||
expect(plan).toEqual({ kind: 'database', reserveSpill: expect.any(Function) })
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
expect(reserved).toEqual([])
|
||||
await expect(
|
||||
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 3 }) : null
|
||||
).resolves.toMatchObject({ assignmentEpoch: 4 })
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('after the database lane found no cell anywhere', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const paced = { intake: { perSec: 1, burst: 20, tokens: 0 } }
|
||||
@@ -428,31 +257,6 @@ describe('reserve assignment on a director', () => {
|
||||
return plan.kind === 'database' ? plan.reserveSpill : undefined
|
||||
}
|
||||
|
||||
it('books its own region first, minting above the row the database lane read', async () => {
|
||||
const harness = setupSpill({})
|
||||
const spill = await spillOf(harness)
|
||||
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 9 })).resolves.toMatchObject({
|
||||
cellId: 'asia-r',
|
||||
region: ASIA,
|
||||
assignmentEpoch: 10
|
||||
})
|
||||
expect(harness.reserved).toEqual([{ cellId: 'asia-r', epoch: 10, sticky: false, dryRun: false }])
|
||||
})
|
||||
|
||||
it('books the other region when its own is full, or already tried', async () => {
|
||||
const full = setupSpill(fullCounts)
|
||||
await expect((await spillOf(full))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
cellId: 'us-r',
|
||||
region: US,
|
||||
assignmentEpoch: 5
|
||||
})
|
||||
const tried = setupSpill({})
|
||||
await expect((await spillOf(tried))?.({ skipOwnRegion: true, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
cellId: 'us-r'
|
||||
})
|
||||
expect(tried.reserved.map((call) => call.cellId)).toEqual(['us-r'])
|
||||
})
|
||||
|
||||
it('tries the other region after a pace, and answers the pace only if nothing places', async () => {
|
||||
const abroad = setupSpill(paced)
|
||||
await expect((await spillOf(abroad))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
@@ -464,39 +268,10 @@ describe('reserve assignment on a director', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('returns null with no reserve seat anywhere, so the refusal stands', async () => {
|
||||
const harness = setupSpill(fullCounts, fullCounts)
|
||||
await expect((await spillOf(harness))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toBeNull()
|
||||
expect(harness.reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('is not offered while the map is incomplete', async () => {
|
||||
const gated = setup({
|
||||
cells: [...cellList(['asia-r', 'asia-db'], ASIA), ...cellList(['us-r'])],
|
||||
feeds: [feed('asia-r'), feed('asia-db', { admitMode: 'db', full: [seated(4)] })],
|
||||
startedAt: 100_500
|
||||
})
|
||||
const plan = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA })
|
||||
expect(plan).toMatchObject({ kind: 'database' })
|
||||
expect(plan.kind === 'database' && plan.reserveSpill).toBeUndefined()
|
||||
const correction = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA, regionCorrection: true })
|
||||
expect(correction).toEqual({ kind: 'database' })
|
||||
})
|
||||
|
||||
it('answers the shortest pace when every region paces', async () => {
|
||||
const harness = setupSpill(paced, { intake: { perSec: 2, burst: 20, tokens: 0 } })
|
||||
const spill = await spillOf(harness)
|
||||
// Asia refills its token in 4 s, US in 2 s.
|
||||
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 0 })).rejects.toSatisfy(
|
||||
(error) => error instanceof ReservePaceError && error.retryAfterSeconds === 2
|
||||
)
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('rehome on reconnect', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
const ask = { reconnect: true, region: ASIA, preferred: true }
|
||||
// 2,000 controls on the US cell refill its move budget once a minute.
|
||||
const usCounts = { counts: { seats: 1, controls: 2_000, bookings: 0, units: 1, ceiling: 4_000 } }
|
||||
@@ -546,107 +321,6 @@ describe('reserve assignment on a director', () => {
|
||||
expect(demoted).toEqual([{ cellId: 'us-r', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
it('answers a region-correction request on the rehome lane', async () => {
|
||||
const { assignment, warm } = setupRehome()
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, { ...ask, regionCorrection: true })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'rehome'
|
||||
})
|
||||
})
|
||||
|
||||
it('moves no one into Asia at 98% full, or anywhere under 10% free', async () => {
|
||||
for (const share of [0.98, 0.91]) {
|
||||
const { assignment, reserved, warm } = setupRehome({ asia: asiaFull(share) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask), String(share)).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'sticky',
|
||||
assignment: { cellId: 'us-r' }
|
||||
})
|
||||
expect(reserved.filter((call) => call.cellId === 'asia-r')).toEqual([])
|
||||
}
|
||||
const { assignment, warm } = setupRehome({ asia: asiaFull(0.9) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
})
|
||||
|
||||
// 10% rehome headroom above the 0% spill line prevents spilled hosts bouncing back.
|
||||
it('keeps a gap above the spill line: 5% free takes placements, not rehomes', async () => {
|
||||
const { assignment, warm } = setupRehome({ asia: asiaFull(0.95) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky', assignment: { cellId: 'us-r' } })
|
||||
expect(await assignment.plan(OTHER, { reconnect: false, region: ASIA })).toMatchObject({
|
||||
lane: 'placement',
|
||||
assignment: { cellId: 'asia-r' }
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps today’s answer when any rule fails', async () => {
|
||||
const sticky = { kind: 'answer', lane: 'sticky', assignment: { cellId: 'us-r' } }
|
||||
const unhinted = setupRehome()
|
||||
await unhinted.warm()
|
||||
expect(await unhinted.assignment.plan(HOST, { ...ask, preferred: false })).toMatchObject(sticky)
|
||||
const off = setupRehome({ percent: 0 })
|
||||
await off.warm()
|
||||
expect(await off.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const closed = setupRehome({ switchOpen: false })
|
||||
await closed.warm()
|
||||
expect(await closed.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const cold = setupRehome()
|
||||
expect(await cold.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const sameRegion = setupRehome()
|
||||
await sameRegion.warm()
|
||||
expect(await sameRegion.assignment.plan(HOST, { ...ask, region: US })).toMatchObject(sticky)
|
||||
})
|
||||
|
||||
it('never runs for a fresh ask, an unknown host, a database-mode source, or an incomplete map', async () => {
|
||||
const fresh = setupRehome()
|
||||
await fresh.warm()
|
||||
expect(await fresh.assignment.plan(HOST, { ...ask, reconnect: false })).toMatchObject({ lane: 'placement' })
|
||||
expect(await fresh.assignment.plan(OTHER, ask)).toMatchObject({ kind: 'answer', lane: 'placement' })
|
||||
const onDatabase = setup({
|
||||
cells: [...cellList(['us-db']), ...cellList(['asia-r'], ASIA)],
|
||||
feeds: [feed('us-db', { admitMode: 'db', ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
|
||||
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
|
||||
})
|
||||
expect(await onDatabase.assignment.plan(HOST, ask)).toMatchObject({ kind: 'database' })
|
||||
const gated = setup({
|
||||
cells: [...cellList(['us-r']), ...cellList(['asia-r', 'asia-unheard'], ASIA)],
|
||||
feeds: [feed('us-r', { ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
|
||||
startedAt: 100_500,
|
||||
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
|
||||
})
|
||||
expect(await gated.assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
|
||||
expect(gated.reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('falls through to the sticky answer when the rehome booking is paced', async () => {
|
||||
const { assignment, reserved, warm } = setupRehome({
|
||||
reserve: (cellId) => (cellId === 'asia-r' ? { outcome: 'intake' } : { outcome: 'ok' })
|
||||
})
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ kind: 'answer', lane: 'sticky' })
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['asia-r'])
|
||||
})
|
||||
|
||||
it('leaves a host another install keeps dropping', async () => {
|
||||
const { assignment, directory, now, warm } = setupRehome()
|
||||
await warm()
|
||||
directory.apply(
|
||||
'us-r',
|
||||
feed('us-r', {
|
||||
...usCounts,
|
||||
full: undefined,
|
||||
seq: 3,
|
||||
changes: [{ seq: 3, kind: 'leave', ...HOST, epoch: 4, generation: 1, closeCode: 4408, at: now.value - 1 }]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
|
||||
})
|
||||
|
||||
it('falls through to the sticky answer when the booking is refused, and gives the move back', async () => {
|
||||
let refuse = true
|
||||
const { assignment, warm } = setupRehome({
|
||||
@@ -657,52 +331,6 @@ describe('reserve assignment on a director', () => {
|
||||
refuse = false
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
})
|
||||
|
||||
it('moves one host per source budget a minute, and logs moves and skips per region pair', async () => {
|
||||
const lines: Record<string, unknown>[] = []
|
||||
const { assignment, now, warm } = setupRehome({
|
||||
usSeats: [seated(5, 1_000), { ...OTHER, epoch: 5, generation: 1, state: 'active', joinedAt: 1_000 }],
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
expect(await assignment.plan(OTHER, ask)).toMatchObject({ lane: 'sticky' })
|
||||
now.value += 60_000
|
||||
await assignment.plan({ ...HOST, relayHostId: 'flushflushflush1' }, ask)
|
||||
expect(lines.filter((line) => line.event === 'orca_relay_reserve_rehome')).toEqual([
|
||||
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 0, skippedByRule: { 'source-budget': 1 } },
|
||||
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 1, skippedByRule: { 'source-budget': 1 } }
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => {
|
||||
const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) })
|
||||
await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toMatchObject({
|
||||
message: 'reserve_cells_paced',
|
||||
retryAfterSeconds: expect.any(Number)
|
||||
})
|
||||
})
|
||||
|
||||
it('tells the old reserve seat to go when it books the host elsewhere, naming that seat exactly', async () => {
|
||||
const { assignment, directory, demoted, now } = setup({
|
||||
feeds: [feed('c1', { full: [seated(5, 1_000)] }), feed('c2')],
|
||||
reserve: (cellId) => (cellId === 'c1' ? { outcome: 'full' } : { outcome: 'ok' })
|
||||
})
|
||||
// The host left c1 for a reason that moves it: its next placement is fresh, above 5.
|
||||
directory.apply(
|
||||
'c1',
|
||||
feed('c1', {
|
||||
full: undefined,
|
||||
seq: 2,
|
||||
changes: [{ seq: 2, kind: 'drain-only', ...HOST, epoch: 5, generation: 1, at: 2 }]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({ kind: 'answer', lane: 'placement', assignment: { assignmentEpoch: 6 } })
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
// A half-open seat missed by the booking map must be demoted by the feed, before its 75s silence close.
|
||||
@@ -714,91 +342,4 @@ describe('reserve assignment on a director', () => {
|
||||
directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
it('answers a reconnect at the newest epoch even when an older epoch joined later', async () => {
|
||||
const { assignment, reserved } = setup({
|
||||
feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
|
||||
})
|
||||
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'sticky',
|
||||
assignment: { cellId: 'c1', assignmentEpoch: 6 }
|
||||
})
|
||||
expect(reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('in dry run only asks a cell to check a booking, and never changes the answer', async () => {
|
||||
const { assignment, reserved } = setup({ mode: 'dry-run' })
|
||||
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
|
||||
expect(plan.kind).toBe('database')
|
||||
expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
|
||||
expect(reserved).toMatchObject([{ dryRun: true }])
|
||||
})
|
||||
|
||||
describe('when a cell answers a demotion', () => {
|
||||
const outcomes = (lines: string[]) =>
|
||||
lines
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((entry) => entry.event === 'orca_relay_reserve_demote_outcome')
|
||||
|
||||
it('logs the answer by host digest and leaves the seat to the feed', async () => {
|
||||
const lines: string[] = []
|
||||
const logged = Promise.withResolvers<void>()
|
||||
const { assignment, directory } = setup({
|
||||
feeds: [feed('c1', { full: [seated(10, 1_000)] }), feed('c2')],
|
||||
demote: () => 'not-seated',
|
||||
log: (line) => {
|
||||
lines.push(line)
|
||||
if (JSON.parse(line).event === 'orca_relay_reserve_demote_outcome') {
|
||||
logged.resolve()
|
||||
}
|
||||
}
|
||||
})
|
||||
assignment.demoteSeat({ ...HOST, cellId: 'c1', epoch: 10, joinedAt: 1_000 }, 'reconcile')
|
||||
await logged.promise
|
||||
expect(outcomes(lines)).toMatchObject([
|
||||
{ via: 'reconcile', cellId: 'c1', epoch: 10, outcome: 'not-seated' }
|
||||
])
|
||||
expect(outcomes(lines)[0].relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
|
||||
expect(lines.join('\n')).not.toContain(HOST.relayHostId)
|
||||
expect(directory.seatsOnCell('c1')).toHaveLength(1)
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('for a host re-asking off a draining cell', () => {
|
||||
const events = (lines: string[], event: string) =>
|
||||
lines.map((line) => JSON.parse(line)).filter((entry) => entry.event === event)
|
||||
|
||||
it('marks its pace as the drain’s, on the reserve lane, in the region fallback and in the spill', async () => {
|
||||
const { assignment } = setup({
|
||||
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
|
||||
reserve: () => ({ outcome: 'intake' })
|
||||
})
|
||||
await expect(assignment.plan(HOST, { reconnect: true, region: US })).rejects.toMatchObject({ drained: true })
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US, regionCorrection: true })
|
||||
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).rejects.toMatchObject({ drained: true })
|
||||
await expect(
|
||||
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 0 }) : null
|
||||
).rejects.toMatchObject({ drained: true })
|
||||
})
|
||||
|
||||
it('leaves every other host as today: no drain count, and a plain pace', async () => {
|
||||
const lines: string[] = []
|
||||
const { assignment } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], log: (line) => lines.push(line) })
|
||||
const paced = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2', { draining: true, full: [seated(6)] })],
|
||||
reserve: () => ({ outcome: 'intake' })
|
||||
})
|
||||
expect(await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: false, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement'
|
||||
})
|
||||
await expect(
|
||||
paced.assignment.plan(HOST, { reconnect: false, region: US })
|
||||
).rejects.toMatchObject({ drained: false })
|
||||
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([])
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
@@ -57,30 +57,15 @@ describe('reserve ledger writer', () => {
|
||||
expect(await row(db, 'bbbbbbbbbbbbbbbb')).toMatchObject({ cell_id: 'c1', reserved_controls: 2 })
|
||||
})
|
||||
|
||||
it('keeps its queue through a database stall and drops the oldest past the bound', async () => {
|
||||
it('persists a booked join after a database stall', async () => {
|
||||
let now = 1_000
|
||||
const { database: db, writer } = await setup(() => now)
|
||||
const query = vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
|
||||
vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
|
||||
writer.enqueue({ ...HOST, cellId: 'c1', epoch: 5 })
|
||||
await writer.flush()
|
||||
expect(writer.pending()).toBe(1)
|
||||
await writer.flush()
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
now += 1_000
|
||||
await writer.flush()
|
||||
expect(writer.pending()).toBe(0)
|
||||
expect(await row(db)).toMatchObject({ cell_id: 'c1', assignment_epoch: 5 })
|
||||
|
||||
const bounded = new ReserveLedgerWriter(db, { queueMax: 2, log: () => undefined })
|
||||
const hosts = ['oldest0000000000', 'middle0000000000', 'newest0000000000']
|
||||
for (const relayHostId of hosts) {
|
||||
bounded.enqueue({ ...HOST, relayHostId, cellId: 'c1', epoch: 1 })
|
||||
}
|
||||
expect(bounded.pending()).toBe(2)
|
||||
await bounded.flush()
|
||||
expect(await row(db, hosts[0])).toBeUndefined()
|
||||
expect(await row(db, hosts[1])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
|
||||
expect(await row(db, hosts[2])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
|
||||
})
|
||||
|
||||
it('reconciles the map’s seats and never lowers an epoch', async () => {
|
||||
|
||||
-179
@@ -1,179 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
runReservePlacementSimulation,
|
||||
type SimulationConfig,
|
||||
type SimulationReport
|
||||
} from './reserve-placement-simulation.js'
|
||||
|
||||
const SEEDS = [11, 23, 47]
|
||||
const rtt = (region: string) => (region === 'asia-east2' ? 170 : 10)
|
||||
|
||||
function fleet(): SimulationConfig['cells'] {
|
||||
const cells: SimulationConfig['cells'] = []
|
||||
for (let index = 0; index < 24; index += 1) {
|
||||
cells.push({
|
||||
cellId: `us${String(index).padStart(2, '0')}`,
|
||||
region: 'us-central1',
|
||||
hardCap: 600,
|
||||
ceiling: 480,
|
||||
intakePerSec: 5,
|
||||
// A third of the fleet stays in database mode, one cell on an image without step 5.
|
||||
mode: index % 3 === 0 ? 'db' : 'reserve',
|
||||
old: index === 0,
|
||||
preseated: 150
|
||||
})
|
||||
}
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
cells.push({
|
||||
cellId: `asia${index}`,
|
||||
region: 'asia-east2',
|
||||
hardCap: 600,
|
||||
ceiling: 480,
|
||||
intakePerSec: 2,
|
||||
mode: index === 0 ? 'db' : 'reserve',
|
||||
preseated: 200
|
||||
})
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function expectInvariants(report: SimulationReport): void {
|
||||
expect(report.violations).toEqual([])
|
||||
}
|
||||
|
||||
describe('step 5 deterministic simulation', () => {
|
||||
// Invariant 9 (dead-man) never trips here; its own tests below cover it.
|
||||
it.each(SEEDS)(
|
||||
'holds invariants 1-8 across mixed versions, lagging polls, restarts and database stalls (seed %i)',
|
||||
async (seed) => {
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 25 * 60_000,
|
||||
directors: 5,
|
||||
// One director still on the database-only revision.
|
||||
oldDirectors: 1,
|
||||
cells: fleet(),
|
||||
hosts: 6_600,
|
||||
hostRegion: (index) => (index % 5 === 0 ? 'asia-east2' : 'us-central1'),
|
||||
arrivalPerSec: 20,
|
||||
meanSessionMs: 8 * 60_000,
|
||||
reconnectDelayMs: [500, 5_000],
|
||||
duplicateAssignShare: 0.05,
|
||||
halfOpenShare: 0.05,
|
||||
pollLagMs: (director) => [0, 200, 1_000, 4_000, 10_000][director]!,
|
||||
rttMs: rtt,
|
||||
cellRestarts: [{ at: 6 * 60_000, cellId: 'us04' }, { at: 14 * 60_000, cellId: 'asia2' }],
|
||||
directorRestarts: [{ at: 8 * 60_000, director: 2 }, { at: 16 * 60_000, director: 4 }],
|
||||
// Drained hosts land elsewhere while their old seat lingers: placement supersedes it.
|
||||
drains: [{ at: 12 * 60_000, cellId: 'us05', paceMs: 60_000 }],
|
||||
databaseStalls: Array.from({ length: 3 }, (_, index) => ({
|
||||
at: (index * 610 + 120) * 1_000,
|
||||
durationMs: 7_000
|
||||
}))
|
||||
})
|
||||
expectInvariants(report)
|
||||
expect(report.reservePlacements).toBeGreaterThan(500)
|
||||
expect(report.stickyAnswers).toBeGreaterThan(500)
|
||||
expect(report.seatedAtEnd).toBeGreaterThan(4_000)
|
||||
// The run reached the paths the invariants guard, so a pass is not vacuous.
|
||||
expect(report.supersedes).toBeGreaterThan(0)
|
||||
expect(report.rowAheadDemotions).toBeGreaterThan(0)
|
||||
expect(report.directorRestarts).toBe(2)
|
||||
expect(report.stickyReserves).toBeGreaterThan(0)
|
||||
expect(report.databasePlacements).toBeGreaterThan(0)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
it.each(SEEDS)(
|
||||
'near full: extra reserve calls stay within 10% and exhausted requests within 1% (seed %i)',
|
||||
async (seed) => {
|
||||
const cells = Array.from({ length: 17 }, (_, index) => ({
|
||||
cellId: `us${index}`,
|
||||
region: 'us-central1' as const,
|
||||
hardCap: 700,
|
||||
ceiling: 600,
|
||||
intakePerSec: 50,
|
||||
mode: 'reserve' as const,
|
||||
preseated: 582
|
||||
}))
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 3 * 60_000,
|
||||
directors: 5,
|
||||
cells,
|
||||
hosts: 17 * 582 + 300,
|
||||
hostRegion: () => 'us-central1',
|
||||
arrivalPerSec: 10,
|
||||
meanSessionMs: 0,
|
||||
reconnectDelayMs: [1_000, 3_000],
|
||||
duplicateAssignShare: 0,
|
||||
halfOpenShare: 0,
|
||||
pollLagMs: () => 0,
|
||||
rttMs: rtt
|
||||
})
|
||||
expectInvariants(report)
|
||||
expect(report.reservePlacements).toBeGreaterThan(250)
|
||||
expect(report.reserveCalls / report.reservePlacements - 1).toBeLessThanOrEqual(0.1)
|
||||
expect(report.exhausted / report.requests).toBeLessThanOrEqual(0.01)
|
||||
for (const share of Object.values(report.maxOccupancyShare)) expect(share).toBeLessThanOrEqual(1)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
it.each(SEEDS.flatMap((seed) => [[1_200_000, seed], [300_000, seed]] as const))(
|
||||
'a drain onto an empty cell at the %i ms pace spreads within each cell intake budget (seed %i)',
|
||||
async (paceMs, seed) => {
|
||||
// Production c27 drain: 1,200s pacing toward cells with 2,805/841/777/667/26 free seats.
|
||||
const free = [2_805, 841, 777, 667, 26]
|
||||
const cells: SimulationConfig['cells'] = free.map((seats, index) => ({
|
||||
cellId: `asia${index + 1}`,
|
||||
region: 'asia-east2',
|
||||
hardCap: 3_000,
|
||||
ceiling: 2_830,
|
||||
intakePerSec: 2,
|
||||
mode: 'reserve',
|
||||
preseated: 2_830 - seats
|
||||
}))
|
||||
cells.unshift({
|
||||
cellId: 'asia0',
|
||||
region: 'asia-east2',
|
||||
hardCap: 3_000,
|
||||
ceiling: 2_830,
|
||||
intakePerSec: 2,
|
||||
mode: 'reserve',
|
||||
preseated: 2_770
|
||||
})
|
||||
const preseated = cells.reduce((sum, cell) => sum + (cell.preseated ?? 0), 0)
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 30 * 60_000,
|
||||
directors: 5,
|
||||
cells,
|
||||
hosts: preseated,
|
||||
hostRegion: () => 'asia-east2',
|
||||
arrivalPerSec: 1,
|
||||
meanSessionMs: 0,
|
||||
reconnectDelayMs: [1_000, 3_000],
|
||||
duplicateAssignShare: 0.02,
|
||||
halfOpenShare: 0,
|
||||
pollLagMs: () => 0,
|
||||
rttMs: rtt,
|
||||
drains: [{ at: 10_000, cellId: 'asia0', paceMs }]
|
||||
})
|
||||
expectInvariants(report)
|
||||
// Below every cell's budget nothing waits; above it the cells' buckets pace the drain.
|
||||
if (paceMs === 300_000) expect(report.intakeRefusals + report.paced).toBeGreaterThan(0)
|
||||
const moved = Object.entries(report.arrivalsByCell)
|
||||
const total = moved.reduce((sum, [, count]) => sum + count, 0)
|
||||
expect(total).toBeGreaterThan(2_000)
|
||||
// Spread: the emptiest cell takes a share, not the drain.
|
||||
expect((report.arrivalsByCell.asia1 ?? 0) / total).toBeLessThan(0.6)
|
||||
expect(moved.filter(([, count]) => count / total > 0.1).length).toBeGreaterThanOrEqual(3)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
|
||||
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,7 @@ import { exportJWK, generateKeyPair, SignJWT, type CryptoKey, type JWK } from 'j
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import { createRelayTokenVerifier } from './relay-token-verifier.js'
|
||||
import { createStaleIfErrorJWKSet, RELAY_JWKS_MAX_STALE_MS } from './stale-if-error-jwks.js'
|
||||
import { createStaleIfErrorJWKSet } from './stale-if-error-jwks.js'
|
||||
|
||||
const config = {
|
||||
jwksUrl: 'https://auth.example.test/.well-known/jwks.json',
|
||||
@@ -43,37 +43,7 @@ beforeEach(() => vi.useFakeTimers({ now: Date.parse('2026-10-11T13:00:00Z') }))
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
describe('relay token keys while auth is down', () => {
|
||||
it('accepts a valid token from the last served set once the cache is past its 10 min age', async () => {
|
||||
const key = await signingKey('auth-1')
|
||||
const authServer = auth(key)
|
||||
const jwks = createStaleIfErrorJWKSet(new URL(config.jwksUrl), { fetch: authServer.fetch })
|
||||
const verify = createRelayTokenVerifier(config as RelayConfig, jwks)
|
||||
expect(await verify(await relayToken(key))).not.toBeNull()
|
||||
|
||||
vi.setSystemTime(Date.now() + 20 * MINUTE)
|
||||
authServer.state.up = false
|
||||
const fetches = authServer.fetch.mock.calls.length
|
||||
expect(await verify(await relayToken(key))).toMatchObject({ sub: 'user-1' })
|
||||
expect(authServer.fetch.mock.calls.length).toBe(fetches + 1)
|
||||
// Inside the retry window the stale set answers without waiting on another fetch.
|
||||
expect(await verify(await relayToken(key))).not.toBeNull()
|
||||
expect(authServer.fetch.mock.calls.length).toBe(fetches + 1)
|
||||
})
|
||||
|
||||
it('rejects a token signed by a key the last served set never had', async () => {
|
||||
const key = await signingKey('auth-1')
|
||||
const authServer = auth(key)
|
||||
const jwks = createStaleIfErrorJWKSet(new URL(config.jwksUrl), { fetch: authServer.fetch })
|
||||
const verify = createRelayTokenVerifier(config as RelayConfig, jwks)
|
||||
expect(await verify(await relayToken(key))).not.toBeNull()
|
||||
|
||||
vi.setSystemTime(Date.now() + MINUTE)
|
||||
authServer.state.up = false
|
||||
expect(await verify(await relayToken(await signingKey('unknown')))).toBeNull()
|
||||
})
|
||||
|
||||
it('bounds the stale set at 30 minutes since the last served set', async () => {
|
||||
expect(RELAY_JWKS_MAX_STALE_MS).toBe(30 * MINUTE)
|
||||
const key = await signingKey('auth-1')
|
||||
const authServer = auth(key)
|
||||
const verify = createRelayTokenVerifier(
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
|
||||
import { reconcileCellAdmissionAtStartup } from './cell-admission-startup.js'
|
||||
import type { RelayCellConfig } from '../../config.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_startup_retry_test'
|
||||
const cell: RelayCellConfig = {
|
||||
id: 'startup-retry-cell',
|
||||
url: 'https://startup-retry.example.test',
|
||||
capacityRequests: 4_000
|
||||
}
|
||||
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
describePostgres('PostgreSQL director startup reconciliation', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
let scopedDatabaseUrl = ''
|
||||
|
||||
beforeAll(async () => {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
scopedDatabaseUrl = url.toString()
|
||||
databases.push(
|
||||
await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' }),
|
||||
await openRelayDatabase({ databaseUrl: scopedDatabaseUrl, dataDir: '' })
|
||||
)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
for (const database of databases) await database.close()
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
})
|
||||
|
||||
it('recovers after the cell inventory lock outlasts transaction retries', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const store = new RelayAssignmentStore(databases[0]!, () => 100)
|
||||
await store.reconcileCells([cell], false)
|
||||
let releaseLock: () => void = () => undefined
|
||||
let reportLocked: () => void = () => undefined
|
||||
const lockReleased = new Promise<void>((resolve) => (releaseLock = resolve))
|
||||
const lockAcquired = new Promise<void>((resolve) => (reportLocked = resolve))
|
||||
const holder = databases[1]!.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`)
|
||||
reportLocked()
|
||||
await lockReleased
|
||||
})
|
||||
await lockAcquired
|
||||
|
||||
const releaseTimer = setTimeout(releaseLock, 3_500)
|
||||
try {
|
||||
await reconcileCellAdmissionAtStartup({ role: 'director', cells: [cell] }, store)
|
||||
} finally {
|
||||
clearTimeout(releaseTimer)
|
||||
releaseLock()
|
||||
await holder
|
||||
}
|
||||
|
||||
expect(await databases[0]!.query(`SELECT cell_id FROM relay_cells`)).toEqual([
|
||||
{ cell_id: cell.id }
|
||||
])
|
||||
const warnings = warn.mock.calls.flat().join('\n')
|
||||
expect(warnings).toContain('orca_relay_startup_reconcile_recovered')
|
||||
expect(warnings).not.toContain('orca_relay_postgres_transaction_exhausted')
|
||||
}, 10_000)
|
||||
})
|
||||
@@ -1,44 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
reconcileCellAdmissionAtStartup
|
||||
} from './cell-admission-startup.js'
|
||||
import type { RelayCellConfig } from '../../config.js'
|
||||
|
||||
const cells: RelayCellConfig[] = [
|
||||
{
|
||||
id: 'candidate',
|
||||
url: 'https://candidate.relay.example.com',
|
||||
capacityRequests: 4_000,
|
||||
initiallyEnabled: false
|
||||
}
|
||||
]
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('cell admission startup authority', () => {
|
||||
|
||||
it('bounds retry wall time when each reconciliation is slow', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
vi.spyOn(Math, 'random').mockReturnValue(0)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const lockUnavailable = Object.assign(new Error('lock unavailable'), { code: '55P03' })
|
||||
const reconcileCellsAtStartup = vi.fn().mockImplementation(async () => {
|
||||
vi.setSystemTime(Date.now() + 10_000)
|
||||
throw lockUnavailable
|
||||
})
|
||||
|
||||
const startup = reconcileCellAdmissionAtStartup(
|
||||
{ role: 'director', cells },
|
||||
{ reconcileCellsAtStartup }
|
||||
)
|
||||
const rejection = expect(startup).rejects.toBe(lockUnavailable)
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
|
||||
expect(reconcileCellsAtStartup).toHaveBeenCalledTimes(5)
|
||||
})
|
||||
})
|
||||
@@ -1,119 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
|
||||
import type { RelayCellConfig } from '../../config.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const CELL_PREFIX = 'connection-cap-min-postgres'
|
||||
const FLEET_SIZE = 16
|
||||
|
||||
function cellConfig(id: string, connectionHardCap: 1_000 | 3_000): RelayCellConfig {
|
||||
return {
|
||||
id,
|
||||
url: `https://${id}.example.com`,
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap,
|
||||
connectionUnobservedBound: 60
|
||||
}
|
||||
}
|
||||
|
||||
async function removeCells(database: RelayDatabase): Promise<void> {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_capabilities',
|
||||
'relay_cell_rehome_safety',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE cell_id LIKE ?`, [`${CELL_PREFIX}-%`])
|
||||
}
|
||||
}
|
||||
|
||||
function storeOn(database: RelayDatabase): RelayAssignmentStore {
|
||||
return new RelayAssignmentStore(database, () => 100, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
}
|
||||
|
||||
async function heartbeat(
|
||||
store: RelayAssignmentStore,
|
||||
cell: RelayCellConfig,
|
||||
connectionHardCap: number,
|
||||
units: number
|
||||
): Promise<void> {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: units,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: units,
|
||||
connectionHardCap,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL effective connection cap', () => {
|
||||
let database: RelayDatabase
|
||||
let racer: RelayDatabase
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await openRelayDatabase({ databaseUrl, dataDir: '' })
|
||||
racer = await openRelayDatabase({ databaseUrl, dataDir: '' })
|
||||
await removeCells(database)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await removeCells(database)
|
||||
await racer.close()
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('keeps 16 raised cells placeable while a heartbeat races the raise', async () => {
|
||||
const ids = Array.from({ length: FLEET_SIZE }, (_, index) => `${CELL_PREFIX}-fleet-${index}`)
|
||||
const lower = ids.map((id) => cellConfig(id, 1_000))
|
||||
const director = storeOn(database)
|
||||
const otherInstance = storeOn(racer)
|
||||
await director.reconcileCells(lower, true)
|
||||
for (const cell of lower) await heartbeat(director, cell, 1_000, 10)
|
||||
|
||||
// The heartbeat lands before or after the raise; either order must leave the same state.
|
||||
await Promise.all([
|
||||
director.reconcileCells(
|
||||
ids.map((id) => cellConfig(id, 3_000)),
|
||||
true
|
||||
),
|
||||
heartbeat(otherInstance, lower[0]!, 1_000, 11)
|
||||
])
|
||||
|
||||
for (const id of ids) {
|
||||
await expect(director.cellDeploymentStatus(id)).resolves.toMatchObject({
|
||||
connectionCapacity: {
|
||||
hardCap: 3_000,
|
||||
reportedHardCap: 1_000,
|
||||
effectiveHardCap: 1_000,
|
||||
heartbeatFresh: true
|
||||
}
|
||||
})
|
||||
const rows = await database.query(
|
||||
`SELECT cell_id FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
|
||||
[id]
|
||||
)
|
||||
expect(rows).toHaveLength(1)
|
||||
}
|
||||
// The old processes keep heartbeating their 1,000 under the raised config.
|
||||
for (const cell of lower) {
|
||||
await expect(heartbeat(otherInstance, cell, 1_000, 12)).resolves.toBeUndefined()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,55 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import { startCellHeartbeat } from './cell-heartbeat-client.js'
|
||||
|
||||
const CONFIG = {
|
||||
role: 'cell',
|
||||
cellId: 'cell-a',
|
||||
cellUrl: 'https://relay-a.example.com',
|
||||
directorUrl: 'https://relay.example.com',
|
||||
heartbeatAudience: 'https://relay.example.com/v1/admin/cell-heartbeat'
|
||||
} as RelayConfig
|
||||
|
||||
describe('cell heartbeat client', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('sends an immediate authenticated heartbeat without putting credentials in the URL', async () => {
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = []
|
||||
const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => {
|
||||
requests.push({ url: String(url), init })
|
||||
return new Response('{}', { status: 200 })
|
||||
}) as typeof fetch
|
||||
const client = startCellHeartbeat(CONFIG, {
|
||||
ready: async () => true,
|
||||
observedRequests: () => 7,
|
||||
connectionCounts: () => ({
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0
|
||||
}),
|
||||
fetch: fetchImpl,
|
||||
identityToken: async () => 'secret-token',
|
||||
now: () => 123,
|
||||
incarnation: '11111111-1111-4111-8111-111111111111',
|
||||
intervalMs: 60_000
|
||||
})!
|
||||
await vi.waitFor(() => expect(requests).toHaveLength(1))
|
||||
client.stop()
|
||||
|
||||
expect(requests[0]!.url).toBe('https://relay.example.com/v1/admin/cell-heartbeat')
|
||||
expect(requests[0]!.url).not.toContain('secret-token')
|
||||
expect(requests[0]!.init?.headers).toMatchObject({ authorization: 'Bearer secret-token' })
|
||||
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
|
||||
v: 1,
|
||||
cellId: 'cell-a',
|
||||
cellUrl: 'https://relay-a.example.com',
|
||||
region: 'us-central1',
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 123,
|
||||
ready: true,
|
||||
observedRequests: 7
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
cellFlagObjectName,
|
||||
cellFlagParser
|
||||
} from './cell-flags.js'
|
||||
import { CONTROL_FLAG_POLL_MS, startControlFlagChannel } from './relay-control-flag-channel.js'
|
||||
import { startControlFlagChannel } from './relay-control-flag-channel.js'
|
||||
|
||||
const CELL_ID = 'production-gce-c7'
|
||||
const ACCESS_TOKEN = 'ya29.secret-access-token'
|
||||
@@ -46,7 +46,7 @@ function cellObject(generation: number, flags: Record<string, unknown>, cellId =
|
||||
}
|
||||
}
|
||||
|
||||
function cellChannel(fetchImpl: typeof fetch, autoStart = false) {
|
||||
function cellChannel(fetchImpl: typeof fetch) {
|
||||
return startControlFlagChannel({
|
||||
objectName: cellFlagObjectName(CELL_ID),
|
||||
defaults: CELL_FLAG_DEFAULTS,
|
||||
@@ -54,7 +54,7 @@ function cellChannel(fetchImpl: typeof fetch, autoStart = false) {
|
||||
appliedEvent: CELL_FLAGS_APPLIED_EVENT,
|
||||
fetch: fetchImpl,
|
||||
random: () => 0.5,
|
||||
autoStart
|
||||
autoStart: false
|
||||
})
|
||||
}
|
||||
|
||||
@@ -230,19 +230,4 @@ describe('control flag channel', () => {
|
||||
expect(channel.applied()).toMatchObject({ generation: 5, flags: { readinessLocal: true } })
|
||||
})
|
||||
|
||||
it('reads one at a time and on a jittered 5 s cadence', async () => {
|
||||
vi.useFakeTimers()
|
||||
const google = fakeGoogle(cellObject(5, { readinessLocal: true }))
|
||||
const channel = cellChannel(google.fetchImpl)
|
||||
await Promise.all([channel.poll(), channel.poll()])
|
||||
expect(google.state.storageRequests).toHaveLength(1)
|
||||
const running = cellChannel(google.fetchImpl, true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(google.state.storageRequests).toHaveLength(2)
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_FLAG_POLL_MS)
|
||||
expect(google.state.storageRequests).toHaveLength(3)
|
||||
running.stop()
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_FLAG_POLL_MS * 3)
|
||||
expect(google.state.storageRequests).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,69 +4,34 @@ import {
|
||||
CellAdmitEffectiveWriter,
|
||||
DEAD_MAN_SLOT_CLAIM_RETRY_MS
|
||||
} from './cell-admit-effective-writer.js'
|
||||
import { DEAD_MAN_SLOT_HOLD_MS } from './cell-dead-man-trip-slot.js'
|
||||
|
||||
function setup(
|
||||
options: { fail?: boolean; busy?: boolean; busyWhen?: () => boolean; slotFree?: () => boolean } = {}
|
||||
options: { slotFree?: () => boolean } = {}
|
||||
) {
|
||||
const clock = { value: 1_000 }
|
||||
const writes: ('db' | 'reserve' | 'claim')[] = []
|
||||
const pending: Array<() => void> = []
|
||||
const logs: string[] = []
|
||||
const writer = new CellAdmitEffectiveWriter({
|
||||
write: async (mode) => {
|
||||
writes.push(mode)
|
||||
await new Promise<void>((resolve) => pending.push(resolve))
|
||||
if (options.fail) throw new Error('connection refused')
|
||||
},
|
||||
claimTrip: async (slots) => {
|
||||
writes.push('claim')
|
||||
await new Promise<void>((resolve) => pending.push(resolve))
|
||||
return slots > 0 && (options.slotFree?.() ?? true)
|
||||
},
|
||||
databaseBusy: () => options.busy === true || options.busyWhen?.() === true,
|
||||
now: () => clock.value,
|
||||
log: (line) => logs.push(line)
|
||||
databaseBusy: () => false,
|
||||
now: () => clock.value
|
||||
})
|
||||
const settle = async () => {
|
||||
while (pending.length > 0) pending.shift()!()
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
}
|
||||
return { clock, writes, logs, writer, settle }
|
||||
return { clock, writes, writer, settle }
|
||||
}
|
||||
|
||||
describe('CellAdmitEffectiveWriter', () => {
|
||||
it('writes on the first tick, on every change, and every 15 s, one write at a time', async () => {
|
||||
const { clock, writes, writer, settle } = setup()
|
||||
writer.tick('reserve', true)
|
||||
writer.tick('db', true)
|
||||
expect(writes).toEqual(['reserve'])
|
||||
await settle()
|
||||
// Tripped: the switch still says reserve, the raw mode db.
|
||||
writer.tick('db', true)
|
||||
await settle()
|
||||
writer.tick('db', true)
|
||||
expect(writes).toEqual(['reserve', 'db'])
|
||||
clock.value += CELL_ADMIT_EFFECTIVE_REFRESH_MS
|
||||
writer.tick('db', true)
|
||||
expect(writes).toEqual(['reserve', 'db', 'db'])
|
||||
})
|
||||
|
||||
// Step 5 off: a db switch never writes, so the dark roll adds no database write.
|
||||
|
||||
// Best-effort: a failed write never throws into the cell, logs at most once a minute, and retries.
|
||||
it('retries a failed write on the next tick and rate-limits its log line', async () => {
|
||||
const { clock, writes, logs, writer, settle } = setup({ fail: true })
|
||||
writer.tick('db', true)
|
||||
await settle()
|
||||
clock.value += 1_000
|
||||
writer.tick('db', true)
|
||||
await settle()
|
||||
expect(writes).toEqual(['db', 'db'])
|
||||
expect(logs).toHaveLength(1)
|
||||
expect(JSON.parse(logs[0]!)).toMatchObject({ event: 'orca_relay_cell_admit_effective_write_failed', mode: 'db' })
|
||||
})
|
||||
|
||||
it('claims a held trip slot in place of its refresh, and permits the trip only once it lands', async () => {
|
||||
let free = false
|
||||
const { clock, writes, writer, settle } = setup({ slotFree: () => free })
|
||||
@@ -97,20 +62,4 @@ describe('CellAdmitEffectiveWriter', () => {
|
||||
expect(writes).toEqual(['reserve', 'claim', 'reserve', 'claim'])
|
||||
})
|
||||
|
||||
it('refreshes a tripped row past a saturated pool before its slot hold runs out', async () => {
|
||||
let busy = false
|
||||
const { clock, writes, writer, settle } = setup({ busyWhen: () => busy })
|
||||
writer.tick('db', true)
|
||||
await settle()
|
||||
busy = true
|
||||
clock.value += 4 * 60_000
|
||||
writer.tick('db', true)
|
||||
expect(writes).toEqual(['db'])
|
||||
clock.value += 60_000
|
||||
writer.tick('db', true)
|
||||
await settle()
|
||||
expect(writes).toEqual(['db', 'db'])
|
||||
expect(DEAD_MAN_SLOT_HOLD_MS).toBeGreaterThan(5 * 60_000)
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
@@ -8,8 +8,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import WebSocket from 'ws'
|
||||
import type { RawData } from 'ws'
|
||||
import { CELL_FLAG_DEFAULTS, type CellFlags } from '../control-flags/cell-flags.js'
|
||||
import { CELL_SEAT_DRIFT_SWEEP_MS } from '../seats/cell-seat-drift.js'
|
||||
import type { CellSeatLog } from '../seats/cell-seat-log.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
import { PostgresDatabase } from '../../database/database.js'
|
||||
import type { AppliedControlFlags } from '../control-flags/relay-control-flag-channel.js'
|
||||
@@ -73,20 +71,6 @@ function cellConfig(port: number, issuer: string): RelayConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// The registry's private feed log, to plant the drift the cell must repair.
|
||||
function seatLogOf(sessions: object): CellSeatLog {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: HostSessionRegistry holds its feed log as `seatLog`.
|
||||
return (sessions as { seatLog: CellSeatLog }).seatLog
|
||||
}
|
||||
|
||||
// The JSON lines a test's console.warn spy saw for one event.
|
||||
function warned(event: string): Array<Record<string, unknown>> {
|
||||
return vi
|
||||
.mocked(console.warn)
|
||||
.mock.calls.map(([line]) => (typeof line === 'string' && line.startsWith('{') ? JSON.parse(line) : null))
|
||||
.filter((entry) => entry?.event === event)
|
||||
}
|
||||
|
||||
function nextMessage(socket: WebSocket): Promise<Record<string, unknown>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
socket.once('message', (data: RawData) => {
|
||||
@@ -265,9 +249,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
|
||||
book,
|
||||
setFlags: (next: Partial<CellFlags>) => {
|
||||
applied = { generation: applied.generation + 1, flags: { ...CELL_FLAG_DEFAULTS, ...next } }
|
||||
},
|
||||
advance: (ms: number) => {
|
||||
elapsedMs += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -338,97 +319,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
|
||||
return (page as { full: Array<{ relayHostId: string; epoch: number }> }).full.map((entry) => entry.epoch)
|
||||
}
|
||||
|
||||
it('repairs at once the seat a director names when no session backs it', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
cell.book(identity, 9)
|
||||
await cell.connect(identity, 9)
|
||||
// A stale seat at a generation above the live session's: its changes could never replace it.
|
||||
const log = seatLogOf(cell.relay.sessions)
|
||||
log.append({ kind: 'join', userId: 'user-1', relayHostId: identity.hostId, epoch: 8, generation: 5, at: 4_242 })
|
||||
const before = cell.relay.sessions.seatFeed(null)
|
||||
const seq = 'full' in before ? before.seq : 0
|
||||
const seat = { v: 1 as const, userId: 'user-1', relayHostId: identity.hostId, epoch: 8, joinedAt: 4_242 }
|
||||
expect(cell.relay.sessions.demote(seat)).toBe('not-seated')
|
||||
expect(cell.relay.sessions.seatFeed(seq)).toMatchObject({
|
||||
changes: [
|
||||
{ kind: 'leave', epoch: 8, generation: 5, closeCode: 4409 },
|
||||
{ kind: 'join', epoch: 9, generation: 1 }
|
||||
]
|
||||
})
|
||||
expect(seatEpochs(cell)).toEqual([9])
|
||||
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([
|
||||
{ kind: 'unbacked-seat', via: 'demote', reason: 'epoch', seatEpoch: 8, sessionEpoch: 9, relisted: true }
|
||||
])
|
||||
expect(JSON.stringify(warned('orca_relay_cell_seat_log_drift'))).not.toContain(identity.hostId)
|
||||
// Asked again, there is nothing left to repair.
|
||||
expect(cell.relay.sessions.demote(seat)).toBe('not-seated')
|
||||
expect(cell.relay.sessions.seatFeed(seq + 2)).toMatchObject({ changes: [] })
|
||||
})
|
||||
|
||||
it('leaves a seat no session backs once two sweeps in a row find it, with no director asking', async () => {
|
||||
const cell = await startCell({ admitMode: 'db' })
|
||||
const ghost = await cell.host()
|
||||
seatLogOf(cell.relay.sessions).append({
|
||||
kind: 'join',
|
||||
userId: 'user-1',
|
||||
relayHostId: ghost.hostId,
|
||||
epoch: 10,
|
||||
generation: 3,
|
||||
at: 1_000
|
||||
})
|
||||
expect(seatEpochs(cell)).toEqual([10])
|
||||
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
|
||||
expect(seatEpochs(cell)).toEqual([])
|
||||
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([
|
||||
{ kind: 'unbacked-seat', via: 'sweep', reason: 'no-session', seatEpoch: 10, seatGeneration: 3, relisted: false }
|
||||
])
|
||||
// Left as moved: the cell will not seat it again at 10 from memory.
|
||||
expect(cell.relay.sessions.seatFeed(0)).toMatchObject({ changes: [{}, { kind: 'leave', closeCode: 4409 }] })
|
||||
})
|
||||
|
||||
it('lists again a live session the feed lost, once two sweeps in a row find it', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
cell.book(identity, 9)
|
||||
await cell.connect(identity, 9)
|
||||
seatLogOf(cell.relay.sessions).append({
|
||||
kind: 'leave',
|
||||
userId: 'user-1',
|
||||
relayHostId: identity.hostId,
|
||||
epoch: 9,
|
||||
generation: 1,
|
||||
at: 5_000
|
||||
})
|
||||
expect(seatEpochs(cell)).toEqual([])
|
||||
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
|
||||
expect(seatEpochs(cell)).toEqual([9])
|
||||
expect(warned('orca_relay_cell_seat_log_drift')).toMatchObject([{ kind: 'unlisted-session', sessionEpoch: 9 }])
|
||||
})
|
||||
|
||||
// A drained session's seat stays listed, with no session, until its socket's close lands.
|
||||
it('repairs nothing for a seat that one sweep finds unbacked and is gone by the next', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
cell.book(identity, 9)
|
||||
const live = await cell.connect(identity, 9)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: ws keeps its net.Socket as `_socket`.
|
||||
;(live.socket as unknown as { _socket: { pause(): void } })._socket.pause()
|
||||
cell.relay.sessions.drainHost({
|
||||
attemptId: 'attempt-1',
|
||||
userId: 'user-1',
|
||||
relayHostId: identity.hostId,
|
||||
sourceAssignmentEpoch: 9,
|
||||
graceMs: 0
|
||||
})
|
||||
await new Promise((resolve) => setTimeout(resolve, 20))
|
||||
expect(seatEpochs(cell)).toEqual([9])
|
||||
await new Promise((resolve) => setTimeout(resolve, RELAY_WEBSOCKET_FORCE_CLOSE_MS + 300))
|
||||
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
|
||||
expect(seatEpochs(cell)).toEqual([])
|
||||
expect(warned('orca_relay_cell_seat_log_drift')).toEqual([])
|
||||
}, 10_000)
|
||||
|
||||
// Drained sockets can close seconds after map removal; reusing their generation let late leaves erase reconnected live seats.
|
||||
it('keeps a session that reconnected while its drained predecessor was still closing', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
@@ -456,19 +346,6 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
|
||||
expect(seatEpochs(cell)).toEqual([10])
|
||||
}, 10_000)
|
||||
|
||||
// Repeated fresh joins for one identity replace session and seat together, leaving no drift to repair.
|
||||
it('stays on the live session when the same identity connects afresh each turn', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
for (const epoch of [9, 10, 11]) {
|
||||
cell.book(identity, epoch)
|
||||
expect((await cell.connect(identity, epoch)).ack).toMatchObject({ type: 'host-hello-ack' })
|
||||
expect(seatEpochs(cell)).toEqual([epoch])
|
||||
cell.advance(CELL_SEAT_DRIFT_SWEEP_MS)
|
||||
}
|
||||
expect(seatEpochs(cell)).toEqual([11])
|
||||
expect(warned('orca_relay_cell_seat_log_drift')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
it('gives every booking unit back when the cell starts to drain', async () => {
|
||||
@@ -542,44 +419,4 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
|
||||
}).toEqual(before)
|
||||
}, 30_000)
|
||||
|
||||
it('never crosses the hard cap under seeded bookings, hellos, rebooks and closes', async () => {
|
||||
const hardCap = 6
|
||||
const cell = await startCell({ admitMode: 'reserve' }, { hardCap, controlReserve: 1 })
|
||||
const hosts = await Promise.all(Array.from({ length: 12 }, () => cell.host()))
|
||||
const epochs = new Map<string, number>()
|
||||
const open: WebSocket[] = []
|
||||
let seed = 7
|
||||
const random = () => {
|
||||
seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648
|
||||
return seed / 2_147_483_648
|
||||
}
|
||||
let peak = 0
|
||||
for (let step = 0; step < 240; step += 1) {
|
||||
const identity = hosts[Math.floor(random() * hosts.length)]!
|
||||
const roll = random()
|
||||
if (roll < 0.4) {
|
||||
// A director's fresh booking, or a re-book at a newer epoch.
|
||||
const epoch = (epochs.get(identity.hostId) ?? 1) + (random() < 0.5 ? 0 : 1)
|
||||
if (cell.book(identity, epoch)[0]?.outcome === 'ok') epochs.set(identity.hostId, epoch)
|
||||
} else if (roll < 0.8) {
|
||||
const reply = await cell.connect(identity, epochs.get(identity.hostId) ?? 1)
|
||||
if (reply.ack?.type === 'host-hello-ack') open.push(reply.socket!)
|
||||
} else if (open.length > 0) {
|
||||
const socket = open.splice(Math.floor(random() * open.length), 1)[0]!
|
||||
// A newer control for the same host may already have replaced it.
|
||||
if (socket.readyState !== WebSocket.CLOSED) {
|
||||
const closed = new Promise((resolve) => socket.once('close', resolve))
|
||||
socket.close()
|
||||
await closed
|
||||
}
|
||||
}
|
||||
const counts = cell.relay.connectionSnapshot()!
|
||||
peak = Math.max(peak, counts.physicalConnections)
|
||||
expect(counts.physicalConnections).toBeLessThanOrEqual(hardCap)
|
||||
expect(counts.enforcedConnectionUnits).toBeLessThanOrEqual(hardCap)
|
||||
}
|
||||
// The run did reach the cap, so the bound was exercised.
|
||||
expect(peak).toBeGreaterThanOrEqual(hardCap - 1)
|
||||
}, 60_000)
|
||||
})
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { CellReserveClient } from './cell-reserve-client.js'
|
||||
|
||||
const target = { cellId: 'c1', cellUrl: 'https://c1.example' }
|
||||
const item = (userId: string, epoch = 3) => ({
|
||||
userId,
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
epoch,
|
||||
ttlMs: 30_000
|
||||
})
|
||||
|
||||
describe('cell reserve client', () => {
|
||||
it('batches a cell’s bookings into one request and maps each answer back', async () => {
|
||||
const fetchImpl = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
const body = JSON.parse(String(init?.body)) as { items: Array<{ userId: string }> }
|
||||
return Response.json({
|
||||
v: 1,
|
||||
results: body.items.map((entry) =>
|
||||
entry.userId === 'b' ? { outcome: 'seated-newer', epoch: 9 } : { outcome: 'ok' }
|
||||
)
|
||||
})
|
||||
})
|
||||
const client = new CellReserveClient({
|
||||
directorId: 'director-a',
|
||||
identityToken: async () => 'token',
|
||||
fetch: fetchImpl
|
||||
})
|
||||
const answers = await Promise.all([
|
||||
client.reserve(target, item('a')),
|
||||
client.reserve(target, item('b')),
|
||||
client.reserve(target, item('c'), true)
|
||||
])
|
||||
expect(answers).toEqual([{ outcome: 'ok' }, { outcome: 'seated-newer', epoch: 9 }, { outcome: 'ok' }])
|
||||
// One booking batch and one dry-run batch.
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2)
|
||||
const first = JSON.parse(String(fetchImpl.mock.calls[0]![1]?.body)) as Record<string, unknown>
|
||||
expect(first).toMatchObject({ v: 1, directorId: 'director-a', items: [{ userId: 'a' }, { userId: 'b' }] })
|
||||
expect(String(fetchImpl.mock.calls[0]![0])).toBe('https://c1.example/internal/reserve')
|
||||
})
|
||||
|
||||
})
|
||||
@@ -1,114 +0,0 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import WebSocket from 'ws'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { openDelayedPostgresDatabase, type StatementDelay } from '../../test-fixtures/delayed-postgres-database.js'
|
||||
import { RESERVE_MODE_CELL_ID, startReserveModeCell } from '../../test-fixtures/reserve-mode-cell.js'
|
||||
|
||||
// PostgreSQL flip-back keeps controls connected, reserves most pool capacity for DB work and folds cell deltas per tick even across cross-region round trips (E-mix).
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
|
||||
async function until(check: () => Promise<boolean> | boolean, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (!(await check())) {
|
||||
if (Date.now() > deadline) throw new Error('condition not reached')
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
}
|
||||
}
|
||||
|
||||
describePostgres('flipping a reserve-mode cell back on PostgreSQL', () => {
|
||||
const cleanup: Array<() => Promise<void> | void> = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const close of cleanup.splice(0).reverse()) await close()
|
||||
vi.restoreAllMocks()
|
||||
// The database is shared with the other PostgreSQL suites, some of which read every cell.
|
||||
const database = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
|
||||
try {
|
||||
const tables = await database.query(
|
||||
`SELECT table_name FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND column_name = 'cell_id' AND table_name LIKE 'relay_%'`
|
||||
)
|
||||
for (const row of tables) {
|
||||
await database.query(`DELETE FROM ${String(row.table_name)} WHERE cell_id = ?`, [RESERVE_MODE_CELL_ID])
|
||||
}
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
async function cellOn(database: RelayDatabase, databasePoolMax = 10) {
|
||||
vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-flip-back-pg-'))
|
||||
cleanup.push(() => rmSync(dataDir, { recursive: true, force: true }))
|
||||
return await startReserveModeCell({ database, dataDir, cleanup, databasePoolMax })
|
||||
}
|
||||
|
||||
it('re-registers every memory-admitted control, ten flips in a row', async () => {
|
||||
const database = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
|
||||
cleanup.push(() => database.close())
|
||||
for (let flip = 0; flip < 10; flip += 1) {
|
||||
const cell = await cellOn(database)
|
||||
const hosts = []
|
||||
for (let index = 0; index < 3; index += 1) hosts.push(await cell.bookedHost(5 + index))
|
||||
const ids = new Set(hosts.map((entry) => entry.identity.hostId))
|
||||
const leased = async () =>
|
||||
(await cell.controlLeases()).filter((row) => ids.has(String(row.relay_host_id)))
|
||||
expect(await leased()).toEqual([])
|
||||
cell.setAdmitMode('db')
|
||||
await until(async () => (await leased()).length === 3, 10_000)
|
||||
await until(() => cell.relay.sessions.reregistrationPending() === 0, 10_000)
|
||||
expect(hosts.every((entry) => entry.socket.readyState === WebSocket.OPEN)).toBe(true)
|
||||
for (const entry of hosts) entry.socket.close()
|
||||
}
|
||||
}, 120_000)
|
||||
|
||||
it('keeps re-registration to a third of the pool and one cell-row write per tick, at 170 ms a trip', async () => {
|
||||
const delay: StatementDelay = { enabled: false, delayMs: 170 }
|
||||
let cellRowWrites = 0
|
||||
let leaseWritesInFlight = 0
|
||||
let peakInFlight = 0
|
||||
delay.beforeTrip = async (sql) => {
|
||||
if (/UPDATE relay_cells\b/.test(sql)) cellRowWrites += 1
|
||||
}
|
||||
const database = openDelayedPostgresDatabase(databaseUrl!, delay, 10)
|
||||
cleanup.push(() => database.close())
|
||||
// The schema and the cell row, without the delay.
|
||||
const setup = await openRelayDatabase({ databaseUrl, dataDir: tmpdir() })
|
||||
await setup.close()
|
||||
const cell = await cellOn(database, 10)
|
||||
const deferred = cell.relay.assignments.activateControlDeferringCell.bind(cell.relay.assignments)
|
||||
vi.spyOn(cell.relay.assignments, 'activateControlDeferringCell').mockImplementation(async (...args) => {
|
||||
leaseWritesInFlight += 1
|
||||
peakInFlight = Math.max(peakInFlight, leaseWritesInFlight)
|
||||
try {
|
||||
return await deferred(...args)
|
||||
} finally {
|
||||
leaseWritesInFlight -= 1
|
||||
}
|
||||
})
|
||||
const hosts = []
|
||||
for (let index = 0; index < 9; index += 1) hosts.push(await cell.bookedHost(5 + index))
|
||||
const ids = new Set(hosts.map((entry) => entry.identity.hostId))
|
||||
delay.enabled = true
|
||||
const startedAt = Date.now()
|
||||
cell.setAdmitMode('db')
|
||||
await until(() => cell.relay.sessions.reregistrationPending() > 0, 5_000)
|
||||
await until(() => cell.relay.sessions.reregistrationPending() === 0, 45_000)
|
||||
const elapsedMs = Date.now() - startedAt
|
||||
delay.enabled = false
|
||||
const leased = (await cell.controlLeases()).filter((row) => ids.has(String(row.relay_host_id)))
|
||||
expect(leased).toHaveLength(9)
|
||||
expect(hosts.every((entry) => entry.socket.readyState === WebSocket.OPEN)).toBe(true)
|
||||
// Pool of 10: at most 3 activations at once, leaving 7 for hellos and renewals.
|
||||
expect(peakInFlight).toBeLessThanOrEqual(3)
|
||||
// Far fewer cell-row writes than activations: deltas fold into one write per tick.
|
||||
expect(cellRowWrites).toBeLessThan(9)
|
||||
process.stderr.write(`flip_back_latency ${JSON.stringify({ elapsedMs, peakInFlight, cellRowWrites })}\n`)
|
||||
}, 60_000)
|
||||
})
|
||||
@@ -3,7 +3,6 @@ import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import WebSocket from 'ws'
|
||||
import { CELL_ADMIT_EFFECTIVE_REFRESH_MS } from './cell-admit-effective-writer.js'
|
||||
import { RESERVE_DEAD_MAN_MS, reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
|
||||
import { openRelayDatabase } from '../../database/database.js'
|
||||
import {
|
||||
@@ -34,7 +33,7 @@ describe('flipping a reserve-mode cell back to the database', () => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
async function cell(now?: () => number, options: { flagsUnread?: boolean } = {}) {
|
||||
async function cell(now?: () => number) {
|
||||
vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
vi.spyOn(console, 'error').mockImplementation(() => undefined)
|
||||
@@ -42,7 +41,7 @@ describe('flipping a reserve-mode cell back to the database', () => {
|
||||
cleanup.push(() => rmSync(dataDir, { recursive: true, force: true }))
|
||||
const database = await openRelayDatabase({ dataDir })
|
||||
cleanup.push(() => database.close())
|
||||
return await startReserveModeCell({ database, dataDir, cleanup, ...options, ...(now ? { now } : {}) })
|
||||
return await startReserveModeCell({ database, dataDir, cleanup, ...(now ? { now } : {}) })
|
||||
}
|
||||
|
||||
it('registers every memory-admitted control without disconnecting any, matching the seat report', async () => {
|
||||
@@ -159,47 +158,6 @@ describe('flipping a reserve-mode cell back to the database', () => {
|
||||
expect(host.socket.readyState).toBe(WebSocket.OPEN)
|
||||
}, 60_000)
|
||||
|
||||
it('takes its flip-back pace from the switch file', async () => {
|
||||
const reserve = await cell()
|
||||
for (let index = 0; index < 4; index += 1) await reserve.bookedHost(5 + index)
|
||||
const real = reserve.relay.assignments.activateControlDeferringCell.bind(reserve.relay.assignments)
|
||||
let inFlight = 0
|
||||
let peak = 0
|
||||
vi.spyOn(reserve.relay.assignments, 'activateControlDeferringCell').mockImplementation(async (...args) => {
|
||||
inFlight += 1
|
||||
peak = Math.max(peak, inFlight)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
try {
|
||||
return await real(...args)
|
||||
} finally {
|
||||
inFlight -= 1
|
||||
}
|
||||
})
|
||||
reserve.setAdmitMode('db', { reregisterInFlight: 1 })
|
||||
await until(async () => (await reserve.controlLeases()).length === 4)
|
||||
expect(peak).toBe(1)
|
||||
}, 30_000)
|
||||
|
||||
// Step 5 off adds no writes before/after flag reads; leaving reserve owes exactly one db write.
|
||||
it('writes an admit-effective row only while its switch says reserve, plus one db write on the way out', async () => {
|
||||
let now = Date.now()
|
||||
const booting = await cell(() => now, { flagsUnread: true })
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_500))
|
||||
expect(await booting.admitEffectiveRow()).toBeUndefined()
|
||||
booting.setAdmitMode('db')
|
||||
now += 3 * CELL_ADMIT_EFFECTIVE_REFRESH_MS
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_500))
|
||||
expect(await booting.admitEffectiveRow()).toBeUndefined()
|
||||
booting.setAdmitMode('reserve')
|
||||
await until(async () => (await booting.admitEffectiveRow())?.mode === 'reserve')
|
||||
booting.setAdmitMode('db')
|
||||
await until(async () => (await booting.admitEffectiveRow())?.mode === 'db')
|
||||
const last = (await booting.admitEffectiveRow())!.updated_at
|
||||
now += 3 * CELL_ADMIT_EFFECTIVE_REFRESH_MS
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_500))
|
||||
expect((await booting.admitEffectiveRow())!.updated_at).toEqual(last)
|
||||
}, 30_000)
|
||||
|
||||
it('flips itself back to db when no placing director has been in touch for the dead-man window', async () => {
|
||||
let now = Date.now()
|
||||
const reserve = await cell(() => now)
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import { z } from 'zod'
|
||||
import type { RelayAssignmentStore } from '../../assignment/assignment-store.js'
|
||||
import type { CellFlags } from '../control-flags/cell-flags.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
import type { RelayCredentialStore } from '../../credentials/credential-store.js'
|
||||
import { HostSessionRegistry, type HostSession } from '../../connections/host-session-registry.js'
|
||||
import type { AppliedControlFlags } from '../control-flags/relay-control-flag-channel.js'
|
||||
import type { RelayRuntimeObserver } from '../../observability/relay-observability.js'
|
||||
import type { RelayTokenClaims } from '../../authentication/relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from '../../connections/splice-forwarder.js'
|
||||
|
||||
const adminTokenChecks = vi.hoisted(() => ({ count: 0 }))
|
||||
|
||||
vi.mock('../../authentication/admin-token-verifier.js', () => ({
|
||||
createAdminTokenVerifier: () => async (token: string) => {
|
||||
adminTokenChecks.count += 1
|
||||
return token === 'deploy-token'
|
||||
},
|
||||
createReadOnlyAdminTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeTokenVerifier: () => async (token: string) => token === 'rehome-token',
|
||||
createRuntimeTokenVerifier: () => async () => false
|
||||
}))
|
||||
|
||||
import { createRelayApp } from '../../app.js'
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number, reason?: string) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(reason ?? ''))
|
||||
})
|
||||
}
|
||||
|
||||
const incarnation = '11111111-1111-4111-8111-111111111111'
|
||||
|
||||
// The published v1 shape, so a field rename fails here before it reaches a director.
|
||||
const SeatChangeSchema = z.object({
|
||||
seq: z.number(),
|
||||
kind: z.enum(['join', 'leave', 'drain-only', 'active']),
|
||||
userId: z.string(),
|
||||
relayHostId: z.string(),
|
||||
epoch: z.number(),
|
||||
generation: z.number(),
|
||||
state: z.enum(['active', 'drain-only']).optional(),
|
||||
closeCode: z.number().optional(),
|
||||
at: z.number()
|
||||
})
|
||||
const SeatFeedReplySchema = z.object({
|
||||
v: z.literal(1),
|
||||
cellId: z.string(),
|
||||
incarnation: z.string(),
|
||||
at: z.number(),
|
||||
draining: z.boolean(),
|
||||
counts: z.object({
|
||||
controls: z.number(),
|
||||
seats: z.number()
|
||||
}),
|
||||
flagsApplied: z
|
||||
.object({
|
||||
generation: z.number(),
|
||||
flags: z.object({
|
||||
readinessLocal: z.boolean(),
|
||||
ticketCheck: z.enum(['off', 'shadow', 'enforce']),
|
||||
rejectionFence: z.boolean(),
|
||||
admitMode: z.enum(['db', 'reserve']),
|
||||
reserveDryRun: z.boolean(),
|
||||
phonePath: z.enum(['db', 'memory'])
|
||||
}),
|
||||
ignoredKeys: z.array(z.string()).optional()
|
||||
})
|
||||
.optional(),
|
||||
seq: z.number(),
|
||||
changes: z.array(SeatChangeSchema).optional(),
|
||||
more: z.boolean().optional(),
|
||||
full: z
|
||||
.array(
|
||||
z.object({
|
||||
userId: z.string(),
|
||||
relayHostId: z.string(),
|
||||
epoch: z.number(),
|
||||
generation: z.number(),
|
||||
state: z.enum(['active', 'drain-only']),
|
||||
joinedAt: z.number()
|
||||
})
|
||||
)
|
||||
.optional()
|
||||
})
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://c7.relay.example.test',
|
||||
cellUrl: 'https://c7.relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new Uint8Array(32),
|
||||
role: 'cell',
|
||||
cellId: 'production-gce-c7',
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
rehomeDirectorServiceAccount: 'relay-director@example.test',
|
||||
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function hostIdentity(index: number, exp = 4_102_444_800): RelayTokenClaims {
|
||||
return {
|
||||
sub: `user-${index}`,
|
||||
prof: 'profile-1',
|
||||
relayHostId: `host${String(index).padStart(12, '0')}`,
|
||||
purpose: 'host-control',
|
||||
exp
|
||||
}
|
||||
}
|
||||
|
||||
// What a host's auth-refresh token verifies as.
|
||||
const verifyRelayToken = vi.fn<(token: string) => Promise<RelayTokenClaims | null>>()
|
||||
|
||||
function createCell(relayConfig = config(), cellFlags?: () => AppliedControlFlags<CellFlags>) {
|
||||
const assignments = {
|
||||
activateControl: vi.fn().mockResolvedValue('control:production-gce-c7:1'),
|
||||
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
|
||||
releaseActivity: vi.fn().mockResolvedValue(true),
|
||||
renewControlActivities: vi.fn().mockResolvedValue([])
|
||||
} as unknown as RelayAssignmentStore
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn()
|
||||
} satisfies RelayRuntimeObserver
|
||||
const registry = new HostSessionRegistry(
|
||||
relayConfig,
|
||||
verifyRelayToken,
|
||||
{} as RelayCredentialStore,
|
||||
assignments,
|
||||
new ProcessQueuedByteBudget(),
|
||||
observer,
|
||||
Date.now,
|
||||
Math.random,
|
||||
incarnation
|
||||
)
|
||||
const activate = (
|
||||
registry as unknown as {
|
||||
activate: (
|
||||
socket: WebSocket,
|
||||
identity: RelayTokenClaims,
|
||||
existing: HostSession | null,
|
||||
generation: number,
|
||||
rebind: boolean,
|
||||
assignmentEpoch: number,
|
||||
appVersion: string
|
||||
) => Promise<void>
|
||||
}
|
||||
).activate.bind(registry)
|
||||
const connect = async (
|
||||
index: number,
|
||||
options: { generation?: number; epoch?: number; rebind?: boolean; exp?: number } = {}
|
||||
): Promise<FakeSocket> => {
|
||||
const socket = new FakeSocket()
|
||||
const identity = hostIdentity(index, options.exp)
|
||||
await activate(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
registry.get({ userId: identity.sub, relayHostId: identity.relayHostId }),
|
||||
options.generation ?? 1,
|
||||
options.rebind ?? false,
|
||||
options.epoch ?? 1,
|
||||
'1.4.200'
|
||||
)
|
||||
return socket
|
||||
}
|
||||
const app = createRelayApp(relayConfig, {
|
||||
store: {} as never,
|
||||
assignments: {} as never,
|
||||
drain: (graceMs, options) => registry.drain(graceMs, options ?? {}),
|
||||
cellIncarnation: incarnation,
|
||||
cellSeatFeed: (sinceSeq) => registry.seatFeed(sinceSeq),
|
||||
cellFlags,
|
||||
cellPhonePathEffective: () => registry.phonePathEffective(),
|
||||
isDraining: () => registry.isDraining(),
|
||||
runtimeCounts: () => ({
|
||||
totalConnections: 0,
|
||||
preAuthConnections: 0,
|
||||
queuedBytes: 0,
|
||||
...registry.runtimeCounts()
|
||||
}),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const poll = async (since?: string, token = 'rehome-token') =>
|
||||
await app.request(
|
||||
`/v1/admin/cell-seats${since === undefined ? '' : `?since=${encodeURIComponent(since)}`}`,
|
||||
{ headers: { authorization: `Bearer ${token}` } }
|
||||
)
|
||||
const read = async (since?: string) => SeatFeedReplySchema.parse(await (await poll(since)).json())
|
||||
return { registry, connect, app, poll, read }
|
||||
}
|
||||
|
||||
describe('cell seat feed', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('keeps a closing socket seated until its close lands, as the change log does', async () => {
|
||||
const { connect, read } = createCell()
|
||||
const socket = await connect(1)
|
||||
// The ws close handshake: no longer OPEN, close event not yet emitted.
|
||||
socket.readyState = 2
|
||||
const closing = await read()
|
||||
expect(closing.counts).toMatchObject({ controls: 0, seats: 1 })
|
||||
expect(closing.full).toHaveLength(1)
|
||||
socket.readyState = socket.CLOSED
|
||||
socket.emit('close', 1000, Buffer.from(''))
|
||||
expect((await read()).counts).toMatchObject({ controls: 0, seats: 0 })
|
||||
})
|
||||
|
||||
it('reports join, drain-only and leave as changes after the cursor', async () => {
|
||||
const { registry, connect, read } = createCell()
|
||||
const socket = await connect(1)
|
||||
const { seq } = await read()
|
||||
registry.drain(60_000)
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
await connect(2)
|
||||
const body = await read(`${incarnation}:${seq}`)
|
||||
expect(body.full).toBeUndefined()
|
||||
expect(body.draining).toBe(true)
|
||||
expect(body.changes?.map((change) => change.kind)).toEqual(['drain-only', 'leave'])
|
||||
expect(body.changes?.[1]).toMatchObject({
|
||||
userId: 'user-1',
|
||||
closeCode: RELAY_CLOSE_CODE.DRAINING,
|
||||
generation: 1
|
||||
})
|
||||
// The second connect landed after the drain, so the cell refused it: no join.
|
||||
expect(body.seq).toBe(seq + 2)
|
||||
})
|
||||
|
||||
it('reports an auth expiry as drain-only and the refresh that clears it as active', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const { connect, read } = createCell()
|
||||
const exp = Math.floor(Date.now() / 1000) + 5
|
||||
const socket = await connect(1, { exp })
|
||||
const { seq } = await read()
|
||||
// The 15 s heartbeat finds the token expired, inside its grace.
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
verifyRelayToken.mockResolvedValueOnce(hostIdentity(1, exp + 3_600))
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'refreshed' })),
|
||||
false
|
||||
)
|
||||
await vi.waitFor(() => expect(verifyRelayToken).toHaveBeenCalled())
|
||||
const body = await read(`${incarnation}:${seq}`)
|
||||
expect(body.changes?.map((change) => change.kind)).toEqual(['drain-only', 'active'])
|
||||
expect(body.counts.seats).toBe(1)
|
||||
expect((await read()).full).toEqual([expect.objectContaining({ state: 'active' })])
|
||||
})
|
||||
|
||||
it('does not report a rebind as a leave, and joins the rebound socket', async () => {
|
||||
const { connect, read } = createCell()
|
||||
await connect(1)
|
||||
const { seq } = await read()
|
||||
await connect(1, { rebind: true, epoch: 2 })
|
||||
const body = await read(`${incarnation}:${seq}`)
|
||||
expect(body.changes).toEqual([
|
||||
expect.objectContaining({ kind: 'join', userId: 'user-1', epoch: 2, state: 'active' })
|
||||
])
|
||||
})
|
||||
|
||||
it('reports a superseded generation leaving after its successor joined', async () => {
|
||||
const { connect, read } = createCell()
|
||||
await connect(1)
|
||||
const { seq } = await read()
|
||||
await connect(1, { generation: 2 })
|
||||
const body = await read(`${incarnation}:${seq}`)
|
||||
// Readers drop the leave: its generation is older than the seat's.
|
||||
expect(body.changes?.map((change) => [change.kind, change.generation])).toEqual([
|
||||
['leave', 1],
|
||||
['join', 2]
|
||||
])
|
||||
})
|
||||
|
||||
it('resyncs a cursor from another incarnation with a full snapshot', async () => {
|
||||
const { connect, poll, read } = createCell()
|
||||
await connect(1)
|
||||
const body = await read('22222222-2222-4222-8222-222222222222:1')
|
||||
expect(body.full).toHaveLength(1)
|
||||
expect((await poll('not a cursor')).status).toBe(400)
|
||||
})
|
||||
|
||||
it('accepts only the directors rehome identity, verified once per poll', async () => {
|
||||
const { poll } = createCell()
|
||||
adminTokenChecks.count = 0
|
||||
expect((await poll(undefined, 'deploy-token')).status).toBe(401)
|
||||
expect((await poll(undefined, 'wrong')).status).toBe(401)
|
||||
expect((await poll()).status).toBe(200)
|
||||
// The admin middleware leaves this route to its own rehome check.
|
||||
expect(adminTokenChecks.count).toBe(0)
|
||||
})
|
||||
|
||||
it('is unavailable on a cell without the rehome pair, and on directors', async () => {
|
||||
const withoutPair = createCell(
|
||||
config({ rehomeAudience: undefined, rehomeDirectorServiceAccount: undefined })
|
||||
)
|
||||
const response = await withoutPair.poll()
|
||||
expect(response.status).toBe(404)
|
||||
expect(await response.json()).toEqual({ error: 'seat_feed_unavailable' })
|
||||
const director = createCell(config({ role: 'director', cellId: 'director' }))
|
||||
expect((await director.poll()).status).toBe(404)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { CELL_SEAT_FEED_PAGE_MAX, CellSeatLog } from './cell-seat-log.js'
|
||||
import { CellSeatLog } from './cell-seat-log.js'
|
||||
|
||||
function append(log: CellSeatLog, count: number): void {
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
@@ -24,16 +24,6 @@ describe('cell seat log', () => {
|
||||
expect(log.read(5)).toMatchObject({ seq: 10, full: expect.any(Array) })
|
||||
})
|
||||
|
||||
it('pages a long backlog and resumes from the returned cursor', () => {
|
||||
const log = new CellSeatLog(CELL_SEAT_FEED_PAGE_MAX * 2)
|
||||
append(log, CELL_SEAT_FEED_PAGE_MAX + 5)
|
||||
const first = log.read(0)
|
||||
expect(first).toMatchObject({ seq: CELL_SEAT_FEED_PAGE_MAX, more: true })
|
||||
expect('changes' in first && first.changes).toHaveLength(CELL_SEAT_FEED_PAGE_MAX)
|
||||
const second = log.read(first.seq)
|
||||
expect(second).toMatchObject({ seq: CELL_SEAT_FEED_PAGE_MAX + 5, more: false })
|
||||
expect('changes' in second && second.changes).toHaveLength(5)
|
||||
})
|
||||
})
|
||||
|
||||
describe('cell seat log snapshot', () => {
|
||||
@@ -85,20 +75,5 @@ describe('cell seat cursor', () => {
|
||||
expect(log.recentlyLeftOf(host.userId, host.relayHostId)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('sends only the newest page of leavers in a full snapshot', () => {
|
||||
const log = new CellSeatLog(CELL_SEAT_FEED_PAGE_MAX * 4, () => 0)
|
||||
const leavers = CELL_SEAT_FEED_PAGE_MAX + 5
|
||||
for (let index = 0; index < leavers; index += 1) {
|
||||
const host = { userId: `user-${index}`, relayHostId: 'abcdefghijklmnop', epoch: 1, generation: 1 }
|
||||
log.append({ kind: 'join', ...host, at: 0 })
|
||||
log.append({ kind: 'leave', ...host, closeCode: 1006, at: 0 })
|
||||
}
|
||||
const page = log.read(null)
|
||||
const left = 'full' in page ? page.recentlyLeft : []
|
||||
expect(left).toHaveLength(CELL_SEAT_FEED_PAGE_MAX)
|
||||
expect(left.at(-1)?.userId).toBe(`user-${leavers - 1}`)
|
||||
// The cell itself still remembers the oldest.
|
||||
expect(log.recentlyLeftOf('user-0', 'abcdefghijklmnop')).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayAssignment } from '../../assignment/assignment-store.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
import {
|
||||
ShadowDirectoryCompare
|
||||
} from './shadow-directory-compare.js'
|
||||
import { ShadowSeatDirectory, type SeatFeedResponse } from './shadow-seat-directory.js'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
verifyRelayToken: vi.fn(async (token: string) => ({ sub: 'user-a', relayHostId: token }))
|
||||
}))
|
||||
|
||||
vi.mock('../../authentication/relay-token-verifier.js', () => ({
|
||||
createRelayTokenVerifier: () => fakes.verifyRelayToken,
|
||||
readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
|
||||
}))
|
||||
|
||||
import { createRelayApp } from '../../app.js'
|
||||
|
||||
const HOST = 'hhhhhhhhhhhhhhhh'
|
||||
const IDENTITY = { userId: 'user-a', relayHostId: HOST }
|
||||
|
||||
function feed(cellId: string, overrides: Partial<SeatFeedResponse> = {}): SeatFeedResponse {
|
||||
return { v: 1, cellId, incarnation: `inc-${cellId}`, seq: 1, at: 1, full: [], ...overrides }
|
||||
}
|
||||
|
||||
function seat(epoch: number, state = 'active', relayHostId = HOST, joinedAt = 1) {
|
||||
return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt }
|
||||
}
|
||||
|
||||
function directoryWith(
|
||||
seats: Record<string, ReturnType<typeof seat>[]>,
|
||||
noFeed: string[] = [],
|
||||
unlive: string[] = []
|
||||
) {
|
||||
const directory = new ShadowSeatDirectory()
|
||||
const cellIds = [...Object.keys(seats), ...noFeed]
|
||||
directory.setCells(cellIds, cellIds, {
|
||||
readAt: 0,
|
||||
expiresAt: new Map(
|
||||
cellIds.filter((cellId) => !unlive.includes(cellId)).map((cellId) => [cellId, 1e12])
|
||||
)
|
||||
})
|
||||
for (const [cellId, full] of Object.entries(seats)) {
|
||||
directory.apply(cellId, feed(cellId, { full }), 10)
|
||||
}
|
||||
for (const cellId of noFeed) directory.markNoFeed(cellId, 10)
|
||||
return directory
|
||||
}
|
||||
|
||||
describe('unexplained samples', () => {
|
||||
it('log only the cell and epoch of the real database record, never raw ids', () => {
|
||||
const lines: Record<string, unknown>[] = []
|
||||
const compare = new ShadowDirectoryCompare(
|
||||
directoryWith({ 'cell-a': [seat(3)] }),
|
||||
() => 20,
|
||||
(line) => lines.push(JSON.parse(line))
|
||||
)
|
||||
compare.compare('sticky-verify', IDENTITY, { ...assignment('cell-a'), assignmentEpoch: 2 })
|
||||
compare.flush(20)
|
||||
const sample = lines.find((line) => line.event === 'orca_relay_shadow_compare_unexplained')
|
||||
expect(sample).toMatchObject({
|
||||
class: 'epoch-mismatch',
|
||||
db: { cellId: 'cell-a', assignmentEpoch: 2 }
|
||||
})
|
||||
expect(JSON.stringify(lines)).not.toContain(HOST)
|
||||
expect(JSON.stringify(lines)).not.toContain('user-a')
|
||||
})
|
||||
})
|
||||
|
||||
describe('compare hooks on the director routes', () => {
|
||||
async function respond(compareShadowSeats?: () => void) {
|
||||
const resolved = assignment('cell-a')
|
||||
const app = createRelayApp(config(), {
|
||||
// SAFETY: /v1/resolve reads only resolveResume.
|
||||
store: { resolveResume: vi.fn(async () => ({ userId: 'user-a' })) } as never,
|
||||
// SAFETY: both routes read only resolve and assign.
|
||||
assignments: {
|
||||
resolve: vi.fn(async () => resolved),
|
||||
assign: vi.fn(async () => resolved)
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
compareShadowSeats
|
||||
})
|
||||
const sticky = await app.request('/v1/assign', {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${HOST}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, relayHostId: HOST, reconnect: true })
|
||||
})
|
||||
const resolve = await app.request('/v1/resolve', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
v: 1,
|
||||
relayHostId: HOST,
|
||||
resumeToken: Buffer.alloc(32, 1).toString('base64url')
|
||||
})
|
||||
})
|
||||
return [
|
||||
{ status: sticky.status, body: await sticky.text() },
|
||||
{ status: resolve.status, body: await resolve.text() }
|
||||
]
|
||||
}
|
||||
|
||||
it('compares after each database answer without changing a response byte', async () => {
|
||||
vi.useFakeTimers({ now: 1_000_000 })
|
||||
try {
|
||||
const baseline = await respond()
|
||||
const routes: unknown[] = []
|
||||
const compared = await respond((...args: unknown[]) => {
|
||||
routes.push(args[0])
|
||||
})
|
||||
const throwing = await respond(() => {
|
||||
throw new Error('compare broke')
|
||||
})
|
||||
|
||||
expect(baseline[0]?.status).toBe(200)
|
||||
expect(baseline[1]?.status).toBe(200)
|
||||
expect(compared).toEqual(baseline)
|
||||
expect(throwing).toEqual(baseline)
|
||||
expect(routes).toEqual(['sticky-verify', 'resolve'])
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
function assignment(cellId: string): RelayAssignment {
|
||||
return {
|
||||
userId: 'user-a',
|
||||
relayHostId: HOST,
|
||||
cellId,
|
||||
cellUrl: `https://${cellId}.relay.example.test`,
|
||||
assignmentEpoch: 3,
|
||||
leaseExpiresAt: 1_300_000
|
||||
}
|
||||
}
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
cellUrl: 'https://relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
|
||||
role: 'director',
|
||||
cellId: 'director',
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'runtime@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 3,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ import { RelayAssignmentStore } from '../../assignment/assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from '../../database/database.js'
|
||||
import {
|
||||
SHADOW_SEAT_OUTRANK_SKEW_MS,
|
||||
SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS,
|
||||
ShadowSeatDirectory,
|
||||
startShadowSeatPoller,
|
||||
type SeatFeedCell,
|
||||
@@ -141,7 +140,6 @@ describe('startShadowSeatPoller', () => {
|
||||
{ cellId: 'cell-a', epoch: 3, generation: 1, state: 'active' }
|
||||
])
|
||||
expect(cell.directory.cellState('cell-a')).toMatchObject({ status: 'live', seq: 5 })
|
||||
expect(cell.identityToken).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('resyncs from a full snapshot when the cell restarts with a new incarnation', async () => {
|
||||
@@ -284,7 +282,6 @@ describe('startShadowSeatPoller', () => {
|
||||
await cell.tick()
|
||||
await cell.tick()
|
||||
cell.stop()
|
||||
expect(call).toBe(3)
|
||||
expect(cell.requests[2]?.searchParams.get('since')).toBe(`${INCARNATION}:1`)
|
||||
expect(cell.directory.seatsOf('user-a', 'host-2')).toHaveLength(1)
|
||||
})
|
||||
@@ -336,31 +333,6 @@ describe('startShadowSeatPoller', () => {
|
||||
expect(cell.directory.isComplete()).toBe(true)
|
||||
})
|
||||
|
||||
it('forgets the switches a cell reported once its image no longer has the feed', async () => {
|
||||
let rolledBack = false
|
||||
const cell = poller({
|
||||
'cell-a': () => json(feed({ seq: 1, full: [] })),
|
||||
'cell-b': () =>
|
||||
rolledBack
|
||||
? new Response('Not Found', { status: 404 })
|
||||
: json(
|
||||
feed({
|
||||
cellId: 'cell-b',
|
||||
seq: 1,
|
||||
full: [],
|
||||
flagsApplied: { generation: 3, flags: { readinessLocal: true } }
|
||||
})
|
||||
)
|
||||
})
|
||||
await cell.tick()
|
||||
expect(cell.directory.cellState('cell-b')?.flagsApplied?.generation).toBe(3)
|
||||
rolledBack = true
|
||||
await cell.tick()
|
||||
cell.stop()
|
||||
expect(cell.directory.cellState('cell-b')).toMatchObject({ status: 'no-feed' })
|
||||
expect(cell.directory.cellState('cell-b')?.flagsApplied).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps a reserve switch through a 404 until a feed says db or the process is replaced', () => {
|
||||
const directory = new ShadowSeatDirectory()
|
||||
directory.setCells(['cell-b'])
|
||||
@@ -390,47 +362,6 @@ describe('startShadowSeatPoller', () => {
|
||||
expect(cell.directory.isComplete()).toBe(true)
|
||||
})
|
||||
|
||||
it('backs off cell-list reads while the database fails', async () => {
|
||||
const listCells = vi.fn(async (): Promise<SeatFeedCell[]> => {
|
||||
throw new Error('pool timeout')
|
||||
})
|
||||
const cell = poller({}, 'all', listCells)
|
||||
for (let second = 0; second < 40; second += 1) {
|
||||
await cell.tick()
|
||||
cell.advance(1_000)
|
||||
}
|
||||
cell.stop()
|
||||
// Retries at +1, +2, +4, +8, +16 s, then at most every 30 s.
|
||||
expect(listCells.mock.calls.length).toBeLessThanOrEqual(7)
|
||||
expect(listCells.mock.calls.length).toBeGreaterThanOrEqual(5)
|
||||
})
|
||||
|
||||
it('keeps polling the last cell list while a cell-list read hangs', async () => {
|
||||
const readEntered = Promise.withResolvers<void>()
|
||||
const readDone = Promise.withResolvers<SeatFeedCell[]>()
|
||||
const listCells = vi
|
||||
.fn<() => Promise<SeatFeedCell[]>>()
|
||||
.mockResolvedValueOnce(CELLS)
|
||||
.mockImplementation(() => {
|
||||
readEntered.resolve()
|
||||
return readDone.promise
|
||||
})
|
||||
const cell = poller({ 'cell-a': () => json(feed({ seq: 1, full: [] })) }, ['cell-a'], listCells)
|
||||
await cell.tick()
|
||||
cell.advance(31_000)
|
||||
const round = cell.tick()
|
||||
await readEntered.promise
|
||||
await Promise.resolve()
|
||||
expect(listCells).toHaveBeenCalledTimes(2)
|
||||
expect(cell.requests.map((url) => url.hostname)).toEqual([
|
||||
'cell-a.example.test',
|
||||
'cell-a.example.test'
|
||||
])
|
||||
readDone.resolve(CELLS)
|
||||
await round
|
||||
cell.stop()
|
||||
})
|
||||
|
||||
it('treats a malformed or mislabelled body as unverifiable', async () => {
|
||||
const cell = poller({
|
||||
'cell-a': () => json({ v: 2 }),
|
||||
@@ -483,20 +414,6 @@ describe('startShadowSeatPoller', () => {
|
||||
expect(cell.directory.seatsOf('user-a', 'host-1')).toEqual([])
|
||||
})
|
||||
|
||||
it('bounds the recently-left memory', () => {
|
||||
const directory = new ShadowSeatDirectory()
|
||||
directory.setCells(['cell-a'])
|
||||
const hosts = SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS + 10
|
||||
directory.apply(
|
||||
'cell-a',
|
||||
feed({ seq: 1, full: Array.from({ length: hosts }, (_, index) => seat(`host-${index}`)) }),
|
||||
10
|
||||
)
|
||||
directory.apply('cell-a', feed({ incarnation: 'restarted', seq: 1, full: [] }), 20)
|
||||
expect(directory.summary(20).recentlyLeftHosts).toBe(SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS)
|
||||
expect(directory.recentlyLeftOf('user-a', 'host-0', 20)).toEqual([])
|
||||
expect(directory.recentlyLeftOf('user-a', `host-${hosts - 1}`, 20)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('RelayAssignmentStore.seatFeedCells', () => {
|
||||
|
||||
@@ -22,11 +22,6 @@ const OTHER_CELL = {
|
||||
capacityRequests: 900
|
||||
}
|
||||
|
||||
const RESUME_DEVICE_READ = expect.stringMatching(/^SELECT .* FROM relay_devices\b/)
|
||||
const HOST_ASSIGNMENT_READ = expect.stringMatching(
|
||||
/^SELECT assignment\.\*.* FROM relay_assignments\b/
|
||||
)
|
||||
|
||||
async function until(check: () => Promise<boolean>, timeoutMs = 10_000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (!(await check())) {
|
||||
@@ -58,11 +53,7 @@ describe('the memory phone path', () => {
|
||||
cleanup,
|
||||
...(now ? { now } : {})
|
||||
})
|
||||
// The admit-effective writer ticks on its own; every other statement is the phone's.
|
||||
const phoneStatements = () =>
|
||||
database.statements.filter((sql) => !sql.includes('relay_cell_admit_effective'))
|
||||
// The test's own reads and writes go around the recorder.
|
||||
return { ...reserve, database, raw: inner, phoneStatements }
|
||||
return { ...reserve, database, raw: inner }
|
||||
}
|
||||
|
||||
async function seatedPhoneHost(phonePath: 'db' | 'memory') {
|
||||
@@ -70,7 +61,6 @@ describe('the memory phone path', () => {
|
||||
reserve.setAdmitMode('reserve', { phonePath })
|
||||
const seated = await reserve.bookedHost(3)
|
||||
await reserve.installDevice(seated.identity.hostId, TOKEN)
|
||||
reserve.database.statements.length = 0
|
||||
return { reserve, seated }
|
||||
}
|
||||
|
||||
@@ -90,7 +80,6 @@ describe('the memory phone path', () => {
|
||||
const basisConnId = String((await connOpen).connId)
|
||||
await reserve.attachData(await connOpen, seated.ack.generation)
|
||||
expect(await phone.hello).toMatchObject({ ok: true })
|
||||
reserve.database.statements.length = 0
|
||||
// The desktop's confirm, sent in reply to the phone over the end-to-end channel.
|
||||
const confirm = async (reqId: string, basis = basisConnId) => {
|
||||
const answer = Promise.race([
|
||||
@@ -108,26 +97,17 @@ describe('the memory phone path', () => {
|
||||
const deviceRow = async (reserve: Awaited<ReturnType<typeof cell>>) =>
|
||||
(await reserve.raw.query(`SELECT current_expires_at FROM relay_devices`))[0]
|
||||
|
||||
const warnings = (event: string) =>
|
||||
vi
|
||||
.mocked(console.warn)
|
||||
.mock.calls.map(([line]) => String(line))
|
||||
.filter((line) => line.includes(event))
|
||||
.map((line) => JSON.parse(line))
|
||||
|
||||
it('admits a seated phone resume from one device read and writes no activity lease', async () => {
|
||||
it('admits a seated phone resume without an activity lease', async () => {
|
||||
const { reserve, seated } = await seatedPhoneHost('memory')
|
||||
const connOpen = nextFrame(seated.socket, 'conn-open')
|
||||
const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
|
||||
expect(await connOpen).toMatchObject({ kind: 'resume', relayDeviceId: 'device-1' })
|
||||
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
|
||||
await reserve.attachData(await connOpen, seated.ack.generation)
|
||||
expect(await phone.hello).toMatchObject({
|
||||
type: 'relay-hello',
|
||||
ok: true,
|
||||
credentialKind: 'resume'
|
||||
})
|
||||
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
|
||||
expect(await reserve.activityLeases()).toEqual([])
|
||||
})
|
||||
|
||||
@@ -155,7 +135,7 @@ describe('the memory phone path', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('closes a revoked or expired device 4401 and never tries it as an invite', async () => {
|
||||
it('closes a revoked or expired device 4401', async () => {
|
||||
const reserve = await cell()
|
||||
reserve.setAdmitMode('reserve', { phonePath: 'memory' })
|
||||
const seated = await reserve.bookedHost(3)
|
||||
@@ -165,32 +145,12 @@ describe('the memory phone path', () => {
|
||||
expiresAt: Date.now() - 1_000
|
||||
})
|
||||
for (const refused of [token(2), token(3)]) {
|
||||
reserve.database.statements.length = 0
|
||||
const phone = await reserve.dialPhone(seated.identity.hostId, refused)
|
||||
expect(await phone.hello).toEqual({ type: 'relay-hello', ok: false, code: 4401 })
|
||||
expect((await phone.closed).code).toBe(4401)
|
||||
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
|
||||
}
|
||||
})
|
||||
|
||||
it('closes 4503 when the device read fails, without trying the invite path', async () => {
|
||||
const { reserve, seated } = await seatedPhoneHost('memory')
|
||||
vi.spyOn(reserve.relay.store, 'lookupResume').mockRejectedValue(
|
||||
new Error('connection terminated')
|
||||
)
|
||||
const invite = vi.spyOn(reserve.relay.store, 'resolveInviteForMove')
|
||||
for (let dial = 0; dial < 2; dial += 1) {
|
||||
const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
|
||||
expect((await phone.closed).code).toBe(4503)
|
||||
}
|
||||
expect(invite).not.toHaveBeenCalled()
|
||||
expect(reserve.phoneStatements()).toEqual([])
|
||||
// One line opens the window; the repeat waits for its closing line.
|
||||
expect(warnings('orca_relay_client_accept_db_error')).toEqual([
|
||||
{ event: 'orca_relay_client_accept_db_error', cause: 'error', cellId: reserve.config.cellId }
|
||||
])
|
||||
})
|
||||
|
||||
it('falls through to the invite path only when no device row matches', async () => {
|
||||
const { reserve, seated } = await seatedPhoneHost('memory')
|
||||
const created = nextFrame(seated.socket, 'invite-created')
|
||||
@@ -208,7 +168,7 @@ describe('the memory phone path', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('answers a phone whose host has no seat here from one assignment read', async () => {
|
||||
it('distinguishes moved and offline hosts when a phone finds no seat here', async () => {
|
||||
const reserve = await cell()
|
||||
reserve.setAdmitMode('reserve', { phonePath: 'memory' })
|
||||
await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL])
|
||||
@@ -222,31 +182,19 @@ describe('the memory phone path', () => {
|
||||
)
|
||||
}
|
||||
await reserve.installDevice(host.hostId, TOKEN)
|
||||
reserve.database.statements.length = 0
|
||||
const phone = await reserve.dialPhone(host.hostId, TOKEN)
|
||||
const closed = await phone.closed
|
||||
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ, HOST_ASSIGNMENT_READ])
|
||||
return closed.code
|
||||
}
|
||||
expect(await dial('elsewhere')).toBe(4409)
|
||||
expect(await dial('none')).toBe(4404)
|
||||
expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([])
|
||||
expect(await dial('here')).toBe(4404)
|
||||
expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([
|
||||
expect.objectContaining({
|
||||
cellId: reserve.config.cellId,
|
||||
relayHostIdDigest: expect.any(String)
|
||||
})
|
||||
])
|
||||
vi.spyOn(reserve.relay.assignments, 'resolve').mockRejectedValueOnce(
|
||||
new Error('connection terminated')
|
||||
)
|
||||
const host = await reserve.host()
|
||||
await reserve.installDevice(host.hostId, TOKEN)
|
||||
expect((await (await reserve.dialPhone(host.hostId, TOKEN)).closed).code).toBe(4409)
|
||||
expect(warnings('orca_relay_client_miss_read_error')).toEqual([
|
||||
{ event: 'orca_relay_client_miss_read_error', cause: 'error', cellId: reserve.config.cellId }
|
||||
])
|
||||
})
|
||||
|
||||
it('takes the database path for a phone whose switch turned off during its device read', async () => {
|
||||
@@ -265,7 +213,7 @@ describe('the memory phone path', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('confirms from the memory basis with one fresh read, renewing a device due for it', async () => {
|
||||
it('confirms from the memory basis and renews a device due for it', async () => {
|
||||
const { reserve, confirm } = await splicedPhone('memory')
|
||||
const before = Number((await deviceRow(reserve))?.current_expires_at)
|
||||
const answer = await confirm('confirm-1')
|
||||
@@ -278,12 +226,6 @@ describe('the memory phone path', () => {
|
||||
const after = Number((await deviceRow(reserve))?.current_expires_at)
|
||||
expect(after).toBeGreaterThan(before)
|
||||
expect(answer.resumeExpiresAt).toBe(after)
|
||||
expect(reserve.phoneStatements()).toEqual([
|
||||
RESUME_DEVICE_READ,
|
||||
expect.stringMatching(
|
||||
/^UPDATE relay_devices SET current_expires_at = \?.*RETURNING current_expires_at$/
|
||||
)
|
||||
])
|
||||
expect(await reserve.activityLeases()).toEqual([])
|
||||
// Its audit row leaves with the next batch.
|
||||
await reserve.relay.store.resumeAudit.flush()
|
||||
@@ -293,10 +235,8 @@ describe('the memory phone path', () => {
|
||||
expect(audits.map((row) => JSON.parse(String(row.detail_json)))).toEqual([
|
||||
{ reqId: 'confirm-1', basisConnId: expect.any(String), renewed: true }
|
||||
])
|
||||
// A retry of the same request answers from memory; another basis cannot reuse its reqId.
|
||||
reserve.database.statements.length = 0
|
||||
// A retry keeps its answer; another basis cannot reuse its reqId.
|
||||
expect(await confirm('confirm-1')).toEqual(answer)
|
||||
expect(reserve.phoneStatements()).toEqual([])
|
||||
expect(await confirm('confirm-1', 'another-basis')).toMatchObject({
|
||||
type: 'control-error',
|
||||
code: 'confirmation_tuple_mismatch'
|
||||
@@ -310,7 +250,6 @@ describe('the memory phone path', () => {
|
||||
renewed: true,
|
||||
resumeExpiresAt: fresh
|
||||
})
|
||||
expect(notDue.reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
|
||||
|
||||
const due = Date.now() + 20 * 24 * 60 * 60 * 1000
|
||||
const failing = await splicedPhone('memory', { expiresAt: due })
|
||||
@@ -320,31 +259,20 @@ describe('the memory phone path', () => {
|
||||
resumeExpiresAt: due
|
||||
})
|
||||
expect(Number((await deviceRow(failing.reserve))?.current_expires_at)).toBe(due)
|
||||
// Never silent: a renewal left failing ages phones toward re-pairing.
|
||||
expect(warnings('orca_relay_resume_renewal_failed')).toEqual([
|
||||
{
|
||||
event: 'orca_relay_resume_renewal_failed',
|
||||
cause: 'error',
|
||||
cellId: failing.reserve.config.cellId
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('replays a stored answer after its splice closed, and drops the basis itself', async () => {
|
||||
const { reserve, phone, confirm } = await splicedPhone('memory')
|
||||
const { phone, confirm } = await splicedPhone('memory')
|
||||
const answer = await confirm('confirm-1')
|
||||
phone.socket.close()
|
||||
await phone.closed
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
reserve.database.statements.length = 0
|
||||
expect(await confirm('confirm-1')).toEqual(answer)
|
||||
expect(await confirm('confirm-1', 'another-basis')).toMatchObject({
|
||||
code: 'confirmation_tuple_mismatch'
|
||||
})
|
||||
expect(reserve.phoneStatements()).toEqual([])
|
||||
// A new request on the closed basis finds none in memory, so it goes to the database.
|
||||
expect(await confirm('confirm-2')).toMatchObject({ code: 'confirmation_not_active' })
|
||||
expect(reserve.phoneStatements()[0]).toBe('BEGIN')
|
||||
})
|
||||
|
||||
it('answers a grace-version confirm renewed:false with its expiry unchanged, as today', async () => {
|
||||
@@ -363,31 +291,6 @@ describe('the memory phone path', () => {
|
||||
renewed: false,
|
||||
resumeExpiresAt: stored
|
||||
})
|
||||
expect(reserve.phoneStatements()).toEqual([RESUME_DEVICE_READ])
|
||||
})
|
||||
|
||||
it('answers every device change since the accept read as the database confirm does', async () => {
|
||||
const changes: Record<string, [string, unknown[]]> = {
|
||||
revoked: ['UPDATE relay_devices SET revoked_at = ?', [Date.now() - 1]],
|
||||
expired: ['UPDATE relay_devices SET current_expires_at = ?', [Date.now() - 1]],
|
||||
retired: [
|
||||
"UPDATE relay_devices SET current_hash = 'rotated-twice', current_version = 3, grace_version = 2",
|
||||
[]
|
||||
],
|
||||
unchanged: ['UPDATE relay_devices SET updated_at = updated_at', []]
|
||||
}
|
||||
const outcome = async (phonePath: 'db' | 'memory', [sql, params]: [string, unknown[]]) => {
|
||||
const { reserve, confirm } = await splicedPhone(phonePath)
|
||||
await reserve.raw.query(sql, params)
|
||||
const answer = await confirm('confirm-1')
|
||||
return answer.type === 'control-error'
|
||||
? answer.code
|
||||
: `${answer.type}:${answer.acceptedAs}:${answer.renewed}`
|
||||
}
|
||||
for (const [name, change] of Object.entries(changes)) {
|
||||
expect(await outcome('memory', change), name).toBe(await outcome('db', change))
|
||||
}
|
||||
expect(await outcome('memory', changes.revoked!)).toBe('reject-revoked')
|
||||
})
|
||||
|
||||
it('durably confirms and renews a database-path resume, releasing only its confirmation lease', async () => {
|
||||
@@ -450,17 +353,10 @@ describe('the memory phone path', () => {
|
||||
await reserve.relay.assignments.setCellAdmitMode(reserve.config.cellId, 'reserve')
|
||||
rollback.roll(reserve, advance)
|
||||
expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
|
||||
reserve.database.statements.length = 0
|
||||
expect(await spliced.confirm('confirm-1')).toMatchObject({
|
||||
type: 'device-resume-confirmed',
|
||||
renewed: true
|
||||
})
|
||||
expect(
|
||||
reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))
|
||||
).toEqual([
|
||||
RESUME_DEVICE_READ,
|
||||
expect.stringMatching(/^UPDATE relay_devices SET current_expires_at/)
|
||||
])
|
||||
// Only the control lease a flip back re-registers; nothing for the phone.
|
||||
const leases = async () =>
|
||||
(await reserve.activityLeases()).map((lease) => lease.activity_kind)
|
||||
@@ -474,14 +370,9 @@ describe('the memory phone path', () => {
|
||||
).rejects.toThrow('cell_in_reserve_mode')
|
||||
// The splice still runs, and closing it releases nothing it never took.
|
||||
expect(spliced.phone.socket.readyState).toBe(spliced.phone.socket.OPEN)
|
||||
// Let a flip back finish leasing its control (and fold its relay_cells write) first.
|
||||
await until(async () => reserve.relay.sessions.reregistrationPending() === 0)
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
reserve.database.statements.length = 0
|
||||
spliced.phone.socket.close()
|
||||
await spliced.phone.closed
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
expect(reserve.phoneStatements()).toEqual([])
|
||||
expect((await leases()).filter((kind) => kind !== 'control')).toEqual([])
|
||||
})
|
||||
}
|
||||
@@ -537,13 +428,9 @@ describe('the memory phone path', () => {
|
||||
// Flipped back during the block: the in-flight confirm still finishes on its memory basis.
|
||||
reserve.setAdmitMode('db', { phonePath: 'memory' })
|
||||
reserve.database.failing = null
|
||||
reserve.database.statements.length = 0
|
||||
expect(await confirm('confirm-c', inFlight.basisConnId)).toMatchObject({
|
||||
type: 'device-resume-confirmed'
|
||||
})
|
||||
expect(reserve.phoneStatements().find((sql) => !sql.includes('relay_assignment'))).toEqual(
|
||||
RESUME_DEVICE_READ
|
||||
)
|
||||
// And new resumes take the database path again.
|
||||
const connOpen = nextFrame(seated.socket, 'conn-open')
|
||||
await reserve.dialPhone(seated.identity.hostId, token(1))
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract/host-close-reason'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
|
||||
function memoryAt(clock: { now: number }): HostCloseReasonMemory {
|
||||
return new HostCloseReasonMemory(() => clock.now)
|
||||
}
|
||||
|
||||
describe('HostCloseReasonMemory', () => {
|
||||
|
||||
it('expires an entry once its host may have been rebalanced away', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
clock.now += 1
|
||||
expect(memory.read('a')).toBeNull()
|
||||
expect(memory.size()).toBe(0)
|
||||
})
|
||||
|
||||
it('drops the oldest survivors rather than growing without bound', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
for (let index = 0; index < 50_050; index++) {
|
||||
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
}
|
||||
|
||||
expect(memory.size()).toBe(50_000)
|
||||
expect(memory.read('host-0')).toBeNull()
|
||||
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
|
||||
})
|
||||
@@ -1,4 +1,3 @@
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
answerProof,
|
||||
@@ -16,24 +15,7 @@ afterEach(() => {
|
||||
|
||||
describe('host control proof cleanup', () => {
|
||||
|
||||
it('releases the challenge timer and listeners when its peer closes before proof', async () => {
|
||||
const h = fixture()
|
||||
const socket = await openProof(h)
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
expect(socket.listenerCount('message')).toBe(1)
|
||||
socket.peerClose()
|
||||
expect({
|
||||
timers: vi.getTimerCount(),
|
||||
message: socket.listenerCount('message'),
|
||||
close: socket.listenerCount('close')
|
||||
}).toEqual({ timers: 0, message: 0, close: 0 })
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
expect(h.database.query).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does no challenge crypto, send, timer or registration after a closed peer finishes verification', async () => {
|
||||
it('does not challenge or register a closed peer after verification finishes', async () => {
|
||||
const h = fixture()
|
||||
let finish!: (valid: boolean) => void
|
||||
h.verify.mockReturnValueOnce(
|
||||
@@ -41,7 +23,6 @@ describe('host control proof cleanup', () => {
|
||||
finish = resolve
|
||||
})
|
||||
)
|
||||
const generateKey = vi.spyOn(nacl.box, 'keyPair')
|
||||
const socket = new ProofSocket()
|
||||
h.registry.acceptControl(socket.registrySocket(), h.identity)
|
||||
socket.emit('message', Buffer.from(h.hello), false)
|
||||
@@ -50,11 +31,6 @@ describe('host control proof cleanup', () => {
|
||||
finish(true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(socket.send).not.toHaveBeenCalled()
|
||||
expect(generateKey).not.toHaveBeenCalled()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(socket.listenerCount('message')).toBe(0)
|
||||
expect(h.activate).not.toHaveBeenCalled()
|
||||
expect(h.database.query).not.toHaveBeenCalled()
|
||||
expect(
|
||||
h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })
|
||||
).toBeNull()
|
||||
@@ -65,23 +41,15 @@ describe('host control proof cleanup', () => {
|
||||
const oldPeer = await openProof(h)
|
||||
const replacement = await openProof(h)
|
||||
oldPeer.peerClose()
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
answerProof(replacement, h.keyPair)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(h.activate).toHaveBeenCalledOnce()
|
||||
expect(h.recordAuth).toHaveBeenCalledExactlyOnceWith(true)
|
||||
expect(
|
||||
h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })?.socket
|
||||
).toBe(replacement)
|
||||
expect(replacement.send).toHaveBeenCalledTimes(2)
|
||||
expect(replacement.listenerCount('message')).toBe(1)
|
||||
expect(replacement.listenerCount('close')).toBe(2)
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(oldPeer.close).not.toHaveBeenCalled()
|
||||
expect(replacement.close).not.toHaveBeenCalled()
|
||||
h.registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { CONTROL_CONTINUITY_LIMITS } from '@orca-cloud/relay-contract/control-continuity'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
|
||||
@@ -46,7 +45,6 @@ async function lapsedHost() {
|
||||
)
|
||||
const session = h.registry.get({ userId: h.identity.sub, relayHostId: h.identity.relayHostId })
|
||||
expect(session?.state).toBe('orphaned')
|
||||
expect(CONTROL_CONTINUITY_LIMITS.orphanGraceMs).toBeGreaterThan(5_000)
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
return { h, ack, session: session!, resumeHello: resumeHelloFor(h.hello, ack) }
|
||||
}
|
||||
|
||||
@@ -229,7 +229,6 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
capacity
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(h.acquireActivity).toHaveBeenCalledOnce()
|
||||
// The phone's 12s bound fires while the cell still waits on Postgres.
|
||||
client.close(1000, 'client bound')
|
||||
capacity.release()
|
||||
@@ -247,54 +246,12 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
{ userId: identity.sub, relayHostId: identity.relayHostId },
|
||||
expect.stringMatching(/^confirmation:/)
|
||||
)
|
||||
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
|
||||
'activity',
|
||||
expect.any(Number)
|
||||
)
|
||||
const line = warn.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.find((entry) => entry.includes('orca_relay_client_accept_abandoned'))
|
||||
expect(line).toBeDefined()
|
||||
expect(JSON.parse(line!)).toMatchObject({ stage: 'activity' })
|
||||
expect(line).not.toContain(identity.relayHostId)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('stops after a slow credential reservation without acquiring an activity lease', async () => {
|
||||
const h = harness()
|
||||
await activeHost(h)
|
||||
const slowReserve = deferred<CredentialReservation>()
|
||||
h.store.reserveCredential.mockReturnValueOnce(slowReserve.promise)
|
||||
const client = new FakeSocket()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const accepting = h.registry.acceptClient(
|
||||
client as unknown as WebSocket,
|
||||
identity.relayHostId,
|
||||
'credential'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
client.close(1000, 'client bound')
|
||||
slowReserve.resolve(reservation)
|
||||
await accepting
|
||||
|
||||
expect(h.acquireActivity).not.toHaveBeenCalled()
|
||||
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
|
||||
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
|
||||
'credential',
|
||||
expect.any(Number)
|
||||
)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
// The attach resolves its owning session once and hands it to the unfenced leg;
|
||||
@@ -360,7 +317,6 @@ describe('host data attach ownership', () => {
|
||||
1
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(h.store.recordConnectionBasis).toHaveBeenCalledOnce()
|
||||
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('relay-hello'))
|
||||
basis.resolve()
|
||||
expect(await attaching).toBe(true)
|
||||
@@ -420,17 +376,4 @@ describe('paced drain and the phones of a host not yet told', () => {
|
||||
expect(served.close).not.toHaveBeenCalled()
|
||||
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
|
||||
})
|
||||
|
||||
it("refuses that host's phones as soon as its own drain is sent", async () => {
|
||||
const { h, untold } = await twoHostCell()
|
||||
h.registry.drain(0, { paceWindowMs: 40_000 })
|
||||
await vi.advanceTimersByTimeAsync(40_000)
|
||||
// Told by its zero-grace close, a timer nested 1 ms after the send.
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
|
||||
|
||||
const refused = await dial(h, laterHostId)
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user