mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Merge origin/main into brennanb2025/workspace-picker-window
Resolve the Cmd+J row shape variant and default-selection start against the windowed options; the default skips disabled actions like keyboard order. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -4,7 +4,7 @@
|
||||
/skill-stubs/*.md text eol=lf
|
||||
/skill-stubs/_shared/*.md text eol=lf
|
||||
/skills/*/SKILL.md text eol=lf
|
||||
/src/cli/bundled-skill-guides.ts text eol=lf
|
||||
/src/cli/skills/bundled-skill-guides.ts text eol=lf
|
||||
# SVG bytes feed embedded data URLs and asset hashes on every build host.
|
||||
*.svg text eol=lf
|
||||
# Compiler input line breaks must not depend on the checkout OS.
|
||||
|
||||
@@ -12,24 +12,57 @@ body:
|
||||
|
||||
Please write your issue in English.
|
||||
|
||||
- type: dropdown
|
||||
id: where
|
||||
attributes:
|
||||
label: Where were you using Orca?
|
||||
description: Where you saw the bug.
|
||||
options:
|
||||
- Desktop only
|
||||
- Desktop + SSH host
|
||||
- Desktop + Orca server
|
||||
- Desktop + WSL
|
||||
- Mobile app
|
||||
- Web browser
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
description: 'The device you were using. WSL: pick Windows. Phone: pick iOS or Android.'
|
||||
options:
|
||||
- macOS
|
||||
- Windows
|
||||
- Linux
|
||||
- iOS
|
||||
- Android
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: remote_os
|
||||
attributes:
|
||||
label: Remote host operating system
|
||||
description: Only for SSH, Orca server or web.
|
||||
options:
|
||||
- Linux
|
||||
- macOS
|
||||
- Windows
|
||||
- Other
|
||||
- Not sure
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: orca_version
|
||||
attributes:
|
||||
label: Orca version
|
||||
description: Include the exact Orca version if you can. You can find it in `Orca > About Orca`.
|
||||
placeholder: v0.14.2
|
||||
description: 'macOS: Orca > About Orca. Windows/Linux: Help > About Orca. Mobile: Settings > About.'
|
||||
placeholder: v1.4.2
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
@@ -1,28 +1,16 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
CloudSqlRolloutLease,
|
||||
LEASE_TTL_MS,
|
||||
LeaseConflict,
|
||||
LeaseUnreadable
|
||||
} from './storage-lease.mjs'
|
||||
import { CloudSqlRolloutLease, LeaseConflict, LeaseUnreadable } from './storage-lease.mjs'
|
||||
|
||||
const BUCKET = 'onorca-cloud-terraform-state'
|
||||
const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock'
|
||||
const NOW = 1_756_000_000_000
|
||||
|
||||
/** Model real GCS generation increments, ifGenerationMatch and 412 conflicts; faults inject failures. */
|
||||
function fakeStorage({ object = null, faults = [] } = {}) {
|
||||
const state = { object, requests: [] }
|
||||
function fakeStorage({ object = null } = {}) {
|
||||
const state = { object }
|
||||
const fetcher = async (rawUrl, init = {}) => {
|
||||
const url = new URL(rawUrl)
|
||||
const method = init.method ?? 'GET'
|
||||
const record = { method, url, path: url.pathname, search: url.searchParams }
|
||||
state.requests.push(record)
|
||||
const fault = faults.find((candidate) => candidate.when(record))
|
||||
if (fault) {
|
||||
return json(fault.status, fault.body ?? {})
|
||||
}
|
||||
|
||||
if (url.pathname.startsWith('/upload/')) {
|
||||
const want = url.searchParams.get('ifGenerationMatch')
|
||||
@@ -63,15 +51,10 @@ function json(status, body) {
|
||||
}
|
||||
}
|
||||
|
||||
function storedRecord({
|
||||
holderKey,
|
||||
expiresAt,
|
||||
repository = 'stablyai/orca',
|
||||
workflow = 'Deploy Relay Production'
|
||||
}) {
|
||||
function storedRecord({ holderKey, expiresAt }) {
|
||||
return {
|
||||
repository,
|
||||
workflow,
|
||||
repository: 'stablyai/orca',
|
||||
workflow: 'Deploy Relay Production',
|
||||
run_id: '9001',
|
||||
run_url: 'https://github.com/stablyai/orca/actions/runs/9001',
|
||||
run_attempt: '1',
|
||||
@@ -81,14 +64,14 @@ function storedRecord({
|
||||
}
|
||||
}
|
||||
|
||||
function leaseFor(storage, { warn = () => {} } = {}) {
|
||||
function leaseFor(storage) {
|
||||
return new CloudSqlRolloutLease({
|
||||
bucket: BUCKET,
|
||||
objectName: OBJECT,
|
||||
accessToken: 'test-token',
|
||||
fetcher: storage.fetcher,
|
||||
now: () => NOW,
|
||||
warn
|
||||
warn: () => {}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -114,77 +97,24 @@ test('refuses a live lease held by another run and never writes', async () => {
|
||||
.catch((thrown) => thrown)
|
||||
|
||||
assert.ok(error instanceof LeaseConflict)
|
||||
assert.equal(error.holder.repository, 'stablyai/orca')
|
||||
assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001')
|
||||
assert.equal(
|
||||
storage.state.requests.filter((request) => request.method !== 'GET').length,
|
||||
0,
|
||||
'a foreign live lease must not be written'
|
||||
)
|
||||
assert.equal(storage.state.object.generation, '1500')
|
||||
})
|
||||
|
||||
test('re-enters a live lease this run already holds and extends it', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
generation: '1500',
|
||||
body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 })
|
||||
}
|
||||
})
|
||||
const warnings = []
|
||||
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
|
||||
HOLDER
|
||||
)
|
||||
|
||||
assert.equal(claim.state, 'reentrant')
|
||||
assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover')
|
||||
assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved')
|
||||
assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS)
|
||||
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
|
||||
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
|
||||
assert.equal(storage.state.object.generation, '1501')
|
||||
})
|
||||
|
||||
test('takes over an expired lease and warns naming the stale holder', async () => {
|
||||
test('takes over an expired lease', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
generation: '1500',
|
||||
body: storedRecord({
|
||||
holderKey: 'stablyai/orca/9001',
|
||||
expiresAt: NOW - 1,
|
||||
repository: 'stablyai/orca',
|
||||
workflow: 'Deploy Relay Production Capacity'
|
||||
expiresAt: NOW - 1
|
||||
})
|
||||
}
|
||||
})
|
||||
const warnings = []
|
||||
const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
|
||||
HOLDER
|
||||
)
|
||||
await leaseFor(storage).acquire(HOLDER)
|
||||
|
||||
assert.equal(claim.state, 'takeover')
|
||||
assert.equal(warnings.length, 1)
|
||||
assert.match(warnings[0], /stablyai\/orca/)
|
||||
assert.match(warnings[0], /Deploy Relay Production Capacity/)
|
||||
assert.match(warnings[0], /actions\/runs\/9001/)
|
||||
const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
|
||||
assert.equal(upload.search.get('ifGenerationMatch'), '1500')
|
||||
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
|
||||
})
|
||||
|
||||
test('releases with a generation match and leaves the object gone', async () => {
|
||||
const storage = fakeStorage()
|
||||
const lease = leaseFor(storage)
|
||||
const claim = await lease.acquire(HOLDER)
|
||||
|
||||
const released = await lease.release(HOLDER.holderKey)
|
||||
|
||||
assert.deepEqual(released, { released: true, generation: claim.generation })
|
||||
const remove = storage.state.requests.find((request) => request.method === 'DELETE')
|
||||
assert.equal(remove.search.get('ifGenerationMatch'), claim.generation)
|
||||
assert.equal(storage.state.object, null)
|
||||
})
|
||||
|
||||
test('refuses to release a lease another run now holds', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
@@ -200,19 +130,6 @@ test('refuses to release a lease another run now holds', async () => {
|
||||
assert.equal(storage.state.object.generation, '1500')
|
||||
})
|
||||
|
||||
test('fails closed when the bucket answers 5xx', async () => {
|
||||
const storage = fakeStorage({
|
||||
faults: [{ when: (request) => request.method === 'GET', status: 503 }]
|
||||
})
|
||||
|
||||
const error = await leaseFor(storage)
|
||||
.acquire(HOLDER)
|
||||
.catch((thrown) => thrown)
|
||||
|
||||
assert.match(error.message, /lease inspection failed: 503/)
|
||||
assert.equal(storage.state.requests.filter((request) => request.method !== 'GET').length, 0)
|
||||
})
|
||||
|
||||
test('treats an unreadable record as held, not free', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: { generation: '1500', body: { holder_key: 'stablyai/orca/9001' } }
|
||||
@@ -225,31 +142,3 @@ test('treats an unreadable record as held, not free', async () => {
|
||||
assert.ok(error instanceof LeaseUnreadable)
|
||||
assert.equal(storage.state.object.generation, '1500')
|
||||
})
|
||||
|
||||
test('renewal rewrites only expires_at on the observed generation', async () => {
|
||||
const storage = fakeStorage()
|
||||
const lease = leaseFor(storage)
|
||||
await lease.acquire(HOLDER)
|
||||
storage.state.object.body.expires_at = NOW - 1
|
||||
|
||||
const renewed = await lease.renew(HOLDER.holderKey)
|
||||
|
||||
assert.equal(renewed.renewed, true)
|
||||
assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS)
|
||||
assert.equal(storage.state.object.body.acquired_at, NOW)
|
||||
assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
|
||||
})
|
||||
|
||||
test('renewal stops once the object belongs to someone else', async () => {
|
||||
const storage = fakeStorage({
|
||||
object: {
|
||||
generation: '1500',
|
||||
body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
|
||||
}
|
||||
})
|
||||
|
||||
assert.deepEqual(await leaseFor(storage).renew(HOLDER.holderKey), {
|
||||
renewed: false,
|
||||
reason: 'foreign'
|
||||
})
|
||||
})
|
||||
|
||||
@@ -385,7 +385,7 @@ jobs:
|
||||
ORCA_RELEASE_JAVASCRIPT_BUILD_IDENTITY: dev
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
# Leave build identity unset: telemetry accepts only official stable/rc, so branch artifacts stay silent.
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
|
||||
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
|
||||
- name: Download the orcad deployment template
|
||||
|
||||
@@ -62,11 +62,24 @@ jobs:
|
||||
current.closed_at === pr.closed_at
|
||||
}
|
||||
if (!Number.isSafeInteger(pr.number) || pr.number <= 0 ||
|
||||
(superseded ? !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? '') :
|
||||
!Number.isFinite(Date.parse(pr.closed_at)))) {
|
||||
(superseded && !/^[a-f0-9]{40}$/.test(pr.head?.sha ?? ''))) {
|
||||
throw new Error('Missing PR identity')
|
||||
}
|
||||
if (!await stillCurrent()) return
|
||||
if (!superseded && !Number.isFinite(Date.parse(pr.closed_at))) {
|
||||
// A close quickly followed by a reopen can deliver a payload rendered after the reopen.
|
||||
const { data: current } = await github.rest.pulls.get({
|
||||
...context.repo, pull_number: pr.number
|
||||
})
|
||||
if (current.state !== 'closed' || current.merged_at !== null) {
|
||||
core.info(`PR #${pr.number} changed after this close event; skipping cancellation`)
|
||||
return
|
||||
}
|
||||
throw new Error('Missing PR identity')
|
||||
}
|
||||
if (!await stillCurrent()) {
|
||||
core.info(`PR #${pr.number} changed after this event; skipping cancellation`)
|
||||
return
|
||||
}
|
||||
for (const [prefix, workflow, suffix = ''] of targets) {
|
||||
const group = prefix === 'cloud-verify-refs/pull'
|
||||
? `${prefix}/${pr.number}${suffix}` : `${prefix}-${pr.number}${suffix}`
|
||||
|
||||
@@ -38,8 +38,8 @@ on:
|
||||
- '!src/main/daemon/executable-host/windows/testing/daemon-host-relocation.test-fixture.ts'
|
||||
- '!src/main/daemon/endpoint/publication/testing/daemon-endpoint-reachability-test-harness.ts'
|
||||
- 'src/main/providers/local-pty/local-pty-provider.ts'
|
||||
- 'src/main/providers/windows-shell-args.ts'
|
||||
- 'src/main/providers/windows-shell-preflight-runtime.windows.test.ts'
|
||||
- 'src/main/providers/shell-launch/windows/shell-args.ts'
|
||||
- 'src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts'
|
||||
- 'native/computer-use-macos/**'
|
||||
- 'native/computer-use-linux/**'
|
||||
- 'native/computer-use-windows/**'
|
||||
@@ -48,12 +48,13 @@ on:
|
||||
- 'src/cli/**'
|
||||
- 'src/main/computer/**'
|
||||
- 'src/main/runtime/rpc/dispatcher.ts'
|
||||
- 'src/main/runtime/rpc/errors.ts'
|
||||
- 'src/main/runtime/rpc/responses/rpc-response.ts'
|
||||
- 'src/main/runtime/rpc/methods/computer/computer*.ts'
|
||||
- 'src/shared/computer-use-*.ts'
|
||||
- 'tests/e2e/computer-linux.e2e.ts'
|
||||
- 'tests/e2e/computer-windows.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-store.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-background.e2e.ts'
|
||||
- 'tests/e2e/helpers/windows-semantic-controls-fixture.*'
|
||||
- 'tests/e2e/helpers/computer-cli-driver.ts'
|
||||
- 'tests/e2e/helpers/computer-driver.ts'
|
||||
- 'tests/e2e/vitest.config.ts'
|
||||
@@ -95,11 +96,10 @@ jobs:
|
||||
config/scripts/performance/macos-computer-owner/macos-computer-helper-owner-loss-processes.test.mjs
|
||||
src/main/computer/sidecar-transport/sidecar-client.test.ts
|
||||
src/main/computer/macos-native/macos-native-provider-client.test.ts
|
||||
src/main/providers/windows-shell-preflight-runtime.windows.test.ts
|
||||
src/main/providers/shell-launch/windows/shell-preflight-runtime.windows.test.ts
|
||||
src/main/computer/desktop-script/actions/desktop-script-provider-actions.test.ts
|
||||
src/cli/format.test.ts
|
||||
src/cli/handlers/computer-action-validation.test.ts
|
||||
src/cli/handlers/computer-state-formatting.test.ts
|
||||
src/cli/output/command-result-output.test.ts
|
||||
src/cli/computer/computer-state-formatting.test.ts
|
||||
src/cli/index.test.ts
|
||||
src/cli/runtime/envelope-schema.test.ts
|
||||
src/shared/remote-runtime/remote-runtime-client.test.ts
|
||||
@@ -152,4 +152,4 @@ jobs:
|
||||
- run: pnpm verify:computer-native
|
||||
- run: pnpm build:cli
|
||||
- run: pnpm run build:electron-vite:parallel
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-background.e2e.ts
|
||||
|
||||
@@ -235,7 +235,7 @@ jobs:
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
# Leave dev identity unset so official-only stable/rc telemetry stays silent.
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
|
||||
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
|
||||
- name: Download the orcad deployment template
|
||||
|
||||
@@ -254,7 +254,7 @@ jobs:
|
||||
ORCA_RELEASE_JAVASCRIPT_SOURCE_SHA: ${{ inputs.javascript_source_sha }}
|
||||
ORCA_RELEASE_JAVASCRIPT_BUILD_IDENTITY: dev
|
||||
# Unset dev build identity so official stable/rc telemetry stays silent.
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
|
||||
# Inline so refs predating the wait can still use the trusted dispatch workflow.
|
||||
- name: Wait for the qualified orcad template
|
||||
|
||||
@@ -204,7 +204,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
environment:
|
||||
name: docs-production
|
||||
url: https://www.onorca.dev/docs
|
||||
url: https://orca.dev/docs
|
||||
env:
|
||||
VERCEL_TELEMETRY_DISABLED: '1'
|
||||
steps:
|
||||
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
# Unset dev identity so official-only stable/rc telemetry stays silent.
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
|
||||
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
|
||||
- name: Download the orcad deployment template
|
||||
|
||||
@@ -14,48 +14,248 @@ jobs:
|
||||
apply-os-label:
|
||||
runs-on: ubuntu-slim
|
||||
timeout-minutes: 5
|
||||
# Serialize runs per issue; each run reconciles from current state, so order does not matter.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-issue-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Apply OS label from issue form
|
||||
- name: Reconcile OS and mobile labels from issue form
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const issue = context.payload.issue;
|
||||
const body = issue.body || "";
|
||||
const osMatch = body.match(/^### Operating system\s+(.+)$/m);
|
||||
// The only labels this workflow may add or remove.
|
||||
const MANAGED_LABELS = ["os:macos", "os:Windows", "os:linux", "os:mobile", "mobile"];
|
||||
const MAX_PASSES = 3;
|
||||
|
||||
if (!osMatch) {
|
||||
core.info("No operating system field found. Skipping.");
|
||||
return;
|
||||
const NOTE_PREFIX =
|
||||
"> \u{1F310} **Translation note (machine-translated by an assistant on ";
|
||||
const NOTE_SUFFIX =
|
||||
"):** This issue's title was originally written in Chinese and has been translated to English (the body is unchanged). The original, untranslated text is preserved intact in the collapsible section below.";
|
||||
const NOTE_DATE = /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/;
|
||||
|
||||
// Metadata headings in accepted order; equal rank = current/legacy names of one field.
|
||||
const HEADING_RANKS = new Map([
|
||||
["Summary", 1],
|
||||
["Where were you using Orca?", 2],
|
||||
["Client", 2],
|
||||
["Operating system", 3],
|
||||
["Host operating system", 3],
|
||||
["Remote host operating system", 4],
|
||||
["Orca version", 5],
|
||||
["Coding harness and version", 6],
|
||||
["Details", 7],
|
||||
["Reproduction steps", 7],
|
||||
]);
|
||||
const FREE_TEXT_RANK = 7;
|
||||
|
||||
const OS_LABELS = new Map([
|
||||
["macos", "os:macos"],
|
||||
["windows", "os:Windows"],
|
||||
["linux", "os:linux"],
|
||||
["ios", "os:mobile"],
|
||||
["android", "os:mobile"],
|
||||
["other", null],
|
||||
]);
|
||||
// Value: whether the choice means the mobile app. Retired option strings stay for open issues.
|
||||
const WHERE_IS_MOBILE = new Map([
|
||||
["desktop only", false],
|
||||
["desktop + ssh host", false],
|
||||
["desktop + orca server", false],
|
||||
["desktop + wsl", false],
|
||||
["mobile app", true],
|
||||
["web browser", false],
|
||||
["other", false],
|
||||
["desktop (this computer)", false],
|
||||
["desktop + ssh", false],
|
||||
["desktop + orca remote server", false],
|
||||
["mobile (ios)", true],
|
||||
["mobile (android)", true],
|
||||
["web / browser", false],
|
||||
]);
|
||||
const CLIENT_IS_MOBILE = new Map([
|
||||
...WHERE_IS_MOBILE,
|
||||
["desktop", false],
|
||||
["mobile", true],
|
||||
["web", false],
|
||||
]);
|
||||
const REMOTE_OS_VALUES = new Set(["linux", "macos", "windows", "other", "not sure"]);
|
||||
|
||||
const isBlank = (line) => /^[ \t]*$/.test(line);
|
||||
const isHeading = (line) => /^###(?:[ \t]|$)/.test(line);
|
||||
const isUnselected = (value) => value === "" || value === "_no response_";
|
||||
const isTranslationNote = (line) =>
|
||||
line.length === NOTE_PREFIX.length + 10 + NOTE_SUFFIX.length &&
|
||||
line.startsWith(NOTE_PREFIX) &&
|
||||
line.endsWith(NOTE_SUFFIX) &&
|
||||
NOTE_DATE.test(line.slice(NOTE_PREFIX.length, NOTE_PREFIX.length + 10));
|
||||
|
||||
// Reads only the leading metadata prefix; null means "not a recognizable bug form".
|
||||
function readFormFields(body) {
|
||||
if (typeof body !== "string") {
|
||||
return null;
|
||||
}
|
||||
const lines = body.replace(/\r\n/g, "\n").split("\n");
|
||||
let index = 0;
|
||||
const skipBlankLines = () => {
|
||||
while (index < lines.length && isBlank(lines[index])) {
|
||||
index += 1;
|
||||
}
|
||||
};
|
||||
|
||||
skipBlankLines();
|
||||
if (index < lines.length && isTranslationNote(lines[index])) {
|
||||
index += 1;
|
||||
if (index >= lines.length || !isBlank(lines[index])) {
|
||||
return null;
|
||||
}
|
||||
skipBlankLines();
|
||||
}
|
||||
|
||||
const fields = new Map();
|
||||
let lastRank = 0;
|
||||
while (index < lines.length) {
|
||||
const line = lines[index];
|
||||
if (!isHeading(line)) {
|
||||
return null;
|
||||
}
|
||||
// Exact "### <heading>" only; any other heading shape is unknown.
|
||||
const name = line.startsWith("### ") ? line.slice(4) : null;
|
||||
const rank = HEADING_RANKS.get(name);
|
||||
if (rank === undefined || rank < lastRank || fields.has(name)) {
|
||||
return null;
|
||||
}
|
||||
if (rank === FREE_TEXT_RANK) {
|
||||
// Free text starts here and is never inspected.
|
||||
return fields;
|
||||
}
|
||||
lastRank = rank;
|
||||
index += 1;
|
||||
|
||||
let value = "";
|
||||
let valueLines = 0;
|
||||
while (index < lines.length && !isHeading(lines[index])) {
|
||||
if (!isBlank(lines[index])) {
|
||||
value = lines[index];
|
||||
valueLines += 1;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
if (valueLines > 1) {
|
||||
return null;
|
||||
}
|
||||
fields.set(name, value.trim().toLowerCase());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const selectedOs = osMatch[1].trim().toLowerCase();
|
||||
const osLabelMap = {
|
||||
macos: "os:macos",
|
||||
windows: "os:Windows",
|
||||
linux: "os:linux",
|
||||
};
|
||||
// Returns the managed labels the body asks for, or null to leave the issue untouched.
|
||||
function desiredLabelsFor(body) {
|
||||
const fields = readFormFields(body);
|
||||
if (!fields || !fields.has("Orca version")) {
|
||||
return null;
|
||||
}
|
||||
const osHeading = ["Operating system", "Host operating system"].find((name) =>
|
||||
fields.has(name),
|
||||
);
|
||||
if (!osHeading) {
|
||||
return null;
|
||||
}
|
||||
const desired = new Set();
|
||||
|
||||
const desiredLabel = osLabelMap[selectedOs];
|
||||
if (!desiredLabel) {
|
||||
core.info(`No OS label configured for "${selectedOs}". Skipping.`);
|
||||
return;
|
||||
const os = fields.get(osHeading);
|
||||
if (!isUnselected(os)) {
|
||||
if (!OS_LABELS.has(os)) {
|
||||
return null;
|
||||
}
|
||||
const osLabel = OS_LABELS.get(os);
|
||||
if (osLabel) {
|
||||
desired.add(osLabel);
|
||||
}
|
||||
}
|
||||
|
||||
// The current heading wins by presence, even when blank.
|
||||
const whereChoices = fields.has("Where were you using Orca?")
|
||||
? [fields.get("Where were you using Orca?"), WHERE_IS_MOBILE]
|
||||
: fields.has("Client")
|
||||
? [fields.get("Client"), CLIENT_IS_MOBILE]
|
||||
: null;
|
||||
if (whereChoices && !isUnselected(whereChoices[0])) {
|
||||
const [where, isMobileByChoice] = whereChoices;
|
||||
if (!isMobileByChoice.has(where)) {
|
||||
return null;
|
||||
}
|
||||
if (isMobileByChoice.get(where)) {
|
||||
desired.add("mobile");
|
||||
}
|
||||
}
|
||||
|
||||
// Validated only; the remote host never contributes a label.
|
||||
const remoteOs = fields.get("Remote host operating system") ?? "";
|
||||
if (!isUnselected(remoteOs) && !REMOTE_OS_VALUES.has(remoteOs)) {
|
||||
return null;
|
||||
}
|
||||
return desired;
|
||||
}
|
||||
|
||||
const managedLabels = Object.values(osLabelMap);
|
||||
const existingLabels = issue.labels.map((label) =>
|
||||
typeof label === "string" ? label : label.name,
|
||||
);
|
||||
const labelsToKeep = existingLabels.filter(
|
||||
(label) => !managedLabels.includes(label),
|
||||
);
|
||||
const nextLabels = [...new Set([...labelsToKeep, desiredLabel])];
|
||||
|
||||
await github.rest.issues.setLabels({
|
||||
const issueRef = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: nextLabels,
|
||||
});
|
||||
issue_number: context.payload.issue.number,
|
||||
};
|
||||
|
||||
core.info(`Applied label ${desiredLabel} to issue #${issue.number}.`);
|
||||
// The event is only a signal: decisions come from the issue as it is now.
|
||||
async function readCurrentIssue() {
|
||||
const { data } = await github.rest.issues.get(issueRef);
|
||||
const labelNames = (data.labels ?? []).map((label) =>
|
||||
typeof label === "string" ? label : label.name,
|
||||
);
|
||||
const desired = desiredLabelsFor(data.body);
|
||||
const present = MANAGED_LABELS.filter((label) => labelNames.includes(label));
|
||||
return {
|
||||
desired,
|
||||
toAdd: desired ? MANAGED_LABELS.filter((l) => desired.has(l) && !present.includes(l)) : [],
|
||||
toRemove: desired ? present.filter((l) => !desired.has(l)) : [],
|
||||
};
|
||||
}
|
||||
|
||||
let state = await readCurrentIssue();
|
||||
let hasWritten = false;
|
||||
for (let pass = 1; ; pass += 1) {
|
||||
if (!state.desired) {
|
||||
if (hasWritten) {
|
||||
throw new Error(
|
||||
`Issue #${issueRef.issue_number} stopped matching the bug form while labels were being updated. Managed labels may be incomplete; rerun this job once the issue body is settled.`,
|
||||
);
|
||||
}
|
||||
core.info("No recognizable bug form at the start of the issue body. Labels left untouched.");
|
||||
return;
|
||||
}
|
||||
if (state.toAdd.length === 0 && state.toRemove.length === 0) {
|
||||
core.info(
|
||||
`Issue #${issueRef.issue_number} managed labels match the form: [${[...state.desired].join(", ")}].`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (pass > MAX_PASSES) {
|
||||
throw new Error(
|
||||
`Issue #${issueRef.issue_number} labels still differ from the form after ${MAX_PASSES} passes (to add: [${state.toAdd.join(", ")}], to remove: [${state.toRemove.join(", ")}]). The issue is probably being edited; rerun this job.`,
|
||||
);
|
||||
}
|
||||
|
||||
hasWritten = true;
|
||||
if (state.toAdd.length > 0) {
|
||||
await github.rest.issues.addLabels({ ...issueRef, labels: state.toAdd });
|
||||
}
|
||||
for (const name of state.toRemove) {
|
||||
try {
|
||||
await github.rest.issues.removeLabel({ ...issueRef, name });
|
||||
} catch (error) {
|
||||
// Already gone is the desired outcome; the re-read below verifies real state.
|
||||
if (error?.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
state = await readCurrentIssue();
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
uses: ./.github/workflows/e2e.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts","tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts","tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts","tests/e2e/headless-paired-remote-terminal-viewer-size.spec.ts","tests/e2e/headless-serve-browser-tab-restart.spec.ts","tests/e2e/headless-serve-cli-terminal-retention-parity.spec.ts","tests/e2e/headless-serve-desktop-activation.spec.ts","tests/e2e/headless-serve-chat-tab-placement.spec.ts","tests/e2e/headless-serve-mobile-native-chat-identity.spec.ts","tests/e2e/layout-compat-old-phone-on-new-server.spec.ts","tests/e2e/layout-compat-serve-restart-keeps-at-at-terminals.spec.ts","tests/e2e/paired-mobile-serve-restart-terminal-tabs.spec.ts","tests/e2e/paired-mobile-desktop-tab-composition.spec.ts","tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts","tests/e2e/paired-desktop-server-layout-frames.spec.ts","tests/e2e/paired-desktop-server-held-pane-reopen.spec.ts","tests/e2e/paired-desktop-server-layout-intents.spec.ts","tests/e2e/paired-desktop-server-layout-writes.spec.ts","tests/e2e/ssh-orcad-"]'
|
||||
test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts","tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts","tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts","tests/e2e/headless-paired-remote-terminal-viewer-size.spec.ts","tests/e2e/headless-serve-browser-tab-restart.spec.ts","tests/e2e/headless-serve-cli-terminal-retention-parity.spec.ts","tests/e2e/headless-serve-desktop-activation.spec.ts","tests/e2e/headless-serve-chat-tab-placement.spec.ts","tests/e2e/headless-serve-mobile-native-chat-identity.spec.ts","tests/e2e/layout-compat-old-phone-on-new-server.spec.ts","tests/e2e/layout-compat-serve-restart-keeps-at-at-terminals.spec.ts","tests/e2e/paired-mobile-serve-restart-terminal-tabs.spec.ts","tests/e2e/paired-mobile-desktop-tab-composition.spec.ts","tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts","tests/e2e/ssh-orcad-"]'
|
||||
run_changed_e2e: true
|
||||
needs_build: true
|
||||
artifact_suffix: -layout-owner-switch-on
|
||||
|
||||
@@ -11,12 +11,12 @@ on:
|
||||
- 'native/macos-shipit-scheduling/**'
|
||||
- 'config/scripts/macos-shipit-scheduling*'
|
||||
- 'config/electron-builder.config.cjs'
|
||||
- 'src/main/startup/main-process-quit*'
|
||||
- 'src/main/window/main-window-state-lifecycle*'
|
||||
- 'src/main/startup/quit/main-process-quit*'
|
||||
- 'src/main/window/main-window/main-window-state-lifecycle*'
|
||||
- 'src/main/window/dashboard-popout-window*'
|
||||
- 'src/packages/process-host/**'
|
||||
- 'src/main/runtime/agent-session/lifecycle/structured-agent-cli-writers*'
|
||||
- 'src/main/providers/macos-tcc-login-shell*'
|
||||
- 'src/main/providers/shell-launch/macos/tcc-login-shell*'
|
||||
- 'src/shared/process-table-snapshot*'
|
||||
- 'src/shared/darwin-process-table-capture*'
|
||||
- 'src/shared/update-status-types.ts'
|
||||
@@ -55,8 +55,8 @@ jobs:
|
||||
src/main/updater/mac-install/updater.mac-install.test.ts
|
||||
src/main/updater/updater.headless-serve-install.test.ts
|
||||
src/main/updater/mac-install/updater-mac-quit-guard.test.ts
|
||||
src/main/startup/main-process-quit-update-veto.test.ts
|
||||
src/main/window/main-window-state-lifecycle.test.ts
|
||||
src/main/startup/quit/main-process-quit-update-veto.test.ts
|
||||
src/main/window/main-window/main-window-state-lifecycle.test.ts
|
||||
- name: Prove login-backed terminal writer census on the disposable runner
|
||||
env:
|
||||
ORCA_MACOS_LOGIN_CENSUS_CI: '1'
|
||||
|
||||
@@ -284,7 +284,7 @@ jobs:
|
||||
"${{ steps.apk.outputs.checksum }}"
|
||||
fi
|
||||
|
||||
# Advisory like the site bump: a failed mirror only leaves onorca.dev's "mirror link" on the prior APK.
|
||||
# Advisory like the site bump: a failed mirror only leaves orca.dev's "mirror link" on the prior APK.
|
||||
release-mirror-r2:
|
||||
needs: android-build
|
||||
if: needs.android-build.outputs.publish_release == 'true'
|
||||
|
||||
@@ -34,18 +34,24 @@ on:
|
||||
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
|
||||
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
|
||||
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
|
||||
- 'src/main/github/issue-field-options.ts'
|
||||
- 'src/main/github/issues/issue-field-options.ts'
|
||||
- 'config/tsconfig.test.mobile-host-declarations.json'
|
||||
- 'config/tsconfig.node.json'
|
||||
- 'src/types/build-constants.d.ts'
|
||||
- 'src/main/native-chat/agent-session-wire/**'
|
||||
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
|
||||
- 'src/main/native-chat/*transcript*.ts'
|
||||
- 'src/main/native-chat/transcript-discovery/**'
|
||||
- 'src/main/native-chat/transcript-filesystem/**'
|
||||
- 'src/main/native-chat/transcript-decoding/**'
|
||||
- 'src/main/native-chat/transcript-streaming/**'
|
||||
- 'src/main/native-chat/transcript-watching/**'
|
||||
- 'src/main/native-chat/transcript-history/**'
|
||||
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
|
||||
- 'src/main/claude/claude-internal-message-content.ts'
|
||||
- 'src/main/claude/claude-stop-hook-feedback.ts'
|
||||
- 'src/main/claude/structured/journal/claude-internal-message-content.ts'
|
||||
- 'src/main/claude/structured/journal/claude-stop-hook-feedback.ts'
|
||||
- 'src/main/claude/structured/journal/**'
|
||||
- 'src/main/claude/__fixtures__/claude-adapter-capture-fold.jsonl'
|
||||
- 'src/main/codex/app-server/codex-item-field-readers.ts'
|
||||
@@ -54,7 +60,55 @@ on:
|
||||
- 'src/main/codex/structured/journal/codex-goal-journal-rows.ts'
|
||||
- 'src/main/codex/__fixtures__/codex-hook-outcomes.jsonl'
|
||||
- 'src/main/ai-vault/session-scanner-*.ts'
|
||||
- 'src/main/ai-vault/ai-vault-scan-cancellation.ts'
|
||||
- 'src/main/ai-vault/discovery/agent-session-directories.ts'
|
||||
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-json-file.ts'
|
||||
- 'src/main/ai-vault/discovery/session-path-absence.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
|
||||
- 'src/main/ai-vault/parsing/session-json-object.ts'
|
||||
- 'src/main/ai-vault/parsing/session-record-fields.ts'
|
||||
- 'src/main/ai-vault/parsing/session-record-time.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
|
||||
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-inventory.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-inventory.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-only.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-walk.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-walk.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope.test.ts'
|
||||
- 'src/main/ai-vault/session-scanner.ts'
|
||||
- 'src/shared/error-message.ts'
|
||||
- 'src/shared/surrogate-safe-text-slice.ts'
|
||||
- 'src/shared/trimmed-string.ts'
|
||||
- 'src/main/ai-vault/scan-lifecycle/ai-vault-scan-cancellation.ts'
|
||||
- 'src/main/ai-vault/scan-service/session-scanner-service-env.ts'
|
||||
- 'src/main/ai-vault/scanners/grok/session-scanner-grok-user-text.ts'
|
||||
- 'src/main/provider-process/provider-json-record.ts'
|
||||
@@ -122,18 +176,24 @@ on:
|
||||
- 'src/main/runtime/agent-session/agent-session-unsaved-*.ts'
|
||||
# The phone/host contract program's unmocked desktop sources and its declaration owner's inputs.
|
||||
- 'src/main/runtime/integrations/github/runtime-github-repository-query-commands.ts'
|
||||
- 'src/main/github/issue-field-options.ts'
|
||||
- 'src/main/github/issues/issue-field-options.ts'
|
||||
- 'config/tsconfig.test.mobile-host-declarations.json'
|
||||
- 'config/tsconfig.node.json'
|
||||
- 'src/types/build-constants.d.ts'
|
||||
- 'src/main/native-chat/agent-session-wire/**'
|
||||
# Recorded phone transcripts exercise these host decoders and their runtime dependencies.
|
||||
- 'src/main/native-chat/*transcript*.ts'
|
||||
- 'src/main/native-chat/transcript-discovery/**'
|
||||
- 'src/main/native-chat/transcript-filesystem/**'
|
||||
- 'src/main/native-chat/transcript-decoding/**'
|
||||
- 'src/main/native-chat/transcript-streaming/**'
|
||||
- 'src/main/native-chat/transcript-watching/**'
|
||||
- 'src/main/native-chat/transcript-history/**'
|
||||
- 'src/main/native-chat/subagent-entry-id-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-payload-bounds.ts'
|
||||
- 'src/main/native-chat/agent-session-journal/item-lifecycle/journal-reasoning-row.ts'
|
||||
- 'src/main/claude/claude-internal-message-content.ts'
|
||||
- 'src/main/claude/claude-stop-hook-feedback.ts'
|
||||
- 'src/main/claude/structured/journal/claude-internal-message-content.ts'
|
||||
- 'src/main/claude/structured/journal/claude-stop-hook-feedback.ts'
|
||||
- 'src/main/claude/structured/journal/**'
|
||||
- 'src/main/claude/__fixtures__/claude-adapter-capture-fold.jsonl'
|
||||
- 'src/main/codex/app-server/codex-item-field-readers.ts'
|
||||
@@ -142,7 +202,55 @@ on:
|
||||
- 'src/main/codex/structured/journal/codex-goal-journal-rows.ts'
|
||||
- 'src/main/codex/__fixtures__/codex-hook-outcomes.jsonl'
|
||||
- 'src/main/ai-vault/session-scanner-*.ts'
|
||||
- 'src/main/ai-vault/ai-vault-scan-cancellation.ts'
|
||||
- 'src/main/ai-vault/discovery/agent-session-directories.ts'
|
||||
- 'src/main/ai-vault/discovery/session-file-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-json-file.ts'
|
||||
- 'src/main/ai-vault/discovery/session-path-absence.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-agent-sources.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-candidates.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-directory-reader.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery-pipeline.test.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-discovery.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-roots.ts'
|
||||
- 'src/main/ai-vault/discovery/session-scanner-source-discovery.ts'
|
||||
- 'src/main/ai-vault/parsing/session-json-object.ts'
|
||||
- 'src/main/ai-vault/parsing/session-record-fields.ts'
|
||||
- 'src/main/ai-vault/parsing/session-record-time.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-accumulator.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-agent-parser.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-candidate-parse.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-core-parser-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-injected-title.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-jsonl-reader.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-messages.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-parse-wsl-stall.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-record-budget.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-recoverable-empty.test.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-text-normalization.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-token-values.ts'
|
||||
- 'src/main/ai-vault/parsing/session-scanner-unlimited-dedup.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt-capture.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.test.ts'
|
||||
- 'src/main/ai-vault/prompts/session-scanner-first-user-prompt.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-incremental-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-prime-agent-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanner-fixtures/session-scanner-test-fixtures.ts'
|
||||
- 'src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.ts'
|
||||
- 'src/main/ai-vault/scanners/opencode/session-scanner-opencode-parser.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-discovery.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-inventory.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-inventory.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-only.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-walk.test.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope-walk.ts'
|
||||
- 'src/main/ai-vault/scope/session-scanner-scope.test.ts'
|
||||
- 'src/main/ai-vault/session-scanner.ts'
|
||||
- 'src/shared/error-message.ts'
|
||||
- 'src/shared/surrogate-safe-text-slice.ts'
|
||||
- 'src/shared/trimmed-string.ts'
|
||||
- 'src/main/ai-vault/scan-lifecycle/ai-vault-scan-cancellation.ts'
|
||||
- 'src/main/ai-vault/scan-service/session-scanner-service-env.ts'
|
||||
- 'src/main/ai-vault/scanners/grok/session-scanner-grok-user-text.ts'
|
||||
- 'src/main/provider-process/provider-json-record.ts'
|
||||
|
||||
@@ -247,7 +247,7 @@ jobs:
|
||||
run: node "$(if [ -f config/scripts/orcad-distribution/template/build-orcad-template.mjs ]; then printf %s config/scripts/orcad-distribution/template/build-orcad-template.mjs; else printf %s config/scripts/build-orcad-template.mjs; fi)" --targets linux-${{ runner.arch == 'ARM64' && 'arm64' || 'x64' }}-glibc
|
||||
- name: Verify a structured chat from the desktop server template
|
||||
if: runner.os == 'Linux'
|
||||
run: pnpm test:node-server --artifact src/main/orcad/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
|
||||
run: pnpm test:node-server --artifact src/main/orcad/testing/structured-chat/orcad-packaged-structured-chat.integration.test.ts --maxWorkers=2
|
||||
# Only a Windows runner compiles it; arm64 cross-compiles here.
|
||||
# Before the Node 18 check below: the build script imports TypeScript, which Node 18 cannot load.
|
||||
# orcad's instance lock and stop proof read a PID's creation time through it; the x64 binary this
|
||||
|
||||
@@ -851,9 +851,10 @@ jobs:
|
||||
src/main/pty/history/terminal-history-fish-session.node-pty.test.ts \
|
||||
src/main/pty/shell-startup/zsh-scoped-histfile.live-shell.test.ts \
|
||||
src/main/pty/shell-startup/zsh-startup-hook-user-config-equivalence.live-shell.test.ts \
|
||||
src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts \
|
||||
src/main/runtime/agent-session/environment/structured-session-cli-login-shell.live-shell.test.ts \
|
||||
src/main/startup/shell-probe-terminal-detach.live-shell.test.ts \
|
||||
src/main/startup/shell-probe-lifetime.live-shell.test.ts \
|
||||
src/main/startup/shell-environment/shell-probe-terminal-detach.live-shell.test.ts \
|
||||
src/main/startup/shell-environment/shell-probe-lifetime.live-shell.test.ts \
|
||||
src/renderer/src/components/terminal-pane/pty-connection/emulation/fish-color-scheme-child-stdin.node-pty.test.ts \
|
||||
src/shared/fish-wsl-cwd-report.node-pty.test.ts \
|
||||
src/main/wsl/filesystem/wsl-sh-cwd-report.node-pty.test.ts \
|
||||
@@ -1116,7 +1117,6 @@ jobs:
|
||||
src/main/browser/cookies/browser-google-cookie-clear.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/presentation/popup-origin-chrome.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/navigation/browser-link-materialization.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-compatibility.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-native-compatibility.electron.test.ts
|
||||
src/main/browser/guest-lifecycle/popups/compatibility/browser-popup-background-compatibility.electron.test.ts
|
||||
src/main/browser/extensions/api-runtime/browser-extension-api.electron.test.ts
|
||||
|
||||
@@ -47,8 +47,8 @@ jobs:
|
||||
src/main/ipc/preflight-runnable-local-cli.test.ts
|
||||
src/main/ipc/preflight-wsl-agent-detection.test.ts
|
||||
src/main/ipc/preflight-windows-path-refresh.repro.test.ts
|
||||
src/main/startup/hydrate-shell-path.windows.test.ts
|
||||
src/main/startup/login-shell-environment.test.ts
|
||||
src/main/startup/shell-environment/hydrate-shell-path.windows.test.ts
|
||||
src/main/startup/shell-environment/login-shell-environment.test.ts
|
||||
src/packages/process-host/src/process-tree-termination.test.ts
|
||||
src/packages/process-host/src/run-process.test.ts
|
||||
src/packages/process-host/src/run-process-termination-failure.test.ts
|
||||
|
||||
@@ -1321,7 +1321,7 @@ jobs:
|
||||
ORCA_BUILD_IDENTITY: ${{ steps.tag-classify.outputs.identity }}
|
||||
SHARED_JAVASCRIPT: ${{ needs.release-javascript.outputs.supported }}
|
||||
ORCA_RELEASE_JAVASCRIPT_SOURCE_SHA: ${{ needs.release-javascript.outputs.source_sha }}
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
ORCA_POSTHOG_WRITE_KEY: ${{ secrets.ORCA_POSTHOG_WRITE_KEY }}
|
||||
|
||||
# After the app build so nothing that cleans out/ can drop it; electron-builder ships it.
|
||||
@@ -2357,7 +2357,7 @@ jobs:
|
||||
needs:
|
||||
- cut
|
||||
- publish-release
|
||||
# Why stable-only: onorca.dev's mirror link tracks GitHub's latest release, which skips RCs.
|
||||
# Why stable-only: orca.dev's mirror link tracks GitHub's latest release, which skips RCs.
|
||||
if: ${{ !cancelled() && needs.publish-release.result == 'success' && needs.cut.outputs.tag != '' && !contains(needs.cut.outputs.tag, '-rc.') && startsWith(needs.cut.outputs.tag, 'v') }}
|
||||
uses: ./.github/workflows/release-mirror-r2.yml
|
||||
with:
|
||||
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
ORCA_CLI_BUILD_COMPILER: ${{ inputs.dev_build && 'tsrs' || 'bun' }}
|
||||
ORCA_BUILD_IDENTITY: ${{ steps.identity.outputs.value }}
|
||||
ORCA_RELEASE_JAVASCRIPT_BUILD_IDENTITY: ${{ inputs.dev_build && 'dev' || '' }}
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
ORCA_POSTHOG_WRITE_KEY: ${{ !inputs.dev_build && secrets.ORCA_POSTHOG_WRITE_KEY || '' }}
|
||||
run: |
|
||||
pnpm run "$JAVASCRIPT_BUILD_SCRIPT"
|
||||
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
ORCA_BUILD_IDENTITY: ${{ steps.tag-classify.outputs.identity }}
|
||||
ORCA_RELEASE_JAVASCRIPT_SOURCE_SHA: ${{ inputs.javascript_source_sha }}
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
|
||||
ORCA_DIAGNOSTICS_TOKEN_URL: https://orca.dev/diagnostics/token
|
||||
ORCA_POSTHOG_WRITE_KEY: ${{ secrets.ORCA_POSTHOG_WRITE_KEY }}
|
||||
|
||||
# Design D2: the parent release-cut run merged it from every node-server lane at this tag.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: Release Mirror (R2)
|
||||
|
||||
# Copy published release downloads to the public R2 bucket behind https://downloads.onorca.dev.
|
||||
# onorca.dev keeps GitHub as the default download and offers this as an opt-in "mirror link"
|
||||
# Copy published release downloads to the public R2 bucket behind https://downloads.orca.dev.
|
||||
# orca.dev keeps GitHub as the default download and offers this as an opt-in "mirror link"
|
||||
# for when GitHub release downloads return 5xx. Called by release-cut (stable desktop tags) and
|
||||
# mobile-android-release after publishing; dispatch manually to backfill a tag.
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
assets:
|
||||
description: Space-separated asset names (blank = the onorca.dev download assets)
|
||||
description: Space-separated asset names (blank = the orca.dev download assets)
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
@@ -24,7 +24,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
assets:
|
||||
description: Space-separated asset names (blank = the onorca.dev download assets)
|
||||
description: Space-separated asset names (blank = the orca.dev download assets)
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
|
||||
@@ -15,8 +15,8 @@ on:
|
||||
- 'src/main/managed-server/**'
|
||||
- 'src/main/ssh/migration-source/**'
|
||||
- 'src/main/persistence/migrating-orcad-catalog/**'
|
||||
- 'src/main/orcad/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/orcad-host-unavailable.ts'
|
||||
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
|
||||
- 'src/main/node-runtime/**'
|
||||
- 'src/main/ssh/remote-exec/**'
|
||||
- 'src/main/ssh/remote-host/**'
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- 'src/main/managed-server/**'
|
||||
- 'src/main/ssh/migration-source/**'
|
||||
- 'src/main/persistence/migrating-orcad-catalog/**'
|
||||
- 'src/main/orcad/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/orcad-host-unavailable.ts'
|
||||
- 'src/main/orcad/artifact-slots/orcad-artifact-materializer.ts'
|
||||
- 'src/main/orcad/diagnostics/orcad-host-unavailable.ts'
|
||||
- 'src/main/node-runtime/**'
|
||||
- 'src/main/ssh/remote-exec/**'
|
||||
- 'src/main/ssh/remote-host/**'
|
||||
@@ -38,7 +38,7 @@ on:
|
||||
- 'config/scripts/native-build/windows/build-windows-cli-launcher.mjs'
|
||||
- 'config/scripts/native-build/windows/windows-rust-executable-build.mjs'
|
||||
- 'native/windows-cli-launcher/**'
|
||||
- 'src/main/orcad/orcad-cli-launcher.ts'
|
||||
- 'src/main/orcad/serve-startup/orcad-cli-launcher.ts'
|
||||
- '.github/actions/prepare-windows-cli-launcher/**'
|
||||
- 'config/patches/node-pty*'
|
||||
- 'config/patches/@vscode__windows-process-tree*'
|
||||
|
||||
@@ -221,8 +221,8 @@ jobs:
|
||||
pnpm-lock.yaml
|
||||
cloud/pnpm-lock.yaml
|
||||
|
||||
# These two tests import the cloud relay workspace directly. Keeping them in one job
|
||||
# avoids installing and building the same workspace once per unit-test shard.
|
||||
# This test imports the cloud relay workspace directly. Its dedicated job avoids
|
||||
# installing and building the same workspace once per unit-test shard.
|
||||
- name: Install relay integration dependencies
|
||||
id: relay_dependencies
|
||||
if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }}
|
||||
@@ -237,5 +237,4 @@ jobs:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
run: >-
|
||||
pnpm exec vitest run --config config/vitest.config.ts
|
||||
tests/e2e/relay-region-compatibility.unit.test.ts
|
||||
tests/e2e/relay-region-correction.unit.test.ts
|
||||
|
||||
@@ -7,8 +7,7 @@ on:
|
||||
- 'src/main/ai-vault/scanners/opencode/**'
|
||||
- 'src/main/ai-vault/scanners/opencode2/**'
|
||||
- 'src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime*'
|
||||
- 'src/main/native-chat/transcript-opencode*'
|
||||
- 'src/main/native-chat/transcript-zcode*'
|
||||
- 'src/main/native-chat/transcript-history/opencode/**'
|
||||
- 'src/shared/native-chat/native-chat-types.ts'
|
||||
- 'src/shared/native-chat/transcript/native-chat-transcript-projection*'
|
||||
- '.github/workflows/zcode-transcript-contract.yml'
|
||||
@@ -41,4 +40,4 @@ jobs:
|
||||
bun-version-file: config/.bun-version
|
||||
no-cache: true
|
||||
- name: Verify SQLite ordering, legacy readers, and host routing
|
||||
run: pnpm test src/main/native-chat/transcript-zcode.test.ts src/main/native-chat/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
|
||||
run: pnpm test src/main/native-chat/transcript-history/opencode/transcript-zcode.test.ts src/main/native-chat/transcript-history/opencode/transcript-opencode-sqlite-query.test.ts src/main/ai-vault/scanners/zcode/session-scanner-zcode.test.ts src/main/ai-vault/scanners/opencode/reader-transport/opencode-wsl-runtime-preparation.test.ts src/shared/native-chat/transcript/native-chat-transcript-projection.test.ts src/renderer/src/components/native-chat/conversation/projection/structured-agent-session-transcript-order.test.ts
|
||||
|
||||
+1
-1
@@ -308,7 +308,7 @@
|
||||
"ignorePatterns": [
|
||||
"src/main/acp/generated/acp-protocol.generated.ts",
|
||||
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
|
||||
"src/cli/bundled-skill-guides.ts",
|
||||
"src/cli/skills/bundled-skill-guides.ts",
|
||||
"resources/plugins/declarative/*.mjs",
|
||||
"**/node_modules",
|
||||
"**/dist",
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ cask "orca" do
|
||||
url "https://github.com/stablyai/orca/releases/download/v#{version}/orca-macos-#{arch}.dmg"
|
||||
name "Orca"
|
||||
desc "IDE for orchestrating AI coding agents across terminals and worktrees"
|
||||
homepage "https://onorca.dev/"
|
||||
homepage "https://orca.dev/"
|
||||
|
||||
livecheck do
|
||||
url :url
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ cask "orca@rc" do
|
||||
url "https://github.com/stablyai/orca/releases/download/v#{version}/orca-macos-#{arch}.dmg"
|
||||
name "Orca RC"
|
||||
desc "IDE for orchestrating AI coding agents across terminals and worktrees"
|
||||
homepage "https://onorca.dev/"
|
||||
homepage "https://orca.dev/"
|
||||
|
||||
livecheck do
|
||||
url "https://github.com/stablyai/orca"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<h1 align="center">
|
||||
<a href="https://onOrca.dev"><img src="resources/build/icon.png" alt="Orca" width="64" valign="middle" /></a> Orca
|
||||
<a href="https://orca.dev"><img src="resources/build/icon.png" alt="Orca" width="64" valign="middle" /></a> Orca
|
||||
</h1>
|
||||
|
||||
<p align="center">
|
||||
@@ -20,7 +20,7 @@
|
||||
Run Claude Code, Codex, and any other coding agent in parallel, each in its own worktree. Chat or terminal, on your laptop or in the cloud, with diffs, a browser, and a CLI in one app.
|
||||
</p>
|
||||
|
||||
<h3 align="center"><a href="https://onorca.dev/download"><ins>Download Orca</ins></a></h3>
|
||||
<h3 align="center"><a href="https://orca.dev/download"><ins>Download Orca</ins></a></h3>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/assets/readme-hero.jpg" alt="Orca desktop app with a coordinator agent running worker agents in parallel workspaces, and the Orca mobile app in the corner" width="960" />
|
||||
@@ -36,11 +36,11 @@
|
||||
|
||||
Run any agent in Chat UI or as the raw CLI, and move a conversation between the two without losing context.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/agents/native-chat)
|
||||
[Docs →](https://orca.dev/docs/agents/native-chat)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/agents/native-chat"><picture><source srcset="docs/assets/readme-features/chat-or-terminal.webp" type="image/webp"><img src="docs/assets/readme-features/chat-or-terminal.poster.jpg" alt="The same agent conversation in Chat UI and in the terminal, side by side" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/agents/native-chat"><picture><source srcset="docs/assets/readme-features/chat-or-terminal.webp" type="image/webp"><img src="docs/assets/readme-features/chat-or-terminal.poster.jpg" alt="The same agent conversation in Chat UI and in the terminal, side by side" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -50,11 +50,11 @@ Run any agent in Chat UI or as the raw CLI, and move a conversation between the
|
||||
|
||||
Hand a big task to one agent. It splits the work, starts worker agents in their own workspaces, and brings their results back to you.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/cli/orchestration)
|
||||
[Docs →](https://orca.dev/docs/cli/orchestration)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/cli/orchestration"><picture><source srcset="docs/assets/readme-features/orchestration.webp" type="image/webp"><img src="docs/assets/readme-features/orchestration.poster.jpg" alt="A coordinator agent splitting a task between two worker agents and reporting their results" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/cli/orchestration"><picture><source srcset="docs/assets/readme-features/orchestration.webp" type="image/webp"><img src="docs/assets/readme-features/orchestration.poster.jpg" alt="A coordinator agent splitting a task between two worker agents and reporting their results" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -64,11 +64,11 @@ Hand a big task to one agent. It splits the work, starts worker agents in their
|
||||
|
||||
Pair a Remote Orca Server, work on any SSH host, or give a worktree its own cloud VM. Your agents keep running when your laptop sleeps.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/ways-to-run)
|
||||
[Docs →](https://orca.dev/docs/ways-to-run)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/ways-to-run"><picture><source srcset="docs/assets/readme-features/remote.webp" type="image/webp"><img src="docs/assets/readme-features/remote.poster.jpg" alt="Choosing which machine a new worktree runs on, from this computer to an SSH host or a per-workspace environment" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/ways-to-run"><picture><source srcset="docs/assets/readme-features/remote.webp" type="image/webp"><img src="docs/assets/readme-features/remote.poster.jpg" alt="Choosing which machine a new worktree runs on, from this computer to an SSH host or a per-workspace environment" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -78,11 +78,11 @@ Pair a Remote Orca Server, work on any SSH host, or give a worktree its own clou
|
||||
|
||||
Pair Orca with the mobile app to get pinged when an agent needs you, reply in chat, and keep work moving on any of your machines when you are away from your desk.
|
||||
|
||||
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [Android APK](https://www.onorca.dev/download/android) · [Docs →](https://www.onorca.dev/docs/mobile)
|
||||
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [Android APK](https://orca.dev/download/android) · [Docs →](https://orca.dev/docs/mobile)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/mobile"><picture><source srcset="docs/assets/readme-features/mobile-companion.webp" type="image/webp"><img src="docs/assets/readme-features/mobile-companion.poster.jpg" alt="The Orca mobile app getting a notification that Claude needs input, opening the session, and replying so the agent opens the PR" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/mobile"><picture><source srcset="docs/assets/readme-features/mobile-companion.webp" type="image/webp"><img src="docs/assets/readme-features/mobile-companion.poster.jpg" alt="The Orca mobile app getting a notification that Claude needs input, opening the session, and replying so the agent opens the PR" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -92,11 +92,11 @@ Pair Orca with the mobile app to get pinged when an agent needs you, reply in ch
|
||||
|
||||
Review diffs, leave notes on exact changed lines, and send focused feedback back to an agent.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/review/annotate-ai-diff)
|
||||
[Docs →](https://orca.dev/docs/review/annotate-ai-diff)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/assets/readme-features/annotate-ai-diffs.webp" type="image/webp"><img src="docs/assets/readme-features/annotate-ai-diffs.poster.jpg" alt="A note on a changed diff line sent back to the agent" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/assets/readme-features/annotate-ai-diffs.webp" type="image/webp"><img src="docs/assets/readme-features/annotate-ai-diffs.poster.jpg" alt="A note on a changed diff line sent back to the agent" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -106,11 +106,11 @@ Review diffs, leave notes on exact changed lines, and send focused feedback back
|
||||
|
||||
Draft commit messages and pull requests from the diff, then send failing checks or PR review comments back to an agent to fix.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/review/commit-push)
|
||||
[Docs →](https://orca.dev/docs/review/commit-push)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/review/commit-push"><picture><source srcset="docs/assets/readme-features/git-integration.webp" type="image/webp"><img src="docs/assets/readme-features/git-integration.poster.jpg" alt="Drafting a commit and pull request, then sending a review comment back to the agent" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/review/commit-push"><picture><source srcset="docs/assets/readme-features/git-integration.webp" type="image/webp"><img src="docs/assets/readme-features/git-integration.poster.jpg" alt="Drafting a commit and pull request, then sending a review comment back to the agent" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -120,11 +120,11 @@ Draft commit messages and pull requests from the diff, then send failing checks
|
||||
|
||||
Quickly split tasks into isolated environments, so multiple agents can work side by side without interfering with each other.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/model/worktrees)
|
||||
[Docs →](https://orca.dev/docs/model/worktrees)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/assets/readme-features/parallel-worktrees.webp" type="image/webp"><img src="docs/assets/readme-features/parallel-worktrees.poster.jpg" alt="Creating a new worktree with its own agent next to other running worktrees" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/model/worktrees"><picture><source srcset="docs/assets/readme-features/parallel-worktrees.webp" type="image/webp"><img src="docs/assets/readme-features/parallel-worktrees.poster.jpg" alt="Creating a new worktree with its own agent next to other running worktrees" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -134,11 +134,11 @@ Quickly split tasks into isolated environments, so multiple agents can work side
|
||||
|
||||
Click any element in Orca's browser to copy its HTML, styles, and selector, then paste it into your agent's chat.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/browser/design-mode)
|
||||
[Docs →](https://orca.dev/docs/browser/design-mode)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/assets/readme-features/design-mode.webp" type="image/webp"><img src="docs/assets/readme-features/design-mode.poster.jpg" alt="Picking a button in Orca's browser and sending it to the agent's chat" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/browser/design-mode"><picture><source srcset="docs/assets/readme-features/design-mode.webp" type="image/webp"><img src="docs/assets/readme-features/design-mode.poster.jpg" alt="Picking a button in Orca's browser and sending it to the agent's chat" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -148,11 +148,11 @@ Click any element in Orca's browser to copy its HTML, styles, and selector, then
|
||||
|
||||
See your GitHub and Linear tasks in one place, and start a workspace for each when you're ready to build.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/review/linear)
|
||||
[Docs →](https://orca.dev/docs/review/linear)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="docs/assets/readme-features/github-linear-tasks.webp" type="image/webp"><img src="docs/assets/readme-features/github-linear-tasks.poster.jpg" alt="GitHub and Linear issues in Orca, with a workspace started from a Linear issue" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/review/linear"><picture><source srcset="docs/assets/readme-features/github-linear-tasks.webp" type="image/webp"><img src="docs/assets/readme-features/github-linear-tasks.poster.jpg" alt="GitHub and Linear issues in Orca, with a workspace started from a Linear issue" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -162,11 +162,11 @@ See your GitHub and Linear tasks in one place, and start a workspace for each wh
|
||||
|
||||
Anything you can do in Orca, your agents can do too: create worktrees, start workers, drive the browser and desktop apps, and schedule automations, all from the command line.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/cli/overview)
|
||||
[Docs →](https://orca.dev/docs/cli/overview)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="docs/assets/readme-features/orca-cli.webp" type="image/webp"><img src="docs/assets/readme-features/orca-cli.poster.jpg" alt="An agent running orca worktree create to start a worker in a new worktree" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/cli/overview"><picture><source srcset="docs/assets/readme-features/orca-cli.webp" type="image/webp"><img src="docs/assets/readme-features/orca-cli.poster.jpg" alt="An agent running orca worktree create to start a worker in a new worktree" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -176,11 +176,11 @@ Anything you can do in Orca, your agents can do too: create worktrees, start wor
|
||||
|
||||
Themes, editors, languages, and panels from the community, installed in a click.
|
||||
|
||||
[Browse plugins →](https://www.onorca.dev/plugins)
|
||||
[Browse plugins →](https://orca.dev/plugins)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/plugins"><picture><source srcset="docs/assets/readme-features/plugins.webp" type="image/webp"><img src="docs/assets/readme-features/plugins.poster.jpg" alt="Installing the Orca Midnight theme from the plugin catalog" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/plugins"><picture><source srcset="docs/assets/readme-features/plugins.webp" type="image/webp"><img src="docs/assets/readme-features/plugins.poster.jpg" alt="Installing the Orca Midnight theme from the plugin catalog" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -190,25 +190,25 @@ Themes, editors, languages, and panels from the community, installed in a click.
|
||||
|
||||
Put agents, terminals, browsers, and diffs side by side in split panes shaped to the task.
|
||||
|
||||
[Docs →](https://www.onorca.dev/docs/model/tabs-panes-splits)
|
||||
[Docs →](https://orca.dev/docs/model/tabs-panes-splits)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
<a href="https://www.onorca.dev/docs/model/tabs-panes-splits"><picture><source srcset="docs/assets/readme-features/split-anything.webp" type="image/webp"><img src="docs/assets/readme-features/split-anything.poster.jpg" alt="Dragging tabs into split panes with an agent, browser, terminal, and diff" width="100%" /></picture></a>
|
||||
<a href="https://orca.dev/docs/model/tabs-panes-splits"><picture><source srcset="docs/assets/readme-features/split-anything.webp" type="image/webp"><img src="docs/assets/readme-features/split-anything.poster.jpg" alt="Dragging tabs into split panes with an agent, browser, terminal, and diff" width="100%" /></picture></a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
**Also in the box:**
|
||||
|
||||
- **[Drag Files to Agents](https://www.onorca.dev/docs/editing/file-explorer)**: drag files from the explorer, or images from your desktop, straight into an agent's prompt.
|
||||
- **[Jump Anywhere](https://www.onorca.dev/docs/model/quick-open)**: one shortcut jumps to any workspace or tab across every repo and host.
|
||||
- **[Rich Previews](https://www.onorca.dev/docs/editing/viewers)**: open images, PDFs, and Markdown right in the workspace, next to the agent that made them.
|
||||
- **[Mobile Emulator](https://www.onorca.dev/docs/cli/overview#mobile-emulator)**: run an iOS Simulator or Android emulator beside your agent, and let it tap, type, and check its own work.
|
||||
- **[Dictation](https://www.onorca.dev/docs/settings#voice)**: press a shortcut and talk. Orca types what you say into the agent's prompt or terminal.
|
||||
- **[Usage & Accounts](https://www.onorca.dev/docs/agents/usage-tracking)**: see how much of your agent limits you've used and when they reset, and switch accounts without logging out.
|
||||
- **[Automations](https://www.onorca.dev/docs/cli/automations)**: schedule prompts to run on any repo: nightly dependency bumps, weekly triage, flaky-test sweeps.
|
||||
- **[Computer Use](https://www.onorca.dev/docs/cli/computer-use)**: when there's no CLI or API, agents can see and drive desktop apps and external browsers.
|
||||
- **[Drag Files to Agents](https://orca.dev/docs/editing/file-explorer)**: drag files from the explorer, or images from your desktop, straight into an agent's prompt.
|
||||
- **[Jump Anywhere](https://orca.dev/docs/model/quick-open)**: one shortcut jumps to any workspace or tab across every repo and host.
|
||||
- **[Rich Previews](https://orca.dev/docs/editing/viewers)**: open images, PDFs, and Markdown right in the workspace, next to the agent that made them.
|
||||
- **[Mobile Emulator](https://orca.dev/docs/cli/overview#mobile-emulator)**: run an iOS Simulator or Android emulator beside your agent, and let it tap, type, and check its own work.
|
||||
- **[Dictation](https://orca.dev/docs/settings#voice)**: press a shortcut and talk. Orca types what you say into the agent's prompt or terminal.
|
||||
- **[Usage & Accounts](https://orca.dev/docs/agents/usage-tracking)**: see how much of your agent limits you've used and when they reset, and switch accounts without logging out.
|
||||
- **[Automations](https://orca.dev/docs/cli/automations)**: schedule prompts to run on any repo: nightly dependency bumps, weekly triage, flaky-test sweeps.
|
||||
- **[Computer Use](https://orca.dev/docs/cli/computer-use)**: when there's no CLI or API, agents can see and drive desktop apps and external browsers.
|
||||
- **And many, many more**: we ship daily, so this list is perpetually behind. The [changelog](https://github.com/stablyai/orca/releases) is the real feature list.
|
||||
|
||||
---
|
||||
@@ -261,7 +261,7 @@ Works with **any CLI agent** — if it runs in a terminal, it runs in Orca.
|
||||
|
||||
### Desktop — macOS, Windows, Linux
|
||||
|
||||
- **[Download from onOrca.dev](https://onorca.dev/download)**
|
||||
- **[Download from orca.dev](https://orca.dev/download)**
|
||||
- Or grab a build directly: [macOS Apple Silicon](https://github.com/stablyai/orca/releases/latest/download/orca-macos-arm64.dmg) · [macOS Intel](https://github.com/stablyai/orca/releases/latest/download/orca-macos-x64.dmg) · [Windows (.exe)](https://github.com/stablyai/orca/releases/latest/download/orca-windows-setup.exe) · [Linux AppImage](https://github.com/stablyai/orca/releases/latest/download/orca-linux.AppImage) · [All builds](https://github.com/stablyai/orca/releases/latest)
|
||||
|
||||
_Or via a package manager:_
|
||||
@@ -284,7 +284,7 @@ yay -S stably-orca-bin
|
||||
Pair with your desktop app to monitor and steer your agents from your phone.
|
||||
|
||||
- **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217)
|
||||
- **Android:** [Download APK](https://www.onorca.dev/download/android) · [Install guide](https://www.onorca.dev/docs/android-apk)
|
||||
- **Android:** [Download APK](https://orca.dev/download/android) · [Install guide](https://orca.dev/docs/android-apk)
|
||||
|
||||
---
|
||||
|
||||
@@ -297,7 +297,7 @@ Pair with your desktop app to monitor and steer your agents from your phone.
|
||||
<img src="docs/assets/wechat-qr.jpg" alt="WeChat QR code for the Orca community" width="160" />
|
||||
|
||||
- **Feedback & Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
|
||||
- **Privacy:** See the [privacy & telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
|
||||
- **Privacy:** See the [privacy & telemetry docs](https://orca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
|
||||
- **Show Support:** [Star](https://github.com/stablyai/orca) this repo to follow along with our daily ships.
|
||||
|
||||
---
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
|
||||
import { DELIVERY_LEASE_MS, DurablePushStore } from './durable-push-store.js'
|
||||
import { DurablePushStore } from './durable-push-store.js'
|
||||
|
||||
import {
|
||||
cleanupDurablePushFixtures,
|
||||
@@ -63,20 +63,6 @@ it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
}
|
||||
)
|
||||
|
||||
it('leases exactly one delivery per device across concurrent claimers', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
for (let seq = 1; seq <= 5; seq++) {
|
||||
await store.accept('host', 'phone', notification(seq))
|
||||
advance(1)
|
||||
}
|
||||
const claims = await Promise.all(
|
||||
Array.from({ length: 8 }, () => new DurablePushStore(db, clock).claim())
|
||||
)
|
||||
const leased = claims.filter((claim) => claim !== null)
|
||||
expect(leased).toHaveLength(1)
|
||||
expect(leased[0]?.notification.notificationSeq).toBe(1)
|
||||
})
|
||||
|
||||
it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
'does not lease the row behind a head another claimer holds',
|
||||
async () => {
|
||||
@@ -138,21 +124,3 @@ it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
expect((await first)?.notification.notificationSeq).toBe(1)
|
||||
}
|
||||
)
|
||||
|
||||
// Only the TTL term keeps the candidate scan off an unpruned backlog; expires_at is not indexed first.
|
||||
|
||||
it('frees a device held by a crashed worker once its lease lapses', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
advance(1)
|
||||
await store.accept('host', 'phone', notification(2))
|
||||
const crashed = (await new DurablePushStore(db, clock).claim())!
|
||||
advance(DELIVERY_LEASE_MS - 1)
|
||||
expect(await store.claim()).toBeNull()
|
||||
advance(1)
|
||||
const reclaimed = (await store.claim())!
|
||||
expect(reclaimed.id).toBe(crashed.id)
|
||||
expect(reclaimed.attempts).toBe(2)
|
||||
await store.finish(reclaimed)
|
||||
expect((await store.claim())?.notification.notificationSeq).toBe(2)
|
||||
})
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openPushDatabase } from '../database/push-database.js'
|
||||
import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js'
|
||||
import {
|
||||
cleanupDurablePushFixtures,
|
||||
durablePushTestDatabaseUrl,
|
||||
fixture,
|
||||
notification
|
||||
} from './durable-push-store.test-fixture.js'
|
||||
@@ -11,20 +9,6 @@ import {
|
||||
afterEach(cleanupDurablePushFixtures)
|
||||
|
||||
describe('durable push acceptance', () => {
|
||||
it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => {
|
||||
const { store, advance } = await fixture()
|
||||
for (let i = 0; i < 300; i++) {
|
||||
expect(await store.accept('host', 'phone1', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone2', notification(i))).toBe('queued')
|
||||
expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued')
|
||||
}
|
||||
expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited')
|
||||
expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited')
|
||||
expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued')
|
||||
advance(15 * 60_000)
|
||||
expect(await store.accept('host', 'phone1', notification(301))).toBe('queued')
|
||||
})
|
||||
|
||||
it('queues one delivery per event and recovers work across service instances', async () => {
|
||||
const { db, store, clock, advance } = await fixture()
|
||||
await store.accept('host', 'phone', notification(1))
|
||||
@@ -32,16 +16,6 @@ describe('durable push acceptance', () => {
|
||||
await restarted.accept('host', 'phone', notification(1))
|
||||
advance(1)
|
||||
await restarted.accept('host', 'phone', notification(2))
|
||||
const rows = await db.query(
|
||||
"SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id",
|
||||
['phone']
|
||||
)
|
||||
expect(rows).toHaveLength(2)
|
||||
expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([
|
||||
notification(1),
|
||||
notification(2)
|
||||
])
|
||||
expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true)
|
||||
const delivery = await restarted.claim()
|
||||
expect(delivery?.notification.notificationSeq).toBe(1)
|
||||
expect(await store.claim()).toBeNull()
|
||||
@@ -93,18 +67,9 @@ describe('durable push acceptance', () => {
|
||||
const { db, store } = await fixture()
|
||||
await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable')
|
||||
try {
|
||||
const databaseUrl = durablePushTestDatabaseUrl
|
||||
if (databaseUrl) {
|
||||
const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' })
|
||||
try {
|
||||
await expect(
|
||||
concurrent.query('SELECT COUNT(*) FROM push_delivery_batches')
|
||||
).resolves.toHaveLength(1)
|
||||
} finally {
|
||||
await concurrent.close()
|
||||
}
|
||||
}
|
||||
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow()
|
||||
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow(
|
||||
/push_delivery_batches/
|
||||
)
|
||||
expect(await db.query('SELECT * FROM push_events')).toEqual([])
|
||||
expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([])
|
||||
} finally {
|
||||
@@ -160,38 +125,3 @@ it('does not resurrect an in-flight alert after a dismissal and transient provid
|
||||
expect(await store.claim()).toBeNull()
|
||||
expect(await store.pendingCount('phone')).toBe(0)
|
||||
})
|
||||
|
||||
it('fences late renew and finish after an expired claim is dismissed', async () => {
|
||||
const { db, store, advance } = await fixture()
|
||||
const alert = notification(1)
|
||||
await store.accept('host', 'phone', alert)
|
||||
const stale = (await store.claim())!
|
||||
advance(DELIVERY_LEASE_MS)
|
||||
await store.accept('host', 'phone', {
|
||||
...notification(2, 'dismiss'),
|
||||
notificationId: alert.notificationId
|
||||
})
|
||||
const read = async () =>
|
||||
(
|
||||
await db.query(
|
||||
'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?',
|
||||
[stale.id]
|
||||
)
|
||||
)[0]
|
||||
expect(await read()).toBeUndefined()
|
||||
await store.renew(stale)
|
||||
await store.finish(stale, 1000)
|
||||
await store.finish(stale)
|
||||
expect(await read()).toBeUndefined()
|
||||
const dismissal = (await store.claim())!
|
||||
expect(dismissal.notification.kind).toBe('dismiss')
|
||||
await store.finish(dismissal)
|
||||
await store.accept('host', 'phone', notification(3))
|
||||
const fresh = (await store.claim())!
|
||||
await store.finish(fresh, 1000)
|
||||
advance(1000)
|
||||
const retry = (await store.claim())!
|
||||
expect(retry.id).toBe(fresh.id)
|
||||
await store.finish(retry)
|
||||
expect(await store.claim()).toBeNull()
|
||||
})
|
||||
|
||||
@@ -13,7 +13,6 @@ import { pauseAfter, within } from './durable-push-store.test-fixture.js'
|
||||
const cleanups: (() => Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0)) await cleanup()
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
const note = (seq: number, overrides: Partial<PushNotification> = {}): PushNotification => ({
|
||||
@@ -42,9 +41,8 @@ async function fixture() {
|
||||
const send = vi.fn(async (_delivery: PushDelivery): Promise<PushProviderOutcome> => ({
|
||||
status: 'sent'
|
||||
}))
|
||||
const onRetry = vi.fn()
|
||||
const dispatcher = new PushDispatcher({ devices, fcm: { send } as never })
|
||||
const worker = new DurablePushWorker(store, dispatcher, { now, onRetry })
|
||||
const worker = new DurablePushWorker(store, dispatcher, { now })
|
||||
cleanups.push(async () => {
|
||||
await worker.stop()
|
||||
await db.close()
|
||||
@@ -57,7 +55,6 @@ async function fixture() {
|
||||
worker,
|
||||
dispatcher,
|
||||
send,
|
||||
onRetry,
|
||||
now,
|
||||
registrationId: device.registrationId,
|
||||
accept: (notification: PushNotification) =>
|
||||
@@ -102,7 +99,6 @@ it('stops queued claim admissions without leasing the remaining device', async (
|
||||
paused.release()
|
||||
await within(Promise.all([pending, stopping]), 2_000)
|
||||
expect(h.send).toHaveBeenCalledOnce()
|
||||
expect(h.send.mock.calls[0]?.[0].orca.notificationSeq).toBe(1)
|
||||
const [remaining] = await h.db.query(
|
||||
'SELECT attempts, lease_until FROM push_delivery_batches WHERE registration_id = ?',
|
||||
[other.registrationId]
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { PushDeviceRegistryStore, type PushDeviceUpsert } from './device-registry-store.js'
|
||||
import { openInMemoryPushDatabase } from '../database/push-database.js'
|
||||
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
|
||||
|
||||
const OWNER = 'abcdefghijklmnop'
|
||||
const OTHER = 'ponmlkjihgfedcba'
|
||||
|
||||
describe('push device registry store', () => {
|
||||
let database: PushDatabase
|
||||
let clock = 1_700_000_000_000
|
||||
let devices: PushDeviceRegistryStore
|
||||
|
||||
beforeEach(async () => {
|
||||
database = await openInMemoryPushDatabase()
|
||||
clock = 1_700_000_000_000
|
||||
devices = new PushDeviceRegistryStore(database, () => clock)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
async function upsertOk(input: PushDeviceUpsert): Promise<string> {
|
||||
const result = await devices.upsert(input)
|
||||
if (!result.ok) throw new Error(`unexpected upsert refusal: ${result.reason}`)
|
||||
return result.registrationId
|
||||
}
|
||||
|
||||
function androidDevice(deviceId: string): PushDeviceUpsert {
|
||||
return {
|
||||
hostFingerprint: OWNER,
|
||||
deviceId,
|
||||
platform: 'android',
|
||||
token: `token-${deviceId}`
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps one registration per host and device while replacing the token', async () => {
|
||||
const first = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: 'a'.repeat(64),
|
||||
apnsEnvironment: 'sandbox'
|
||||
})
|
||||
clock += 1_000
|
||||
const second = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'ios',
|
||||
token: 'b'.repeat(64),
|
||||
apnsEnvironment: 'production'
|
||||
})
|
||||
expect(second).toBe(first)
|
||||
const registration = await devices.findById(first)
|
||||
expect(registration).toMatchObject({
|
||||
token: 'b'.repeat(64),
|
||||
apnsEnvironment: 'production',
|
||||
dead: false
|
||||
})
|
||||
expect(await devices.list(OWNER)).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('revives a registration that a re-registered token replaces', async () => {
|
||||
const registrationId = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
await devices.markDead((await devices.findById(registrationId))!)
|
||||
expect((await devices.findById(registrationId))?.dead).toBe(true)
|
||||
await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-two'
|
||||
})
|
||||
expect(await devices.findById(registrationId)).toMatchObject({
|
||||
token: 'token-two',
|
||||
dead: false
|
||||
})
|
||||
})
|
||||
|
||||
it('lets only the owning host delete a registration', async () => {
|
||||
const registrationId = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
expect(await devices.deleteOwned(OTHER, registrationId)).toBe(false)
|
||||
expect(await devices.findById(registrationId)).not.toBeNull()
|
||||
expect(await devices.deleteOwned(OWNER, registrationId)).toBe(true)
|
||||
expect(await devices.findById(registrationId)).toBeNull()
|
||||
})
|
||||
|
||||
it('scopes lookups and listings to the owning host', async () => {
|
||||
const owned = await upsertOk({
|
||||
hostFingerprint: OWNER,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'token-one'
|
||||
})
|
||||
const foreign = await upsertOk({
|
||||
hostFingerprint: OTHER,
|
||||
deviceId: 'device-2',
|
||||
platform: 'android',
|
||||
token: 'token-two'
|
||||
})
|
||||
const found = await devices.findOwned(OWNER, [owned, foreign])
|
||||
expect([...found.keys()]).toEqual([owned])
|
||||
expect(await devices.list(OTHER)).toEqual([
|
||||
{ registrationId: foreign, deviceId: 'device-2', platform: 'android', dead: false }
|
||||
])
|
||||
expect(await devices.findOwned(OWNER, [])).toEqual(new Map())
|
||||
})
|
||||
|
||||
it('counts the cap per host, not across the whole table', async () => {
|
||||
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
|
||||
await upsertOk(androidDevice(`device-${index}`))
|
||||
}
|
||||
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false)
|
||||
expect(
|
||||
(await devices.upsert({ ...androidDevice('device-0'), hostFingerprint: OTHER })).ok
|
||||
).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -1,4 +1,3 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
answerPushHostChallenge,
|
||||
@@ -27,33 +26,10 @@ describe('push host challenge store', () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('rejects a challenge the moment its own ttl elapses', async () => {
|
||||
const host = createPushHostKeypair(4)
|
||||
const challenge = await store.issue(hostPublicKeyB64(host))
|
||||
const proof = answerPushHostChallenge(challenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: host,
|
||||
now: () => clock
|
||||
})!
|
||||
clock += PUSH_LIMITS.challengeTtlMs + 1
|
||||
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'expired'
|
||||
})
|
||||
})
|
||||
|
||||
it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => {
|
||||
it('rejects a proof issued for another challenge', async () => {
|
||||
const owner = createPushHostKeypair(6)
|
||||
const intruder = createPushHostKeypair(7)
|
||||
const ownerChallenge = await store.issue(hostPublicKeyB64(owner))
|
||||
expect(
|
||||
answerPushHostChallenge(ownerChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
keypair: intruder,
|
||||
now: () => clock
|
||||
})
|
||||
).toBeNull()
|
||||
|
||||
const intruderChallenge = await store.issue(hostPublicKeyB64(intruder))
|
||||
const intruderProof = answerPushHostChallenge(intruderChallenge!, {
|
||||
gatewayOrigin: GATEWAY_ORIGIN,
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { PushHostSessionStore } from './host-session-store.js'
|
||||
import { openInMemoryPushDatabase } from '../database/push-database.js'
|
||||
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
|
||||
|
||||
const HOST = 'abcdefghijklmnop'
|
||||
|
||||
describe('push host session store', () => {
|
||||
let database: PushDatabase
|
||||
let clock = 1_700_000_000_000
|
||||
let sessions: PushHostSessionStore
|
||||
|
||||
beforeEach(async () => {
|
||||
database = await openInMemoryPushDatabase()
|
||||
clock = 1_700_000_000_000
|
||||
sessions = new PushHostSessionStore(database, () => clock)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('reports expiry separately from an unknown token', async () => {
|
||||
const session = await sessions.create(HOST)
|
||||
clock += PUSH_LIMITS.sessionTtlMs + 1
|
||||
await expect(sessions.resolve(session.sessionToken)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'session_expired'
|
||||
})
|
||||
await expect(sessions.resolve('not-a-session')).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'unknown_session'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps one live session per host and prunes it once expired', async () => {
|
||||
const first = await sessions.create(HOST)
|
||||
const second = await sessions.create(HOST)
|
||||
// The earlier session is gone the moment its host proves again, so a flood
|
||||
// of proofs leaves one row per host rather than one per proof.
|
||||
await expect(sessions.resolve(first.sessionToken)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: 'unknown_session'
|
||||
})
|
||||
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
|
||||
const other = await sessions.create('ponmlkjihgfedcba')
|
||||
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
|
||||
clock += PUSH_LIMITS.sessionTtlMs + 1
|
||||
expect(await sessions.pruneExpired()).toBe(2)
|
||||
await expect(sessions.resolve(other.sessionToken)).resolves.toMatchObject({ ok: false })
|
||||
})
|
||||
})
|
||||
@@ -19,11 +19,6 @@ async function concurrentSessions(db: PushDatabase) {
|
||||
sessions.map((session) => store.resolve(session.sessionToken))
|
||||
)
|
||||
expect(decisions.filter((decision) => decision.ok)).toHaveLength(1)
|
||||
const [row] = await db.query(
|
||||
'SELECT COUNT(*) AS count FROM push_sessions WHERE host_fingerprint = ?',
|
||||
[host]
|
||||
)
|
||||
expect(Number(row?.count)).toBe(1)
|
||||
} finally {
|
||||
await db.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [host])
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { Hono } from 'hono'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { ClientIpRateLimiter, clientIpRateLimit } from './client-ip-rate-limit.js'
|
||||
|
||||
const CAPACITY = PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp
|
||||
|
||||
function limiterApp(limiter: ClientIpRateLimiter, trustedProxyHops = 0): Hono {
|
||||
const app = new Hono()
|
||||
app.post('/probe', clientIpRateLimit(limiter, { trustedProxyHops }), (context) =>
|
||||
context.json({ ok: true })
|
||||
)
|
||||
return app
|
||||
}
|
||||
|
||||
describe('client ip rate limiter', () => {
|
||||
it('buckets on the last forwarded hop, the only one the platform appended', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
|
||||
for (let index = 0; index < CAPACITY; index++) {
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': `10.0.0.${index}, 203.0.113.7` }
|
||||
})
|
||||
}
|
||||
const sameClient = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '10.9.9.9, 203.0.113.7' }
|
||||
})
|
||||
expect(sameClient.status).toBe(429)
|
||||
const otherClient = await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '10.0.0.1, 198.51.100.9' }
|
||||
})
|
||||
expect(otherClient.status).toBe(200)
|
||||
})
|
||||
|
||||
it('skips the configured trusted proxies when counting from the right', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
|
||||
// <client>, <cloud run>, <load balancer>: one trusted hop after the client.
|
||||
const headers = { 'x-forwarded-for': '203.0.113.7, 10.0.0.1' }
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(429)
|
||||
expect(
|
||||
(
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9, 10.0.0.1' }
|
||||
})
|
||||
).status
|
||||
).toBe(200)
|
||||
})
|
||||
|
||||
it('trusts nothing when the header is shorter than the configured depth', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
|
||||
// Only one hop, so the client value the depth points at does not exist.
|
||||
const headers = { 'x-forwarded-for': '203.0.113.7' }
|
||||
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
|
||||
expect(
|
||||
(
|
||||
await app.request('/probe', {
|
||||
method: 'POST',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9' }
|
||||
})
|
||||
).status
|
||||
).toBe(429)
|
||||
})
|
||||
|
||||
it('ignores spoofable x-real-ip and uses a single shared bucket', async () => {
|
||||
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }))
|
||||
expect(
|
||||
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '198.51.100.9' } }))
|
||||
.status
|
||||
).toBe(200)
|
||||
expect(
|
||||
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '203.0.113.7' } }))
|
||||
.status
|
||||
).toBe(429)
|
||||
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
|
||||
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
|
||||
})
|
||||
})
|
||||
@@ -1,36 +0,0 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
const harnesses: Awaited<ReturnType<typeof createPushServerHarness>>[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(harnesses.splice(0).map((h) => h.close()))
|
||||
})
|
||||
|
||||
it('returns queued for concurrent retries without double quota or delivery', async () => {
|
||||
const h = await createPushServerHarness()
|
||||
harnesses.push(h)
|
||||
const token = await h.signIn(createPushHostKeypair(2))
|
||||
const registrationId = await h.registerAndroid(token)
|
||||
const body = { v: 1, registrationIds: [registrationId], notification: notification() }
|
||||
const responses = await Promise.all(
|
||||
Array.from({ length: 10 }, () => h.post('/v1/send', body, token))
|
||||
)
|
||||
for (const response of responses)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1)
|
||||
await h.flushDeliveries()
|
||||
await h.post('/v1/send', body, token)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined()
|
||||
expect(
|
||||
Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count)
|
||||
).toBe(1)
|
||||
await h.post(
|
||||
'/v1/send',
|
||||
{ ...body, notification: notification({ notificationEpoch: 'new-epoch' }) },
|
||||
token
|
||||
)
|
||||
await h.flushDeliveries()
|
||||
expect(h.fcmRequests).toHaveLength(2)
|
||||
})
|
||||
@@ -1,9 +1,6 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
import type { CloudDatabase as PushDatabase } from '@orca-cloud/cloud-database/cloud-database'
|
||||
import { createPushServer } from '../push-server.js'
|
||||
import { createPushServerHarness, testPushConfig } from './push-server-harness.test-fixture.js'
|
||||
import { createPushServerHarness } from './push-server-harness.test-fixture.js'
|
||||
|
||||
describe('push gateway authentication and device routes', () => {
|
||||
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
|
||||
@@ -16,63 +13,6 @@ describe('push gateway authentication and device routes', () => {
|
||||
await harness.close()
|
||||
})
|
||||
|
||||
it('reports not ready when the database is unreachable', async () => {
|
||||
const unreachable: PushDatabase = {
|
||||
dialect: 'sqlite',
|
||||
query: async () => {
|
||||
throw new Error('no connection')
|
||||
},
|
||||
transaction: async (operation) => await operation(unreachable),
|
||||
lockQuotaScope: async () => undefined,
|
||||
tryLockScope: async () => true,
|
||||
tryLockSharedScope: async () => true,
|
||||
close: async () => undefined
|
||||
}
|
||||
const broken = createPushServer(testPushConfig(), unreachable, {
|
||||
fcmAccessToken: async () => 'token',
|
||||
fcmTransport: async () => ({ status: 200, body: '{}' })
|
||||
})
|
||||
expect((await broken.app.request('/health')).status).toBe(200)
|
||||
expect((await broken.app.request('/ready')).status).toBe(503)
|
||||
await broken.worker.stop()
|
||||
})
|
||||
|
||||
it('completes challenge, session, register, list, delete', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(11))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
|
||||
const list = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(await list.json()).toEqual({
|
||||
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: false }]
|
||||
})
|
||||
|
||||
const deleted = await harness.authorized(
|
||||
`/v1/devices/${registrationId}`,
|
||||
{ method: 'DELETE' },
|
||||
sessionToken
|
||||
)
|
||||
expect(deleted.status).toBe(204)
|
||||
expect(await harness.server.devices.findById(registrationId)).toBeNull()
|
||||
})
|
||||
|
||||
it('refuses a request with no bearer, a bogus bearer, and an expired session', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(12))
|
||||
expect((await harness.server.app.request('/v1/devices')).status).toBe(401)
|
||||
const bogus = await harness.authorized('/v1/devices', {}, 'nonsense')
|
||||
expect(bogus.status).toBe(401)
|
||||
expect(await bogus.json()).toEqual({ error: 'invalid_token' })
|
||||
|
||||
harness.advanceClock(PUSH_LIMITS.sessionTtlMs + 1)
|
||||
const expired = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(expired.status).toBe(401)
|
||||
expect(await expired.json()).toEqual({ error: 'session_expired' })
|
||||
})
|
||||
|
||||
it('never returns the host fingerprint on the challenge itself', async () => {
|
||||
const challenge = await harness.issueChallenge(createPushHostKeypair(22))
|
||||
expect(challenge).not.toHaveProperty('hostFingerprint')
|
||||
})
|
||||
|
||||
it('lets only the owning host delete a registration', async () => {
|
||||
const ownerToken = await harness.signIn(createPushHostKeypair(14))
|
||||
const intruderToken = await harness.signIn(createPushHostKeypair(15))
|
||||
@@ -87,34 +27,4 @@ describe('push gateway authentication and device routes', () => {
|
||||
expect(await forbidden.json()).toEqual({ error: 'not_found' })
|
||||
expect(await harness.server.devices.findById(registrationId)).not.toBeNull()
|
||||
})
|
||||
|
||||
it('replaces the token on a re-registration and keeps one registration id', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(23))
|
||||
const first = await harness.registerAndroid(sessionToken)
|
||||
const again = await harness.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: 'device-1',
|
||||
platform: 'android',
|
||||
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(await again.json()).toEqual({ registrationId: first })
|
||||
expect(await harness.server.devices.findById(first)).toMatchObject({
|
||||
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a malformed registration body', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(16))
|
||||
const bad = await harness.post(
|
||||
'/v1/devices',
|
||||
{ v: 1, deviceId: 'device-1', platform: 'ios', token: 'not-hex' },
|
||||
sessionToken
|
||||
)
|
||||
expect(bad.status).toBe(400)
|
||||
expect(await bad.json()).toEqual({ error: 'invalid_request' })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
createPushHostKeypair,
|
||||
hostPublicKeyB64
|
||||
} from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
|
||||
import { createPushServerHarness } from './push-server-harness.test-fixture.js'
|
||||
|
||||
const CLIENT_IP = '203.0.113.7'
|
||||
const OTHER_CLIENT_IP = '198.51.100.9'
|
||||
@@ -48,42 +48,6 @@ describe('push gateway request limits', () => {
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('still refuses an oversized body that declares a content length', async () => {
|
||||
const body = oversizedChallengeBody()
|
||||
const response = await harness.server.app.request('/v1/host/challenge', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(Buffer.byteLength(body))
|
||||
},
|
||||
body
|
||||
})
|
||||
expect(response.status).toBe(413)
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('caps an authenticated oversized send as well', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(61))
|
||||
const response = await harness.server.app.request(
|
||||
new Request('http://push.test/v1/send', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${sessionToken}`
|
||||
},
|
||||
body: new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(oversizedChallengeBody()))
|
||||
controller.close()
|
||||
}
|
||||
}),
|
||||
duplex: 'half'
|
||||
} as RequestInit)
|
||||
)
|
||||
expect(response.status).toBe(413)
|
||||
expect(await response.json()).toEqual({ error: 'request_too_large' })
|
||||
})
|
||||
|
||||
it('rate limits one client ip across both unauthenticated routes', async () => {
|
||||
const body = JSON.stringify({
|
||||
v: 1,
|
||||
@@ -135,27 +99,6 @@ describe('push gateway request limits', () => {
|
||||
expect(spoofed.status).toBe(429)
|
||||
})
|
||||
|
||||
it('lets a throttled client back in once the window refills', async () => {
|
||||
const body = JSON.stringify({
|
||||
v: 1,
|
||||
hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(63))
|
||||
})
|
||||
const headers = { 'content-type': 'application/json', 'x-forwarded-for': CLIENT_IP }
|
||||
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
|
||||
await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body })
|
||||
}
|
||||
expect(
|
||||
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
|
||||
.status
|
||||
).toBe(429)
|
||||
|
||||
harness.advanceClock(60_000)
|
||||
expect(
|
||||
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
|
||||
.status
|
||||
).toBe(200)
|
||||
})
|
||||
|
||||
it('limits authenticated hosts independently behind the same IP', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(64))
|
||||
const headers = { 'x-forwarded-for': CLIENT_IP }
|
||||
@@ -175,43 +118,4 @@ describe('push gateway request limits', () => {
|
||||
})
|
||||
expect(challenge.status).toBe(200)
|
||||
})
|
||||
|
||||
// Why: a database error carries the failing row in its message. The response
|
||||
// and the log must both stop at the error's name.
|
||||
it('answers an unexpected route failure with a bare 500 and logs only the name', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(66))
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
await harness.database.close()
|
||||
const response = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(response.status).toBe(500)
|
||||
expect(await response.json()).toEqual({ error: 'internal' })
|
||||
const logged = warn.mock.calls.map((call) => String(call[0])).join('\n')
|
||||
expect(logged).toContain('"event":"orca_push_request_failed"')
|
||||
expect(logged).not.toContain('SELECT')
|
||||
expect(logged).not.toContain('push_devices')
|
||||
expect(harness.server.observability.consume().request_error).toBe(1)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('charges a repeated registration id once and returns one result', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(65))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
|
||||
const response = await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId, registrationId, registrationId],
|
||||
notification: notification()
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(1)
|
||||
const [row] = await harness.database.query('SELECT COUNT(*) AS sends FROM push_events')
|
||||
expect(Number(row?.sends)).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
import { PushNotificationSchema } from '@orca-cloud/push-contract/send-messages'
|
||||
import { PUSH_LIMITS } from '@orca-cloud/push-contract/push-limits'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
import {
|
||||
APNS_TOKEN,
|
||||
createPushServerHarness,
|
||||
FCM_TOKEN,
|
||||
notification
|
||||
} from './push-server-harness.test-fixture.js'
|
||||
|
||||
@@ -20,24 +16,6 @@ describe('push gateway send route', () => {
|
||||
await harness.close()
|
||||
})
|
||||
|
||||
it('rejects a batch over the registration cap', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(16))
|
||||
const oversized = await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: Array.from(
|
||||
{ length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 },
|
||||
(_, index) => `reg-${index}`
|
||||
),
|
||||
notification: notification()
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
expect(oversized.status).toBe(400)
|
||||
expect(await oversized.json()).toEqual({ error: 'invalid_request' })
|
||||
})
|
||||
|
||||
it('queues a send, delivers it to fcm, and reports a dead token on the next send', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(17))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
@@ -55,12 +33,6 @@ describe('push gateway send route', () => {
|
||||
})
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.fcmRequests).toHaveLength(1)
|
||||
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
|
||||
message: {
|
||||
token: FCM_TOKEN,
|
||||
data: { title: 'Agent needs input', tag: expect.stringMatching(/^[a-f0-9]{64}$/) }
|
||||
}
|
||||
})
|
||||
|
||||
const afterDeath = await harness.post(
|
||||
'/v1/send',
|
||||
@@ -68,106 +40,6 @@ describe('push gateway send route', () => {
|
||||
sessionToken
|
||||
)
|
||||
expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] })
|
||||
|
||||
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
|
||||
expect(await listed.json()).toEqual({
|
||||
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }]
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a live registration alone when the provider reports a transient failure', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(24))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
harness.setFcmResponse({
|
||||
status: 503,
|
||||
body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } })
|
||||
})
|
||||
await harness.flushDeliveries()
|
||||
expect(await harness.server.devices.findById(registrationId)).toMatchObject({ dead: false })
|
||||
})
|
||||
|
||||
it('reports retries from the durable worker after the provider delay', async () => {
|
||||
const token = await harness.signIn(createPushHostKeypair(26))
|
||||
const registrationId = await harness.registerAndroid(token)
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId],
|
||||
notification: notification()
|
||||
},
|
||||
token
|
||||
)
|
||||
harness.setFcmResponse({ status: 503, body: '{}' })
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.server.observability.consume()).toMatchObject({
|
||||
delivery_error: 1,
|
||||
delivery_retry: 0
|
||||
})
|
||||
harness.advanceClock(10_000)
|
||||
harness.setFcmResponse({ status: 200, body: '{}' })
|
||||
await harness.server.worker.runDue()
|
||||
expect(harness.server.observability.consume()).toMatchObject({
|
||||
delivery_sent: 1,
|
||||
delivery_retry: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('sends a burst as individual APNs alerts grouped by the host thread', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(18))
|
||||
const registration = await harness.post(
|
||||
'/v1/devices',
|
||||
{
|
||||
v: 1,
|
||||
deviceId: 'iphone-1',
|
||||
platform: 'ios',
|
||||
token: APNS_TOKEN,
|
||||
apnsEnvironment: 'sandbox'
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
const { registrationId } = (await registration.json()) as { registrationId: string }
|
||||
for (const seq of [1, 2, 3]) {
|
||||
await harness.post(
|
||||
'/v1/send',
|
||||
{
|
||||
v: 1,
|
||||
registrationIds: [registrationId],
|
||||
notification: notification({ notificationId: `note-${seq}`, notificationSeq: seq })
|
||||
},
|
||||
sessionToken
|
||||
)
|
||||
}
|
||||
await harness.flushDeliveries()
|
||||
expect(harness.apnsRequests).toHaveLength(3)
|
||||
const bodies = harness.apnsRequests.map(
|
||||
(request) =>
|
||||
JSON.parse(request.body) as {
|
||||
aps: { alert: { title: string; body: string }; 'thread-id': string }
|
||||
orca: Record<string, unknown> & { notificationSeq: number }
|
||||
}
|
||||
)
|
||||
expect(
|
||||
harness.apnsRequests.every((request) => request.host === 'api.sandbox.push.apple.com')
|
||||
).toBe(true)
|
||||
expect(bodies.map((body) => body.aps.alert)).toEqual(
|
||||
Array.from({ length: 3 }, () => ({
|
||||
title: 'Agent needs input',
|
||||
body: 'Waiting on your answer'
|
||||
}))
|
||||
)
|
||||
expect(new Set(bodies.map((body) => body.aps['thread-id'])).size).toBe(1)
|
||||
expect(bodies.map((body) => body.orca.notificationSeq).sort((a, b) => a - b)).toEqual([1, 2, 3])
|
||||
expect(bodies.every((body) => !('coalescedCount' in body.orca))).toBe(true)
|
||||
expect(bodies.every((body) => !('summaryMembers' in body.orca))).toBe(true)
|
||||
expect(
|
||||
new Set(harness.apnsRequests.map((request) => request.headers['apns-collapse-id'])).size
|
||||
).toBe(3)
|
||||
})
|
||||
|
||||
it('reports an error for a registration the host does not own', async () => {
|
||||
@@ -188,29 +60,4 @@ describe('push gateway send route', () => {
|
||||
})
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(0)
|
||||
})
|
||||
|
||||
it('rate limits a host that exhausted its 15-minute allowance', async () => {
|
||||
const sessionToken = await harness.signIn(createPushHostKeypair(21))
|
||||
const registrationId = await harness.registerAndroid(sessionToken)
|
||||
const hostFingerprint = (await harness.server.devices.findById(registrationId))!.hostFingerprint
|
||||
for (let index = 0; index < PUSH_LIMITS.hostEventsPerWindow; index++) {
|
||||
expect(
|
||||
await harness.server.deliveryStore.accept(
|
||||
hostFingerprint,
|
||||
registrationId,
|
||||
PushNotificationSchema.parse(
|
||||
notification({ notificationId: `note-${index + 1000}`, notificationSeq: index + 1000 })
|
||||
)
|
||||
)
|
||||
).toBe('queued')
|
||||
}
|
||||
const limited = await harness.post(
|
||||
'/v1/send',
|
||||
{ v: 1, registrationIds: [registrationId], notification: notification() },
|
||||
sessionToken
|
||||
)
|
||||
expect(limited.status).toBe(200)
|
||||
expect(await limited.json()).toEqual({ results: [{ registrationId, status: 'rate_limited' }] })
|
||||
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(300)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { startPushBackground } from '../push-background.js'
|
||||
import { createPushHostKeypair } from '../host-auth/host-challenge-answering.test-fixture.js'
|
||||
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
|
||||
|
||||
const cleanups: (() => Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0)) {
|
||||
await cleanup()
|
||||
}
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
async function fixture() {
|
||||
const h = await createPushServerHarness()
|
||||
const token = await h.signIn(createPushHostKeypair(31))
|
||||
const registrationId = await h.registerAndroid(token)
|
||||
let stop = async () => {}
|
||||
cleanups.push(async () => {
|
||||
await stop()
|
||||
await h.close()
|
||||
})
|
||||
vi.useFakeTimers()
|
||||
return {
|
||||
...h,
|
||||
registrationId,
|
||||
start: () => {
|
||||
stop = startPushBackground({ mode: 'active' }, h.server)
|
||||
},
|
||||
send: (event = notification(), ids = [registrationId]) =>
|
||||
h.post('/v1/send', { v: 1, registrationIds: ids, notification: event }, token),
|
||||
invalid: () => h.post('/v1/send', { v: 1 }, token)
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps concurrent accepted retries idempotent while automatic delivery runs', async () => {
|
||||
const h = await fixture()
|
||||
h.start()
|
||||
await Promise.all(Array.from({ length: 10 }, () => h.send()))
|
||||
await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1))
|
||||
await h.send()
|
||||
await h.server.worker.runDue()
|
||||
expect(h.fcmRequests).toHaveLength(1)
|
||||
expect(await h.server.deliveryStore.pendingCount(h.registrationId)).toBe(0)
|
||||
})
|
||||
@@ -36,7 +36,6 @@ it('delivers again after a topic error without re-registering the phone', async
|
||||
expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
|
||||
await h.flushDeliveries()
|
||||
if (seq === 1) {
|
||||
expect(h.server.observability.consume().delivery_error).toBe(1)
|
||||
h.setApnsResponse({ status: 200, body: '' })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,11 +67,10 @@ describe('fcm client', () => {
|
||||
error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' }
|
||||
})
|
||||
})
|
||||
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
|
||||
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toMatchObject({
|
||||
status: 'error',
|
||||
reason: 'INVALID_ARGUMENT',
|
||||
retryable: false,
|
||||
retryAfterMs: 10000
|
||||
retryable: false
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,8 +5,8 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs
|
||||
import { SignJWT, exportJWK, generateKeyPair, importJWK, type JWK } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/relay-transport'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// client wrapper, the echo service, phone sockets and workerd memory sampling.
|
||||
import { execSync } from 'node:child_process'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/relay-transport'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const PORT = 8787
|
||||
export const ORIGIN = `http://127.0.0.1:${PORT}`
|
||||
|
||||
@@ -39,7 +39,7 @@ export async function runLifecycleChecks(ctx: Ctx, main: Control, mainAck: Ack):
|
||||
mute.on('close', (c) => (muteClose = c))
|
||||
await new Promise((res) => mute.on('open', res))
|
||||
// Complete the handshake by hand with the shipped proof answerer, then never pong.
|
||||
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/relay-host-proof')
|
||||
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-host-proof')
|
||||
mute.send(JSON.stringify({ type: 'host-hello', v: 1, relayHostId: ctx.relayHostId, assignmentEpoch: 7,
|
||||
hostPublicKeyB64: ctx.keypair.publicKeyB64, appVersion: 'spike' }))
|
||||
mute.on('message', (raw) => {
|
||||
|
||||
@@ -12,7 +12,7 @@ const ERASER = 'https://erasure.test.invalid'
|
||||
const BODY = JSON.stringify({ relayHostId: 'AAAAAAAAAAAAAAAA' })
|
||||
const JTI = 'jti-0123456789abcdef'
|
||||
|
||||
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string; iat?: number; exp?: number }
|
||||
type Sign = { key?: string; issuer?: string; op?: AdminOp; jti?: unknown; body?: string }
|
||||
|
||||
async function sign(opts: Sign = {}): Promise<string> {
|
||||
const claims: Record<string, unknown> = {
|
||||
@@ -26,8 +26,8 @@ async function sign(opts: Sign = {}): Promise<string> {
|
||||
.setProtectedHeader({ alg: 'HS256' })
|
||||
.setIssuer(opts.issuer ?? ISSUER)
|
||||
.setAudience('orca-relay-edge-admin')
|
||||
.setIssuedAt(opts.iat ?? now)
|
||||
.setExpirationTime(opts.exp ?? now + 60)
|
||||
.setIssuedAt(now)
|
||||
.setExpirationTime(now + 60)
|
||||
.sign(new TextEncoder().encode(opts.key ?? KEY))
|
||||
}
|
||||
|
||||
@@ -45,24 +45,19 @@ const baseEnv = { ADMIN_SIGNING_KEY: KEY, ADMIN_ISSUER: ISSUER } as Env
|
||||
|
||||
describe('admin token verify (jti)', () => {
|
||||
|
||||
it('still binds op, body, key and age', async () => {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
it('binds the operation and exact body', async () => {
|
||||
expect(await verifyWith(await sign({ jti: JTI, op: 'erase' }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, body: '{}' }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, key: NEXT }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
expect(await verifyWith(await sign({ jti: JTI, iat: now - 180, exp: now + 60 }), KEY, ISSUER, 'drain', raw())).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('admin request verify (keys)', () => {
|
||||
|
||||
it('accepts the primary or the NEXT key, nothing else', async () => {
|
||||
it('accepts the primary key and only a configured NEXT key', async () => {
|
||||
const env = { ...baseEnv, ADMIN_SIGNING_KEY_NEXT: NEXT } as Env
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI })), 'drain')).not.toBeNull()
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: NEXT })), 'drain')).not.toBeNull()
|
||||
expect(await verifyAdminRequest(baseEnv, request(await sign({ jti: JTI, key: NEXT })), 'drain')).toBeNull()
|
||||
const wrong = randomBytes(36).toString('base64url')
|
||||
expect(await verifyAdminRequest(env, request(await sign({ jti: JTI, key: wrong })), 'drain')).toBeNull()
|
||||
})
|
||||
|
||||
it('the erasure keys (primary or NEXT) verify only erase, under their own issuer', async () => {
|
||||
|
||||
@@ -5,8 +5,8 @@ import { readFileSync } from 'node:fs'
|
||||
import { SignJWT, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/relay-transport'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
|
||||
export const PORT = 8789
|
||||
export const REMOTE = process.env.MVP_URL?.replace(/\/$/, '')
|
||||
|
||||
@@ -8,8 +8,8 @@ import { join } from 'node:path'
|
||||
import { SignJWT, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/relay-transport'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
|
||||
|
||||
export const DIRECTOR_APP = process.env.DIRECTOR_APP ?? ''
|
||||
|
||||
@@ -28,7 +28,7 @@ const mint = (key: CryptoKey = privateKey, kid = 'k1') =>
|
||||
.sign(key)
|
||||
|
||||
// A Cache API stand-in shared by every "isolate" (module instance) in the test.
|
||||
function stubCache(): Map<string, Response> {
|
||||
function stubCache(): void {
|
||||
const store = new Map<string, Response>()
|
||||
vi.stubGlobal('caches', {
|
||||
default: {
|
||||
@@ -38,7 +38,6 @@ function stubCache(): Map<string, Response> {
|
||||
match: async (request: Request) => store.get(request.url)?.clone()
|
||||
}
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
// Each call is a fresh isolate: a new module instance with an empty in-memory key cache.
|
||||
@@ -64,24 +63,21 @@ afterEach(() => {
|
||||
|
||||
describe('relay JWT key availability', () => {
|
||||
|
||||
it('still says invalid for a bad token when keys are available', async () => {
|
||||
it('rejects an unknown key when the key set is available', async () => {
|
||||
stubCache()
|
||||
authUp(() => true)
|
||||
const verify = await coldVerifier()
|
||||
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
|
||||
expect(await verify(env, await mint(otherKey))).toEqual({ ok: false, reason: 'invalid' })
|
||||
expect(await verify(env, 'not-a-jwt')).toEqual({ ok: false, reason: 'invalid' })
|
||||
expect(await verify(env, await mint(privateKey, 'unknown-kid'))).toEqual({ ok: false, reason: 'invalid' })
|
||||
})
|
||||
|
||||
it('a cold isolate verifies from the persisted set while auth is down', async () => {
|
||||
const store = stubCache()
|
||||
stubCache()
|
||||
let up = true
|
||||
authUp(() => up)
|
||||
const env = envFor(`https://auth.test.invalid/jwks/${++urlSeq}`)
|
||||
const warm = await coldVerifier()
|
||||
expect((await warm(env, await mint())).ok).toBe(true)
|
||||
expect(store.size).toBe(1)
|
||||
up = false
|
||||
const cold = await coldVerifier()
|
||||
const result = await cold(env, await mint())
|
||||
|
||||
@@ -15,8 +15,8 @@ import { SignJWT, exportJWK, generateKeyPair, type KeyLike } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import pg from 'pg'
|
||||
import WebSocket from 'ws'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/relay-transport'
|
||||
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
|
||||
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
|
||||
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
|
||||
|
||||
const PORT = 8788
|
||||
|
||||
@@ -10,4 +10,3 @@ vi.mock('../../relay/src/database/database.js', async (importOriginal) => ({
|
||||
}))
|
||||
|
||||
await import('../../relay/src/credentials/credential-store.test.ts')
|
||||
await import('../../relay/src/credentials/credential-store-cleanup.test.ts')
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { ASSIGNMENT_CONNECTION_HEADROOM_QUERY } from './assignment-connection-headroom-query.js'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
@@ -9,7 +7,6 @@ import {
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const headroomIndexName = 'relay_control_connection_reservation_headroom'
|
||||
const cell = {
|
||||
id: 'connection-headroom-postgres',
|
||||
url: 'https://connection-headroom-postgres.example.com',
|
||||
@@ -160,39 +157,6 @@ describePostgres('PostgreSQL assignment connection headroom', () => {
|
||||
expect(Number(reservations[0]!.count)).toBe(1)
|
||||
}, 15_000)
|
||||
|
||||
it('uses the composite headroom index when released history dominates', async () => {
|
||||
await databases[0]!.query(
|
||||
`INSERT INTO relay_control_connection_reservations
|
||||
(reservation_id, idempotency_key, user_id, relay_host_id,
|
||||
assignment_epoch, cell_id, state, created_at, timeout_at, updated_at)
|
||||
SELECT 'headroom-plan-' || value, 'headroom-plan-' || value,
|
||||
'connection-headroom-postgres-plan', 'plan-host-' || value,
|
||||
1, ?, 'released', 1, 2, 1
|
||||
FROM generate_series(1, 50000) AS value`,
|
||||
[cell.id]
|
||||
)
|
||||
await databases[0]!.query(`ANALYZE relay_control_connection_reservations`)
|
||||
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
let reservationIndexPlan: Record<string, unknown> | undefined
|
||||
try {
|
||||
const plan = await client.query(
|
||||
`EXPLAIN (FORMAT JSON) ${ASSIGNMENT_CONNECTION_HEADROOM_QUERY}`
|
||||
)
|
||||
reservationIndexPlan = findReservationHeadroomIndexPlan(
|
||||
plan.rows[0]?.['QUERY PLAN']
|
||||
)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
|
||||
expect(reservationIndexPlan).toBeDefined()
|
||||
expect(String(reservationIndexPlan?.['Index Cond'])).toContain('cell_id')
|
||||
expect(String(reservationIndexPlan?.['Index Cond'])).toContain('state = ANY')
|
||||
expect(reservationIndexPlan?.['Filter']).toBeUndefined()
|
||||
})
|
||||
|
||||
it('deduplicates expired debt after a fresh PostgreSQL snapshot', async () => {
|
||||
const now = 200
|
||||
const store = new RelayAssignmentStore(databases[0]!, () => now, {
|
||||
@@ -337,16 +301,3 @@ describePostgres('PostgreSQL assignment connection headroom', () => {
|
||||
).toEqual([{ state: 'released' }])
|
||||
})
|
||||
})
|
||||
|
||||
function findReservationHeadroomIndexPlan(
|
||||
value: unknown
|
||||
): Record<string, unknown> | undefined {
|
||||
if (value === null || typeof value !== 'object') return undefined
|
||||
const record = value as Record<string, unknown>
|
||||
if (!Array.isArray(value) && record['Index Name'] === headroomIndexName) return record
|
||||
for (const child of Object.values(value)) {
|
||||
const match = findReservationHeadroomIndexPlan(child)
|
||||
if (match) return match
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const cell = {
|
||||
id: 'control-supersession-postgres',
|
||||
url: 'https://control-supersession-postgres.example.com',
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
const identity = {
|
||||
userId: 'control-supersession-postgres-user',
|
||||
relayHostId: 'supersedehost001'
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL control supersession', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
databases.push(
|
||||
await openRelayDatabase({ databaseUrl, dataDir: '' }),
|
||||
await openRelayDatabase({ databaseUrl, dataDir: '' })
|
||||
)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
const database = databases[0]
|
||||
if (database) {
|
||||
await database.query(
|
||||
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
|
||||
// A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
|
||||
await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
|
||||
cell.id
|
||||
])
|
||||
await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
|
||||
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id])
|
||||
}
|
||||
for (const connection of databases) await connection.close()
|
||||
})
|
||||
|
||||
it('serializes parallel generations into one durable control', async () => {
|
||||
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
|
||||
await stores[0]!.reconcileCells([cell])
|
||||
await stores[0]!.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionInclusionWatermark: 1,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 50
|
||||
})
|
||||
const assignment = await stores[0]!.assign(identity)
|
||||
|
||||
await Promise.all([
|
||||
stores[0]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1,
|
||||
connectionInclusionWatermark: 10
|
||||
}),
|
||||
stores[1]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 2,
|
||||
connectionInclusionWatermark: 11
|
||||
})
|
||||
])
|
||||
await databases[0]!.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
|
||||
request_units, expires_at, updated_at)
|
||||
VALUES (?, ?, ?, 'control', ?, 1, 90100, 100),
|
||||
(?, ?, ?, 'control', ?, 1, 90100, 100)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`control:${cell.id}:100`,
|
||||
cell.id,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`control:${cell.id}:101`,
|
||||
cell.id
|
||||
]
|
||||
)
|
||||
// The store reserves a unit per control lease, so a hand-written pair has to
|
||||
// carry its own reservation or the fixture starts out of balance.
|
||||
await databases[0]!.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`,
|
||||
[cell.id]
|
||||
)
|
||||
await stores[0]!.activateControl(identity, {
|
||||
cellId: cell.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 102,
|
||||
connectionInclusionWatermark: 12
|
||||
})
|
||||
|
||||
const controls = await databases[0]!.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND activity_kind = 'control'`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(Number(controls[0]!.count)).toBe(1)
|
||||
const assignments = await databases[0]!.query(
|
||||
`SELECT reserved_controls FROM relay_assignments WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(Number(assignments[0]!.reserved_controls)).toBe(1)
|
||||
const cells = await databases[0]!.query(
|
||||
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
|
||||
[cell.id]
|
||||
)
|
||||
expect(Number(cells[0]!.reserved_requests)).toBe(1)
|
||||
const claims = await databases[0]!.query(
|
||||
`SELECT claim_activity_id FROM relay_control_connection_reservations
|
||||
WHERE user_id = ? AND state = 'claimed'`,
|
||||
[identity.userId]
|
||||
)
|
||||
expect(claims).toEqual([{ claim_activity_id: `control:${cell.id}:102` }])
|
||||
}, 15_000)
|
||||
})
|
||||
@@ -157,14 +157,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
|
||||
expect(await reservedOn(dead)).toEqual(reservedBefore)
|
||||
})
|
||||
|
||||
it('hard crash with no dead-man trip: a ledger-only row moves too', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 7)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
await killDeadCell()
|
||||
expect(await hostAssign(host)).toMatchObject({ cellId: live.id, assignmentEpoch: 8 })
|
||||
})
|
||||
|
||||
it('a seat ahead of the row: the move mints above it, and the revived cell\'s seat never wins the row back', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 4)
|
||||
@@ -177,7 +169,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
|
||||
const ledger = new ReserveLedgerWriter(database, { now: () => now, log: () => undefined })
|
||||
for (const epoch of [6, 7]) ledger.enqueue({ userId: USER, relayHostId: host, cellId: dead.id, epoch })
|
||||
await ledger.flush()
|
||||
expect(ledger.pending()).toBe(0)
|
||||
expect(await rowOf(host)).toEqual({ cell_id: live.id, assignment_epoch: 7 })
|
||||
})
|
||||
|
||||
@@ -189,63 +180,6 @@ describePostgres('a host leaving a dead reserve cell, on PostgreSQL', () => {
|
||||
await expect(hostAssign(host, true, null)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
})
|
||||
|
||||
it('a withheld verdict (fleet or region guard) keeps the fence', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
await killDeadCell()
|
||||
store = new RelayAssignmentStore(database, () => now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: HEARTBEAT_TTL_MS,
|
||||
deadCellVerdict: () => ({
|
||||
dead: false,
|
||||
withheld: 'region',
|
||||
fleetFailing: 6,
|
||||
fleetPolled: 34,
|
||||
regionFailing: 6,
|
||||
regionPolled: 6
|
||||
})
|
||||
})
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
})
|
||||
|
||||
// Only a heartbeat that stopped (cause 'unheard') is death; polls failing for 45 s add nothing else.
|
||||
it('a cell heartbeating but not ready keeps its hosts, however long its polls fail', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
now += HEARTBEAT_TTL_MS + 15_000
|
||||
await heartbeat(dead, false)
|
||||
await heartbeat(live)
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
})
|
||||
|
||||
it('drain then death keeps the fence: a disabled cell is draining, not unheard', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
await database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [dead.id])
|
||||
try {
|
||||
await killDeadCell()
|
||||
await expect(hostAssign(host)).rejects.toThrow('relay_home_cell_unavailable')
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
} finally {
|
||||
await database.query(`UPDATE relay_cells SET enabled = 1 WHERE cell_id = ?`, [dead.id])
|
||||
}
|
||||
})
|
||||
|
||||
it('LB drain: heartbeat fresh while polls fail, so the host is pinned to its cell', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
await store.setCellAdmitMode(dead.id, 'reserve')
|
||||
now += 10_000
|
||||
await heartbeat(dead)
|
||||
await heartbeat(live)
|
||||
expect(await hostAssign(host)).toMatchObject({ cellId: dead.id, assignmentEpoch: 3 })
|
||||
expect(await rowOf(host)).toEqual({ cell_id: dead.id, assignment_epoch: 3 })
|
||||
})
|
||||
|
||||
it('DB outage or director freeze: heartbeat stale but polls answered (no verdict), so 503', async () => {
|
||||
const host = nextHost()
|
||||
await ledgerRow(host, 3)
|
||||
|
||||
@@ -84,27 +84,4 @@ describePostgres('PostgreSQL deployment status', () => {
|
||||
restartBlockingReservedRequests: -1
|
||||
})
|
||||
})
|
||||
|
||||
it('reads the same counts through the reserve-aware status, and none for a reserve-mode cell', async () => {
|
||||
const store = new RelayAssignmentStore(database!, () => 100)
|
||||
const reserveCell = { ...cell, id: `${cell.id}-reserve`, url: 'https://deployment-status-reserve.example.com' }
|
||||
await store.reconcileCells([cell, reserveCell])
|
||||
await store.assign({ userId: 'aware-user', relayHostId: 'd1e2f3a4b5c6' })
|
||||
expect(await store.reserveAwareCellDeploymentStatus(cell.id)).toEqual({
|
||||
admitMode: 'db',
|
||||
status: await store.cellDeploymentStatus(cell.id)
|
||||
})
|
||||
|
||||
await store.setCellAdmitMode(reserveCell.id, 'reserve')
|
||||
await expect(store.cellDeploymentStatus(reserveCell.id)).rejects.toThrow('cell_in_reserve_mode')
|
||||
const reserve = await store.reserveAwareCellDeploymentStatus(reserveCell.id)
|
||||
expect(reserve).toMatchObject({
|
||||
admitMode: 'reserve',
|
||||
status: { cellId: reserveCell.id, cellUrl: reserveCell.url, enabled: true, capacityRequests: 20 }
|
||||
})
|
||||
for (const field of ['assignments', 'activityLeases', 'reservedRequests', 'restartBlockingReservedRequests']) {
|
||||
expect(reserve.status).not.toHaveProperty(field)
|
||||
}
|
||||
await expect(store.reserveAwareCellDeploymentStatus('missing-cell')).rejects.toThrow('cell_not_found')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,42 +3,27 @@ import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const NOW = 100_000
|
||||
const CELL_COUNT = 32
|
||||
const identity = { userId: 'headroom-user', relayHostId: 'headroomhost0001' }
|
||||
|
||||
function observeHeadroom(database: RelayDatabase, rowCounts: number[]): RelayDatabase {
|
||||
return {
|
||||
dialect: database.dialect,
|
||||
async query(sql, params) {
|
||||
const rows = await database.query(sql, params)
|
||||
if (sql.includes('FROM relay_cell_connection_limits limits')) rowCounts.push(rows.length)
|
||||
return rows
|
||||
},
|
||||
queryLocked: (sql, params, options) => database.queryLocked(sql, params, options),
|
||||
transaction: (operation, options) =>
|
||||
database.transaction((transaction) => operation(observeHeadroom(transaction, rowCounts)), options),
|
||||
close: () => database.close()
|
||||
}
|
||||
}
|
||||
|
||||
describe('assignment headroom query scope', () => {
|
||||
describe('assignment headroom admission', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
async function setup() {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
const rowCounts: number[] = []
|
||||
const store = new RelayAssignmentStore(observeHeadroom(database, rowCounts), () => NOW, {
|
||||
const store = new RelayAssignmentStore(database, () => NOW, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
const cells = Array.from({ length: CELL_COUNT }, (_, index) => ({
|
||||
id: `cell-${String(index).padStart(2, '0')}`,
|
||||
url: `https://cell-${index}.example.com`,
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}))
|
||||
const cells = [
|
||||
{
|
||||
id: 'cell-00',
|
||||
url: 'https://cell-0.example.com',
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
]
|
||||
await store.reconcileCells(cells)
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
@@ -69,29 +54,9 @@ describe('assignment headroom query scope', () => {
|
||||
kind: 'splice',
|
||||
cellId: 'cell-00'
|
||||
})
|
||||
rowCounts.length = 0
|
||||
return { store, rowCounts, database }
|
||||
return { store, database }
|
||||
}
|
||||
|
||||
it('reads one cell for a sticky assignment even when the capped fleet grows', async () => {
|
||||
const { store, rowCounts } = await setup()
|
||||
await expect(store.assign(identity)).resolves.toMatchObject({
|
||||
...identity,
|
||||
cellId: 'cell-00',
|
||||
assignmentEpoch: 1
|
||||
})
|
||||
console.info('sticky headroom rows', { fleetCells: CELL_COUNT, rowCounts })
|
||||
expect(rowCounts).toEqual([1])
|
||||
})
|
||||
|
||||
it('still considers the full fleet for a new placement', async () => {
|
||||
const { store, rowCounts } = await setup()
|
||||
await expect(
|
||||
store.assign({ userId: 'new-user', relayHostId: 'newheadroomhost1' })
|
||||
).resolves.toMatchObject({ cellId: 'cell-01', assignmentEpoch: 1 })
|
||||
expect(rowCounts).toEqual([CELL_COUNT])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['missing snapshot', `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = 'cell-00'`],
|
||||
['expired snapshot', `UPDATE relay_cell_connection_snapshots SET snapshot_at = ${NOW - 45_000} WHERE cell_id = 'cell-00'`],
|
||||
@@ -104,10 +69,9 @@ describe('assignment headroom query scope', () => {
|
||||
})
|
||||
|
||||
it('preserves admission for cells without a connection limit', async () => {
|
||||
const { store, rowCounts, database } = await setup()
|
||||
const { store, database } = await setup()
|
||||
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = 'cell-00'`)
|
||||
await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'cell-00' })
|
||||
expect(rowCounts).toEqual([0])
|
||||
})
|
||||
|
||||
it('counts outstanding reservations at the admission boundary', async () => {
|
||||
|
||||
@@ -106,15 +106,6 @@ describe('home cell unavailable', () => {
|
||||
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
|
||||
})
|
||||
|
||||
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
|
||||
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
|
||||
await store.assign(IDENTITY)
|
||||
|
||||
await expect(
|
||||
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
|
||||
).rejects.toThrow('relay_capacity_exhausted')
|
||||
})
|
||||
|
||||
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
|
||||
const home: RelayCellConfig = {
|
||||
id: 'home',
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
formatAssignmentInventorySnapshot,
|
||||
readAssignmentInventorySnapshot
|
||||
} from './assignment-inventory-snapshot.js'
|
||||
import { openInMemoryRelayDatabase } from '../database/database.js'
|
||||
|
||||
describe('assignment inventory snapshot', () => {
|
||||
it('reports per-cell counters, lease backlog, and reservation debt', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const now = 1_000_000
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 3999, 5, ?, ?)`,
|
||||
[now, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-a', 'general', ?)`,
|
||||
[now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_runtime
|
||||
(cell_id, cell_url, cell_incarnation, started_at, ready, observed_requests,
|
||||
last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 'inc-1', ?, 1, 5, ?, ?)`,
|
||||
[now - 60_000, now - 10_000, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units,
|
||||
expires_at, updated_at)
|
||||
VALUES
|
||||
('user-1', 'host-1', 'control:1', 'control', 'cell-a', 1, ?, ?),
|
||||
('user-1', 'host-2', 'control:1', 'control', 'cell-a', 3, ?, ?)`,
|
||||
[now - 1, now, now + 90_000, now]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_control_connection_reservations
|
||||
(reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch,
|
||||
cell_id, state, created_at, timeout_at, updated_at)
|
||||
VALUES
|
||||
('r1', 'k1', 'user-1', 'host-1', 1, 'cell-a', 'late-arrival-debt', ?, ?, ?),
|
||||
('r2', 'k2', 'user-1', 'host-2', 1, 'cell-a', 'reserved', ?, ?, ?),
|
||||
('r3', 'k3', 'user-1', 'host-3', 1, 'cell-a', 'claimed', ?, ?, ?),
|
||||
('r4', 'k4', 'user-1', 'host-4', 1, 'cell-a', 'released', ?, ?, ?)`,
|
||||
[now, now, now, now, now, now, now, now, now, now, now, now]
|
||||
)
|
||||
|
||||
const snapshot = await readAssignmentInventorySnapshot(database, now)
|
||||
|
||||
expect(snapshot.cells).toEqual([
|
||||
{
|
||||
cellId: 'cell-a',
|
||||
region: 'us-central1',
|
||||
admissionState: 'general',
|
||||
enabled: true,
|
||||
capacityRequests: 4000,
|
||||
reservedRequests: 3999,
|
||||
runtimeReady: true,
|
||||
heartbeatAgeMs: 10_000
|
||||
}
|
||||
])
|
||||
expect(snapshot.activityLeases).toEqual({ total: 2, expired: 1, requestUnits: 4 })
|
||||
expect(snapshot.connectionReservations).toEqual({ outstanding: 3, lateArrivalDebt: 1 })
|
||||
expect(snapshot.regionalRehomes).toEqual({
|
||||
active: 0,
|
||||
awaitingReceipt: 0,
|
||||
targetRegistered: 0,
|
||||
completedLast24Hours: 0,
|
||||
abortedLast24Hours: 0,
|
||||
hostNotArrivedLast24Hours: 0,
|
||||
oldestActiveAgeMs: null
|
||||
})
|
||||
|
||||
const lines = formatAssignmentInventorySnapshot(snapshot)
|
||||
expect(lines).toHaveLength(3)
|
||||
expect(lines[0]).toContain('cellId=cell-a')
|
||||
expect(lines[0]).toContain('reserved=3999')
|
||||
expect(lines[1]).toContain('expiredLeases=1')
|
||||
expect(lines[1]).toContain('lateArrivalDebt=1')
|
||||
expect(lines[2]).toContain('active=0')
|
||||
})
|
||||
|
||||
it('reports cells missing runtime and admission rows without failing', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 0)`
|
||||
)
|
||||
|
||||
const snapshot = await readAssignmentInventorySnapshot(database, 5_000)
|
||||
|
||||
expect(snapshot.cells[0]).toMatchObject({
|
||||
cellId: 'cell-b',
|
||||
admissionState: 'unset',
|
||||
enabled: false,
|
||||
runtimeReady: null,
|
||||
heartbeatAgeMs: null
|
||||
})
|
||||
expect(snapshot.activityLeases).toEqual({ total: 0, expired: 0, requestUnits: 0 })
|
||||
expect(snapshot.regionalRehomes.active).toBe(0)
|
||||
})
|
||||
})
|
||||
+1
-4
@@ -302,7 +302,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
|
||||
).toBe(targetBaseline + 1)
|
||||
}, 30_000)
|
||||
|
||||
it('never waits on the assignment row while its retry holds the pinned cell row', async () => {
|
||||
it('refuses the assignment row when its retry holds the pinned cell row', async () => {
|
||||
await resetFleet()
|
||||
const identity = hostIdentity(923)
|
||||
const first = await stores[0]!.assign(identity, 'us-central1')
|
||||
@@ -324,10 +324,7 @@ describePostgres('PostgreSQL re-placement off a cell isolated for a roll', () =>
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
await releaseCell()
|
||||
const cellFreedAt = performance.now()
|
||||
expect(await regrant).toBeInstanceOf(RelayAssignmentRowBusyError)
|
||||
// NOWAIT once the cell row is held: nowhere near the 1s bounded wait.
|
||||
expect(performance.now() - cellFreedAt).toBeLessThan(300)
|
||||
} finally {
|
||||
await releaseAssignment?.()
|
||||
await releaseCell()
|
||||
|
||||
@@ -1,22 +1,10 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract/close-codes'
|
||||
import { SignJWT } from 'jose'
|
||||
import nacl from 'tweetnacl'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
ASSIGNMENT_LEASE_AUDIENCE,
|
||||
assignmentLeaseKeyId,
|
||||
classifyAssignmentLease
|
||||
} from './assignment-lease.js'
|
||||
import { AssignmentLeaseShadow } from './assignment-lease-shadow.js'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from '../config.js'
|
||||
import type { RelayCredentialStore } from '../credentials/credential-store.js'
|
||||
import { HostSessionRegistry } from '../connections/host-session-registry.js'
|
||||
import type { RelayTokenClaims } from '../authentication/relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from '../connections/splice-forwarder.js'
|
||||
|
||||
const KEY = new TextEncoder().encode('0123456789abcdef0123456789abcdef')
|
||||
const OTHER_KEY = new TextEncoder().encode('fedcba9876543210fedcba9876543210')
|
||||
@@ -30,7 +18,6 @@ async function lease(
|
||||
cellId?: string
|
||||
assignmentEpoch?: number
|
||||
sub?: string
|
||||
relayHostId?: string
|
||||
expiresAt?: string | number
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
@@ -40,7 +27,7 @@ async function lease(
|
||||
cellId: overrides.cellId ?? CELL_ID,
|
||||
cellUrl: 'https://c7.relay.example.test',
|
||||
assignmentEpoch: overrides.assignmentEpoch ?? 4,
|
||||
relayHostId: overrides.relayHostId ?? HOST.relayHostId
|
||||
relayHostId: HOST.relayHostId
|
||||
})
|
||||
.setProtectedHeader(kid === null ? { alg: 'HS256' } : { alg: 'HS256', kid })
|
||||
.setIssuer('https://relay.example.test')
|
||||
@@ -73,7 +60,6 @@ describe('assignment lease classification', () => {
|
||||
expect(await classify(await lease({ kid: assignmentLeaseKeyId(OTHER_KEY) }))).toBe(
|
||||
'bad-signature'
|
||||
)
|
||||
expect(await classify('not.a.jwt')).toBe('bad-signature')
|
||||
expect(await classify('x'.repeat(5_000))).toBe('bad-signature')
|
||||
expect(await classify(await lease({ sub: 'user-2' }))).toBe('wrong-host')
|
||||
expect(await classify(await lease({ cellId: 'production-gce-c8' }))).toBe('wrong-cell')
|
||||
@@ -81,127 +67,3 @@ describe('assignment lease classification', () => {
|
||||
expect(await classify(await lease({ assignmentEpoch: 5 }))).toBe('epoch-ahead')
|
||||
})
|
||||
})
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(''))
|
||||
})
|
||||
}
|
||||
|
||||
describe('host hello with the lease shadow on', () => {
|
||||
async function hello(input: { withLease: boolean; databaseSays: boolean }) {
|
||||
const keyPair = nacl.box.keyPair()
|
||||
const relayHostId = createHash('sha256')
|
||||
.update(keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
const identity: RelayTokenClaims = {
|
||||
sub: HOST.userId,
|
||||
prof: 'profile-1',
|
||||
relayHostId,
|
||||
purpose: 'host-control',
|
||||
exp: 4_102_444_800
|
||||
}
|
||||
const recorded: string[] = []
|
||||
const shadow = new AssignmentLeaseShadow({ enabled: () => true, key: KEY, cellId: CELL_ID })
|
||||
const record = shadow.record.bind(shadow)
|
||||
vi.spyOn(shadow, 'record').mockImplementation((check, dbValid, dbReadMs) => {
|
||||
void check.then((leaseClass) => recorded.push(`${leaseClass}:${dbValid}`))
|
||||
record(check, dbValid, dbReadMs)
|
||||
})
|
||||
const verifyCellAssignment = vi.fn(async () => input.databaseSays)
|
||||
const registry = new HostSessionRegistry(
|
||||
config(),
|
||||
vi.fn(),
|
||||
{} as RelayCredentialStore,
|
||||
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
|
||||
new ProcessQueuedByteBudget(),
|
||||
{
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn()
|
||||
},
|
||||
Date.now,
|
||||
Math.random,
|
||||
'incarnation-1',
|
||||
shadow
|
||||
)
|
||||
const socket = new FakeSocket()
|
||||
const hostLease = input.withLease ? await lease({ relayHostId }) : undefined
|
||||
registry.acceptControl(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
undefined,
|
||||
undefined,
|
||||
hostLease
|
||||
)
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId,
|
||||
assignmentEpoch: 4,
|
||||
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
|
||||
appVersion: '1.4.200'
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
await vi.waitFor(() => expect(recorded).toHaveLength(1))
|
||||
return { socket, recorded, verifyCellAssignment }
|
||||
}
|
||||
|
||||
it('still refuses a host the database refuses, whatever its lease says', async () => {
|
||||
const { socket, recorded } = await hello({ withLease: true, databaseSays: false })
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
|
||||
expect(recorded).toEqual(['valid:false'])
|
||||
})
|
||||
|
||||
it('still admits a host the database admits, with no lease at all', async () => {
|
||||
const { socket, recorded, verifyCellAssignment } = await hello({
|
||||
withLease: false,
|
||||
databaseSays: true
|
||||
})
|
||||
expect(verifyCellAssignment).toHaveBeenCalledTimes(1)
|
||||
expect(socket.close).not.toHaveBeenCalled()
|
||||
expect(String(socket.send.mock.calls[0]?.[0])).toContain('"type":"host-challenge"')
|
||||
expect(recorded).toEqual(['absent:true'])
|
||||
})
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://c7.relay.example.test',
|
||||
cellUrl: 'https://c7.relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: KEY,
|
||||
role: 'cell',
|
||||
cellId: CELL_ID,
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const CELL = { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
|
||||
const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
|
||||
describe('assign marks only the answers it minted as placed', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
async function setup(now: () => number): Promise<RelayAssignmentStore> {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayAssignmentStore(database, now)
|
||||
await store.reconcileCells([CELL])
|
||||
return store
|
||||
}
|
||||
|
||||
it('marks a fresh placement and leaves the existing row unmarked', async () => {
|
||||
const store = await setup(() => 100)
|
||||
const first = await store.assign(IDENTITY)
|
||||
const again = await store.assign(IDENTITY)
|
||||
|
||||
expect(first.placed).toBe(true)
|
||||
expect(again).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
|
||||
expect(again.placed).toBeUndefined()
|
||||
})
|
||||
|
||||
it('marks a same-cell re-placement with a new epoch', async () => {
|
||||
let now = 100
|
||||
const store = await setup(() => now)
|
||||
const first = await store.assign(IDENTITY)
|
||||
await store.changeActivity(IDENTITY, 'control', -1)
|
||||
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.releaseExpiredActivity()
|
||||
now += ASSIGNMENT_LIMITS.dormantTtlMs
|
||||
const replaced = await store.assign(IDENTITY)
|
||||
|
||||
expect(replaced).toMatchObject({
|
||||
cellId: first.cellId,
|
||||
assignmentEpoch: first.assignmentEpoch + 1
|
||||
})
|
||||
expect(replaced.placed).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -36,28 +36,8 @@ describe('assignment rejection log window', () => {
|
||||
|
||||
expect(window.admit('assign:placement:host-rate-limited', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:sticky:host-rate-limited', 'host-a')).toBe(true)
|
||||
expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(false)
|
||||
})
|
||||
|
||||
it('bounds the tracked keys and reports what an evicted window suppressed', () => {
|
||||
const timers = manualTimers()
|
||||
const closed: { key: string; suppressed: number }[] = []
|
||||
const window = new AssignmentRejectionLogWindow<string>({
|
||||
windowMs: 10_000,
|
||||
schedule: timers.schedule,
|
||||
onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed })
|
||||
})
|
||||
|
||||
window.admit('key-0', 'host-a')
|
||||
window.admit('key-0', 'host-b')
|
||||
for (let index = 1; index < 200; index++) window.admit(`key-${index}`, 'host-a')
|
||||
|
||||
expect(closed[0]).toEqual({ key: 'key-0', suppressed: 1 })
|
||||
// Evicted keys emit again rather than staying silently suppressed.
|
||||
expect(window.admit('key-0', 'host-a')).toBe(true)
|
||||
expect(window.admit('key-199', 'host-a')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
function manualTimers(): {
|
||||
|
||||
@@ -15,11 +15,9 @@ vi.mock('../authentication/relay-token-verifier.js', () => ({
|
||||
}))
|
||||
|
||||
import { createRelayApp } from '../app.js'
|
||||
import { relayHostLogDigest } from '../observability/relay-host-log-digest.js'
|
||||
|
||||
describe('assignment rejection logging', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
@@ -49,44 +47,9 @@ describe('assignment rejection logging', () => {
|
||||
expect(line).toContain('lane=sticky')
|
||||
expect(line).toContain('hinted=true')
|
||||
expect(line).toContain('reason=relay_capacity_exhausted')
|
||||
expect(line).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
expect(line).toContain('host=af342ad00ee4')
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('names the throttled host once per window and closes it with the suppressed count', async () => {
|
||||
vi.useFakeTimers()
|
||||
const host = 'mmmmmmmmmmmmmmmm'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
assign: vi.fn(async () => assignment('cell-r', host)),
|
||||
resolve: vi.fn(async () => null)
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
|
||||
expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
|
||||
|
||||
const throttled = (): string[] =>
|
||||
warn.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.filter((entry) => entry.includes('reason=host-rate-limited'))
|
||||
expect(throttled()).toHaveLength(1)
|
||||
expect(throttled()[0]).toContain('route=assign')
|
||||
expect(throttled()[0]).toContain('lane=placement')
|
||||
expect(throttled()[0]).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
expect(throttled()[0]).not.toContain('suppressed=')
|
||||
expect(throttled()[0]).not.toContain(host)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(throttled()).toHaveLength(2)
|
||||
expect(throttled()[1]).toContain('suppressed=1')
|
||||
expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`)
|
||||
})
|
||||
})
|
||||
|
||||
function assignmentRequest(
|
||||
|
||||
@@ -202,49 +202,5 @@ describePostgres('reserve-mode census on PostgreSQL', () => {
|
||||
await holder.close()
|
||||
}
|
||||
}, 20_000)
|
||||
|
||||
// A stall must not double connection demand: concurrent assigns share one reserve-set read,
|
||||
// and after it fails the last answer stands for the backoff without another query.
|
||||
it('reads the reserve set once for concurrent assigns during a stall, and not again in the backoff', async () => {
|
||||
const timed = await openRelayDatabase({ databaseUrl, dataDir: '/tmp', statementTimeoutMs: 300 })
|
||||
const holder = await openRelayDatabase({ databaseUrl, dataDir: '/tmp' })
|
||||
let release!: () => void
|
||||
const released = new Promise<void>((resolve) => (release = resolve))
|
||||
let held!: () => void
|
||||
const holding = new Promise<void>((resolve) => (held = resolve))
|
||||
const lock = holder.transaction(async (transaction) => {
|
||||
await transaction.query(`LOCK TABLE relay_cell_admit_modes IN ACCESS EXCLUSIVE MODE`)
|
||||
held()
|
||||
await released
|
||||
})
|
||||
try {
|
||||
await holding
|
||||
const store = new RelayAssignmentStore(timed, () => 500_000)
|
||||
await store.reconcileCells([cellA, cellB], false)
|
||||
await heartbeat(store, cellA)
|
||||
await heartbeat(store, cellB)
|
||||
const query = timed.query.bind(timed)
|
||||
let reads = 0
|
||||
timed.query = async (sql, params) => {
|
||||
if (sql.includes('FROM relay_cell_admit_modes m')) reads += 1
|
||||
return await query(sql, params)
|
||||
}
|
||||
const placed = await Promise.all(
|
||||
Array.from({ length: 6 }, async (_, index) =>
|
||||
await store.assign({ userId: USER, relayHostId: `hoststall0000${String(index).padStart(3, '0')}` })
|
||||
)
|
||||
)
|
||||
expect(placed).toHaveLength(6)
|
||||
expect(reads).toBe(1)
|
||||
await store.assign({ userId: USER, relayHostId: 'hoststall0000100' })
|
||||
expect(await store.reserveModeCells()).toBeNull()
|
||||
expect(reads).toBe(1)
|
||||
} finally {
|
||||
release()
|
||||
await lock
|
||||
await timed.close()
|
||||
await holder.close()
|
||||
}
|
||||
}, 20_000)
|
||||
})
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ describe('assignment reserve-mode behavior', () => {
|
||||
const { unknown, restore } = await unknownSet(() => now)
|
||||
expect(await unknown.reserveModeCells()).toBeNull()
|
||||
expect(await unknown.releaseExpiredActivityLeases()).toBe(0)
|
||||
expect(await unknown.evacuateDeadCells()).toBe(0)
|
||||
await restore()
|
||||
await setReserve(store, [])
|
||||
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
|
||||
@@ -125,7 +124,7 @@ describe('assignment reserve-mode behavior', () => {
|
||||
}
|
||||
return await query(sql, params)
|
||||
}
|
||||
return { counter, release, restore: () => (database!.query = query) }
|
||||
return { release, restore: () => (database!.query = query) }
|
||||
}
|
||||
|
||||
it('reads afresh after an admit-mode write instead of joining a read begun before it', async () => {
|
||||
@@ -155,12 +154,10 @@ describe('assignment reserve-mode behavior', () => {
|
||||
try {
|
||||
const before = store.reserveModeCells()
|
||||
await store.setCellAdmitMode('cell-a', 'reserve')
|
||||
const reads = held.counter.reads
|
||||
held.release()
|
||||
await before
|
||||
// A write cleared the cache: the next caller reads now, not after a 5 s backoff.
|
||||
expect([...((await store.reserveModeCells()) ?? [])]).toEqual(['cell-a'])
|
||||
expect(held.counter.reads).toBeGreaterThan(reads)
|
||||
} finally {
|
||||
held.release()
|
||||
held.restore()
|
||||
@@ -197,13 +194,8 @@ describe('assignment reserve-mode behavior', () => {
|
||||
|
||||
it('never places a fresh host on a reserve-mode cell through the database', async () => {
|
||||
const store = await setup(() => 1_000, ['cell-a'])
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
const placed = await store.assign({
|
||||
userId: `user-${index}`,
|
||||
relayHostId: `host${String(index).padStart(12, '0')}`
|
||||
})
|
||||
expect(placed.cellId).toBe('cell-b')
|
||||
}
|
||||
const placed = await store.assign({ userId: 'user-0', relayHostId: 'host000000000000' })
|
||||
expect(placed.cellId).toBe('cell-b')
|
||||
})
|
||||
|
||||
it('refuses drains, evacuations and status reads that would treat a reserve-mode cell as empty', async () => {
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
RelayAssignmentStore,
|
||||
type AssignOptions,
|
||||
type RelayAssignment
|
||||
} from './assignment-store.js'
|
||||
import type { RelayCellConfig } from '../config.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
type RelayDatabase,
|
||||
type RelayLockOptions,
|
||||
type RelayTransactionOptions,
|
||||
type SqlRow
|
||||
} from '../database/database.js'
|
||||
import { ReservePaceError } from './reserve-placement/reserve-assignment.js'
|
||||
|
||||
const CELLS: RelayCellConfig[] = [
|
||||
{ id: 'us-c1', url: 'https://us-c1.relay.example.com', region: 'us-central1', capacityRequests: 100 },
|
||||
{ id: 'asia-c1', url: 'https://asia-c1.relay.example.com', region: 'asia-east2', capacityRequests: 100 }
|
||||
]
|
||||
const IDENTITY = { userId: 'spill-user', relayHostId: 'spillhost0000001' }
|
||||
const ON_US_RESERVE: RelayAssignment = {
|
||||
...IDENTITY,
|
||||
cellId: 'us-reserve',
|
||||
cellUrl: 'https://us-reserve.relay.example.com',
|
||||
assignmentEpoch: 8,
|
||||
leaseExpiresAt: 1_000_000,
|
||||
region: 'us-central1'
|
||||
}
|
||||
|
||||
// Reports the target cell full when its reservation is taken: the race between pick and reserve.
|
||||
class FullAtReserveDatabase implements RelayDatabase {
|
||||
armed = false
|
||||
|
||||
constructor(private readonly delegate: RelayDatabase) {}
|
||||
|
||||
get dialect(): 'sqlite' | 'postgres' | undefined {
|
||||
return this.delegate.dialect
|
||||
}
|
||||
|
||||
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
||||
return await this.delegate.query(sql, params)
|
||||
}
|
||||
|
||||
async queryLocked(sql: string, params?: unknown[], options?: RelayLockOptions): Promise<SqlRow[]> {
|
||||
const rows = await this.delegate.queryLocked(sql, params, options)
|
||||
if (!this.armed || sql.trim() !== 'SELECT * FROM relay_cells WHERE cell_id = ?') {
|
||||
return rows
|
||||
}
|
||||
return rows.map((row) => ({ ...row, reserved_requests: row.capacity_requests }))
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>, options?: RelayTransactionOptions): Promise<T> {
|
||||
return await this.delegate.transaction(async (inner) => {
|
||||
const wrapped = new FullAtReserveDatabase(inner)
|
||||
wrapped.armed = this.armed
|
||||
return await operation(wrapped)
|
||||
}, options)
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.delegate.close()
|
||||
}
|
||||
}
|
||||
|
||||
describe('reserve spill before relay_capacity_exhausted', () => {
|
||||
let database: FullAtReserveDatabase
|
||||
let store: RelayAssignmentStore
|
||||
let now: number
|
||||
|
||||
beforeEach(async () => {
|
||||
now = 100
|
||||
database = new FullAtReserveDatabase(await openInMemoryRelayDatabase())
|
||||
store = new RelayAssignmentStore(database, () => now)
|
||||
await store.reconcileCells(CELLS)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await database.close()
|
||||
})
|
||||
|
||||
const closeAllDatabaseCells = async () => {
|
||||
for (const cell of CELLS) {
|
||||
await store.configureCell(cell, 'migration-only')
|
||||
}
|
||||
}
|
||||
const assignAsia = async (step5: AssignOptions) =>
|
||||
await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, step5)
|
||||
|
||||
it('books a reserve seat when no database cell has room anywhere', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ reserveSpill })).resolves.toEqual({ ...ON_US_RESERVE, placed: true })
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: 0 })
|
||||
expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
|
||||
})
|
||||
|
||||
it('skips the own region after the in-region reserve fallback already tried it', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const regionReserve = vi.fn(async () => null)
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ regionReserve, reserveSpill })).resolves.toMatchObject({ cellId: 'us-reserve' })
|
||||
expect(regionReserve).toHaveBeenCalledOnce()
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: true, rowEpoch: 0 })
|
||||
})
|
||||
|
||||
it('hands the spill the epoch of the host row it read', async () => {
|
||||
const first = await assignAsia({})
|
||||
// Dormant: its next assign is a placement, which reads the row and finds no cell.
|
||||
await store.changeActivity(IDENTITY, 'control', -1)
|
||||
now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.releaseExpiredActivity()
|
||||
now += ASSIGNMENT_LIMITS.dormantTtlMs
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await assignAsia({ reserveSpill })
|
||||
expect(reserveSpill).toHaveBeenCalledWith({ skipOwnRegion: false, rowEpoch: first.assignmentEpoch })
|
||||
})
|
||||
|
||||
it('refuses with relay_capacity_exhausted when no reserve seat exists either', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
await expect(assignAsia({ reserveSpill: async () => null })).rejects.toThrow('relay_capacity_exhausted')
|
||||
})
|
||||
|
||||
it('answers a paced spill with its pace, not a refusal', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => {
|
||||
throw new ReservePaceError(4)
|
||||
})
|
||||
await expect(assignAsia({ reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
|
||||
})
|
||||
|
||||
it('answers an in-region reserve pace before any spill', async () => {
|
||||
await closeAllDatabaseCells()
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
const regionReserve = async () => {
|
||||
throw new ReservePaceError(2)
|
||||
}
|
||||
await expect(assignAsia({ regionReserve, reserveSpill })).rejects.toBeInstanceOf(ReservePaceError)
|
||||
expect(reserveSpill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not spill a reservation race: the host retries and re-picks', async () => {
|
||||
database.armed = true
|
||||
const reserveSpill = vi.fn(async () => ON_US_RESERVE)
|
||||
await expect(assignAsia({ reserveSpill })).rejects.toThrow('relay_capacity_exhausted')
|
||||
expect(reserveSpill).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -143,18 +143,12 @@ async function setup(cells: RelayCellConfig[] = [SOURCE, TARGET]) {
|
||||
// The old process stopped heartbeating at START_MS, so it is unheard from here.
|
||||
const OLD_UNHEARD_AT = START_MS + HEARTBEAT_TTL_MS + 1_000
|
||||
|
||||
// Both routes that refused in production: a hinted assign (placement lane after its
|
||||
// verification read finds the cell not live) and resolve, which then assigns.
|
||||
async function hintedAssign(store: RelayAssignmentStore) {
|
||||
return await store.assign(IDENTITY, 'us-central1', 'us-central1', undefined, {
|
||||
hostRequested: true
|
||||
})
|
||||
}
|
||||
|
||||
async function resolveRoute(store: RelayAssignmentStore) {
|
||||
return (await store.resolve(IDENTITY)) ?? (await store.assign(IDENTITY))
|
||||
}
|
||||
|
||||
async function expectRefused(
|
||||
attempt: Promise<unknown>,
|
||||
cause: 'not_ready' | 'unheard'
|
||||
@@ -164,10 +158,7 @@ async function expectRefused(
|
||||
expect(error).toMatchObject({ unavailableCause: cause })
|
||||
}
|
||||
|
||||
describe.each([
|
||||
['hinted assign', hintedAssign],
|
||||
['resolve', resolveRoute]
|
||||
] as const)('a host offline through a roll redials via %s', (_route, redial) => {
|
||||
describe('a host offline through a roll redials via hinted assign', () => {
|
||||
describe('while the replacement process boots (not ready)', () => {
|
||||
it('is re-placed when that process started after the roll isolated the cell', async () => {
|
||||
const { store, first, heartbeat, at, isolateForRoll } = await setup()
|
||||
@@ -175,7 +166,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
const moved = await redial(store)
|
||||
const moved = await hintedAssign(store)
|
||||
expect(moved).toMatchObject({ cellId: TARGET.id })
|
||||
expect(moved.assignmentEpoch).toBeGreaterThan(first.assignmentEpoch)
|
||||
})
|
||||
@@ -186,7 +177,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused while the new process reports holding connections', async () => {
|
||||
@@ -200,7 +191,7 @@ describe.each([
|
||||
totalConnections: 1
|
||||
})
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused once the new process itself goes unheard', async () => {
|
||||
@@ -210,7 +201,7 @@ describe.each([
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
await at(20_000 + HEARTBEAT_TTL_MS + 1_000)
|
||||
|
||||
await expectRefused(redial(store), 'unheard')
|
||||
await expectRefused(hintedAssign(store), 'unheard')
|
||||
})
|
||||
|
||||
it('is refused on an ordinary readiness dip with no roll stamp', async () => {
|
||||
@@ -218,7 +209,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
|
||||
it('is refused on a migration-only cell parked without a roll stamp', async () => {
|
||||
@@ -227,7 +218,7 @@ describe.each([
|
||||
await at(20_000)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: 15_000, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -238,7 +229,7 @@ describe.each([
|
||||
await isolateForRoll()
|
||||
await at(OLD_UNHEARD_AT)
|
||||
|
||||
await expectRefused(redial(store), 'unheard')
|
||||
await expectRefused(hintedAssign(store), 'unheard')
|
||||
})
|
||||
|
||||
it('is refused when the roll stamp is stale, even with a new process', async () => {
|
||||
@@ -248,7 +239,7 @@ describe.each([
|
||||
await at(stale)
|
||||
await heartbeat(SOURCE, { incarnation: NEW_INCARNATION, startedAt: stale - 1, ready: false })
|
||||
|
||||
await expectRefused(redial(store), 'not_ready')
|
||||
await expectRefused(hintedAssign(store), 'not_ready')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
|
||||
const cells = [
|
||||
{
|
||||
id: 'row-lock-order-a',
|
||||
url: 'https://row-lock-order-a.example.com',
|
||||
capacityRequests: 200,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
},
|
||||
{
|
||||
id: 'row-lock-order-b',
|
||||
url: 'https://row-lock-order-b.example.com',
|
||||
capacityRequests: 200,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
]
|
||||
const userId = 'row-lock-order-user'
|
||||
const hosts = ['rowlockhost00001', 'rowlockhost00002', 'rowlockhost00003', 'rowlockhost00004'].map(
|
||||
(relayHostId) => ({ userId, relayHostId })
|
||||
)
|
||||
|
||||
// pg_stat_database exposes cycles hidden by three 40P01 retries; eventual success alone misses deadlocks that exhaust retries under load.
|
||||
describePostgres('PostgreSQL row lock order', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
for (let index = 0; index < 4; index++) {
|
||||
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
|
||||
}
|
||||
})
|
||||
|
||||
async function removeTestRows(database: RelayDatabase): Promise<void> {
|
||||
await database.query(
|
||||
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
|
||||
[userId]
|
||||
)
|
||||
for (const table of [
|
||||
'relay_control_capabilities',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_post_drain_migration_pins',
|
||||
'relay_assignment_migration_incarnations',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [userId])
|
||||
}
|
||||
for (const cell of cells) {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_runtime',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterAll(async () => {
|
||||
if (databases[0]) await removeTestRows(databases[0])
|
||||
for (const connection of databases) await connection.close()
|
||||
})
|
||||
|
||||
async function deadlockCount(): Promise<number> {
|
||||
const rows = await databases[0]!.query(
|
||||
`SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()`
|
||||
)
|
||||
return Number(rows[0]!.deadlocks)
|
||||
}
|
||||
|
||||
it('runs the cell accept and the placement retry concurrently without a cycle', async () => {
|
||||
await removeTestRows(databases[0]!)
|
||||
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
|
||||
await stores[0]!.reconcileCells(cells)
|
||||
for (const cell of cells) {
|
||||
await stores[0]!.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionInclusionWatermark: 1,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 50
|
||||
})
|
||||
}
|
||||
const placements = new Map<string, { cellId: string; assignmentEpoch: number }>()
|
||||
for (const identity of hosts) {
|
||||
const assignment = await stores[0]!.assign(identity)
|
||||
placements.set(identity.relayHostId, assignment)
|
||||
}
|
||||
|
||||
const before = await deadlockCount()
|
||||
// The accept path (host rows, then the cell row last) against the paths
|
||||
// that must read the inventory first: placement and evacuation.
|
||||
for (let round = 0; round < 12; round++) {
|
||||
await Promise.allSettled(
|
||||
hosts.flatMap((identity, index) => {
|
||||
const placement = placements.get(identity.relayHostId)!
|
||||
const store = stores[index % stores.length]!
|
||||
const other = stores[(index + 1) % stores.length]!
|
||||
return [
|
||||
store.activateControl(identity, {
|
||||
cellId: placement.cellId,
|
||||
assignmentEpoch: placement.assignmentEpoch,
|
||||
generation: round + 2
|
||||
}),
|
||||
other.assign(identity),
|
||||
other.startEvacuation(
|
||||
identity,
|
||||
placement.cellId === cells[0]!.id ? cells[1]!.id : cells[0]!.id
|
||||
)
|
||||
]
|
||||
})
|
||||
)
|
||||
}
|
||||
const after = await deadlockCount()
|
||||
|
||||
expect(after - before).toBe(0)
|
||||
}, 120_000)
|
||||
})
|
||||
@@ -5,46 +5,16 @@ import type { RelayCellConfig } from '../../config.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
openRelayDatabase,
|
||||
type RelayDatabase,
|
||||
type SqlRow
|
||||
type RelayDatabase
|
||||
} from '../../database/database.js'
|
||||
|
||||
// Both dialects must skip host scans with no actionable cells; stale existing-only rows previously scanned every assignment for nothing.
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const schema = 'relay_dead_cell_precheck_test'
|
||||
const HOST_QUERY = 'FROM relay_assignments assignment'
|
||||
const CELLS: RelayCellConfig[] = [
|
||||
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
|
||||
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
|
||||
]
|
||||
|
||||
class RecordingDatabase implements RelayDatabase {
|
||||
readonly statements: string[] = []
|
||||
|
||||
constructor(private readonly delegate: RelayDatabase) {}
|
||||
|
||||
get dialect() {
|
||||
return this.delegate.dialect
|
||||
}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
this.statements.push(sql)
|
||||
return await this.delegate.query(sql, params)
|
||||
}
|
||||
|
||||
async queryLocked(...args: Parameters<RelayDatabase['queryLocked']>): Promise<SqlRow[]> {
|
||||
return await this.delegate.queryLocked(...args)
|
||||
}
|
||||
|
||||
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
|
||||
return await this.delegate.transaction(operation)
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.delegate.close()
|
||||
}
|
||||
}
|
||||
|
||||
function scopedUrl(): string {
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
@@ -91,13 +61,12 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
|
||||
async function setup(now: () => number) {
|
||||
database = await open()
|
||||
const recording = new RecordingDatabase(database)
|
||||
const store = new RelayAssignmentStore(recording, now, {
|
||||
const store = new RelayAssignmentStore(database, now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
await store.reconcileCells(CELLS)
|
||||
return { store, recording }
|
||||
return { store }
|
||||
}
|
||||
|
||||
async function heartbeat(store: RelayAssignmentStore, cell: RelayCellConfig): Promise<void> {
|
||||
@@ -111,31 +80,9 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
})
|
||||
}
|
||||
|
||||
it('skips the host query when the only dead cell is unfenced and existing-only', async () => {
|
||||
let now = 100
|
||||
const { store, recording } = await setup(() => now)
|
||||
for (const cell of CELLS) await heartbeat(store, cell)
|
||||
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
expect(await store.assign(identity)).toMatchObject({ cellId: 'cell-a' })
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
await store.setCellEnabled('cell-a', false)
|
||||
now += 45_001
|
||||
await heartbeat(store, CELLS[1]!)
|
||||
recording.statements.length = 0
|
||||
|
||||
expect(await store.evacuateDeadCells()).toBe(0)
|
||||
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(false)
|
||||
expect(
|
||||
await database!.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
|
||||
identity.userId
|
||||
])
|
||||
).toEqual([{ cell_id: 'cell-a' }])
|
||||
})
|
||||
|
||||
it('still evacuates hosts from a dead uncapped cell that admits', async () => {
|
||||
let now = 100
|
||||
const { store, recording } = await setup(() => now)
|
||||
const { store } = await setup(() => now)
|
||||
for (const cell of CELLS) await heartbeat(store, cell)
|
||||
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
@@ -143,10 +90,8 @@ describe.each(dialects)('dead-cell evacuation pre-check (%s)', (_dialect, open)
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
now += 45_001
|
||||
await heartbeat(store, CELLS[1]!)
|
||||
recording.statements.length = 0
|
||||
|
||||
expect(await store.evacuateDeadCells()).toBe(1)
|
||||
expect(recording.statements.some((sql) => sql.includes(HOST_QUERY))).toBe(true)
|
||||
expect((await store.resolve(identity))?.cellId).toBe('cell-b')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import { expect, it } from 'vitest'
|
||||
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
|
||||
import type { SqlRow } from '../../database/database.js'
|
||||
|
||||
function text(row: SqlRow, field: string): string {
|
||||
const value = row[field]
|
||||
if (typeof value !== 'string') {
|
||||
throw new Error(`invalid_${field}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
it('keeps first assignment matches, lease order, row identity, and separate identity components', () => {
|
||||
const rows = [
|
||||
{ user_id: 'a:b', relay_host_id: 'c', value: 1 },
|
||||
{ user_id: 'a', relay_host_id: 'b:c', value: 2 },
|
||||
{ user_id: 'a:b', relay_host_id: 'c', value: 3 },
|
||||
{ user_id: '', relay_host_id: '', value: 4 }
|
||||
]
|
||||
const lookup = createDrainMigrationRowLookup(rows, text)
|
||||
for (const identity of [
|
||||
{ userId: 'a:b', relayHostId: 'c' },
|
||||
{ userId: 'a', relayHostId: 'b:c' },
|
||||
{ userId: '', relayHostId: '' },
|
||||
{ userId: 'missing', relayHostId: 'c' }
|
||||
]) {
|
||||
const expected = rows.filter(
|
||||
(row) =>
|
||||
row.user_id === identity.userId && row.relay_host_id === identity.relayHostId
|
||||
)
|
||||
expect(lookup.first(identity)).toBe(expected[0])
|
||||
expect(lookup.all(identity)).toEqual(expected)
|
||||
lookup.all(identity).forEach((row, index) => expect(row).toBe(expected[index]))
|
||||
}
|
||||
})
|
||||
|
||||
it('retains lazy validation and short circuiting when an inventory is malformed', () => {
|
||||
const first = { user_id: 'user', relay_host_id: 'host' }
|
||||
const identity = { userId: 'user', relayHostId: 'host' }
|
||||
const lookup = createDrainMigrationRowLookup(
|
||||
[first, { user_id: null, relay_host_id: 'bad' }],
|
||||
text
|
||||
)
|
||||
expect(lookup.first(identity)).toBe(first)
|
||||
expect(() => lookup.all(identity)).toThrow('invalid_user_id')
|
||||
const unrelated = createDrainMigrationRowLookup(
|
||||
[first, { user_id: 'other', relay_host_id: null }],
|
||||
text
|
||||
)
|
||||
expect(unrelated.all(identity)).toEqual([first])
|
||||
expect(() => unrelated.first({ userId: 'other', relayHostId: 'host' })).toThrow(
|
||||
'invalid_relay_host_id'
|
||||
)
|
||||
})
|
||||
@@ -1,122 +0,0 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract/assignment-invariants'
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from '../../database/database.js'
|
||||
|
||||
let database: RelayDatabase | undefined
|
||||
afterEach(async () => await database?.close())
|
||||
|
||||
it.each([false, true])(
|
||||
'looks up a whole drain inventory with linear identity reads (expired: %s)',
|
||||
async (expired) => {
|
||||
database = await openInMemoryRelayDatabase()
|
||||
let measuring = false
|
||||
let identityReads = 0
|
||||
let indexedRows = 0
|
||||
const instrument = (delegate: RelayDatabase): RelayDatabase => ({
|
||||
dialect: delegate.dialect,
|
||||
query: (sql, params) => delegate.query(sql, params),
|
||||
queryLocked: async (sql, params, options) => {
|
||||
const rows = await delegate.queryLocked(sql, params, options)
|
||||
if (
|
||||
!measuring ||
|
||||
!sql.includes('WHERE EXISTS') ||
|
||||
!(sql.includes('SELECT assignment.*') || sql.includes('SELECT lease.*'))
|
||||
) {
|
||||
return rows
|
||||
}
|
||||
indexedRows += rows.length
|
||||
return rows.map(
|
||||
(row): SqlRow =>
|
||||
new Proxy(row, {
|
||||
get(target, key) {
|
||||
if (key === 'user_id' || key === 'relay_host_id') {
|
||||
identityReads++
|
||||
}
|
||||
return Reflect.get(target, key)
|
||||
}
|
||||
})
|
||||
)
|
||||
},
|
||||
transaction: (operation, options) =>
|
||||
delegate.transaction((tx) => operation(instrument(tx)), options),
|
||||
close: () => delegate.close()
|
||||
})
|
||||
let now = 100
|
||||
const store = new RelayAssignmentStore(instrument(database), () => now, {
|
||||
requireLiveCells: true
|
||||
})
|
||||
const cells = ['a', 'b'].map((id) => ({
|
||||
id: `cell-${id}`,
|
||||
url: `https://relay-${id}.example.com`,
|
||||
capacityRequests: 500
|
||||
}))
|
||||
await store.reconcileCells(cells)
|
||||
const incarnation = '11111111-1111-4111-8111-111111111111'
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: incarnation,
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
}
|
||||
await store.setCellEnabled('cell-b', false)
|
||||
const identities = Array.from({ length: 50 }, (_, index) => ({
|
||||
userId: `user-${index % 5}`,
|
||||
relayHostId: `host${String(index).padStart(12, '0')}`
|
||||
}))
|
||||
for (const identity of identities) {
|
||||
await store.assign(identity)
|
||||
}
|
||||
await store.setCellEnabled('cell-b', true)
|
||||
await store.setCellEnabled('cell-a', false)
|
||||
for (const identity of identities) {
|
||||
const migration = await store.startEvacuation(identity, 'cell-b')
|
||||
await store.markMigrationTargetRegistered(identity, {
|
||||
cellId: 'cell-b',
|
||||
assignmentEpoch: migration.assignmentEpoch
|
||||
})
|
||||
}
|
||||
const attempt = {
|
||||
attemptId: '55555555-5555-4555-8555-555555555555',
|
||||
cellId: 'cell-a',
|
||||
cellIncarnation: incarnation,
|
||||
traceValue: '66666666-6666-4666-8666-666666666666',
|
||||
plannedGraceMs: 120_000
|
||||
}
|
||||
await store.prepareCellDrainAttempt(attempt)
|
||||
if (expired) {
|
||||
now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
await store.releaseExpiredActivityLeases()
|
||||
for (const cell of cells) {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: incarnation,
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
}
|
||||
}
|
||||
measuring = true
|
||||
await expect(store.beginCellDrainSend(attempt)).resolves.toMatchObject({
|
||||
state: 'send-may-have-started',
|
||||
shouldSend: true
|
||||
})
|
||||
expect(indexedRows).toBeGreaterThanOrEqual(100)
|
||||
expect(identityReads).toBeLessThanOrEqual(indexedRows * 2)
|
||||
const migrations = await database.query(
|
||||
'SELECT expires_at FROM relay_assignment_migrations'
|
||||
)
|
||||
expect(migrations).toHaveLength(50)
|
||||
expect(
|
||||
migrations.every(
|
||||
(row) => row.expires_at === now + ASSIGNMENT_LIMITS.migrationLeaseMs
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
)
|
||||
-282
@@ -1,282 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import {
|
||||
encodeMembership,
|
||||
type CellAdmissionMembership,
|
||||
type CellAdmissionState
|
||||
} from '../../cell/admission/cell-admission-selector.js'
|
||||
import type { RelayCellConfig } from '../../config.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const DIRECTOR_INSTANCES = 5
|
||||
const DIRECTOR_POOL_MAX = 3
|
||||
const DIRECTOR_APPLICATION = 'drain-release-contention-director'
|
||||
|
||||
const USER_PREFIX = 'drain-release-contention'
|
||||
const NOW = 1_000_000
|
||||
const CAPPED = {
|
||||
capacityRequests: 6_000,
|
||||
connectionHardCap: 3_000,
|
||||
connectionUnobservedBound: 60
|
||||
} as const
|
||||
const SOURCE: RelayCellConfig = {
|
||||
id: 'drain-release-contention-a-source',
|
||||
url: 'https://drain-release-contention-source.example.test',
|
||||
region: 'asia-east2',
|
||||
...CAPPED
|
||||
}
|
||||
const TARGETS: RelayCellConfig[] = ['b', 'c', 'd'].map((suffix) => ({
|
||||
id: `drain-release-contention-${suffix}-target`,
|
||||
url: `https://drain-release-contention-${suffix}-target.example.test`,
|
||||
region: 'asia-east2',
|
||||
...CAPPED
|
||||
}))
|
||||
// Next to the database. Taken only once every Asia neighbour is at its cap.
|
||||
const US_CELL: RelayCellConfig = {
|
||||
id: 'drain-release-contention-e-us',
|
||||
url: 'https://drain-release-contention-us.example.test',
|
||||
region: 'us-central1',
|
||||
...CAPPED
|
||||
}
|
||||
const CELLS = [SOURCE, ...TARGETS, US_CELL]
|
||||
|
||||
type Identity = { userId: string; relayHostId: string }
|
||||
type Instance = { database: RelayDatabase; store: RelayAssignmentStore }
|
||||
|
||||
function hostIdentity(index: number): Identity {
|
||||
return {
|
||||
userId: `${USER_PREFIX}-${index}`,
|
||||
// Relay host ids are fixed-width opaque ids.
|
||||
relayHostId: `drainrel${String(index).padStart(8, '0')}`
|
||||
}
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL drain releases against director placement', () => {
|
||||
const storeOptions = { requireLiveCells: true, heartbeatTtlMs: 45_000 }
|
||||
const directors: Instance[] = []
|
||||
let observer: RelayDatabase
|
||||
|
||||
function admin(): Instance {
|
||||
return directors[0]!
|
||||
}
|
||||
|
||||
async function applySelector(
|
||||
states: Record<string, CellAdmissionState>,
|
||||
rollIsolatedCells?: string[]
|
||||
): Promise<void> {
|
||||
const current = await admin().store.inspectCellAdmissionSelector()
|
||||
// Built from relay_cells: the apply requires exact coverage of a shared fleet.
|
||||
const fleet = await admin().database.query(
|
||||
`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`
|
||||
)
|
||||
const membership: CellAdmissionMembership = {
|
||||
existingOnly: [],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
for (const row of fleet) {
|
||||
const cellId = String(row['cell_id'])
|
||||
const state =
|
||||
states[cellId] ??
|
||||
(current.selector.membership.existingOnly.includes(cellId)
|
||||
? 'existing-only'
|
||||
: current.selector.membership.migrationOnly.includes(cellId)
|
||||
? 'migration-only'
|
||||
: 'general')
|
||||
if (state === 'existing-only') membership.existingOnly.push(cellId)
|
||||
else if (state === 'migration-only') membership.migrationOnly.push(cellId)
|
||||
else membership.general.push(cellId)
|
||||
}
|
||||
await admin().store.applyCellAdmissionSelector({
|
||||
attemptId: `drain-release-${current.selector.generation}`,
|
||||
expectedGeneration: current.selector.generation,
|
||||
...(current.selector.generation === 0
|
||||
? {
|
||||
expectedMembershipSha256: createHash('sha256')
|
||||
.update(encodeMembership(current.selector.membership))
|
||||
.digest('hex')
|
||||
}
|
||||
: {}),
|
||||
membership,
|
||||
...(rollIsolatedCells ? { rollIsolatedCells } : {})
|
||||
})
|
||||
}
|
||||
|
||||
// Other Postgres files write admission through the generation-0 helpers, so
|
||||
// the selector goes back to generation 0 exactly as found.
|
||||
async function resetSelectorBoundary(): Promise<void> {
|
||||
await admin().database.query(
|
||||
`UPDATE relay_admission_selectors SET generation = 0, attempt_id = NULL
|
||||
WHERE selector_id = 'general'`
|
||||
)
|
||||
await admin().database.query(
|
||||
`DELETE FROM relay_admission_selector_intents WHERE attempt_id LIKE 'drain-release-%'`
|
||||
)
|
||||
await admin().store.reconcileCells([], false)
|
||||
}
|
||||
|
||||
async function deleteHostRows(): Promise<void> {
|
||||
for (const table of [
|
||||
'relay_control_capabilities',
|
||||
'relay_control_connection_reservations',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await admin().database.query(`DELETE FROM ${table} WHERE user_id LIKE '${USER_PREFIX}-%'`)
|
||||
}
|
||||
}
|
||||
|
||||
// `atCap` reports those cells at their connection cap: no headroom left.
|
||||
async function heartbeatAll(atCap: RelayCellConfig[] = []): Promise<void> {
|
||||
for (const [index, config] of CELLS.entries()) {
|
||||
await admin().store.recordCellHeartbeat({
|
||||
cellId: config.id,
|
||||
cellUrl: config.url,
|
||||
cellIncarnation: `2222222${index}-2222-4222-8222-222222222222`,
|
||||
startedAt: NOW - 1_000,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
region: config.region,
|
||||
// The cell checks enforced = total + in-flight + reserved.
|
||||
totalConnections: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
|
||||
connectionHardCap: CAPPED.connectionHardCap,
|
||||
connectionUnobservedBound: CAPPED.connectionUnobservedBound
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Holds `cellIds`' relay_cells rows in an open transaction until the returned release runs.
|
||||
async function holdCellRows(cellIds: string[]): Promise<() => Promise<void>> {
|
||||
let release!: () => void
|
||||
const released = new Promise<void>((resolve) => (release = resolve))
|
||||
let held!: () => void
|
||||
const acquired = new Promise<void>((resolve) => (held = resolve))
|
||||
const holder = observer.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT cell_id FROM relay_cells WHERE cell_id IN (${cellIds.map(() => '?').join(', ')})
|
||||
ORDER BY cell_id ASC`,
|
||||
cellIds
|
||||
)
|
||||
held()
|
||||
await released
|
||||
})
|
||||
await acquired
|
||||
return async () => {
|
||||
release()
|
||||
await holder
|
||||
}
|
||||
}
|
||||
|
||||
// Every host holds a live control lease on the source, then the source is
|
||||
// isolated for a roll: the state an Asia drain starts from.
|
||||
async function seed(hosts: Identity[]): Promise<void> {
|
||||
await deleteHostRows()
|
||||
await admin().database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id LIKE '${USER_PREFIX}-%'`
|
||||
)
|
||||
// Restoring the source to general also clears the previous run's roll stamp.
|
||||
await applySelector({
|
||||
[SOURCE.id]: 'general',
|
||||
...Object.fromEntries(TARGETS.map((target) => [target.id, 'migration-only' as const]))
|
||||
})
|
||||
for (const identity of hosts) {
|
||||
const grant = await admin().store.assign(identity, 'asia-east2', 'asia-east2')
|
||||
expect(grant.cellId).toBe(SOURCE.id)
|
||||
await admin().store.activateControl(identity, {
|
||||
cellId: SOURCE.id,
|
||||
assignmentEpoch: grant.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
}
|
||||
await applySelector(
|
||||
Object.fromEntries(TARGETS.map((target) => [target.id, 'general' as const]))
|
||||
)
|
||||
await applySelector({ [SOURCE.id]: 'migration-only' }, [SOURCE.id])
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
for (let index = 0; index < DIRECTOR_INSTANCES; index += 1) {
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl,
|
||||
dataDir: '',
|
||||
poolMax: DIRECTOR_POOL_MAX,
|
||||
applicationName: DIRECTOR_APPLICATION
|
||||
})
|
||||
directors.push({
|
||||
database,
|
||||
store: new RelayAssignmentStore(database, () => NOW, storeOptions)
|
||||
})
|
||||
}
|
||||
observer = await openRelayDatabase({ databaseUrl, dataDir: '', poolMax: 1 })
|
||||
await deleteHostRows()
|
||||
await admin().store.reconcileCells(CELLS, false)
|
||||
await resetSelectorBoundary()
|
||||
await heartbeatAll()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (directors.length > 0) {
|
||||
await deleteHostRows()
|
||||
// Cells before the selector rebuild, or its membership names missing rows.
|
||||
for (const config of CELLS) {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await admin().database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [config.id])
|
||||
}
|
||||
}
|
||||
await resetSelectorBoundary()
|
||||
}
|
||||
for (const instance of directors) {
|
||||
await instance.database.close()
|
||||
}
|
||||
await observer?.close()
|
||||
})
|
||||
|
||||
it('places a reconnecting host off the isolated source while that source row is held', async () => {
|
||||
await heartbeatAll()
|
||||
const host = hostIdentity(9_000)
|
||||
await seed([host])
|
||||
// A drain release holds the source row for its whole transaction; target-only placement never asks for it.
|
||||
const sourceHeld = await holdCellRows([SOURCE.id])
|
||||
try {
|
||||
const grant = await Promise.race([
|
||||
directors[1]!.store.assign(host, 'asia-east2', 'asia-east2'),
|
||||
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 3_000))
|
||||
])
|
||||
expect(grant).not.toBe('blocked')
|
||||
expect(TARGETS.map(({ id }) => id)).toContain(grant === 'blocked' ? null : grant.cellId)
|
||||
} finally {
|
||||
await sourceHeld()
|
||||
}
|
||||
// Control: the same hold on the target rows does block a placement, so the hold above was real.
|
||||
const other = hostIdentity(9_001)
|
||||
await seed([other])
|
||||
const targetsHeld = await holdCellRows(TARGETS.map(({ id }) => id))
|
||||
const placement = directors[2]!.store.assign(other, 'asia-east2', 'asia-east2')
|
||||
try {
|
||||
const raced = await Promise.race([
|
||||
placement.then(() => 'placed' as const),
|
||||
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 1_000))
|
||||
])
|
||||
expect(raced).toBe('blocked')
|
||||
} finally {
|
||||
await targetsHeld()
|
||||
}
|
||||
expect((await placement).cellId).not.toBe(SOURCE.id)
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
type RelayAssignmentMigration
|
||||
} from '../assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -245,39 +244,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
)
|
||||
}
|
||||
|
||||
it('reaps an abandoned registered migration onto its healthy target', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(
|
||||
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
),
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('reaps immediately after the retired source has a durable completed fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -290,7 +256,7 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
(migration.completed_at IS NOT NULL) AS completed, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
@@ -303,177 +269,12 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
|
||||
completed: true,
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('reaps immediately after the retired source has an adopted legacy fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
expect(
|
||||
(
|
||||
await readRegisteredMigrationInventory(
|
||||
database,
|
||||
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
)
|
||||
).abandoned
|
||||
).toBe(1)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
cell_id: input.cells.failed.id,
|
||||
assignment_epoch: '2',
|
||||
completed_at: String(100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1),
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('does not reap after temporary legacy adoption without durable commit', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
expect(
|
||||
(
|
||||
await readRegisteredMigrationInventory(
|
||||
database,
|
||||
100 + 45_001 + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
|
||||
)
|
||||
).abandoned
|
||||
).toBe(0)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('invalidates an adopted legacy fence when the source heartbeats', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.heartbeat(input.cells.source)
|
||||
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('invalidates an adopted legacy fence when the source restarts', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.store.recordCellHeartbeat({
|
||||
cellId: input.cells.source.id,
|
||||
cellUrl: input.cells.source.url,
|
||||
cellIncarnation: '99999999-9999-4999-8999-999999999999',
|
||||
startedAt: 51,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('serializes legacy adoption with a returning source heartbeat', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.store.adoptLegacyCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
|
||||
const [commit, heartbeatResult] = await Promise.allSettled([
|
||||
input.store.commitLegacyCellFenceAdoption(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
),
|
||||
input.heartbeat(input.cells.source)
|
||||
])
|
||||
expect(heartbeatResult.status).toBe('fulfilled')
|
||||
if (commit.status === 'rejected') {
|
||||
expect(commit.reason).toMatchObject({ message: 'legacy_cell_fence_not_active' })
|
||||
}
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
|
||||
[input.cells.source.id]
|
||||
)
|
||||
).toEqual([])
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('restores the 24-hour guard when the fenced source heartbeats again', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -495,83 +296,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('restores the 24-hour guard when the fenced source restarts', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await completeSourceFence(input)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
await input.store.recordCellHeartbeat({
|
||||
cellId: input.cells.source.id,
|
||||
cellUrl: input.cells.source.url,
|
||||
cellIncarnation: '99999999-9999-4999-8999-999999999999',
|
||||
startedAt: 51,
|
||||
ready: true,
|
||||
observedRequests: 0
|
||||
})
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('serializes fenced cleanup with a returning source heartbeat', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await completeSourceFence(input)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
const [cleanup, heartbeatResult] = await Promise.allSettled([
|
||||
input.store.abortExpiredEvacuations(),
|
||||
input.heartbeat(input.cells.source)
|
||||
])
|
||||
expect(cleanup.status).toBe('fulfilled')
|
||||
expect(heartbeatResult.status).toBe('fulfilled')
|
||||
if (cleanup.status !== 'fulfilled') throw cleanup.reason
|
||||
if (heartbeatResult.status !== 'fulfilled') throw heartbeatResult.reason
|
||||
expect([0, 1]).toContain(cleanup.value)
|
||||
const migrations = await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
if (cleanup.value === 1) {
|
||||
expect(migrations[0]?.completed_at).not.toBeNull()
|
||||
} else {
|
||||
expect(migrations).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a freshly retired source protected without a durable fence', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
input.advance(ASSIGNMENT_LIMITS.migrationLeaseMs + 1)
|
||||
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(0)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT completed_at, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
).toEqual([{ completed_at: null, aborted_at: null }])
|
||||
input.advance(STRANDED_MIGRATION_ABANDON_MS)
|
||||
await expect(input.store.abortExpiredEvacuations()).resolves.toBe(1)
|
||||
})
|
||||
|
||||
it('rolls an abandoned disabled target back to its source', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
@@ -585,7 +309,7 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment.cell_id, assignment.assignment_epoch,
|
||||
migration.completed_at, migration.aborted_at
|
||||
migration.completed_at, (migration.aborted_at IS NOT NULL) AS aborted
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
@@ -599,103 +323,11 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
cell_id: input.cells.source.id,
|
||||
assignment_epoch: '3',
|
||||
completed_at: null,
|
||||
aborted_at: String(
|
||||
100 + ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
aborted: true
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('serializes cleanup against supersession without reporting a false target', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
await input.store.releaseActivity(input.identity, input.targetControlId)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advance(
|
||||
ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
|
||||
)
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
|
||||
const [cleanup, supersession] = await Promise.allSettled([
|
||||
input.store.abortExpiredEvacuations(),
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
])
|
||||
expect(cleanup.status).toBe('fulfilled')
|
||||
if (cleanup.status !== 'fulfilled') throw cleanup.reason
|
||||
if (supersession.status === 'fulfilled') {
|
||||
expect([0, 1]).toContain(supersession.value)
|
||||
const assignment = await database.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
expect(assignment).toEqual([{
|
||||
cell_id:
|
||||
supersession.value === 1
|
||||
? input.cells.replacement.id
|
||||
: input.cells.source.id
|
||||
}])
|
||||
if (supersession.value === 0) expect(cleanup.value).toBeGreaterThanOrEqual(1)
|
||||
} else {
|
||||
expect(cleanup.value).toBeGreaterThanOrEqual(1)
|
||||
expect(supersession.reason).toMatchObject({ message: 'migration_already_superseded' })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ cell_id: input.cells.source.id }])
|
||||
}
|
||||
})
|
||||
|
||||
it('completes a registered migration from a dead source idempotently', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.failed)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.source.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
const completion = {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
targetCellId: input.cells.failed.id
|
||||
}
|
||||
|
||||
await expect(
|
||||
input.store.cellEvacuationStatus(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
true
|
||||
)
|
||||
).resolves.toMatchObject({ inProgress: 0, completed: 1 })
|
||||
await expect(
|
||||
input.store.completeEvacuationFromDeadSource(input.identity, completion)
|
||||
).resolves.toMatchObject({ changed: false })
|
||||
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 1, replacement: 0 })
|
||||
})
|
||||
|
||||
it('requires fence evidence for dead-source completion', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.releaseActivity(input.identity, input.sourceControlId)
|
||||
|
||||
await expect(
|
||||
input.store.completeEvacuationFromDeadSource(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
targetCellId: input.cells.failed.id
|
||||
})
|
||||
).rejects.toThrow('cell_fence_attestation_missing')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 0, failed: 2, replacement: 0 })
|
||||
})
|
||||
|
||||
it('supersedes a registered migration with exact accounting and idempotency', async () => {
|
||||
const input = await fixture()
|
||||
await database.query(
|
||||
@@ -760,416 +392,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('reconciles durable cell accounting before aggregate supersession', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells
|
||||
SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END
|
||||
WHERE cell_id IN (?, ?)`,
|
||||
[input.cells.replacement.id, input.cells.source.id, input.cells.replacement.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
})
|
||||
|
||||
it('rebuilds an expired registered migration lease before supersession', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
|
||||
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '3',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('retires an obsolete row without discarding replacement activity', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await fenceFailedCell(input)
|
||||
await input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET aborted_at = NULL
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
const activityBefore = await assignmentActivities(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await assignmentActivities(input)).toEqual(activityBefore)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{ assignment_epoch: '2', aborted_at: '45101' },
|
||||
{ assignment_epoch: '3', aborted_at: null }
|
||||
])
|
||||
})
|
||||
|
||||
it('anchors a newer dormant failed-cell epoch and preserves drain evidence', async () => {
|
||||
const input = await fixture()
|
||||
const drainAttemptId = await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, source_request_units, target_reserved_units, aborted_at
|
||||
FROM relay_assignment_migrations WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '4',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: '45101'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '5',
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1',
|
||||
aborted_at: null
|
||||
}
|
||||
])
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, drain_attempt_id, source_request_units,
|
||||
target_reserved_units
|
||||
FROM relay_post_drain_migration_pins
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
assignment_epoch: '2',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '1',
|
||||
target_reserved_units: '2'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '4',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1'
|
||||
},
|
||||
{
|
||||
assignment_epoch: '5',
|
||||
drain_attempt_id: drainAttemptId,
|
||||
source_request_units: '0',
|
||||
target_reserved_units: '1'
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('uses an existing current-epoch migration once when retiring an older row', async () => {
|
||||
const input = await fixture()
|
||||
await pinMigration(input, input.migration.assignmentEpoch)
|
||||
await clearActivities(input)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id,
|
||||
previous_epoch, assignment_epoch, source_request_units,
|
||||
target_reserved_units, expires_at, target_registered_at,
|
||||
completed_at, aborted_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 2, 4, 0, 1, 0, 100, NULL, NULL, 100, 100)`,
|
||||
[
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id
|
||||
]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migration_incarnations
|
||||
(user_id, relay_host_id, assignment_epoch, source_cell_incarnation,
|
||||
target_cell_incarnation)
|
||||
VALUES (?, ?, 4, ?, ?)`,
|
||||
[
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
input.cellIncarnation,
|
||||
input.cellIncarnation
|
||||
]
|
||||
)
|
||||
await pinMigration(input, 4)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? ORDER BY assignment_epoch`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([
|
||||
{ assignment_epoch: '2', aborted_at: '45101' },
|
||||
{ assignment_epoch: '4', aborted_at: '45101' },
|
||||
{ assignment_epoch: '5', aborted_at: null }
|
||||
])
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 0,
|
||||
failed: 0,
|
||||
replacement: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a newer failed-cell epoch with ambiguous activity', async () => {
|
||||
const input = await fixture()
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET assignment_epoch = 4
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).rejects.toThrow(
|
||||
'migration_activity_topology_mismatch'
|
||||
)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 2,
|
||||
replacement: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves lease repair retryable when supersession later fails', async () => {
|
||||
const input = await fixture(1)
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET migration_leases = 0
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests - 1
|
||||
WHERE cell_id = ?`,
|
||||
[input.cells.failed.id]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_migrations SET expires_at = 0
|
||||
WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.migration.assignmentEpoch]
|
||||
)
|
||||
await fenceFailedCell(input)
|
||||
|
||||
await expect(supersedeAggregate(input)).rejects.toThrow(
|
||||
'relay_capacity_exhausted'
|
||||
)
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT activity_id FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ activity_id: 'migration:2' }])
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND assignment_epoch = 2`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ aborted_at: null }])
|
||||
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET capacity_requests = 2 WHERE cell_id = ?`,
|
||||
[input.cells.replacement.id]
|
||||
)
|
||||
await expect(supersedeAggregate(input)).resolves.toBe(1)
|
||||
expect(await reservations(input.cells)).toEqual({
|
||||
source: 1,
|
||||
failed: 0,
|
||||
replacement: 2
|
||||
})
|
||||
})
|
||||
|
||||
it('serializes concurrent supersession retries without duplicating capacity', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
const supersession = {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
}
|
||||
|
||||
const results = await Promise.all([
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, supersession),
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, supersession)
|
||||
])
|
||||
expect(results[0]).toEqual(results[1])
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 0, replacement: 2 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ count: '2' }])
|
||||
})
|
||||
|
||||
it('fails a competing aggregate supersession instead of reporting the wrong target', async () => {
|
||||
const input = await fixture()
|
||||
const alternate = {
|
||||
id: `recovery-cell-${sequence}-alternate`,
|
||||
url: `https://recovery-${sequence}-alternate.example.com`,
|
||||
capacityRequests: 20
|
||||
}
|
||||
await input.store.reconcileCells([...Object.values(input.cells), alternate], false)
|
||||
await input.heartbeat(alternate)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.heartbeat(alternate)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
),
|
||||
input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
alternate.id,
|
||||
100
|
||||
)
|
||||
])
|
||||
const fulfilled = results.filter(
|
||||
(result): result is PromiseFulfilledResult<number> => result.status === 'fulfilled'
|
||||
)
|
||||
const rejected = results.filter(
|
||||
(result): result is PromiseRejectedResult => result.status === 'rejected'
|
||||
)
|
||||
expect(fulfilled).toHaveLength(1)
|
||||
expect(fulfilled[0]!.value).toBe(1)
|
||||
expect(rejected).toHaveLength(1)
|
||||
expect(rejected[0]!.reason).toMatchObject({ message: 'migration_already_superseded' })
|
||||
const successor = await database.query(
|
||||
`SELECT assignment.cell_id, migration.target_cell_id
|
||||
FROM relay_assignments assignment
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = assignment.user_id
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE assignment.user_id = ? AND migration.previous_epoch = ?`,
|
||||
[input.identity.userId, input.migration.assignmentEpoch]
|
||||
)
|
||||
expect(successor).toHaveLength(1)
|
||||
expect(successor[0]!.cell_id).toBe(successor[0]!.target_cell_id)
|
||||
expect([input.cells.replacement.id, alternate.id]).toContain(
|
||||
successor[0]!.target_cell_id
|
||||
)
|
||||
})
|
||||
|
||||
it('rolls back every supersession change when replacement capacity is insufficient', async () => {
|
||||
const input = await fixture(1)
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
@@ -1205,153 +427,6 @@ describePostgres('PostgreSQL migration recovery', () => {
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
})
|
||||
|
||||
it('rolls back supersession when migration request-unit shape drifted', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
await database.query(
|
||||
`UPDATE relay_assignment_activity_leases
|
||||
SET request_units = request_units + 1
|
||||
WHERE user_id = ? AND activity_kind = 'migration'`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = reserved_requests + 1 WHERE cell_id = ?`,
|
||||
[input.cells.failed.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
).rejects.toThrow('migration_activity_lease_shape_mismatch')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 3, replacement: 0 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ cell_id: input.cells.failed.id, assignment_epoch: '2' }])
|
||||
})
|
||||
|
||||
it('rolls back supersession when locked cell reservation accounting drifted', async () => {
|
||||
const input = await fixture()
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(input.cells.failed.id, input.cellIncarnation)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`,
|
||||
[input.cells.replacement.id]
|
||||
)
|
||||
|
||||
await expect(
|
||||
input.store.supersedeRegisteredEvacuation(input.identity, {
|
||||
assignmentEpoch: input.migration.assignmentEpoch,
|
||||
sourceCellId: input.cells.source.id,
|
||||
currentTargetCellId: input.cells.failed.id,
|
||||
replacementTargetCellId: input.cells.replacement.id
|
||||
})
|
||||
).rejects.toThrow('migration_cell_reservation_accounting_mismatch')
|
||||
expect(await reservations(input.cells)).toEqual({ source: 1, failed: 2, replacement: 1 })
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT assignment_epoch, aborted_at FROM relay_assignment_migrations
|
||||
WHERE user_id = ?`,
|
||||
[input.identity.userId]
|
||||
)
|
||||
).toEqual([{ assignment_epoch: '2', aborted_at: null }])
|
||||
})
|
||||
|
||||
async function fenceFailedCell(input: RecoveryFixture): Promise<void> {
|
||||
await input.store.setCellEnabled(input.cells.failed.id, false)
|
||||
input.advancePastHeartbeat()
|
||||
await input.heartbeat(input.cells.source)
|
||||
await input.heartbeat(input.cells.replacement)
|
||||
await input.store.attestCellFence(
|
||||
input.cells.failed.id,
|
||||
input.cellIncarnation
|
||||
)
|
||||
}
|
||||
|
||||
async function supersedeAggregate(input: RecoveryFixture): Promise<number> {
|
||||
return await input.store.supersedeRegisteredCellEvacuations(
|
||||
input.cells.source.id,
|
||||
input.cells.failed.id,
|
||||
input.cells.replacement.id,
|
||||
100
|
||||
)
|
||||
}
|
||||
|
||||
async function clearActivities(input: RecoveryFixture): Promise<void> {
|
||||
await database.query(
|
||||
`DELETE FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0,
|
||||
reserved_invites = 0, pending_installs = 0, pending_confirmations = 0,
|
||||
migration_leases = 0
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
await database.query(
|
||||
`UPDATE relay_cells SET reserved_requests = 0
|
||||
WHERE cell_id IN (?, ?, ?)`,
|
||||
[input.cells.source.id, input.cells.failed.id, input.cells.replacement.id]
|
||||
)
|
||||
}
|
||||
|
||||
async function pinMigration(
|
||||
input: RecoveryFixture,
|
||||
assignmentEpoch: number
|
||||
): Promise<string> {
|
||||
const drainAttemptId = `${input.identity.userId}-drain-${assignmentEpoch}`
|
||||
await database.query(
|
||||
`INSERT INTO relay_post_drain_migration_pins
|
||||
(user_id, relay_host_id, assignment_epoch, drain_attempt_id,
|
||||
source_cell_id, source_cell_incarnation, target_cell_id,
|
||||
target_cell_incarnation, source_request_units, target_reserved_units,
|
||||
pinned_at)
|
||||
SELECT migration.user_id, migration.relay_host_id,
|
||||
migration.assignment_epoch, ?, migration.source_cell_id,
|
||||
incarnation.source_cell_incarnation, migration.target_cell_id,
|
||||
incarnation.target_cell_incarnation, migration.source_request_units,
|
||||
migration.target_reserved_units, 100
|
||||
FROM relay_assignment_migrations migration
|
||||
JOIN relay_assignment_migration_incarnations incarnation
|
||||
ON incarnation.user_id = migration.user_id
|
||||
AND incarnation.relay_host_id = migration.relay_host_id
|
||||
AND incarnation.assignment_epoch = migration.assignment_epoch
|
||||
WHERE migration.user_id = ? AND migration.relay_host_id = ?
|
||||
AND migration.assignment_epoch = ?`,
|
||||
[
|
||||
drainAttemptId,
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
assignmentEpoch
|
||||
]
|
||||
)
|
||||
return drainAttemptId
|
||||
}
|
||||
|
||||
async function assignmentActivities(input: RecoveryFixture): Promise<unknown[]> {
|
||||
return await database.query(
|
||||
`SELECT activity_id, activity_kind, cell_id, request_units
|
||||
FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`,
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
}
|
||||
|
||||
async function reservations(cells: RecoveryFixture['cells']): Promise<Record<string, number>> {
|
||||
const rows = await database.query(
|
||||
`SELECT cell_id, reserved_requests FROM relay_cells
|
||||
|
||||
@@ -184,7 +184,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
|
||||
})
|
||||
|
||||
now = migration.expiresAt + 1
|
||||
expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
|
||||
await store.releaseExpiredActivityLeases()
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: source.id,
|
||||
cellUrl: source.url,
|
||||
@@ -207,7 +207,7 @@ describePostgres('PostgreSQL drain-send locking', () => {
|
||||
cellId: source.id,
|
||||
cellIncarnation: sourceIncarnation
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'send-may-have-started', shouldSend: true })
|
||||
).resolves.toMatchObject({ shouldSend: true })
|
||||
await expect(
|
||||
database.query(
|
||||
`SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
|
||||
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
|
||||
import { readRegisteredMigrationInventory } from './registered-migration-inventory.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_migration_inventory_test'
|
||||
|
||||
describePostgres('PostgreSQL registered migration inventory', () => {
|
||||
let database: RelayDatabase | undefined
|
||||
|
||||
beforeAll(async () => {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await database?.close()
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
})
|
||||
|
||||
it('counts a disabled-target migration even while its source is active', async () => {
|
||||
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-a', 'https://a.example.test', 1, 4000, 1, 0, 0, 1),
|
||||
('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-a', 'general', 1), ('cell-b', 'existing-only', 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
|
||||
request_units, expires_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'control:source:1', 'control', 'cell-a', 1, ?, ?)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, 2, 2, 1, 2)`
|
||||
)
|
||||
|
||||
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES ('cell-c', 'https://c.example.test', 1, 4000, 0, 0, 0, 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES ('cell-c', 'migration-only', 1)`
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-2', '222222222222', 'cell-c', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[now, now]
|
||||
)
|
||||
await database!.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-2', '222222222222', 'cell-a', 'cell-c', 1, 2, 1, 2, ?, NULL, 1, 2)`,
|
||||
[now + 60_000]
|
||||
)
|
||||
|
||||
expect(await readRegisteredMigrationInventory(database!, now)).toEqual({
|
||||
open: 2,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
},
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-c',
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,120 +0,0 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
import { REGISTERED_MIGRATION_ABANDON_MS } from './registered-migration-abandonment.js'
|
||||
import {
|
||||
formatRegisteredMigrationInventory,
|
||||
readRegisteredMigrationInventory
|
||||
} from './registered-migration-inventory.js'
|
||||
|
||||
describe('registered migration inventory', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
afterEach(async () => {
|
||||
for (const database of databases.splice(0)) {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
it('does not restart abandonment when an old source migration lease is refreshed', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const now = REGISTERED_MIGRATION_ABANDON_MS + 10_000
|
||||
await insertCell(database, 'cell-a', false, 'existing-only')
|
||||
await insertCell(database, 'cell-b', true, 'migration-only')
|
||||
await insertMigration(database, now - 1)
|
||||
|
||||
const fresh = await readRegisteredMigrationInventory(database, now)
|
||||
|
||||
expect(fresh).toEqual({
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 1,
|
||||
abandoned: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
expect(formatRegisteredMigrationInventory(fresh)).toEqual([
|
||||
'[orca-relay] migration inventory open=1 expiredRegisteredInactive=1 abandonedRegistered=1',
|
||||
'[orca-relay] migration pair sourceCellId=cell-a targetCellId=cell-b open=1 expiredRegisteredInactive=1 abandoned=1'
|
||||
])
|
||||
await database.query(`UPDATE relay_assignment_migrations SET expires_at = ?, updated_at = ?`, [
|
||||
now + 1,
|
||||
now
|
||||
])
|
||||
expect((await readRegisteredMigrationInventory(database, now + 1)).abandoned).toBe(1)
|
||||
})
|
||||
|
||||
it('includes unregistered open migrations without logging a host identity', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
const now = 10_000
|
||||
await insertCell(database, 'cell-a', false, 'existing-only')
|
||||
await insertCell(database, 'cell-b', true, 'migration-only')
|
||||
await insertMigration(database, now + 60_000, null)
|
||||
|
||||
const inventory = await readRegisteredMigrationInventory(database, now)
|
||||
|
||||
expect(inventory).toEqual({
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0,
|
||||
pairs: [
|
||||
{
|
||||
sourceCellId: 'cell-a',
|
||||
targetCellId: 'cell-b',
|
||||
open: 1,
|
||||
inactive: 0,
|
||||
abandoned: 0
|
||||
}
|
||||
]
|
||||
})
|
||||
expect(formatRegisteredMigrationInventory(inventory).join('\n')).not.toContain('111111111111')
|
||||
})
|
||||
})
|
||||
|
||||
async function insertCell(
|
||||
database: RelayDatabase,
|
||||
cellId: string,
|
||||
enabled: boolean,
|
||||
admissionState: string
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_cells
|
||||
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
||||
observed_requests, last_heartbeat_at, updated_at)
|
||||
VALUES (?, ?, ?, 4000, 0, 0, 0, 1)`,
|
||||
[cellId, `https://${cellId}.example.test`, enabled ? 1 : 0]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
|
||||
VALUES (?, ?, 1)`,
|
||||
[cellId, admissionState]
|
||||
)
|
||||
}
|
||||
|
||||
async function insertMigration(
|
||||
database: RelayDatabase,
|
||||
expiresAt: number,
|
||||
targetRegisteredAt: number | null = 2
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignments
|
||||
(user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
|
||||
last_activity_at, reserved_controls, reserved_splices, reserved_invites,
|
||||
pending_installs, pending_confirmations, migration_leases)
|
||||
VALUES ('user-1', '111111111111', 'cell-b', 2, ?, ?, 0, 0, 0, 0, 0, 0)`,
|
||||
[expiresAt, expiresAt]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_migrations
|
||||
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
|
||||
assignment_epoch, source_request_units, target_reserved_units, expires_at,
|
||||
target_registered_at, created_at, updated_at)
|
||||
VALUES ('user-1', '111111111111', 'cell-a', 'cell-b', 1, 2, 1, 2, ?, ?, 1, 2)`,
|
||||
[expiresAt, targetRegisteredAt]
|
||||
)
|
||||
}
|
||||
@@ -1,23 +1,16 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS,
|
||||
RelayDrainReturnAdmission,
|
||||
type DrainReturnDeferral,
|
||||
type DrainReturnGrant
|
||||
} from './drain-return-admission.js'
|
||||
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
|
||||
|
||||
type Timer = { at: number; callback: () => void; cancelled: boolean }
|
||||
|
||||
function harness(
|
||||
overrides: {
|
||||
maxQueued?: number
|
||||
maxRetryAfterSeconds?: number
|
||||
onServiceMs?: (durationMs: number) => void
|
||||
} = {}
|
||||
) {
|
||||
let now = 0
|
||||
const timers: Timer[] = []
|
||||
const placement = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 2,
|
||||
maxQueued: 8,
|
||||
@@ -26,32 +19,18 @@ function harness(
|
||||
maxDrainReturnConcurrent: 1,
|
||||
maxDrainReturnQueued: overrides.maxQueued ?? 0,
|
||||
drainReturnWaitMs: 3_000,
|
||||
drainReturnMinIntervalMs: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS * 1_000,
|
||||
now: () => now,
|
||||
schedule: (callback, delayMs) => {
|
||||
const timer = { at: now + delayMs, callback, cancelled: false }
|
||||
timers.push(timer)
|
||||
return () => {
|
||||
timer.cancelled = true
|
||||
}
|
||||
}
|
||||
drainReturnMinIntervalMs: 2_000,
|
||||
now: () => 0,
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const lane = new RelayDrainReturnAdmission(placement, {
|
||||
maxConcurrent: 1,
|
||||
maxRetryAfterSeconds: overrides.maxRetryAfterSeconds ?? 300,
|
||||
now: () => now,
|
||||
onServiceMs: overrides.onServiceMs
|
||||
maxRetryAfterSeconds: 300,
|
||||
now: () => 0
|
||||
})
|
||||
return {
|
||||
lane,
|
||||
placement,
|
||||
advance: (ms: number) => {
|
||||
now += ms
|
||||
for (const timer of timers.filter((entry) => !entry.cancelled && entry.at <= now)) {
|
||||
timer.cancelled = true
|
||||
timer.callback()
|
||||
}
|
||||
}
|
||||
placement
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,28 +48,6 @@ const host = (index: number): string => `host${String(index).padStart(12, '0')}`
|
||||
|
||||
describe('drain-return admission', () => {
|
||||
|
||||
it('shortens the pacing once the measured service time is short', async () => {
|
||||
const { lane, advance } = harness()
|
||||
for (let index = 0; index < 30; index++) {
|
||||
const grant = admitted(await lane.acquire(host(index)))
|
||||
advance(50)
|
||||
grant.lease.release()
|
||||
advance(2_000)
|
||||
}
|
||||
admitted(await lane.acquire(host(100)))
|
||||
|
||||
const retries = []
|
||||
for (let index = 101; index <= 120; index++) {
|
||||
retries.push(deferred(await lane.acquire(host(index))).retryAfterSeconds)
|
||||
}
|
||||
|
||||
// ~50 ms per re-placement: twenty deferrals span about one second, where the
|
||||
// 860 ms starting estimate would have spread them over seventeen.
|
||||
expect(Math.max(...retries)).toBe(3)
|
||||
})
|
||||
|
||||
// The unclamped sample: the EWMA's floor and ceiling would hide the real tail.
|
||||
|
||||
it('answers a host’s own early retry with its interval, not a place behind the cohort', async () => {
|
||||
const { lane } = harness()
|
||||
admitted(await lane.acquire(host(0))).lease.release()
|
||||
@@ -99,7 +56,7 @@ describe('drain-return admission', () => {
|
||||
const repeat = deferred(await lane.acquire(host(0)))
|
||||
expect(repeat).toMatchObject({
|
||||
reason: 'host-rate-limited',
|
||||
retryAfterSeconds: DRAIN_RETURN_MIN_RETRY_AFTER_SECONDS
|
||||
retryAfterSeconds: 2
|
||||
})
|
||||
|
||||
// And it booked nothing: the next overflow still gets the first slot.
|
||||
@@ -107,17 +64,6 @@ describe('drain-return admission', () => {
|
||||
expect(deferred(await lane.acquire(host(2))).retryAfterSeconds).toBe(2)
|
||||
})
|
||||
|
||||
it('keeps a host’s reserved slot when it comes back early', async () => {
|
||||
const { lane, advance } = harness()
|
||||
admitted(await lane.acquire(host(0)))
|
||||
for (let index = 1; index <= 4; index++) await lane.acquire(host(index))
|
||||
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(6)
|
||||
|
||||
advance(1_000)
|
||||
// Early and still full: the same instant, not a new slot after everyone else.
|
||||
expect(deferred(await lane.acquire(host(5))).retryAfterSeconds).toBe(5)
|
||||
})
|
||||
|
||||
it('lets a queued placement go before a queued drain return', async () => {
|
||||
const { lane, placement } = harness({ maxQueued: 1 })
|
||||
const drain = admitted(await lane.acquire(host(0)))
|
||||
|
||||
+3
-42
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
|
||||
|
||||
describe('public assignment admission', () => {
|
||||
@@ -7,7 +7,7 @@ describe('public assignment admission', () => {
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 1,
|
||||
minIntervalMs: 0,
|
||||
now: vi.fn(() => 0)
|
||||
now: () => 0
|
||||
})
|
||||
const lease = await admission.acquire('host-a')
|
||||
expect(lease).not.toBeNull()
|
||||
@@ -46,19 +46,12 @@ describe('public assignment admission', () => {
|
||||
})
|
||||
|
||||
it('gives a bounded reserved waiter the next public slot', async () => {
|
||||
let expire!: () => void
|
||||
let cancelled = false
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 2,
|
||||
maxReservedConcurrent: 1,
|
||||
reservedWaitMs: 1_000,
|
||||
minIntervalMs: 0,
|
||||
schedule: (callback) => {
|
||||
expire = callback
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const first = await admission.acquire('host-a')
|
||||
const second = await admission.acquire('host-b')
|
||||
@@ -70,12 +63,10 @@ describe('public assignment admission', () => {
|
||||
const reserved = await reservedPromise
|
||||
|
||||
expect(reserved).not.toBeNull()
|
||||
expect(cancelled).toBe(true)
|
||||
await expect(admission.acquire('host-d')).resolves.toBeNull()
|
||||
reserved?.release()
|
||||
await expect(admission.acquire('host-d')).resolves.not.toBeNull()
|
||||
second?.release()
|
||||
expire()
|
||||
})
|
||||
|
||||
it('serializes same-host assignment and reserved recovery without losing priority', async () => {
|
||||
@@ -100,34 +91,4 @@ describe('public assignment admission', () => {
|
||||
await expect(admission.acquire('host-b')).resolves.not.toBeNull()
|
||||
})
|
||||
|
||||
it('tells each caller its own rejection reason without crossing requests', async () => {
|
||||
const admission = new RelayPublicAssignmentAdmission({
|
||||
maxConcurrent: 1,
|
||||
maxQueued: 1,
|
||||
waitMs: 1_000,
|
||||
maxReservedConcurrent: 1,
|
||||
reservedWaitMs: 1_000,
|
||||
minIntervalMs: 5_000,
|
||||
now: () => 0,
|
||||
schedule: () => () => undefined
|
||||
})
|
||||
const queued: string[] = []
|
||||
const displaced: string[] = []
|
||||
const throttled: string[] = []
|
||||
|
||||
const active = await admission.acquire('host-a')
|
||||
// 'host-b' waits, then reserved recovery for the same host displaces it.
|
||||
const ordinary = admission.acquire('host-b', (reason) => displaced.push(reason))
|
||||
const reserved = admission.acquireReserved('host-b', (reason) => queued.push(reason))
|
||||
await expect(ordinary).resolves.toBeNull()
|
||||
active?.release()
|
||||
const recovered = await reserved
|
||||
recovered?.release()
|
||||
await admission.acquire('host-a', (reason) => throttled.push(reason))
|
||||
|
||||
expect(displaced).toEqual(['superseded'])
|
||||
expect(queued).toEqual([])
|
||||
expect(throttled).toEqual(['host-rate-limited'])
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createRelayApp } from '../../app.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
|
||||
describe('public assignment circuit breaker', () => {
|
||||
it('rejects assign and resolve without invoking relay state', async () => {
|
||||
const assign = vi.fn()
|
||||
const resolveResume = vi.fn()
|
||||
const app = createRelayApp(config(), {
|
||||
store: { resolveResume } as never,
|
||||
assignments: { assign } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
for (const path of ['/v1/assign', '/v1/resolve']) {
|
||||
const response = await app.request(path, { method: 'POST' })
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('retry-after')).toBe('5')
|
||||
expect(await response.json()).toEqual({ error: 'assignments_temporarily_unavailable' })
|
||||
}
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
expect((await app.request('/health')).status).toBe(200)
|
||||
expect((await app.request('/v1/admin/drain', { method: 'POST' })).status).toBe(401)
|
||||
})
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
cellUrl: 'https://relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
|
||||
role: 'director',
|
||||
cellId: 'director',
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'runtime@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 3,
|
||||
publicAssignmentsEnabled: false,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
+3
-123
@@ -1,6 +1,5 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
RelayAssignmentRowBusyError,
|
||||
type RelayAssignment,
|
||||
type ResolvedRelayAssignment
|
||||
} from '../assignment-store.js'
|
||||
@@ -21,7 +20,6 @@ vi.mock('../../authentication/relay-token-verifier.js', () => ({
|
||||
import { createRelayApp } from '../../app.js'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
@@ -36,15 +34,11 @@ describe('drain-return lane', () => {
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
relayHostId === drained ? isolatedHome(relayHostId) : assignment('cell-o', relayHostId)
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const servedLanes: string[] = []
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
|
||||
recordAdmissionServiceMs: (lane) => servedLanes.push(lane)
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
// The re-placement holds the drain lane; the sticky lane's only slot is free.
|
||||
@@ -56,125 +50,12 @@ describe('drain-return lane', () => {
|
||||
)
|
||||
|
||||
expect(reconnect.status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: drained },
|
||||
{ classifyHomeRollIsolation: true }
|
||||
)
|
||||
replacement.resolve(assignment('cell-new', drained))
|
||||
expect((await moving).status).toBe(200)
|
||||
expect(outcomes).toEqual(['drain-return', 'sticky'])
|
||||
// The drained host held a sticky slot for its verification read too.
|
||||
expect(servedLanes).toEqual(['sticky', 'sticky', 'drain-return'])
|
||||
})
|
||||
|
||||
it('defers an overflowing drain return with a paced Retry-After and says why', async () => {
|
||||
const holder = 'hhhhhhhhhhhhhhhh'
|
||||
const replacement = deferred<RelayAssignment>()
|
||||
const assign = vi.fn(async () => await replacement.promise)
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
isolatedHome(relayHostId)
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const reasons: string[] = []
|
||||
const retryAfters: number[] = []
|
||||
const causes: string[] = []
|
||||
const app = createRelayApp(config({ drainReturnQueueMax: 1, drainReturnWaitMs: 50 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome),
|
||||
recordAssignmentRejectionReason: (lane, reason) => reasons.push(`${lane}:${reason}`),
|
||||
recordDrainReturnRetryAfter: (seconds) => retryAfters.push(seconds),
|
||||
recordAssignmentUnavailable: (cause) => causes.push(cause)
|
||||
})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
const holding = app.request('/v1/assign', assignmentRequest(holder, { reconnect: true }))
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
|
||||
const queued = app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('qqqqqqqqqqqqqqqq', { reconnect: true })
|
||||
)
|
||||
const overflow = await app.request(
|
||||
'/v1/assign',
|
||||
assignmentRequest('ffffffffffffffff', { reconnect: true })
|
||||
)
|
||||
|
||||
expect(overflow.status).toBe(503)
|
||||
expect(overflow.headers.get('retry-after')).toBe('2')
|
||||
const timedOut = await queued
|
||||
expect(timedOut.status).toBe(503)
|
||||
expect(Number(timedOut.headers.get('retry-after'))).toBeGreaterThanOrEqual(2)
|
||||
expect(reasons).toEqual(['drain-return:queue-full', 'drain-return:wait-timeout'])
|
||||
expect(retryAfters).toEqual([2, Number(timedOut.headers.get('retry-after'))])
|
||||
expect(outcomes.filter((outcome) => outcome === 'drain-return-deferred')).toHaveLength(2)
|
||||
expect(causes).toEqual(['drain-return-deferred', 'drain-return-deferred'])
|
||||
expect(warn.mock.calls.flat().join('\n')).toMatch(
|
||||
/lane=drain-return hinted=true reason=queue-full .*retryAfter=2/
|
||||
)
|
||||
replacement.resolve(assignment('cell-new', holder))
|
||||
expect((await holding).status).toBe(200)
|
||||
})
|
||||
|
||||
it('answers a drained host’s row-busy redial with the short interval', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const holder = 'hhhhhhhhhhhhhhhh'
|
||||
const host = 'bbbbbbbbbbbbbbbb'
|
||||
const holding = deferred<RelayAssignment>()
|
||||
let busy = true
|
||||
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
if (relayHostId === holder) return await holding.promise
|
||||
if (relayHostId === host && busy) {
|
||||
busy = false
|
||||
throw new RelayAssignmentRowBusyError()
|
||||
}
|
||||
return assignment('cell-new', relayHostId)
|
||||
})
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
isolatedHome(relayHostId)
|
||||
)
|
||||
const app = createRelayApp(config({ drainReturnQueueMax: 1 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const request = (relayHostId: string) =>
|
||||
Promise.resolve(
|
||||
app.request('/v1/assign', assignmentRequest(relayHostId, { reconnect: true }))
|
||||
)
|
||||
|
||||
const held = request(holder)
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(1))
|
||||
const queued = request(host)
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
// A cohort behind it books the next ten service slots.
|
||||
for (let index = 0; index < 10; index++) {
|
||||
const overflow = await request(hostId('f', index))
|
||||
expect(overflow.status).toBe(503)
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(2_490)
|
||||
holding.resolve(assignment('cell-new', holder))
|
||||
expect((await held).status).toBe(200)
|
||||
const refused = await queued
|
||||
expect(refused.headers.get('retry-after')).toBe('1')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
const redial = await request(host)
|
||||
|
||||
// Its own lane interval, not a slot after the ten hosts that came later.
|
||||
expect(redial.status).toBe(503)
|
||||
expect(redial.headers.get('retry-after')).toBe('2')
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
function hostId(prefix: string, index: number): string {
|
||||
return `${prefix}${String(index).padStart(15, '0')}`
|
||||
}
|
||||
|
||||
function assignmentRequest(relayHostId: string, extra: { reconnect?: boolean } = {}): RequestInit {
|
||||
return {
|
||||
method: 'POST',
|
||||
@@ -209,7 +90,7 @@ function deferred<T>() {
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
@@ -233,7 +114,6 @@ function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data',
|
||||
...overrides
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
|
||||
+2
-69
@@ -18,68 +18,10 @@ import { createRelayApp } from '../../app.js'
|
||||
|
||||
describe('public assignment overload', () => {
|
||||
|
||||
it('fairly drains a bounded incident-scale burst without raising database concurrency', async () => {
|
||||
const pending = new Map<string, ReturnType<typeof deferred<RelayAssignment>>>()
|
||||
const admittedHosts: string[] = []
|
||||
let active = 0
|
||||
let highWater = 0
|
||||
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
admittedHosts.push(relayHostId)
|
||||
active++
|
||||
highWater = Math.max(highWater, active)
|
||||
const operation = deferred<RelayAssignment>()
|
||||
pending.set(relayHostId, operation)
|
||||
try {
|
||||
return await operation.promise
|
||||
} finally {
|
||||
active--
|
||||
pending.delete(relayHostId)
|
||||
}
|
||||
})
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
const hosts = Array.from(
|
||||
{ length: 430 },
|
||||
(_, index) => `host-${String(index).padStart(11, '0')}`
|
||||
)
|
||||
|
||||
const firstWave = hosts.map((host) =>
|
||||
app.request('/v1/assign', assignmentRequest(host))
|
||||
)
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
|
||||
const retryWave = await Promise.all(
|
||||
hosts.map((host) => app.request('/v1/assign', assignmentRequest(host)))
|
||||
)
|
||||
|
||||
expect(retryWave.every((response) => response.status === 503)).toBe(true)
|
||||
expect(assign).toHaveBeenCalledTimes(2)
|
||||
while (assign.mock.calls.length < 130) {
|
||||
const activeOperations = [...pending]
|
||||
const expectedCalls = assign.mock.calls.length + activeOperations.length
|
||||
for (const [host, operation] of activeOperations) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
}
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(expectedCalls))
|
||||
}
|
||||
for (const [host, operation] of pending) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
}
|
||||
const firstResponses = await Promise.all(firstWave)
|
||||
expect(firstResponses.filter((response) => response.status === 200)).toHaveLength(130)
|
||||
expect(firstResponses.filter((response) => response.status === 503)).toHaveLength(300)
|
||||
expect(admittedHosts).toEqual(hosts.slice(0, 130))
|
||||
expect(highWater).toBe(2)
|
||||
})
|
||||
|
||||
it('refuses an oversized chunked body before parsing it, sparing only the reserve routes', async () => {
|
||||
const resolve = vi.fn()
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { resolve } as never,
|
||||
assignments: {} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
@@ -98,15 +40,11 @@ describe('public assignment overload', () => {
|
||||
const refused = await app.request('/v1/resolve', chunked(64 * 1024))
|
||||
expect(refused.status).toBe(413)
|
||||
expect(await refused.json()).toEqual({ error: 'request_too_large' })
|
||||
expect(resolve).not.toHaveBeenCalled()
|
||||
// Reserve bodies may reach 16 KiB, so the 4 KiB app limit must not answer for them. An
|
||||
// unavailable route refuses before its own limit, including a body past 16 KiB.
|
||||
// unavailable route refuses before its own limit.
|
||||
const reserve = await app.request('/internal/reserve', chunked(6 * 1024))
|
||||
expect(reserve.status).toBe(404)
|
||||
expect(await reserve.json()).toEqual({ error: 'reserve_unavailable' })
|
||||
const oversized = await app.request('/internal/reserve', chunked(20 * 1024))
|
||||
expect(oversized.status).toBe(404)
|
||||
expect(await oversized.json()).toEqual({ error: 'reserve_unavailable' })
|
||||
})
|
||||
|
||||
it('reserves one bounded resolve lane while assignment work is saturated', async () => {
|
||||
@@ -133,8 +71,6 @@ describe('public assignment overload', () => {
|
||||
await vi.waitFor(() => expect(assign).toHaveBeenCalledTimes(2))
|
||||
|
||||
const recovery = app.request('/v1/resolve', resolveRequest('cccccccccccccccc', 1))
|
||||
await Promise.resolve()
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
const excessResolve = await app.request(
|
||||
'/v1/resolve',
|
||||
resolveRequest('dddddddddddddddd', 2)
|
||||
@@ -143,8 +79,6 @@ describe('public assignment overload', () => {
|
||||
|
||||
expect(excessResolve.status).toBe(503)
|
||||
expect(excessAssign.status).toBe(503)
|
||||
expect(assign).toHaveBeenCalledTimes(2)
|
||||
expect(resolveResume).not.toHaveBeenCalled()
|
||||
|
||||
pendingAssignments
|
||||
.get('aaaaaaaaaaaaaaaa')
|
||||
@@ -153,7 +87,6 @@ describe('public assignment overload', () => {
|
||||
await vi.waitFor(() => expect(resolveResume).toHaveBeenCalledTimes(1))
|
||||
pendingResolve.resolve({ userId: 'user-1', relayDeviceId: 'device-1' })
|
||||
expect((await recovery).status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledTimes(1)
|
||||
|
||||
for (const [host, operation] of pendingAssignments) {
|
||||
operation.resolve(assignment(`cell-${host}`, host))
|
||||
|
||||
+1
-39
@@ -37,13 +37,11 @@ describe('public assignment reconnect lane', () => {
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) =>
|
||||
relayHostId === reconnecting ? assignment('cell-r', reconnecting) : null
|
||||
)
|
||||
const outcomes: string[] = []
|
||||
const app = createRelayApp(config({ publicAssignmentQueueMax: 0 }), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordAssignmentAdmission: (outcome) => outcomes.push(outcome)
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
// Saturate the placement lane with two in-flight newcomers, zero queue.
|
||||
@@ -59,18 +57,6 @@ describe('public assignment reconnect lane', () => {
|
||||
)
|
||||
|
||||
expect(reconnected.status).toBe(200)
|
||||
expect(resolve).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: reconnecting },
|
||||
{ classifyHomeRollIsolation: true }
|
||||
)
|
||||
expect(assign).toHaveBeenCalledWith(
|
||||
{ userId: 'user-1', relayHostId: reconnecting },
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{ hostRequested: true }
|
||||
)
|
||||
expect(outcomes).toContain('sticky')
|
||||
|
||||
pending.get(newcomerA)?.resolve(assignment('cell-a', newcomerA))
|
||||
pending.get(newcomerB)?.resolve(assignment('cell-b', newcomerB))
|
||||
@@ -78,28 +64,6 @@ describe('public assignment reconnect lane', () => {
|
||||
expect((await second).status).toBe(200)
|
||||
})
|
||||
|
||||
it('rate-limits repeat fast-lane attempts with the short retry-after', async () => {
|
||||
const host = 'rrrrrrrrrrrrrrrr'
|
||||
const assign = vi.fn(async () => assignment('cell-r', host))
|
||||
const resolve = vi.fn(async () => assignment('cell-r', host))
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
expect(
|
||||
(await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))).status
|
||||
).toBe(200)
|
||||
const repeat = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
|
||||
|
||||
expect(repeat.status).toBe(503)
|
||||
expect(repeat.headers.get('retry-after')).toBe('2')
|
||||
expect(resolve).toHaveBeenCalledTimes(1)
|
||||
expect(assign).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rejects on the fast lane when the verification probe fails transiently', async () => {
|
||||
const host = 'tttttttttttttttt'
|
||||
const assign = vi.fn()
|
||||
@@ -117,7 +81,6 @@ describe('public assignment reconnect lane', () => {
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('retry-after')).toBe('2')
|
||||
expect(assign).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses a dial whose assignment row is busy at once, and frees the fast lane', async () => {
|
||||
@@ -143,7 +106,6 @@ describe('public assignment reconnect lane', () => {
|
||||
expect(refused.status).toBe(503)
|
||||
expect(refused.headers.get('retry-after')).toBe('1')
|
||||
expect(await refused.json()).toEqual({ error: 'assignment_row_busy' })
|
||||
expect(warn.mock.calls.flat().join('\n')).toContain('reason=relay_assignment_row_busy')
|
||||
// The single fast-lane slot went back with the refusal.
|
||||
expect(
|
||||
(await app.request('/v1/assign', assignmentRequest(next, { reconnect: true }))).status
|
||||
|
||||
+9
-131
@@ -111,51 +111,15 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('constrained idle regional assignment transaction', () => {
|
||||
it.each(['missing', 'disabled', 'future'] as const)(
|
||||
'does only one read per tick with %s durable control and sees later enablement',
|
||||
async (state) => {
|
||||
const { store, database, safety } = await setup()
|
||||
const control = (await database.query(
|
||||
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
|
||||
))[0]!
|
||||
if (state === 'missing') {
|
||||
await database.query('DELETE FROM relay_region_rehome_control')
|
||||
} else {
|
||||
await database.query(
|
||||
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
|
||||
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
|
||||
)
|
||||
}
|
||||
const query = vi.spyOn(database, 'query')
|
||||
const transaction = vi.spyOn(database, 'transaction')
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
|
||||
expect(transaction).not.toHaveBeenCalled()
|
||||
}
|
||||
if (state === 'missing') {
|
||||
const columns = Object.keys(control)
|
||||
await database.query(
|
||||
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
|
||||
Object.values(control)
|
||||
)
|
||||
} else {
|
||||
await database.query(
|
||||
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
|
||||
[safety.observedAt]
|
||||
)
|
||||
}
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
expect(query.mock.calls.length).toBeGreaterThan(1)
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
}
|
||||
)
|
||||
it('sees durable control changes between candidate polls', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 1 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
})
|
||||
|
||||
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
|
||||
const { store, database, safety, request } = await setup()
|
||||
@@ -184,33 +148,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['next_dispatch_at', 'paused_until'] as const)(
|
||||
'skips the candidate join while %s holds the durable dispatch budget closed',
|
||||
async (column) => {
|
||||
const { store, database, safety } = await setup()
|
||||
const query = vi.spyOn(database, 'query')
|
||||
// One assignment only: setup leaves both fields at 0, and naming the other
|
||||
// one too would assign this column twice, which Postgres rejects.
|
||||
await database.query(
|
||||
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
|
||||
[safety.observedAt + 1]
|
||||
)
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
expect(query.mock.calls[1]![0]).toMatch(/FROM relay_region_rehome_worker_state/s)
|
||||
}
|
||||
await database.query(
|
||||
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
|
||||
[safety.observedAt]
|
||||
)
|
||||
query.mockClear()
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
|
||||
expect(query.mock.calls.length).toBeGreaterThan(2)
|
||||
}
|
||||
)
|
||||
|
||||
// sourceRequestUnits reads a reserve cell's missing phone leases as zero units; selection must drop it.
|
||||
it.each(['source', 'target'])('drops every candidate while the %s cell is recorded as reserve', async (cellId) => {
|
||||
const { store, safety } = await setup()
|
||||
@@ -267,51 +204,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
expect(first.some((c) => c.relayHostId === next[0]!.relayHostId)).toBe(false)
|
||||
})
|
||||
|
||||
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
for (const table of [
|
||||
'relay_assignments',
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_control_capabilities',
|
||||
'relay_region_decisions'
|
||||
]) {
|
||||
const template = (
|
||||
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
])
|
||||
)[0]!
|
||||
const columns = Object.keys(template)
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const values = columns.map((column) =>
|
||||
column === 'user_id' || column === 'relay_host_id' ? '?' : column
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
`idle-store-test-${String(index).padStart(3, '0')}`,
|
||||
`pagehost${String(index).padStart(8, '0')}`,
|
||||
identity.userId,
|
||||
identity.relayHostId
|
||||
]
|
||||
)
|
||||
}
|
||||
}
|
||||
const first = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
const next = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(first).toHaveLength(100)
|
||||
expect(next).toHaveLength(1)
|
||||
expect(next[0]!.relayHostId).toBe('pagehost00000099')
|
||||
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
|
||||
regionalRehomeCohortPercent: 100
|
||||
})
|
||||
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
|
||||
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
|
||||
})
|
||||
|
||||
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
|
||||
'rechecks generation when replacement wins after the initial authority lookup',
|
||||
async () => {
|
||||
@@ -430,20 +322,6 @@ describe('constrained idle regional assignment transaction', () => {
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
|
||||
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
|
||||
const { store, safety, database } = await setup()
|
||||
const before = await database.query('SELECT * FROM relay_assignments')
|
||||
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(candidates).toHaveLength(1)
|
||||
expect(candidates[0]).toMatchObject({
|
||||
sourceGeneration: 7,
|
||||
sourceCellId: 'source',
|
||||
targetCellId: 'target'
|
||||
})
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
|
||||
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
|
||||
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
function holdStatement(database: RelayDatabase, fragment: string) {
|
||||
|
||||
@@ -151,52 +151,6 @@ describe('idle regional rehome candidate window', () => {
|
||||
expect(new Set(candidates.map((candidate) => candidate.attemptId)).size).toBe(3)
|
||||
})
|
||||
|
||||
it('drops only the targets without room for the host plus its source activity', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await seedHost(store, identity)
|
||||
await database.query(
|
||||
'UPDATE relay_assignment_activity_leases SET request_units = 4 WHERE user_id = ?',
|
||||
[identity.userId]
|
||||
)
|
||||
// Five units needed: four source units plus the assignment the move reserves.
|
||||
await database.query("UPDATE relay_cells SET capacity_requests = 4 WHERE cell_id = 'asia-idle'")
|
||||
const short = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(short.map((candidate) => candidate.targetCellId)).toEqual(['asia-mid', 'asia-busy'])
|
||||
// Exactly enough room is enough; it ranks last because the ratio is per capacity.
|
||||
await database.query("UPDATE relay_cells SET capacity_requests = 5 WHERE cell_id = 'asia-idle'")
|
||||
const exact = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(exact.map((candidate) => candidate.targetCellId)).toEqual([
|
||||
'asia-mid',
|
||||
'asia-busy',
|
||||
'asia-idle'
|
||||
])
|
||||
})
|
||||
|
||||
it('reads a bounded window of decisions however many hosts are eligible', async () => {
|
||||
const { store, database, safety } = await setup()
|
||||
await seedHost(store, identity)
|
||||
await cloneHosts(database, identity, 700)
|
||||
const query = database.query.bind(database)
|
||||
const decisionRows: number[] = []
|
||||
vi.spyOn(database, 'query').mockImplementation(async (sql, params) => {
|
||||
const rows = await query(sql, params)
|
||||
if (/^SELECT user_id, relay_host_id FROM relay_region_decisions/.test(sql)) {
|
||||
decisionRows.push(rows.length)
|
||||
}
|
||||
return rows
|
||||
})
|
||||
const first = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
const next = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(decisionRows.length).toBeGreaterThan(0)
|
||||
for (const count of decisionRows) {
|
||||
expect(count).toBeLessThanOrEqual(500)
|
||||
}
|
||||
expect(first).toHaveLength(102)
|
||||
expect(next).toHaveLength(102)
|
||||
const firstKeys = new Set(first.map((c) => `${c.userId}/${c.targetCellId}`))
|
||||
expect(next.every((c) => !firstKeys.has(`${c.userId}/${c.targetCellId}`))).toBe(true)
|
||||
})
|
||||
|
||||
it("keeps the window's last host when a decision turns eligible between the two reads", async () => {
|
||||
const { store, database, safety, now } = await setup()
|
||||
await seedHost(store, identity)
|
||||
@@ -252,7 +206,6 @@ describe('idle regional rehome candidate window', () => {
|
||||
for (let poll = 0; poll < 20 && !wrapped; poll++) {
|
||||
const page = await store.selectIdleRegionalRehomeCandidates(safety)
|
||||
expect(page.length).toBeGreaterThan(0)
|
||||
expect(page.length).toBeLessThanOrEqual(102)
|
||||
for (const candidate of page) {
|
||||
const key = `${candidate.userId}/${candidate.targetCellId}`
|
||||
if (seen.has(key)) {
|
||||
@@ -286,23 +239,4 @@ describe('idle regional rehome candidate window', () => {
|
||||
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('excludes a host outside the cohort', async () => {
|
||||
const { store, database } = await setup()
|
||||
await seedHost(store, identity)
|
||||
const now = 100_000_000 + 86_400_000
|
||||
const safety = {
|
||||
observedAt: now,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
await database.query('UPDATE relay_region_decisions SET cohort_bucket = 40')
|
||||
const narrow = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 40 })
|
||||
expect(await narrow.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
|
||||
const wide = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 41 })
|
||||
expect(await wide.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -80,7 +80,7 @@ describe('persisted region decisions across director restart', () => {
|
||||
expect(row.outcome).toBe('inconclusive')
|
||||
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(successor.generation).toBe(2)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
})
|
||||
@@ -111,7 +111,7 @@ describe('persisted region decisions across director restart', () => {
|
||||
context.setNow(1_000_000_000 - 60_000)
|
||||
const restarted = await context.restart()
|
||||
const successor = (await context.issue(restarted.store)).window!
|
||||
expect(successor.generation).toBe(context.window.generation + 1)
|
||||
expect(successor.generation).toBe(2)
|
||||
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
|
||||
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
|
||||
.toMatchObject({ reportStatus: 'stale' })
|
||||
|
||||
@@ -124,104 +124,7 @@ async function window(context: Awaited<ReturnType<typeof setup>>) {
|
||||
return result.window!
|
||||
}
|
||||
|
||||
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
|
||||
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
|
||||
const attemptId = '33333333-3333-4333-8333-333333333333'
|
||||
await context.database.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
|
||||
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
|
||||
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
|
||||
[
|
||||
attemptId,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
cells[0]!.id,
|
||||
incarnations[0],
|
||||
cells[1]!.id,
|
||||
incarnations[1],
|
||||
migration.previousEpoch,
|
||||
migration.assignmentEpoch,
|
||||
context.now(),
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
return { migration }
|
||||
}
|
||||
|
||||
describe('ordered region decisions and migration outcomes', () => {
|
||||
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
|
||||
const context = await setup()
|
||||
const { migration } = await regionalMigration(context)
|
||||
context.advance(1_000)
|
||||
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
|
||||
const outcomes = await context.store.regionCorrectionOutcomes()
|
||||
expect(outcomes).toEqual([
|
||||
expect.objectContaining({
|
||||
sourceCellId: cells[0]!.id,
|
||||
targetCellId: cells[1]!.id,
|
||||
state: 'registering',
|
||||
count: 1,
|
||||
oldestOpenMs: 1_000
|
||||
})
|
||||
])
|
||||
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
|
||||
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
|
||||
before
|
||||
)
|
||||
await context.store.activateControl(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.markMigrationTargetRegistered(identity, {
|
||||
cellId: cells[1]!.id,
|
||||
assignmentEpoch: migration.assignmentEpoch
|
||||
})
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({ state: 'registered' })
|
||||
])
|
||||
await context.store.releaseActivity(identity, context.activityId)
|
||||
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
|
||||
expect(await context.store.regionCorrectionOutcomes()).toEqual([
|
||||
expect.objectContaining({
|
||||
state: 'completed',
|
||||
targetReservedUnits: 0,
|
||||
oldestOpenMs: 0
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
|
||||
const context = await setup()
|
||||
const query = context.database.query.bind(context.database)
|
||||
const transaction = context.database.transaction.bind(context.database)
|
||||
const queryLocked = context.database.queryLocked.bind(context.database)
|
||||
context.database.query = async (sql, params) => {
|
||||
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
|
||||
return query(sql, params)
|
||||
}
|
||||
context.database.transaction = async () => {
|
||||
throw new Error('preview_must_not_open_mutating_transaction')
|
||||
}
|
||||
context.database.queryLocked = async () => {
|
||||
throw new Error('preview_must_not_lock')
|
||||
}
|
||||
try {
|
||||
const preview = await context.store.previewRegionalRehomeEligibility()
|
||||
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
|
||||
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
|
||||
expect(JSON.stringify(preview)).not.toContain(identity.userId)
|
||||
} finally {
|
||||
context.database.query = query
|
||||
context.database.transaction = transaction
|
||||
context.database.queryLocked = queryLocked
|
||||
}
|
||||
})
|
||||
|
||||
it('allocates distinct ordered generations for concurrent window issuers', async () => {
|
||||
const context = await setup()
|
||||
const replies = await Promise.all([window(context), window(context), window(context)])
|
||||
|
||||
+4
-26
@@ -1,10 +1,6 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
type RelayDatabase
|
||||
} from '../../database/database.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from '../../database/database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
@@ -127,25 +123,15 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
await expect(
|
||||
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
|
||||
).rejects.toMatchObject({ code: '23514' })
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'
|
||||
ORDER BY conname`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
// The existing control row keeps its tuning and gains the cooldown.
|
||||
// Preserve existing control tuning across the upgrade.
|
||||
const control = await client.query(
|
||||
`SELECT generation, preference_max_age_ms, host_cooldown_ms
|
||||
`SELECT generation, preference_max_age_ms
|
||||
FROM relay_region_rehome_control WHERE control_id = 'global'`
|
||||
)
|
||||
expect(control.rows).toEqual([
|
||||
{
|
||||
generation: '3',
|
||||
preference_max_age_ms: '86400000',
|
||||
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
|
||||
preference_max_age_ms: '86400000'
|
||||
}
|
||||
])
|
||||
})
|
||||
@@ -182,14 +168,6 @@ describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
})
|
||||
}, 60_000)
|
||||
})
|
||||
|
||||
+2
-14
@@ -1,4 +1,4 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
|
||||
|
||||
@@ -134,10 +134,6 @@ const moves = (candidates: { userId: string; sourceCellId: string; targetCellId:
|
||||
to: cells.find(({ id }) => id === targetCellId)!.region
|
||||
}))
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('rehome source-region filter', () => {
|
||||
it('keeps US to Asia moves and drops Asia to US moves', async () => {
|
||||
const context = await setup('us-central1')
|
||||
@@ -151,13 +147,5 @@ describe('rehome source-region filter', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('reports the filtered hosts in the preview, agreeing with selection', async () => {
|
||||
const context = await setup('us-central1')
|
||||
const preview = await context.store.previewRegionalRehomeEligibility(context.safety())
|
||||
expect(preview.counts).toEqual({
|
||||
'eligible:us-central1-to-asia-east2': 1,
|
||||
'source-outside-director-region': 1
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-99
@@ -1,7 +1,6 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import {
|
||||
consumeRelayCellInventoryHold,
|
||||
openRelayDatabase,
|
||||
type PostgresDatabase,
|
||||
type RelayDatabase
|
||||
@@ -18,21 +17,6 @@ const describePostgres = databaseUrl ? describe : describe.skip
|
||||
// Roughly the round trip from an Asia cell to the us-central1 database.
|
||||
const STATEMENT_DELAY_MS = 150
|
||||
|
||||
// Probe the formerly fleet-locked tables plus admission/region tables read by commit.
|
||||
const CELL_TABLES = [
|
||||
'relay_cells',
|
||||
'relay_cell_runtime',
|
||||
'relay_cell_capabilities',
|
||||
'relay_cell_rehome_safety',
|
||||
'relay_cell_admission',
|
||||
'relay_cell_regions'
|
||||
] as const
|
||||
|
||||
// The target-row statement locks exactly these, held from it to COMMIT.
|
||||
const TARGET_LOCKED = ['target:relay_cells', 'target:relay_cell_admission']
|
||||
|
||||
type Trip = { sql: string; lockable: Record<string, boolean>; probeMs: number }
|
||||
|
||||
type DelayControl = StatementDelay & { beforeTrip: (sql: string) => Promise<void> }
|
||||
|
||||
describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
@@ -97,27 +81,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function lockable(table: string, cellId: string): Promise<boolean> {
|
||||
try {
|
||||
await observer.transaction(
|
||||
async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT cell_id FROM ${table} WHERE cell_id = ?`,
|
||||
[cellId],
|
||||
{
|
||||
failIfUnavailable: true
|
||||
}
|
||||
)
|
||||
},
|
||||
{ reportRetries: false }
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'database_lock_unavailable') return false
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function reservedRequests(cellId: string): Promise<number> {
|
||||
const row = (
|
||||
await primary.query(`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`, [cellId])
|
||||
@@ -152,59 +115,6 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
)
|
||||
}
|
||||
|
||||
it('holds only the target cell and admission rows until the next COMMIT boundary', async () => {
|
||||
const context = await fixture()
|
||||
const request = await context.select()
|
||||
const trips: Trip[] = []
|
||||
const probed = {
|
||||
source: context.source.id,
|
||||
target: context.target.id,
|
||||
bystander: context.bystander.id
|
||||
}
|
||||
control.beforeTrip = async (sql) => {
|
||||
const probeStartedAt = performance.now()
|
||||
const state: Record<string, boolean> = {}
|
||||
for (const [role, cellId] of Object.entries(probed)) {
|
||||
for (const table of CELL_TABLES) state[`${role}:${table}`] = await lockable(table, cellId)
|
||||
}
|
||||
trips.push({ sql, lockable: state, probeMs: performance.now() - probeStartedAt })
|
||||
}
|
||||
consumeRelayCellInventoryHold(delayed)
|
||||
control.enabled = true
|
||||
|
||||
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
|
||||
control.enabled = false
|
||||
|
||||
expect(result).toEqual({ outcome: 'committed' })
|
||||
expect(trips.length).toBeGreaterThanOrEqual(15)
|
||||
expect(trips[trips.length - 1]!.sql).toBe('COMMIT')
|
||||
for (const trip of trips) {
|
||||
for (const [key, free] of Object.entries(trip.lockable)) {
|
||||
if (TARGET_LOCKED.includes(key)) continue
|
||||
expect({ sql: trip.sql, key, free }).toEqual({ sql: trip.sql, key, free: true })
|
||||
}
|
||||
}
|
||||
for (const key of TARGET_LOCKED) {
|
||||
expect(trips.filter((trip) => !trip.lockable[key]).map((trip) => trip.sql)).toEqual([
|
||||
'COMMIT'
|
||||
])
|
||||
}
|
||||
const counts = consumeRelayCellInventoryHold(delayed)
|
||||
const commitProbeMs = trips.at(-1)!.probeMs
|
||||
console.info(
|
||||
JSON.stringify({
|
||||
event: 'rehome_target_row_hold',
|
||||
trips: trips.length,
|
||||
commitProbeMs,
|
||||
...counts
|
||||
})
|
||||
)
|
||||
expect(counts.rehomeTargetRowHolds).toBe(1)
|
||||
expect(counts.cellInventoryHoldMaxSite).toBe('rehome-target-row')
|
||||
expect(counts.rehomeTargetRowHoldMsMax).toBeGreaterThanOrEqual(STATEMENT_DELAY_MS)
|
||||
expect(await reservedRequests(context.target.id)).toBe(context.targetReservedBefore + 2)
|
||||
}, 45_000)
|
||||
|
||||
it('defers when the target leaves general admission after selection', async () => {
|
||||
const context = await fixture()
|
||||
const request = await context.select()
|
||||
@@ -240,23 +150,14 @@ describePostgres('PostgreSQL regional rehome target-row lock', () => {
|
||||
})
|
||||
await lockedPromise
|
||||
}
|
||||
consumeRelayCellInventoryHold(delayed)
|
||||
control.enabled = true
|
||||
|
||||
const startedAt = performance.now()
|
||||
const result = await context.delayedStore.commitIdleRegionalRehome(request, context.safety())
|
||||
const elapsedMs = performance.now() - startedAt
|
||||
control.enabled = false
|
||||
release()
|
||||
await writer
|
||||
|
||||
expect(result).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
|
||||
expect(consumeRelayCellInventoryHold(delayed)).toMatchObject({
|
||||
cellInventoryLockUnavailable: 1,
|
||||
rehomeTargetRowHolds: 0
|
||||
})
|
||||
// NOWAIT: no 1s lock_timeout wait and no transaction retry behind the writer.
|
||||
expect(elapsedMs).toBeLessThan(40 * STATEMENT_DELAY_MS)
|
||||
await expectNothingCommitted(context)
|
||||
})
|
||||
|
||||
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from '../../database/database.js'
|
||||
import { REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT } from './regional-rehome-safety.js'
|
||||
|
||||
const cell = (id: string, region: 'us-central1' | 'asia-east2') => ({
|
||||
id,
|
||||
url: `https://${id}.relay.example.test`,
|
||||
region,
|
||||
capacityRequests: 100,
|
||||
connectionHardCap: 1_000 as const,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
const source = cell('us-c1', 'us-central1')
|
||||
const noHeadroom = cell('asia-a', 'asia-east2')
|
||||
const unclean = cell('asia-b', 'asia-east2')
|
||||
const highLoad = cell('asia-c', 'asia-east2')
|
||||
const lowLoad = cell('asia-d', 'asia-east2')
|
||||
|
||||
const incarnation = (n: number) =>
|
||||
`${String(n).repeat(8)}-${String(n).repeat(4)}-4${String(n).repeat(3)}` +
|
||||
`-8${String(n).repeat(3)}-${String(n).repeat(12)}`
|
||||
|
||||
async function setup() {
|
||||
let clock = 1_000_000
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayAssignmentStore(database, () => clock, {
|
||||
regionalRehomeCohortPercent: 100,
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
})
|
||||
await store.inspectRegionalRehomeControl()
|
||||
clock += 24 * 60 * 60_000
|
||||
await store.applyRegionalRehomeControl({
|
||||
expectedGeneration: 0,
|
||||
enabled: true,
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
|
||||
const beat = async (
|
||||
config: typeof source,
|
||||
n: number,
|
||||
protocol: number,
|
||||
state: { observedRequests: number; enforcedConnections: number; sqlFailures: number }
|
||||
) => {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: config.id,
|
||||
cellUrl: config.url,
|
||||
region: config.region,
|
||||
cellIncarnation: incarnation(n),
|
||||
startedAt: 900_000,
|
||||
ready: true,
|
||||
observedRequests: state.observedRequests,
|
||||
totalConnections: state.enforcedConnections,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: state.enforcedConnections,
|
||||
connectionInclusionWatermark: clock,
|
||||
connectionHardCap: 1_000,
|
||||
connectionUnobservedBound: 60
|
||||
})
|
||||
await store.recordCellRegionalRehomeStatus({
|
||||
cellId: config.id,
|
||||
cellIncarnation: incarnation(n),
|
||||
regionalRehomeProtocol: protocol,
|
||||
safety: {
|
||||
observedAt: clock,
|
||||
sqlFailures: state.sqlFailures,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
})
|
||||
}
|
||||
const activatePreferredSource = async () => {
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1,
|
||||
idleRegionalRehome: true,
|
||||
cellIncarnation: incarnation(1)
|
||||
})
|
||||
const { window } = await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{ v: 1, action: 'issue-window' },
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
expect(window).toBeDefined()
|
||||
await store.exchangeRegionCorrection(
|
||||
identity,
|
||||
{
|
||||
v: 1,
|
||||
action: 'report',
|
||||
generation: window!.generation,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
policyVersion: 1,
|
||||
outcome: 'conclusive',
|
||||
measurements: { 'us-central1': 180, 'asia-east2': 40 }
|
||||
},
|
||||
assignment.assignmentEpoch
|
||||
)
|
||||
}
|
||||
const safety = () => ({
|
||||
observedAt: clock,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
return { database, store, beat, activatePreferredSource, safety }
|
||||
}
|
||||
|
||||
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
|
||||
|
||||
describe('regional rehome target selection', () => {
|
||||
it('never selects a target without connection headroom, even at lowest load', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
// Lowest load but the connection hard cap is exhausted.
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(lowLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('falls to the next clean target when the load winner goes unclean', async () => {
|
||||
const context = await setup()
|
||||
await context.beat(source, 1, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(noHeadroom, 2, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 3, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 3, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 3, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.activatePreferredSource()
|
||||
|
||||
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
|
||||
const attempt = candidates[0]
|
||||
expect(attempt?.targetCellId).toBe(highLoad.id)
|
||||
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
|
||||
outcome: 'committed'
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
})
|
||||
@@ -1,189 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import type { RelayConfig } from '../../config.js'
|
||||
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
|
||||
|
||||
describe('regional rehome worker', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
|
||||
vi.useFakeTimers()
|
||||
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
|
||||
const worker = startRegionalRehomeWorker(
|
||||
config(),
|
||||
{
|
||||
selectIdleRegionalRehomeCandidates
|
||||
} as unknown as RelayAssignmentStore,
|
||||
{
|
||||
safetySnapshot: () => safety(Date.now()),
|
||||
random: () => 0
|
||||
}
|
||||
)!
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(5_999)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
|
||||
worker.stop()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
|
||||
} finally {
|
||||
worker.stop()
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('stops walking the page when the source names a deferral no candidate can pass', async () => {
|
||||
const fetchImpl = respondWith([{ outcome: 'deferred', reason: 'concurrency-limit' }])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 3, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1)
|
||||
expect(summaries.entries).toEqual([
|
||||
{
|
||||
event: 'orca_relay_idle_rehome_dispatch_summary',
|
||||
candidates: 3,
|
||||
dispatched: 1,
|
||||
stoppedBy: 'concurrency-limit',
|
||||
outcomes: { 'deferred:concurrency-limit': 1 }
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps its whole-page walk when the source sends no reason at all', async () => {
|
||||
const fetchImpl = respondWith([{ outcome: 'deferred' }])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 3, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3)
|
||||
expect(summaries.entries[0]).toMatchObject({
|
||||
stoppedBy: null,
|
||||
outcomes: { deferred: 3 }
|
||||
})
|
||||
})
|
||||
|
||||
it('walks past a deferral that only concerns the one candidate', async () => {
|
||||
const fetchImpl = respondWith([
|
||||
{ outcome: 'deferred', reason: 'host-unsupported' },
|
||||
{ outcome: 'busy' },
|
||||
{ outcome: 'committed' }
|
||||
])
|
||||
const summaries = collectSummaries()
|
||||
try {
|
||||
await runOnePoll(fetchImpl, 4, summaries)
|
||||
} finally {
|
||||
summaries.restore()
|
||||
}
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3)
|
||||
expect(summaries.entries[0]).toMatchObject({
|
||||
dispatched: 3,
|
||||
stoppedBy: 'committed',
|
||||
outcomes: { 'deferred:host-unsupported': 1, busy: 1, committed: 1 }
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function respondWith(bodies: { outcome: string; reason?: string }[]) {
|
||||
let index = 0
|
||||
return vi.fn(async () => {
|
||||
const body = bodies[Math.min(index++, bodies.length - 1)]!
|
||||
return new Response(JSON.stringify({ v: 1, ...body }), {
|
||||
headers: { 'content-type': 'application/json' }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function collectSummaries() {
|
||||
const entries: Record<string, unknown>[] = []
|
||||
let arrived: (() => void) | undefined
|
||||
// Construction immediately polls; await first instead of waiting for a timer tick.
|
||||
const first = new Promise<void>((resolve) => {
|
||||
arrived = resolve
|
||||
})
|
||||
const original = console.warn
|
||||
console.warn = (line: unknown, ...rest: unknown[]) => {
|
||||
try {
|
||||
const parsed = JSON.parse(line as string) as Record<string, unknown>
|
||||
if (parsed.event === 'orca_relay_idle_rehome_dispatch_summary') {
|
||||
entries.push(parsed)
|
||||
arrived?.()
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// Not a JSON log line; fall through to the original writer.
|
||||
}
|
||||
original(line as string, ...rest)
|
||||
}
|
||||
return { entries, first, restore: () => (console.warn = original) }
|
||||
}
|
||||
|
||||
async function runOnePoll(
|
||||
fetchImpl: typeof fetch,
|
||||
candidates: number,
|
||||
summaries: { first: Promise<void> }
|
||||
): Promise<void> {
|
||||
const assignments = {
|
||||
selectIdleRegionalRehomeCandidates: vi.fn(async () =>
|
||||
Array.from({ length: candidates }, (_, index) => ({
|
||||
v: 1 as const,
|
||||
attemptId: `00000000-0000-4000-8000-00000000000${index}`,
|
||||
userId: `user-${index}`,
|
||||
relayHostId: 'abcdefghijklmnop',
|
||||
sourceCellId: 'us-c1',
|
||||
sourceCellUrl: 'https://us-c1.relay.example.test',
|
||||
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
sourceAssignmentEpoch: 1,
|
||||
sourceGeneration: 1,
|
||||
targetCellId: 'asia-c1'
|
||||
}))
|
||||
)
|
||||
} as unknown as RelayAssignmentStore
|
||||
const worker = startRegionalRehomeWorker(config(), assignments, {
|
||||
fetch: fetchImpl,
|
||||
identityToken: async () => 'token',
|
||||
safetySnapshot: () => safety(Date.now()),
|
||||
intervalMs: 60_000
|
||||
})!
|
||||
try {
|
||||
await summaries.first
|
||||
} finally {
|
||||
worker.stop()
|
||||
}
|
||||
}
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
return {
|
||||
role: 'director',
|
||||
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
|
||||
rehomeDirectorServiceAccount: 'relay-director@example.test',
|
||||
...overrides
|
||||
} as RelayConfig
|
||||
}
|
||||
|
||||
function safety(observedAt: number) {
|
||||
return {
|
||||
requiredCells: 2,
|
||||
missingCells: 0,
|
||||
observedAt,
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
databasePoolTotal: 3,
|
||||
databasePoolIdle: 3,
|
||||
databasePoolWaiting: 0,
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolOldestWaitMs: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { PLACEMENT_INTAKE_SHARE, PlacementLoadBand } from './placement-load-band.js'
|
||||
import { PlacementLoadBand } from './placement-load-band.js'
|
||||
import { RelayAssignmentStore } from '../assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from '../../database/database.js'
|
||||
|
||||
@@ -13,74 +13,8 @@ function seededRandom(seed: number): () => number {
|
||||
}
|
||||
}
|
||||
|
||||
type SimulatedCell = { cellId: string; reserved: number; capacity: number }
|
||||
|
||||
const quiet = () => undefined
|
||||
|
||||
// Each pick goes to one of the directors at random, as the load balancer spreads requests.
|
||||
function simulate(
|
||||
directors: PlacementLoadBand[],
|
||||
cells: SimulatedCell[],
|
||||
picks: number,
|
||||
at: (index: number) => number,
|
||||
route: () => number = seededRandom(99)
|
||||
) {
|
||||
const counts = new Map(cells.map((cell) => [cell.cellId, 0]))
|
||||
for (let index = 0; index < picks; index += 1) {
|
||||
const ranked = cells
|
||||
.filter((cell) => cell.reserved < cell.capacity)
|
||||
.map((cell) => ({ cellId: cell.cellId, loadRatio: cell.reserved / cell.capacity, cell }))
|
||||
.sort(
|
||||
(left, right) => left.loadRatio - right.loadRatio || left.cellId.localeCompare(right.cellId)
|
||||
)
|
||||
const director = directors[Math.floor(route() * directors.length)]
|
||||
const picked = director?.pick(ranked, at(index))
|
||||
if (!picked) throw new Error('no candidate')
|
||||
picked.cell.reserved += 1
|
||||
counts.set(picked.cellId, (counts.get(picked.cellId) ?? 0) + 1)
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// c27's drain on 10-08: 2,770 hosts onto one empty Asia cell (2,805 free)
|
||||
// and four fuller ones (2,311 free between them).
|
||||
function c27DrainCells(): SimulatedCell[] {
|
||||
return [
|
||||
{ cellId: 'cell-empty', reserved: 0, capacity: 4000 },
|
||||
...['cell-b', 'cell-c', 'cell-d', 'cell-e'].map((cellId, index) => ({
|
||||
cellId,
|
||||
reserved: 4000 - 577 - (index === 0 ? 3 : 0),
|
||||
capacity: 4000
|
||||
}))
|
||||
]
|
||||
}
|
||||
|
||||
function drainShare(paceMs: number, seed: number) {
|
||||
const cells = c27DrainCells()
|
||||
const directors = Array.from(
|
||||
{ length: 5 },
|
||||
(_, index) => new PlacementLoadBand(seededRandom(seed + index), quiet)
|
||||
)
|
||||
const hosts = 2770
|
||||
const counts = simulate(directors, cells, hosts, (index) => (index * paceMs) / hosts)
|
||||
return { counts, cells, share: (counts.get('cell-empty') ?? 0) / hosts }
|
||||
}
|
||||
|
||||
describe('PlacementLoadBand', () => {
|
||||
it.each([
|
||||
['1,200 s', 1_200_000],
|
||||
['300 s', 300_000]
|
||||
])('keeps the empty cell near the share across five directors at the %s pace', (_, paceMs) => {
|
||||
for (const seed of [1, 2, 3]) {
|
||||
const { counts, cells, share } = drainShare(paceMs, seed * 10)
|
||||
expect([...counts.values()].reduce((sum, count) => sum + count, 0)).toBe(2770)
|
||||
expect(share).toBeGreaterThan(0.3)
|
||||
expect(share).toBeLessThan(0.5)
|
||||
for (const cell of cells) expect(cell.reserved).toBeLessThanOrEqual(cell.capacity)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('leastLoadedCell with the load band', () => {
|
||||
async function setupRegions() {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
@@ -121,20 +55,6 @@ describe('leastLoadedCell with the load band', () => {
|
||||
return counts
|
||||
}
|
||||
|
||||
it('spreads within the preferred region and keeps an empty US cell out', async () => {
|
||||
const { database, store } = await setupRegions()
|
||||
try {
|
||||
const counts = await placements(store, 200)
|
||||
|
||||
expect(counts.get('us-a')).toBeUndefined()
|
||||
expect(counts.get('asia-a')).toBeLessThanOrEqual(PLACEMENT_INTAKE_SHARE * 200 + 1)
|
||||
const used = [...counts.values()].filter((count) => count >= 20)
|
||||
expect(used.length).toBeGreaterThanOrEqual(3)
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('fills the preferred region to its hard cap before spilling, never past it', async () => {
|
||||
const { database, store } = await setupRegions()
|
||||
try {
|
||||
|
||||
@@ -111,25 +111,6 @@ function seated(epoch: number, joinedAt = 0) {
|
||||
|
||||
describe('reserve assignment on a director', () => {
|
||||
|
||||
it('reads what each cell acts on: a tripped dead-man is db, a flip back still leasing is reserve', async () => {
|
||||
const tripped = setup({
|
||||
feeds: [feed('c1', { admitModeEffective: 'db' }), feed('c2', { admitModeEffective: 'db' })]
|
||||
})
|
||||
expect(await tripped.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
|
||||
const leasing = setup({ feeds: [feed('c1', { admitMode: 'db', admitModeEffective: 'reserve' }), feed('c2')] })
|
||||
expect(leasing.directory.admitModeOf('c1')).toBe('reserve')
|
||||
// Raw db must immediately stop booking/pacing even while effective mode stays reserve during re-registration.
|
||||
const reregistering = setup({
|
||||
feeds: [
|
||||
feed('c1', { admitModeEffective: 'reserve', admitModeRaw: 'db' }),
|
||||
feed('c2', { admitModeEffective: 'reserve', admitModeRaw: 'db' })
|
||||
]
|
||||
})
|
||||
expect(await reregistering.assignment.plan(HOST, { reconnect: false, region: US })).toEqual({ kind: 'database' })
|
||||
expect(reregistering.reserved).toEqual([])
|
||||
expect(reregistering.directory.admitModeOf('c1')).toBe('reserve')
|
||||
})
|
||||
|
||||
it('before every live cell has answered, holds back only hosts that may be on an unheard cell', async () => {
|
||||
const cells = cellList(['c1', 'c2'])
|
||||
const onUnheard = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
@@ -153,18 +134,6 @@ describe('reserve assignment on a director', () => {
|
||||
expect((await gated.assignment.plan(onUnheard, { reconnect: true, region: US })).kind).not.toBe('retry')
|
||||
})
|
||||
|
||||
it('answers a reconnect from memory at the same epoch with no booking and no database read', async () => {
|
||||
const row = vi.fn(async () => null)
|
||||
const { assignment, reserved } = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')],
|
||||
row
|
||||
})
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({ kind: 'answer', lane: 'sticky', assignment: { cellId: 'c1', assignmentEpoch: 7 } })
|
||||
expect(reserved).toEqual([])
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('re-books a host on a cell that restarted since its seat, and only then answers it', async () => {
|
||||
const { assignment, directory, reserved, now } = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2')]
|
||||
@@ -198,57 +167,6 @@ describe('reserve assignment on a director', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('never sends a drained or demoted host back, and books it elsewhere above every known epoch', async () => {
|
||||
const { assignment, directory, reserved, now } = setup()
|
||||
directory.apply(
|
||||
'c1',
|
||||
feed('c1', {
|
||||
full: undefined,
|
||||
seq: 3,
|
||||
changes: [
|
||||
{ seq: 2, kind: 'join', ...HOST, epoch: 7, generation: 1, at: 1 },
|
||||
{ seq: 3, kind: 'leave', ...HOST, epoch: 7, generation: 1, closeCode: 4503, at: 2 }
|
||||
]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement',
|
||||
assignment: { assignmentEpoch: 8 }
|
||||
})
|
||||
expect(reserved.length).toBeGreaterThan(0)
|
||||
expect(reserved.every((call) => !call.sticky && call.epoch === 8)).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves a cell without a feed (c17/c18) on today’s path without starving the reserve pool', async () => {
|
||||
const cells = cellList(['c1', 'c2', 'c17'])
|
||||
let rowOnOld = false
|
||||
const { assignment, directory, reserved, now } = setup({
|
||||
cells,
|
||||
feeds: [feed('c1'), feed('c2', { admitMode: 'db' })],
|
||||
row: async () => (rowOnOld ? { cellId: 'c17', assignmentEpoch: 5 } : null)
|
||||
})
|
||||
directory.markNoFeed('c17', now.value)
|
||||
expect(directory.isComplete()).toBe(true)
|
||||
// An old image reports no ceiling; counting it as a database cell would zero the share.
|
||||
expect(await assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement',
|
||||
assignment: { cellId: 'c1' }
|
||||
})
|
||||
expect(reserved.map((call) => call.cellId)).not.toContain('c17')
|
||||
rowOnOld = true
|
||||
expect(
|
||||
await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: true, region: US })
|
||||
).toMatchObject({ kind: 'database', epochFloor: 5 })
|
||||
directory.markNoFeed('c1', now.value)
|
||||
expect(directory.cellState('c1')?.ceiling).toBeUndefined()
|
||||
expect(assignment.placementCells().find((cell) => cell.cellId === 'c1')?.general).toBe(false)
|
||||
})
|
||||
|
||||
it('reads the row once for a host the map never saw, and refuses it while the database is down', async () => {
|
||||
const down = setup({ row: async () => Promise.reject(new Error('timeout')) })
|
||||
expect(await down.assignment.plan(HOST, { reconnect: false, region: US })).toMatchObject({
|
||||
@@ -318,95 +236,6 @@ describe('reserve assignment on a director', () => {
|
||||
}
|
||||
})
|
||||
|
||||
describe('for a share-draw loser', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) =>
|
||||
setup({
|
||||
...(reserve ? { reserve } : {}),
|
||||
cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])],
|
||||
feeds: [
|
||||
reserveCell,
|
||||
// Database room far above the reserve cell's, so the 0.25 draw loses.
|
||||
feed('c2', {
|
||||
admitMode: 'db',
|
||||
counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 }
|
||||
}),
|
||||
feed('c3')
|
||||
]
|
||||
})
|
||||
const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => {
|
||||
const plan = await assignment.plan(identity, { reconnect: false, region: ASIA })
|
||||
expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 })
|
||||
return plan.kind === 'database' ? plan.regionReserve : undefined
|
||||
}
|
||||
|
||||
it('offers the reserve cell in its region only, only while it has a seat', async () => {
|
||||
const { assignment, reserved, directory, now } = setupAsia()
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
expect(reserved).toEqual([])
|
||||
await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA })
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } })
|
||||
directory.apply('c1', full, now.value, now.value)
|
||||
expect(await regionReserveOf(assignment, OTHER)).toBeUndefined()
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
})
|
||||
|
||||
it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => {
|
||||
const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } }))
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError)
|
||||
expect(reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('gives the region up when the reserve cell refuses with a seat still on the map', async () => {
|
||||
for (const outcome of ['full', 'unreachable']) {
|
||||
const { assignment, reserved } = setupAsia(feed('c1'), (cellId) =>
|
||||
cellId === 'c1' ? { outcome } : { outcome: 'ok' }
|
||||
)
|
||||
const regionReserve = await regionReserveOf(assignment)
|
||||
// Null sends the host to the database lane's cross-region fallback, not a 1 s retry.
|
||||
await expect(regionReserve?.()).resolves.toBeNull()
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('for a region-correction request', () => {
|
||||
const request = { reconnect: true, region: US, regionCorrection: true }
|
||||
|
||||
it('leaves the answer to the database lane, offering only the in-region reserve fallback and the spill', async () => {
|
||||
const row = vi.fn(async () => null)
|
||||
// A memory answer would skip the window exchange; the share draw is not offered either.
|
||||
const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row })
|
||||
const plan = await assignment.plan(HOST, request)
|
||||
expect(plan).toEqual({
|
||||
kind: 'database',
|
||||
regionReserve: expect.any(Function),
|
||||
reserveSpill: expect.any(Function)
|
||||
})
|
||||
expect(reserved).toEqual([])
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({
|
||||
assignmentEpoch: 8
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps today’s path for a host the map never saw, without reading its row, but may spill', async () => {
|
||||
const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 }))
|
||||
const { assignment, reserved } = setup({ row })
|
||||
const plan = await assignment.plan(HOST, request)
|
||||
expect(plan).toEqual({ kind: 'database', reserveSpill: expect.any(Function) })
|
||||
expect(row).not.toHaveBeenCalled()
|
||||
expect(reserved).toEqual([])
|
||||
await expect(
|
||||
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 3 }) : null
|
||||
).resolves.toMatchObject({ assignmentEpoch: 4 })
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('after the database lane found no cell anywhere', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const paced = { intake: { perSec: 1, burst: 20, tokens: 0 } }
|
||||
@@ -428,31 +257,6 @@ describe('reserve assignment on a director', () => {
|
||||
return plan.kind === 'database' ? plan.reserveSpill : undefined
|
||||
}
|
||||
|
||||
it('books its own region first, minting above the row the database lane read', async () => {
|
||||
const harness = setupSpill({})
|
||||
const spill = await spillOf(harness)
|
||||
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 9 })).resolves.toMatchObject({
|
||||
cellId: 'asia-r',
|
||||
region: ASIA,
|
||||
assignmentEpoch: 10
|
||||
})
|
||||
expect(harness.reserved).toEqual([{ cellId: 'asia-r', epoch: 10, sticky: false, dryRun: false }])
|
||||
})
|
||||
|
||||
it('books the other region when its own is full, or already tried', async () => {
|
||||
const full = setupSpill(fullCounts)
|
||||
await expect((await spillOf(full))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
cellId: 'us-r',
|
||||
region: US,
|
||||
assignmentEpoch: 5
|
||||
})
|
||||
const tried = setupSpill({})
|
||||
await expect((await spillOf(tried))?.({ skipOwnRegion: true, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
cellId: 'us-r'
|
||||
})
|
||||
expect(tried.reserved.map((call) => call.cellId)).toEqual(['us-r'])
|
||||
})
|
||||
|
||||
it('tries the other region after a pace, and answers the pace only if nothing places', async () => {
|
||||
const abroad = setupSpill(paced)
|
||||
await expect((await spillOf(abroad))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toMatchObject({
|
||||
@@ -464,39 +268,10 @@ describe('reserve assignment on a director', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('returns null with no reserve seat anywhere, so the refusal stands', async () => {
|
||||
const harness = setupSpill(fullCounts, fullCounts)
|
||||
await expect((await spillOf(harness))?.({ skipOwnRegion: false, rowEpoch: 0 })).resolves.toBeNull()
|
||||
expect(harness.reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('is not offered while the map is incomplete', async () => {
|
||||
const gated = setup({
|
||||
cells: [...cellList(['asia-r', 'asia-db'], ASIA), ...cellList(['us-r'])],
|
||||
feeds: [feed('asia-r'), feed('asia-db', { admitMode: 'db', full: [seated(4)] })],
|
||||
startedAt: 100_500
|
||||
})
|
||||
const plan = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA })
|
||||
expect(plan).toMatchObject({ kind: 'database' })
|
||||
expect(plan.kind === 'database' && plan.reserveSpill).toBeUndefined()
|
||||
const correction = await gated.assignment.plan(HOST, { reconnect: true, region: ASIA, regionCorrection: true })
|
||||
expect(correction).toEqual({ kind: 'database' })
|
||||
})
|
||||
|
||||
it('answers the shortest pace when every region paces', async () => {
|
||||
const harness = setupSpill(paced, { intake: { perSec: 2, burst: 20, tokens: 0 } })
|
||||
const spill = await spillOf(harness)
|
||||
// Asia refills its token in 4 s, US in 2 s.
|
||||
await expect(spill?.({ skipOwnRegion: false, rowEpoch: 0 })).rejects.toSatisfy(
|
||||
(error) => error instanceof ReservePaceError && error.retryAfterSeconds === 2
|
||||
)
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('rehome on reconnect', () => {
|
||||
const ASIA = 'asia-east2' as const
|
||||
const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
|
||||
const ask = { reconnect: true, region: ASIA, preferred: true }
|
||||
// 2,000 controls on the US cell refill its move budget once a minute.
|
||||
const usCounts = { counts: { seats: 1, controls: 2_000, bookings: 0, units: 1, ceiling: 4_000 } }
|
||||
@@ -546,107 +321,6 @@ describe('reserve assignment on a director', () => {
|
||||
expect(demoted).toEqual([{ cellId: 'us-r', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
it('answers a region-correction request on the rehome lane', async () => {
|
||||
const { assignment, warm } = setupRehome()
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, { ...ask, regionCorrection: true })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'rehome'
|
||||
})
|
||||
})
|
||||
|
||||
it('moves no one into Asia at 98% full, or anywhere under 10% free', async () => {
|
||||
for (const share of [0.98, 0.91]) {
|
||||
const { assignment, reserved, warm } = setupRehome({ asia: asiaFull(share) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask), String(share)).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'sticky',
|
||||
assignment: { cellId: 'us-r' }
|
||||
})
|
||||
expect(reserved.filter((call) => call.cellId === 'asia-r')).toEqual([])
|
||||
}
|
||||
const { assignment, warm } = setupRehome({ asia: asiaFull(0.9) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
})
|
||||
|
||||
// 10% rehome headroom above the 0% spill line prevents spilled hosts bouncing back.
|
||||
it('keeps a gap above the spill line: 5% free takes placements, not rehomes', async () => {
|
||||
const { assignment, warm } = setupRehome({ asia: asiaFull(0.95) })
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky', assignment: { cellId: 'us-r' } })
|
||||
expect(await assignment.plan(OTHER, { reconnect: false, region: ASIA })).toMatchObject({
|
||||
lane: 'placement',
|
||||
assignment: { cellId: 'asia-r' }
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps today’s answer when any rule fails', async () => {
|
||||
const sticky = { kind: 'answer', lane: 'sticky', assignment: { cellId: 'us-r' } }
|
||||
const unhinted = setupRehome()
|
||||
await unhinted.warm()
|
||||
expect(await unhinted.assignment.plan(HOST, { ...ask, preferred: false })).toMatchObject(sticky)
|
||||
const off = setupRehome({ percent: 0 })
|
||||
await off.warm()
|
||||
expect(await off.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const closed = setupRehome({ switchOpen: false })
|
||||
await closed.warm()
|
||||
expect(await closed.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const cold = setupRehome()
|
||||
expect(await cold.assignment.plan(HOST, ask)).toMatchObject(sticky)
|
||||
const sameRegion = setupRehome()
|
||||
await sameRegion.warm()
|
||||
expect(await sameRegion.assignment.plan(HOST, { ...ask, region: US })).toMatchObject(sticky)
|
||||
})
|
||||
|
||||
it('never runs for a fresh ask, an unknown host, a database-mode source, or an incomplete map', async () => {
|
||||
const fresh = setupRehome()
|
||||
await fresh.warm()
|
||||
expect(await fresh.assignment.plan(HOST, { ...ask, reconnect: false })).toMatchObject({ lane: 'placement' })
|
||||
expect(await fresh.assignment.plan(OTHER, ask)).toMatchObject({ kind: 'answer', lane: 'placement' })
|
||||
const onDatabase = setup({
|
||||
cells: [...cellList(['us-db']), ...cellList(['asia-r'], ASIA)],
|
||||
feeds: [feed('us-db', { admitMode: 'db', ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
|
||||
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
|
||||
})
|
||||
expect(await onDatabase.assignment.plan(HOST, ask)).toMatchObject({ kind: 'database' })
|
||||
const gated = setup({
|
||||
cells: [...cellList(['us-r']), ...cellList(['asia-r', 'asia-unheard'], ASIA)],
|
||||
feeds: [feed('us-r', { ...usCounts, full: [seated(5, 1_000)] }), feed('asia-r', asiaFull(0.5))],
|
||||
startedAt: 100_500,
|
||||
rehome: new ReserveRehome({ percent: 100, switchOpen: () => true })
|
||||
})
|
||||
expect(await gated.assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
|
||||
expect(gated.reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('falls through to the sticky answer when the rehome booking is paced', async () => {
|
||||
const { assignment, reserved, warm } = setupRehome({
|
||||
reserve: (cellId) => (cellId === 'asia-r' ? { outcome: 'intake' } : { outcome: 'ok' })
|
||||
})
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ kind: 'answer', lane: 'sticky' })
|
||||
expect(reserved.map((call) => call.cellId)).toEqual(['asia-r'])
|
||||
})
|
||||
|
||||
it('leaves a host another install keeps dropping', async () => {
|
||||
const { assignment, directory, now, warm } = setupRehome()
|
||||
await warm()
|
||||
directory.apply(
|
||||
'us-r',
|
||||
feed('us-r', {
|
||||
...usCounts,
|
||||
full: undefined,
|
||||
seq: 3,
|
||||
changes: [{ seq: 3, kind: 'leave', ...HOST, epoch: 4, generation: 1, closeCode: 4408, at: now.value - 1 }]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'sticky' })
|
||||
})
|
||||
|
||||
it('falls through to the sticky answer when the booking is refused, and gives the move back', async () => {
|
||||
let refuse = true
|
||||
const { assignment, warm } = setupRehome({
|
||||
@@ -657,52 +331,6 @@ describe('reserve assignment on a director', () => {
|
||||
refuse = false
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
})
|
||||
|
||||
it('moves one host per source budget a minute, and logs moves and skips per region pair', async () => {
|
||||
const lines: Record<string, unknown>[] = []
|
||||
const { assignment, now, warm } = setupRehome({
|
||||
usSeats: [seated(5, 1_000), { ...OTHER, epoch: 5, generation: 1, state: 'active', joinedAt: 1_000 }],
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
await warm()
|
||||
expect(await assignment.plan(HOST, ask)).toMatchObject({ lane: 'rehome' })
|
||||
expect(await assignment.plan(OTHER, ask)).toMatchObject({ lane: 'sticky' })
|
||||
now.value += 60_000
|
||||
await assignment.plan({ ...HOST, relayHostId: 'flushflushflush1' }, ask)
|
||||
expect(lines.filter((line) => line.event === 'orca_relay_reserve_rehome')).toEqual([
|
||||
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 0, skippedByRule: { 'source-budget': 1 } },
|
||||
{ event: 'orca_relay_reserve_rehome', fromRegion: US, toRegion: ASIA, count: 1, skippedByRule: { 'source-budget': 1 } }
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => {
|
||||
const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) })
|
||||
await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toMatchObject({
|
||||
message: 'reserve_cells_paced',
|
||||
retryAfterSeconds: expect.any(Number)
|
||||
})
|
||||
})
|
||||
|
||||
it('tells the old reserve seat to go when it books the host elsewhere, naming that seat exactly', async () => {
|
||||
const { assignment, directory, demoted, now } = setup({
|
||||
feeds: [feed('c1', { full: [seated(5, 1_000)] }), feed('c2')],
|
||||
reserve: (cellId) => (cellId === 'c1' ? { outcome: 'full' } : { outcome: 'ok' })
|
||||
})
|
||||
// The host left c1 for a reason that moves it: its next placement is fresh, above 5.
|
||||
directory.apply(
|
||||
'c1',
|
||||
feed('c1', {
|
||||
full: undefined,
|
||||
seq: 2,
|
||||
changes: [{ seq: 2, kind: 'drain-only', ...HOST, epoch: 5, generation: 1, at: 2 }]
|
||||
}),
|
||||
now.value,
|
||||
now.value
|
||||
)
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US })
|
||||
expect(plan).toMatchObject({ kind: 'answer', lane: 'placement', assignment: { assignmentEpoch: 6 } })
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
// A half-open seat missed by the booking map must be demoted by the feed, before its 75s silence close.
|
||||
@@ -714,91 +342,4 @@ describe('reserve assignment on a director', () => {
|
||||
directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
it('answers a reconnect at the newest epoch even when an older epoch joined later', async () => {
|
||||
const { assignment, reserved } = setup({
|
||||
feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
|
||||
})
|
||||
expect(await assignment.plan(HOST, { reconnect: true, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'sticky',
|
||||
assignment: { cellId: 'c1', assignmentEpoch: 6 }
|
||||
})
|
||||
expect(reserved).toEqual([])
|
||||
})
|
||||
|
||||
it('in dry run only asks a cell to check a booking, and never changes the answer', async () => {
|
||||
const { assignment, reserved } = setup({ mode: 'dry-run' })
|
||||
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
|
||||
expect(plan.kind).toBe('database')
|
||||
expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
|
||||
expect(reserved).toMatchObject([{ dryRun: true }])
|
||||
})
|
||||
|
||||
describe('when a cell answers a demotion', () => {
|
||||
const outcomes = (lines: string[]) =>
|
||||
lines
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((entry) => entry.event === 'orca_relay_reserve_demote_outcome')
|
||||
|
||||
it('logs the answer by host digest and leaves the seat to the feed', async () => {
|
||||
const lines: string[] = []
|
||||
const logged = Promise.withResolvers<void>()
|
||||
const { assignment, directory } = setup({
|
||||
feeds: [feed('c1', { full: [seated(10, 1_000)] }), feed('c2')],
|
||||
demote: () => 'not-seated',
|
||||
log: (line) => {
|
||||
lines.push(line)
|
||||
if (JSON.parse(line).event === 'orca_relay_reserve_demote_outcome') {
|
||||
logged.resolve()
|
||||
}
|
||||
}
|
||||
})
|
||||
assignment.demoteSeat({ ...HOST, cellId: 'c1', epoch: 10, joinedAt: 1_000 }, 'reconcile')
|
||||
await logged.promise
|
||||
expect(outcomes(lines)).toMatchObject([
|
||||
{ via: 'reconcile', cellId: 'c1', epoch: 10, outcome: 'not-seated' }
|
||||
])
|
||||
expect(outcomes(lines)[0].relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
|
||||
expect(lines.join('\n')).not.toContain(HOST.relayHostId)
|
||||
expect(directory.seatsOnCell('c1')).toHaveLength(1)
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('for a host re-asking off a draining cell', () => {
|
||||
const events = (lines: string[], event: string) =>
|
||||
lines.map((line) => JSON.parse(line)).filter((entry) => entry.event === event)
|
||||
|
||||
it('marks its pace as the drain’s, on the reserve lane, in the region fallback and in the spill', async () => {
|
||||
const { assignment } = setup({
|
||||
feeds: [feed('c1', { draining: true, full: [seated(7)] }), feed('c2')],
|
||||
reserve: () => ({ outcome: 'intake' })
|
||||
})
|
||||
await expect(assignment.plan(HOST, { reconnect: true, region: US })).rejects.toMatchObject({ drained: true })
|
||||
const plan = await assignment.plan(HOST, { reconnect: true, region: US, regionCorrection: true })
|
||||
await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).rejects.toMatchObject({ drained: true })
|
||||
await expect(
|
||||
plan.kind === 'database' ? plan.reserveSpill?.({ skipOwnRegion: false, rowEpoch: 0 }) : null
|
||||
).rejects.toMatchObject({ drained: true })
|
||||
})
|
||||
|
||||
it('leaves every other host as today: no drain count, and a plain pace', async () => {
|
||||
const lines: string[] = []
|
||||
const { assignment } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], log: (line) => lines.push(line) })
|
||||
const paced = setup({
|
||||
feeds: [feed('c1', { full: [seated(7)] }), feed('c2', { draining: true, full: [seated(6)] })],
|
||||
reserve: () => ({ outcome: 'intake' })
|
||||
})
|
||||
expect(await assignment.plan({ ...HOST, relayHostId: 'qrstuvwxyzabcdef' }, { reconnect: false, region: US })).toMatchObject({
|
||||
kind: 'answer',
|
||||
lane: 'placement'
|
||||
})
|
||||
await expect(
|
||||
paced.assignment.plan(HOST, { reconnect: false, region: US })
|
||||
).rejects.toMatchObject({ drained: false })
|
||||
expect(events(lines, 'orca_relay_reserve_drain_replaced')).toEqual([])
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
@@ -57,30 +57,15 @@ describe('reserve ledger writer', () => {
|
||||
expect(await row(db, 'bbbbbbbbbbbbbbbb')).toMatchObject({ cell_id: 'c1', reserved_controls: 2 })
|
||||
})
|
||||
|
||||
it('keeps its queue through a database stall and drops the oldest past the bound', async () => {
|
||||
it('persists a booked join after a database stall', async () => {
|
||||
let now = 1_000
|
||||
const { database: db, writer } = await setup(() => now)
|
||||
const query = vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
|
||||
vi.spyOn(db, 'query').mockRejectedValueOnce(new Error('timeout'))
|
||||
writer.enqueue({ ...HOST, cellId: 'c1', epoch: 5 })
|
||||
await writer.flush()
|
||||
expect(writer.pending()).toBe(1)
|
||||
await writer.flush()
|
||||
expect(query).toHaveBeenCalledTimes(1)
|
||||
now += 1_000
|
||||
await writer.flush()
|
||||
expect(writer.pending()).toBe(0)
|
||||
expect(await row(db)).toMatchObject({ cell_id: 'c1', assignment_epoch: 5 })
|
||||
|
||||
const bounded = new ReserveLedgerWriter(db, { queueMax: 2, log: () => undefined })
|
||||
const hosts = ['oldest0000000000', 'middle0000000000', 'newest0000000000']
|
||||
for (const relayHostId of hosts) {
|
||||
bounded.enqueue({ ...HOST, relayHostId, cellId: 'c1', epoch: 1 })
|
||||
}
|
||||
expect(bounded.pending()).toBe(2)
|
||||
await bounded.flush()
|
||||
expect(await row(db, hosts[0])).toBeUndefined()
|
||||
expect(await row(db, hosts[1])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
|
||||
expect(await row(db, hosts[2])).toMatchObject({ cell_id: 'c1', assignment_epoch: 1 })
|
||||
})
|
||||
|
||||
it('reconciles the map’s seats and never lowers an epoch', async () => {
|
||||
|
||||
-179
@@ -1,179 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
runReservePlacementSimulation,
|
||||
type SimulationConfig,
|
||||
type SimulationReport
|
||||
} from './reserve-placement-simulation.js'
|
||||
|
||||
const SEEDS = [11, 23, 47]
|
||||
const rtt = (region: string) => (region === 'asia-east2' ? 170 : 10)
|
||||
|
||||
function fleet(): SimulationConfig['cells'] {
|
||||
const cells: SimulationConfig['cells'] = []
|
||||
for (let index = 0; index < 24; index += 1) {
|
||||
cells.push({
|
||||
cellId: `us${String(index).padStart(2, '0')}`,
|
||||
region: 'us-central1',
|
||||
hardCap: 600,
|
||||
ceiling: 480,
|
||||
intakePerSec: 5,
|
||||
// A third of the fleet stays in database mode, one cell on an image without step 5.
|
||||
mode: index % 3 === 0 ? 'db' : 'reserve',
|
||||
old: index === 0,
|
||||
preseated: 150
|
||||
})
|
||||
}
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
cells.push({
|
||||
cellId: `asia${index}`,
|
||||
region: 'asia-east2',
|
||||
hardCap: 600,
|
||||
ceiling: 480,
|
||||
intakePerSec: 2,
|
||||
mode: index === 0 ? 'db' : 'reserve',
|
||||
preseated: 200
|
||||
})
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
function expectInvariants(report: SimulationReport): void {
|
||||
expect(report.violations).toEqual([])
|
||||
}
|
||||
|
||||
describe('step 5 deterministic simulation', () => {
|
||||
// Invariant 9 (dead-man) never trips here; its own tests below cover it.
|
||||
it.each(SEEDS)(
|
||||
'holds invariants 1-8 across mixed versions, lagging polls, restarts and database stalls (seed %i)',
|
||||
async (seed) => {
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 25 * 60_000,
|
||||
directors: 5,
|
||||
// One director still on the database-only revision.
|
||||
oldDirectors: 1,
|
||||
cells: fleet(),
|
||||
hosts: 6_600,
|
||||
hostRegion: (index) => (index % 5 === 0 ? 'asia-east2' : 'us-central1'),
|
||||
arrivalPerSec: 20,
|
||||
meanSessionMs: 8 * 60_000,
|
||||
reconnectDelayMs: [500, 5_000],
|
||||
duplicateAssignShare: 0.05,
|
||||
halfOpenShare: 0.05,
|
||||
pollLagMs: (director) => [0, 200, 1_000, 4_000, 10_000][director]!,
|
||||
rttMs: rtt,
|
||||
cellRestarts: [{ at: 6 * 60_000, cellId: 'us04' }, { at: 14 * 60_000, cellId: 'asia2' }],
|
||||
directorRestarts: [{ at: 8 * 60_000, director: 2 }, { at: 16 * 60_000, director: 4 }],
|
||||
// Drained hosts land elsewhere while their old seat lingers: placement supersedes it.
|
||||
drains: [{ at: 12 * 60_000, cellId: 'us05', paceMs: 60_000 }],
|
||||
databaseStalls: Array.from({ length: 3 }, (_, index) => ({
|
||||
at: (index * 610 + 120) * 1_000,
|
||||
durationMs: 7_000
|
||||
}))
|
||||
})
|
||||
expectInvariants(report)
|
||||
expect(report.reservePlacements).toBeGreaterThan(500)
|
||||
expect(report.stickyAnswers).toBeGreaterThan(500)
|
||||
expect(report.seatedAtEnd).toBeGreaterThan(4_000)
|
||||
// The run reached the paths the invariants guard, so a pass is not vacuous.
|
||||
expect(report.supersedes).toBeGreaterThan(0)
|
||||
expect(report.rowAheadDemotions).toBeGreaterThan(0)
|
||||
expect(report.directorRestarts).toBe(2)
|
||||
expect(report.stickyReserves).toBeGreaterThan(0)
|
||||
expect(report.databasePlacements).toBeGreaterThan(0)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
it.each(SEEDS)(
|
||||
'near full: extra reserve calls stay within 10% and exhausted requests within 1% (seed %i)',
|
||||
async (seed) => {
|
||||
const cells = Array.from({ length: 17 }, (_, index) => ({
|
||||
cellId: `us${index}`,
|
||||
region: 'us-central1' as const,
|
||||
hardCap: 700,
|
||||
ceiling: 600,
|
||||
intakePerSec: 50,
|
||||
mode: 'reserve' as const,
|
||||
preseated: 582
|
||||
}))
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 3 * 60_000,
|
||||
directors: 5,
|
||||
cells,
|
||||
hosts: 17 * 582 + 300,
|
||||
hostRegion: () => 'us-central1',
|
||||
arrivalPerSec: 10,
|
||||
meanSessionMs: 0,
|
||||
reconnectDelayMs: [1_000, 3_000],
|
||||
duplicateAssignShare: 0,
|
||||
halfOpenShare: 0,
|
||||
pollLagMs: () => 0,
|
||||
rttMs: rtt
|
||||
})
|
||||
expectInvariants(report)
|
||||
expect(report.reservePlacements).toBeGreaterThan(250)
|
||||
expect(report.reserveCalls / report.reservePlacements - 1).toBeLessThanOrEqual(0.1)
|
||||
expect(report.exhausted / report.requests).toBeLessThanOrEqual(0.01)
|
||||
for (const share of Object.values(report.maxOccupancyShare)) expect(share).toBeLessThanOrEqual(1)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
it.each(SEEDS.flatMap((seed) => [[1_200_000, seed], [300_000, seed]] as const))(
|
||||
'a drain onto an empty cell at the %i ms pace spreads within each cell intake budget (seed %i)',
|
||||
async (paceMs, seed) => {
|
||||
// Production c27 drain: 1,200s pacing toward cells with 2,805/841/777/667/26 free seats.
|
||||
const free = [2_805, 841, 777, 667, 26]
|
||||
const cells: SimulationConfig['cells'] = free.map((seats, index) => ({
|
||||
cellId: `asia${index + 1}`,
|
||||
region: 'asia-east2',
|
||||
hardCap: 3_000,
|
||||
ceiling: 2_830,
|
||||
intakePerSec: 2,
|
||||
mode: 'reserve',
|
||||
preseated: 2_830 - seats
|
||||
}))
|
||||
cells.unshift({
|
||||
cellId: 'asia0',
|
||||
region: 'asia-east2',
|
||||
hardCap: 3_000,
|
||||
ceiling: 2_830,
|
||||
intakePerSec: 2,
|
||||
mode: 'reserve',
|
||||
preseated: 2_770
|
||||
})
|
||||
const preseated = cells.reduce((sum, cell) => sum + (cell.preseated ?? 0), 0)
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
durationMs: 30 * 60_000,
|
||||
directors: 5,
|
||||
cells,
|
||||
hosts: preseated,
|
||||
hostRegion: () => 'asia-east2',
|
||||
arrivalPerSec: 1,
|
||||
meanSessionMs: 0,
|
||||
reconnectDelayMs: [1_000, 3_000],
|
||||
duplicateAssignShare: 0.02,
|
||||
halfOpenShare: 0,
|
||||
pollLagMs: () => 0,
|
||||
rttMs: rtt,
|
||||
drains: [{ at: 10_000, cellId: 'asia0', paceMs }]
|
||||
})
|
||||
expectInvariants(report)
|
||||
// Below every cell's budget nothing waits; above it the cells' buckets pace the drain.
|
||||
if (paceMs === 300_000) expect(report.intakeRefusals + report.paced).toBeGreaterThan(0)
|
||||
const moved = Object.entries(report.arrivalsByCell)
|
||||
const total = moved.reduce((sum, [, count]) => sum + count, 0)
|
||||
expect(total).toBeGreaterThan(2_000)
|
||||
// Spread: the emptiest cell takes a share, not the drain.
|
||||
expect((report.arrivalsByCell.asia1 ?? 0) / total).toBeLessThan(0.6)
|
||||
expect(moved.filter(([, count]) => count / total > 0.1).length).toBeGreaterThanOrEqual(3)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
|
||||
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user