starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-24 00:02:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into principals-01-columns; renumber the principal migration to v42

Browse Source 
Main took the v41 slot for the derived-delivery migration, so the principal-column
migration moves to v42 and runs after it.

- migrate-v41.ts is main's file unchanged; the principal columns, trigger recreation
  and backfill move to migrate-v42.ts with their version guards renumbered.
- SCHEMA_VERSION is 42. The version-history comment keeps main's per-version lines and
  records v40 and v42, which the two sides had each dropped.
- run-binding keeps the principal argument the four-placeholder UPDATE requires while
  taking main's renamed fenceUnacknowledgedMailboxDeliveries.
- The migration test targets migrateV42 from v41; its real-chain case still seeds a v40
  stamp, so it now exercises main's v41 before v42 as a real upgrade would.
This commit is contained in:
Brennan Benson
2026-09-16 16:26:01 -07:00
parent 64e470af0b 533b0bd02e
commit 49a7083176
4977 changed files with 813625 additions and 57188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+7
View File
@@ -31,6 +31,13 @@
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
.github/CONTRIBUTING.md
+5
View File
@@ -21,6 +21,11 @@ pnpm install
pnpm dev
```
Ordinary installs include native optional dependencies for the current OS and CPU only.
Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and
arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants.
See [the install policy](../docs/reference/pnpm-install-policy.md).
## Branch Naming
Use a clear, descriptive branch name that reflects the change.
.github/actions/install-node-dependencies/action.yml
+13 -15
View File
@@ -10,6 +10,10 @@ inputs:
description: Node.js version override; defaults to the version declared in package.json.
required: false
default: ''
cache-dependency-path:
description: Lockfiles for the pnpm download store; include mobile/pnpm-lock.yaml only when the job installs mobile dependencies.
required: false
default: pnpm-lock.yaml
persist-native-cache:
description: Save restored native modules at job end. Set false when a later step overwrites the same path with a different ABI.
required: false
@@ -39,9 +43,7 @@ runs:
with:
install: false
# Why both lockfiles: setup-node keys the pnpm store on the root lockfile alone, so
# jobs that also install mobile restored a store with none of the React Native tree
# in it and re-downloaded the lot on every run.
# Desktop-only jobs should not miss their download cache when mobile dependencies change.
- name: Setup Node.js
id: default-node
if: inputs.node-version == ''
@@ -49,9 +51,7 @@ runs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Setup requested Node.js
id: requested-node
@@ -60,9 +60,7 @@ runs:
with:
node-version: ${{ inputs.node-version }}
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Validate native runtime
shell: bash
@@ -152,9 +150,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Restore compiled native modules without saving
id: native-cache-restore-only
@@ -163,9 +161,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
.github/pull_request_template.md
+4 -4
View File
@@ -1,14 +1,14 @@
## ELI5
<!-- Simple high-level explanation -->
<!-- Simple high-level explanation, in plain language. No jargon. -->
## What Changed
<!-- Describe the change clearly and keep scope tight. -->
<!-- Describe the change clearly and keep scope tight. Cover the before and after as the user experiences it, and the mechanism you changed — not just the symptom. -->
## Why
<!-- What problem does this solve, and why is this approach right? -->
<!-- What problem does this solve, and why is this approach better than the alternatives you considered? -->
## Linked Issue
@@ -47,7 +47,7 @@ Ensure no issues in: Security, Cross-platoform support (Linux, Windows, Mac), Re
## Checklist
- [ ] This PR is small and focused
- [ ] I explained what changed and why (including ELI5)
- [ ] I explained what changed and why (ELI5, the user-facing before/after, the mechanism, and why over the alternatives)
- [ ] Before/after screenshots or videos attached for UI changes, or `N/A` with reason
- [ ] Self-reviewed for correctness, security, and performance
- [ ] Cross-platform, SSH/remote, and path/shortcut impact considered (or N/A)
.github/workflows/adhoc-mac-build.yml
+13 -10
View File
@@ -160,16 +160,14 @@ jobs:
- name: Checkout the requested ref
uses: actions/checkout@v6
env:
# Full-history checkout must also preserve case-twin branch and tag names.
GIT_DEFAULT_REF_FORMAT: reftable
with:
# Why an input at all rather than just github.ref: the whole point is to
# build code that has not landed, and the workflow definition itself
# always comes from the dispatch ref — naming the branch here instead
# applies main's current copy of this file to an arbitrary branch.
ref: ${{ steps.vetted.outputs.sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# This job only reads stablyai/orca and never pushes; every write goes
# to the adhoc repo through a minted App token passed by env. Not
# persisting the checkout credential shrinks the blast radius if a build
@@ -197,13 +195,15 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
@@ -235,11 +235,14 @@ jobs:
fi
done
echo "head_sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT"
# Why the main repo's tags: package.json on a branch is as stale as the
# main it forked from, and stable patches never merge back into it.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag, which still owns that number.
# Releases-only let adhoc sit on a number already taken, so the updater
# would not install it. Empty on failure — the script then falls back
# to package.json.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \
node config/scripts/adhoc-build-version.mjs \
>"$RUNNER_TEMP/adhoc-identity.txt"
.github/workflows/cloud-deploy-relay-production-director.yml
+12 -2
View File
@@ -13,6 +13,11 @@ on:
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
@@ -62,6 +67,7 @@ jobs:
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
@@ -106,8 +112,11 @@ jobs:
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
@@ -218,7 +227,8 @@ jobs:
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}"
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+3 -3
View File
@@ -67,8 +67,8 @@ jobs:
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
@@ -599,7 +599,7 @@ jobs:
| jq -e '.control.enabled == false' >/dev/null
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+3 -3
View File
@@ -26,13 +26,13 @@ on:
required: true
default: '1'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
@@ -62,7 +62,7 @@ on:
required: false
type: string
canary-run-id:
description: Successful same-commit canary run required for batch-apply
description: Successful same-code canary in this rehome control generation; reusable across batches
required: false
type: string
confirmation:
.github/workflows/computer-e2e.yml
+9
View File
@@ -6,6 +6,9 @@ on:
- '.github/workflows/computer-e2e.yml'
- 'config/electron-builder.config.cjs'
- 'config/scripts/build-computer-macos.mjs'
- 'config/scripts/build-native-for-platform.mjs'
- 'config/scripts/build-native-for-platform.test.mjs'
- 'config/scripts/pnpm-cli-invocation.mjs'
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/build-windows-cli-launcher.test.mjs'
- 'config/scripts/computer-e2e-workflow.test.mjs'
@@ -157,6 +160,12 @@ jobs:
pnpm vitest run
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
# Why: the parallel launcher's cancellation tests are darwin-only and no
# other PR job runs on macOS.
- name: Parallel native build launcher cancellation
run: >-
pnpm vitest run --config config/vitest.config.ts
config/scripts/build-native-for-platform.test.mjs
- name: Authenticated helper owner-loss smoke
run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1
- name: Swift tests and signed universal helper verification
.github/workflows/daily-mac-build.yml
+17 -10
View File
@@ -90,7 +90,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the daily repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -167,6 +168,8 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
@@ -174,7 +177,7 @@ jobs:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
@@ -206,17 +209,21 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$DAILY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. That dragged hourlies backwards so
# electron-updater stopped offering them; dailies would do the same. A
# separate token because GH_TOKEN above is the App's, scoped to the
# daily repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor so unpublishing a
# buggy main release cannot drag this series below a daily already out.
channel_tags="$(gh release list --repo "$DAILY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_DAILY_RELEASE_NAMES="$names" \
node config/scripts/daily-build-version.mjs \
>"$RUNNER_TEMP/daily-identity.txt"
.github/workflows/dev-channel-win-build.yml
+2
View File
@@ -219,6 +219,8 @@ jobs:
# Why retried: pnpm install triggers electron's postinstall, which pulls the
# Electron binary from GitHub release assets, and that CDN returns transient
# 504s often enough to lose a build to it.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
.github/workflows/e2e.yml
+32 -1
View File
@@ -170,8 +170,39 @@ jobs:
# artifact instead of starting five concurrent electron-vite builds.
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
# launches but never creates a BrowserWindow.
- name: Balance E2E shard from timing evidence
env:
ORCA_BACKGROUND_LAUNCH: '1'
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
ORCA_E2E_WEB_CLIENT: '1'
ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay
run: |
mkdir -p ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --list --reporter=json > ci-shards/discovery.json
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
node config/scripts/ci-e2e-shard-plan.mjs ci-shards/discovery.json '${{ matrix.shard }}' ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --test-list=ci-shards/selected.txt --list --reporter=json > ci-shards/selected-discovery.json
node config/scripts/ci-e2e-shard-plan.mjs --verify ci-shards/assignment.json ci-shards/selected-discovery.json
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --test-list=ci-shards/selected.txt
- name: Upload E2E shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: e2e-shard-${{ matrix.shard_name }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
# The frame benchmark needs a mapped window, which the headless shards exclude.
- name: Run worktree first-paint benchmark
if: matrix.shard == '1/14'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
.github/workflows/git-command-termination-runtime.yml
+22
View File
@@ -0,0 +1,22 @@
name: Git command termination runtime
on:
pull_request:
paths:
- 'src/main/git/command-runner/spawned-command-tree-kill*'
- '.github/workflows/git-command-termination-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
windows-exit:
runs-on: windows-latest
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Verify exited native child does not trigger taskkill
run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/git/command-runner/spawned-command-tree-kill.test.ts
.github/workflows/hourly-mac-build.yml
+24 -13
View File
@@ -137,7 +137,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: ${{ needs.preflight.outputs.head_sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -174,13 +175,15 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
@@ -210,17 +213,25 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# separate token because GH_TOKEN above is the App's, scoped to the
# hourly repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. On 2026-09-14 we deleted v1.4.202's
# release for a bug; hourlies had already climbed to 1.4.203, then
# `gh release list` fell back to v1.4.201 and the next hourlies shipped
# as 1.4.202-hourly — which electron-updater will not install over
# 1.4.203-hourly or over the still-tagged 1.4.202. A separate token
# because GH_TOKEN above is the App's, scoped to the hourly repo. Empty
# on failure — the script then falls back to package.json, which is
# stale but never wrong enough to fail a build.
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor: even if main's tag
# list is empty this run, a 1.4.203-hourly already out must not be
# followed by a 1.4.202-hourly.
channel_tags="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_HOURLY_RELEASE_NAMES="$names" \
node config/scripts/hourly-build-version.mjs \
>"$RUNNER_TEMP/hourly-identity.txt"
.github/workflows/mobile-ios-release.yml
+2
View File
@@ -94,6 +94,8 @@ jobs:
run: node -e 'const fs = require("node:fs"); const { expo } = require("./app.json"); fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${expo.version}\nbuild_number=${expo.ios.buildNumber}\n`)'
- name: Expo prebuild
env:
ORCA_IOS_APS_ENVIRONMENT: production
run: npx expo prebuild --platform ios --no-install
- name: Install CocoaPods
.github/workflows/mobile.yml
+18
View File
@@ -12,6 +12,20 @@ on:
# Why: the mobile terminal link parsers are conformance-tested against
# these shared fixtures; desktop-side fixture edits must re-run this suite.
- 'src/shared/terminal-file-link-conformance.ts'
# Why: mobile imports the negotiated capability names directly and records
# the whole capability read verbatim in its goldens, so a capability added
# desktop-side rewrites a mobile fixture and must re-run this suite.
- 'src/shared/protocol-version.ts'
# Why: mobile's rpc-params-contract.ts is a type-only re-export of the
# generated params catalog, and mobile/tsconfig.json includes **/*.ts. A
# schema edit anywhere under here changes mobile's types, so a desktop-only
# change can break mobile's typecheck with no other mobile signal.
- 'src/shared/rpc-contract/**'
# Why: the catalog above holds params only. This file is the sole holder of
# the agent.launch RESULT shape, and mobile imports it as a value, not just
# a type. CROSS_VERSION_WIRE_PREFIXES already treats it as wire-critical, so
# without this one gate classes it that way while this one cannot see it.
- 'src/shared/agent-launch-intent.ts'
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
@@ -41,6 +55,10 @@ jobs:
uses: actions/checkout@v6
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
.github/workflows/pi-owner-runtime.yml
+29
View File
@@ -0,0 +1,29 @@
name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/agent-status-handler-source.ts'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
.github/workflows/pi-provider-runtime.yml
+28
View File
@@ -0,0 +1,28 @@
name: Pi extension provider verification
on:
pull_request:
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
.github/workflows/pr.yml
+70 -35
View File
@@ -126,10 +126,16 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Lint
run: pnpm exec oxlint --format github
- name: Reject low-evidence patterns
run: pnpm run audit:anti-slop
- name: Enforce focused code-quality plugins
run: pnpm run audit:code-quality:native
@@ -167,6 +173,9 @@ jobs:
- name: Check reliability gate manifest
run: pnpm run check:reliability-gates
- name: Enforce dead design-system classes
run: pnpm run check:dead-classes
- name: Check VM runtime rollback compatibility
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
@@ -203,6 +212,9 @@ jobs:
- name: Boot orcad and round-trip a terminal
run: pnpm run smoke:orcad-terminal
- name: Verify the generated RPC params catalog
run: pnpm run verify:rpc-params-catalog
- name: Verify bundled skill guides
run: pnpm run verify:bundled-skill-guides
@@ -270,6 +282,12 @@ jobs:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node .github/scripts/check-root-directory-entries.mjs "$BASE_SHA" "$HEAD_SHA"
# Why here: the READMEs embed media owned by docs/site and resources/onboarding,
# and the classifier skips static_analysis for docs-only diffs. This job runs
# on every PR and needs no install.
- name: Check README local links
run: node config/scripts/check-readme-local-links.mjs
typecheck:
needs: [code_paths]
if: needs.code_paths.outputs.typecheck == 'true'
@@ -314,40 +332,59 @@ jobs:
# Why: the 2.25.5 lane is a source build of a pinned tarball, so it produced the
# same binary on every PR for minutes of runner time. The key carries the version
# because that is the only input; the sha256 assertion below still guards the
# tarball on the miss path that actually builds.
# tarball on the miss path that actually builds. Only this PR's own later pushes
# can restore it — GitHub scopes a cache written from a pull_request run to that
# ref — so a first push always takes the build path below.
- name: Cache baseline Git build
uses: actions/cache@v5
with:
path: ~/.cache/orca-git-compat/git-2.25.5
key: git-compat-baseline-${{ runner.os }}-${{ runner.arch }}-2.25.5
# Why its own step: this is `make -j$(nproc)` on every core, and the lanes below
# spend their wall clock waiting on container starts, not on Git. Sharing a runner
# with the build stretched one ~1.5s boundary case past Vitest's 30s timeout, so
# the build has to finish before anything timed starts.
- name: Build the baseline Git binary
run: |
archive="$RUNNER_TEMP/git-2.25.5.tar.gz"
source="$HOME/.cache/orca-git-compat/git-2.25.5"
if [ -x "$source/git" ]; then
exit 0
fi
curl -fsSL https://www.kernel.org/pub/software/scm/git/git-2.25.5.tar.gz -o "$archive"
echo "41662c52fc16fec4963bfc41075e71f8ead6b5e386797eb6f9a1111ff95a8ddf $archive" \
| sha256sum --check
mkdir -p "$source"
tar -xzf "$archive" -C "$source" --strip-components=1
make -C "$source" -j"$(nproc)" \
NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PYTHON=YesPlease git
# Why: the linked binaries are what the next run needs; the objects that
# produced them are most of the tree and would bloat the cache entry.
find "$source" -name '*.o' -delete
- name: Verify Git binary compatibility matrix
run: |
specs=(
"alpine/git:edge-2.38.1|2.38.1"
"alpine/git:v2.49.1|2.49.1"
)
# Why pull up front: a lane's first `docker run` otherwise pulls its image
# while the sibling lane is mid-test, and that stall is charged to the test.
for spec in "${specs[@]}"; do
docker pull --quiet "${spec%%|*}"
done
pids=()
(
archive="$RUNNER_TEMP/git-2.25.5.tar.gz"
source="$HOME/.cache/orca-git-compat/git-2.25.5"
if [ ! -x "$source/git" ]; then
curl -fsSL https://www.kernel.org/pub/software/scm/git/git-2.25.5.tar.gz -o "$archive"
echo "41662c52fc16fec4963bfc41075e71f8ead6b5e386797eb6f9a1111ff95a8ddf $archive" \
| sha256sum --check
mkdir -p "$source"
tar -xzf "$archive" -C "$source" --strip-components=1
make -C "$source" -j"$(nproc)" \
NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PYTHON=YesPlease git
# Why: the linked binaries are what the next run needs; the objects that
# produced them are most of the tree and would bloat the cache entry.
find "$source" -name '*.o' -delete
fi
ORCA_GIT_COMPAT_BINARY="$source/git" ORCA_GIT_COMPAT_VERSION="2.25.5" \
ORCA_GIT_COMPAT_BINARY="$HOME/.cache/orca-git-compat/git-2.25.5/git" \
ORCA_GIT_COMPAT_VERSION="2.25.5" \
pnpm exec vitest run --config config/vitest.config.ts \
src/shared/git-binary-compatibility.test.ts
) &
pids+=("$!")
for spec in \
"alpine/git:edge-2.38.1|2.38.1" \
"alpine/git:v2.49.1|2.49.1"; do
for spec in "${specs[@]}"; do
(
image="${spec%%|*}"
version="${spec#*|}"
@@ -772,18 +809,9 @@ jobs:
[[ "$rpm_marker" == rpm ]] || { echo "Expected rpm marker, got: $rpm_marker"; exit 1; }
- name: Verify headless serve signal shutdown
run: node config/scripts/run-headless-serve-shutdown-docker.mjs --appimage dist/orca-linux.AppImage
- name: Verify extracted launcher serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint launcher
- name: Verify AppImage CLI registration and serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint appimage
--signal-target serving-electron --int-delivery pid
--appimage dist/orca-linux.AppImage --all-entrypoints
# A default container reproduces the hostile AppImage launch environment.
- name: Verify Linux CLI launch contract
@@ -832,9 +860,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# vitest runs here directly rather than through `pnpm test`, so the addon
# assertions only hold once install-node-dependencies has rebuilt natives.
@@ -843,6 +871,11 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
src/main/windows-registry-addon.test.ts
config/scripts/windows-process-tree-gyp-path.test.mjs
config/scripts/windows-process-tree-gyp-rebuild.test.mjs
config/scripts/package-electron-runtime-contract.test.mjs
config/scripts/electron-builder-runtime-resources.test.mjs
src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts
src/main/browser/browser-route-tcp-egress.electron.test.ts
src/main/browser/browser-route-webrtc-egress.electron.test.ts
@@ -855,6 +888,8 @@ jobs:
src/shared/child-process/windows-cmd-shim-resolution.win32.test.ts
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
src/main/codex/windows-hook-command.test.ts
src/main/codex/windows-hook-upgrade.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-msys-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
@@ -900,9 +935,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Prepare Electron native runtime
run: node config/scripts/ensure-native-runtime.mjs --runtime=electron
.github/workflows/release-cut.yml
+19
View File
@@ -1262,6 +1262,8 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
@@ -1309,6 +1311,20 @@ jobs:
echo "identity=$identity" >>"$GITHUB_OUTPUT"
echo "Classified $TAG as $identity"
# Why here and not in build:relay: only a Windows runner can compile it, and
# arm64 cross-compiles from this same x64 agent. Mirrors dev-channel-win-build.yml,
# which had it while release-cut did not — so every stable installer through
# v1.4.203 shipped Windows relays with no windows-process-tree.node, silently
# falling back to the PowerShell scan on every Windows SSH host.
# Why no run_attempt guard, unlike the artifact steps below: Build app is ungated,
# so a rerun would reach the required-addon check with nothing staged and fail.
- name: Build Windows process-table addon for the relay
if: matrix.platform == 'win'
shell: bash
run: |
node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=x64
node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=arm64
# Why ORCA_POSTHOG_WRITE_KEY here: this is the only build that
# produces a published binary, so this is the only place the secret
# needs to be in scope. The key is a PostHog *project* API key, not
@@ -1331,6 +1347,9 @@ jobs:
ORCA_BUILD_IDENTITY: ${{ steps.tag-classify.outputs.identity }}
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
ORCA_POSTHOG_WRITE_KEY: ${{ secrets.ORCA_POSTHOG_WRITE_KEY }}
# Fail the release rather than ship a relay that silently falls back to
# the PowerShell scan on every Windows SSH host.
ORCA_REQUIRE_RELAY_NATIVE_ADDONS: ${{ matrix.platform == 'win' && 'x64,arm64' || '' }}
- name: Gate runtime file-watcher process isolation
if: runner.os == 'Linux'
.github/workflows/release-mac-build.yml
+3 -1
View File
@@ -64,13 +64,15 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
- name: Verify macOS signing environment
run: node config/scripts/verify-macos-release-env.mjs
.github/workflows/unit-tests.yml
+24 -1
View File
@@ -37,14 +37,26 @@ jobs:
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
- name: Test shard
# The real two-cell transport test imports cloud relay source and its contracts.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test shard
env:
ORCA_BALANCE_UNIT_SHARDS: '1'
ORCA_BACKGROUND_LAUNCH: '1'
run: |
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
pnpm exec vitest run --config config/vitest.config.ts \
--exclude=src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts \
--exclude=src/main/daemon/shell-ready.test.ts \
--exclude=src/main/daemon/node-pty-fd-leak.test.ts \
--exclude=src/main/providers/local-pty-shell-ready-zsh-launch-environment.test.ts \
--exclude=src/main/providers/__tests__/shell-ready-framework-example.test.ts \
--exclude=src/main/pty/omp-shell-wrapper-alias-safety.test.ts \
--exclude=src/main/pty/omp-shell-wrapper.node-pty.test.ts \
--exclude=src/main/shell-startup-feature-channel.test.ts \
--exclude=src/main/terminal-history-fish-session.node-pty.test.ts \
@@ -58,3 +70,14 @@ jobs:
--exclude=src/shared/posix-command-path-lookup.test.ts \
--exclude=tests/e2e/cross-version-wire/** \
--shard=${{ matrix.shard }}/${{ matrix.shard_total }}
- name: Upload unit shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: unit-shard-node-${{ matrix.node }}-${{ matrix.shard }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
.github/workflows/windows-signing-rehearsal.yml
+2
View File
@@ -68,6 +68,8 @@ jobs:
restore-keys: |
electron-builder-win-
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
.gitignore
+12
View File
@@ -28,6 +28,9 @@ out/
/build/
release/
native/**/.build/
# node-gyp output for the vendored Windows registry addon; generated per host and ABI.
native/windows-registry/build/
native/windows-registry/bin/
# pnpm
.pnpm-store/
@@ -103,13 +106,18 @@ docs/**
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-pty-transcript-capture.md
!docs/reference/agent-session-search-query-tuning.md
!docs/reference/agent-session-search-contract.md
!docs/reference/agent-status-store.md
!docs/reference/antigravity-readiness-evidence.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
!docs/reference/ime-regression-checklist.md
!docs/reference/linux-glibc-compatibility.md
!docs/reference/macos-press-and-hold.md
!docs/reference/orcad-operations.md
!docs/reference/pnpm-install-policy.md
!docs/reference/relay-grace-time-reconfiguration.md
!docs/reference/windows-cmd-shim-resolution.md
!docs/reference/windows-daemon-host-relocation.md
@@ -122,6 +130,7 @@ docs/**
!docs/reference/ssh-host-key-verification.md
!docs/reference/ssh-reconnect-source-recovery.md
!docs/reference/windows-setup-shell.md
!docs/reference/windows-terminal-shell-selection.md
!docs/reference/worktree-scan-fingerprint.md
!docs/reference/wsl-command-execution.md
!docs/reference/wsl-probe-failure-semantics.md
@@ -173,3 +182,6 @@ tests/e2e/.cross-version-checkouts/
# IS committed). Also keeps oxfmt/oxlint, which honor this file, from walking
# vendored gems.
/mobile/vendor/
# Generated by config/scripts/sync-anti-slop-plugin.mjs from the pinned oxlint-plugin-anti-slop
.anti-slop-plugin/
.oxfmtrc.json
+5 -1
View File
@@ -4,5 +4,9 @@
"semi": false,
"printWidth": 100,
"trailingComma": "none",
"ignorePatterns": ["cloud/**", ".github/actions/cloud-sql-rollout-lease/**"]
"ignorePatterns": [
"cloud/**",
".github/actions/cloud-sql-rollout-lease/**",
".anti-slop-plugin/**"
]
}
.oxlintrc.json
+1
View File
@@ -180,6 +180,7 @@
}
],
"ignorePatterns": [
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
"**/node_modules",
"**/dist",
"**/out",
AGENTS.md
+30 -1
View File
@@ -1,6 +1,6 @@
# Design System
All UI work — layout, color, typography, spacing, component selection, UX behavior — must follow [`docs/STYLEGUIDE.md`](./docs/STYLEGUIDE.md). Use the tokens defined in `src/renderer/src/assets/main.css` (the canonical source) and the shadcn primitives in `src/renderer/src/components/ui/`. Don't invent new color values, font sizes, or shadow tiers when a documented one already covers the role. When STYLEGUIDE.md is silent, follow the resolution order in its final section.
All UI work — layout, color, typography, spacing, component selection, UX behavior — must follow [`docs/STYLEGUIDE.md`](./docs/STYLEGUIDE.md). Most of it is linted: `pnpm run check:code-quality:changed` fails on new restyles of a `components/ui/` primitive, raw palette colors, and computed `className` strings; `pnpm lint` fails on any class Tailwind cannot generate. See the Enforcement section of the style guide before suppressing either. Use the tokens defined in `src/renderer/src/assets/main.css` (the canonical source) and the shadcn primitives in `src/renderer/src/components/ui/`. Don't invent new color values, font sizes, or shadow tiers when a documented one already covers the role. When STYLEGUIDE.md is silent, follow the resolution order in its final section.
## Electron UI Validation
@@ -33,11 +33,31 @@ Never use vague names like `helpers`, `utils`, `common`, `misc`, or `shared-stuf
## Type Declarations: Prefer `.ts` Over `.d.ts`
## Type Assertions: Prefer Checked Types
Avoid type assertions except `as const`. Unavoidable casts need a line-specific `SAFETY:` explanation:
```ts
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: Explain the verified invariant here.
```
# Verifying Changes
- **Typecheck**: `pnpm tc` (or `tc:node` / `tc:cli` / `tc:web`)
- **Test**: `pnpm test [path/to/file.test.ts]`
- **Lint**: `oxlint`, or `pnpm run check:code-quality:changed` for changed files (full `pnpm lint` is slow); format with `pnpm format`
- **Design system**: `pnpm run lint:design-system` for the full renderer report (not a gate); the changed-lines gate above is what CI enforces
# Writing Pull Requests
Fill in [`.github/pull_request_template.md`](./.github/pull_request_template.md), written for a reviewer who has never seen this code:
- No jargon — plain language, no internal shorthand.
- The before and after as the user experiences it.
- The mechanism you changed, not just the symptom.
- Why this approach over the alternatives you considered.
Cover all four concisely. Don't pad or walk the diff.
# Considerations
@@ -52,6 +72,7 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **Keyboard shortcuts**: Never hardcode `e.metaKey`. Use a platform check (`navigator.userAgent.includes('Mac')`) to pick `metaKey` on Mac and `ctrlKey` on Linux/Windows. Electron menu accelerators should use `CmdOrCtrl`.
- **Shortcut labels in UI**: Display `⌘` / `⇧` on Mac and `Ctrl+` / `Shift+` on other platforms.
- **File paths**: Use `path.join` or Electron/Node path utilities — never assume `/` or `\`.
- **Windows terminal shells**: `--shell` picks the shell a terminal *is*; `--command` is typed into whatever shell the host spawned, so a shell choice routed through `command` silently becomes a child process. See [`docs/reference/windows-terminal-shell-selection.md`](./docs/reference/windows-terminal-shell-selection.md).
- **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import. Recognised npm/pnpm `.cmd` shims are resolved to their real target so the spawn skips `cmd.exe` entirely; see [`docs/reference/windows-cmd-shim-resolution.md`](./docs/reference/windows-cmd-shim-resolution.md) before adding a shim shape or debugging one.
- **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
@@ -60,6 +81,10 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
- **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.
## Native Dependency Installs
Ordinary `pnpm install` covers the host OS and CPU only. Before packaging for another architecture — including `pnpm build:mac`, which builds x64 and arm64 by default — run `pnpm install:release`. electron-builder only warns on a missing `extraResources` source, so the `beforePack` guard is what turns a thin install into a build failure instead of a silently broken artifact; see [`docs/reference/pnpm-install-policy.md`](./docs/reference/pnpm-install-policy.md).
## SSH Use Case
All changes must consider the SSH use case. Don't assume local-only execution. Before changing anything that reports on, stops, or lists remote work, follow [`docs/reference/ssh-execution-boundary.md`](./docs/reference/ssh-execution-boundary.md): the execution host owns everything that touches execution, and loss of contact is never evidence of process death — the verdict vocabulary is `live` / `unverifiable` / `exited`, with no synonyms.
@@ -72,6 +97,10 @@ All changes must consider folder workspaces as well as git worktrees. Don't assu
The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy.
## Agent Terminal Screens
A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md).
## Remote Wire Compatibility
Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change.
README.md
+8 -8
View File
@@ -54,7 +54,7 @@ Fan one prompt across five agents, each in its own isolated git worktree — com
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/assets/feature-wall/parallel-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/parallel-worktrees.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/site/public/docs/tab-split.gif" type="image/gif"><img src="docs/site/public/docs/posters/tab-split.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -68,7 +68,7 @@ Ghostty-class terminals with WebGL rendering, infinite splits, and scrollback th
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="docs/assets/feature-wall/terminal-splits.gif" type="image/gif"><img src="docs/assets/feature-wall/terminal-splits.jpg" alt="Terminal splits" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="resources/onboarding/feature-wall/tile-02.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-02.poster.jpg" alt="Terminal splits" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -82,7 +82,7 @@ Click any UI element in a real Chromium window to send its HTML, CSS, and a crop
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/assets/feature-wall/design-mode.gif" type="image/gif"><img src="docs/assets/feature-wall/design-mode.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/site/public/docs/orca-design-mode.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-05.poster.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -96,7 +96,7 @@ Browse PRs, issues, and project boards in-app — open a worktree from any task
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="docs/assets/feature-wall/github-linear.gif" type="image/gif"><img src="docs/assets/feature-wall/github-linear.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="resources/onboarding/feature-wall/tile-03.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-03.poster.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -110,7 +110,7 @@ Run agents on a beefy remote box with full file editing, git, and terminals —
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="docs/assets/feature-wall/ssh-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/ssh-worktrees.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="resources/onboarding/feature-wall/tile-06.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-06.poster.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -124,7 +124,7 @@ Drop comments on any diff line and ship them back to the agent — review, edit,
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/assets/feature-wall/annotate-diff.gif" type="image/gif"><img src="docs/assets/feature-wall/annotate-diff.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/site/public/docs/annotate-ai-diff.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-08.poster.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -138,7 +138,7 @@ VS Code's editor with autosave everywhere — drag files or images straight into
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="docs/assets/feature-wall/file-drag.gif" type="image/gif"><img src="docs/assets/feature-wall/file-drag.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="resources/onboarding/feature-wall/tile-07.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-07.poster.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -152,7 +152,7 @@ Agents drive Orca too — script every workflow with `orca worktree create`, `sn
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="docs/assets/feature-wall/orca-cli.gif" type="image/gif"><img src="docs/assets/feature-wall/orca-cli.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="resources/onboarding/feature-wall/tile-09.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-09.poster.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
</td>
</tr>
</table>
cloud/apps/push/src/fcm-client.test.ts
+13 -24
View File
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
const NOW = 1_700_000_000_000
@@ -20,6 +20,7 @@ function delivery(agentState: 'needs-input' | null = 'needs-input') {
agentState,
title: 'Agent needs input',
body: 'Waiting on your answer',
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1'
}
})
@@ -59,27 +60,14 @@ describe('fcm client', () => {
expect(JSON.parse(request.body)).toEqual({
message: {
token: TOKEN,
notification: { title: 'Agent needs input', body: 'Waiting on your answer' },
android: {
priority: 'HIGH',
ttl: '300s',
collapse_key: createHash('sha256')
.update(
createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
)
.digest('hex')
.slice(0, 32),
notification: {
channel_id: 'orca-desktop',
tag: createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
}
},
android: { priority: 'HIGH', ttl: '300s' },
data: {
title: 'Agent needs input',
message: 'Waiting on your answer',
tag: delivery().collapseId,
channelId: 'orca-desktop',
hostFingerprint: HOST,
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: '7',
@@ -96,7 +84,7 @@ describe('fcm client', () => {
await fcm.send(delivery(null), { token: TOKEN })
const message = JSON.parse(fake.requests[0]!.body) as {
message: {
android: { collapse_key: string; notification: { tag: string } }
android: Record<string, unknown>
data: Record<string, string>
}
}
@@ -108,9 +96,10 @@ describe('fcm client', () => {
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
expect(message.message.data.coalescedCount).toBeUndefined()
expect(message.message.android.notification.tag).toBe(tag)
expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag))
expect(message.message.android.collapse_key).toHaveLength(32)
expect(message.message.data.tag).toBe(tag)
expect(message.message.android).not.toHaveProperty('collapse_key')
expect(message.message).not.toHaveProperty('notification')
expect(message.message.data).not.toHaveProperty('body')
})
it('marks an unregistered token dead from the status or the error detail', async () => {
cloud/apps/push/src/fcm-client.ts
+11 -19
View File
@@ -1,5 +1,4 @@
import { providerRetryAfter } from './provider-retry-delay.js'
import { createHash } from 'node:crypto'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
@@ -22,12 +21,6 @@ type FcmErrorBody = {
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
}
// FCM collapse_key is a short opaque string, so the collapse id is hashed
// rather than truncated: truncation would merge unrelated notifications.
export function fcmCollapseKey(collapseId: string): string {
return createHash('sha256').update(collapseId).digest('hex').slice(0, 32)
}
export function fcmMessageBody(input: {
delivery: PushDelivery
token: string
@@ -39,24 +32,23 @@ export function fcmMessageBody(input: {
return JSON.stringify({
message: {
token: input.token,
...(delivery.orca.kind === 'dismiss'
? {}
: { notification: { title: delivery.title, body: delivery.body } }),
android: {
priority: 'HIGH',
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`,
collapse_key: fcmCollapseKey(delivery.collapseId),
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`
},
// Notification payloads collapse offline; Expo renders these data messages natively.
data: {
...orcaDataStrings(delivery.orca),
...(delivery.orca.kind === 'dismiss'
? {}
: {
notification: {
channel_id:
delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
tag: delivery.collapseId
}
title: delivery.title,
message: delivery.body,
tag: delivery.collapseId,
channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
...(delivery.sound === false ? { sound: '' } : {})
})
},
data: orcaDataStrings(delivery.orca)
}
}
})
}
cloud/apps/push/src/push-delivery-message.ts
+2
View File
@@ -5,6 +5,7 @@ export type PushOrcaData = {
kind?: 'alert' | 'dismiss'
hostFingerprint: string
worktreeId?: string
paneKey?: string
notificationId?: string
notificationSeq: number
notificationEpoch: string
@@ -51,6 +52,7 @@ export function buildPushDelivery(input: {
orca: {
...(notification.kind ? { kind: notification.kind } : {}),
hostFingerprint,
...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }),
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
...(notification.notificationId === undefined
? {}
cloud/apps/push/src/push-dismissal-provider.test.ts
+4
View File
@@ -23,5 +23,9 @@ it('dismissal provider payloads cannot display a new alert or play a sound', ()
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
expect(android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('collapse_key')
expect(android.data).not.toHaveProperty('title')
expect(android.data).not.toHaveProperty('message')
expect(android.data).not.toHaveProperty('sound')
expect(android.data.kind).toBe('dismiss')
})
cloud/apps/push/src/push-notification-sound.test.ts
+5 -1
View File
@@ -23,7 +23,11 @@ it('carries a silent preference through validation to APNs and Android payloads'
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.android.notification.channel_id
.data.channelId
).toBe('orca-desktop-silent')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.sound
).toBe('')
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
})
cloud/apps/push/src/push-pane-routing.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { expect, it } from 'vitest'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js'
it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => {
const base = {
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
worktreeId: 'folder:/work'
}
const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111'
for (const extra of [{}, { paneKey }]) {
const notification = PushNotificationSchema.parse({ ...base, ...extra })
const delivery = buildPushDelivery({
notification,
hostFingerprint: 'host',
registrationId: 'phone',
expiresAt: Date.now() + 300000
})
expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined)
expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined)
}
})
cloud/apps/push/src/push-server-send.test.ts
+2 -2
View File
@@ -56,7 +56,7 @@ describe('push gateway send route', () => {
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } }
message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } }
})
const afterDeath = await harness.post(
@@ -179,7 +179,7 @@ describe('push gateway send route', () => {
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
message: { android: { notification: { tag: string } }; data: Record<string, string> }
}
expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.coalescedCount).toBeUndefined()
})
cloud/apps/relay-ops/src/incident-monitor.test.ts
+20
View File
@@ -273,6 +273,26 @@ describe('incident monitor evaluator', () => {
)
})
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
for (const errors of [1, 2, 3]) {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
}
const excess = healthySample()
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
)
const auth = healthySample()
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
expect(evaluateIncidentSample(auth, startedAt).status).toBe('freeze')
const pressure = healthySample()
pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1)
pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze')
})
it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
cloud/apps/relay-ops/src/incident-monitor.ts
+2 -1
View File
@@ -101,7 +101,8 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
directorCpuUtilization: 0.8,
directorMemoryUtilization: 0.8,
directorConcurrency: 64,
directorErrors: 0,
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
directorErrors: 3,
authErrors: 0,
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
cloud/apps/relay/src/admin-token-verifier.ts
+1
View File
@@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
'/v1/admin/cell-status',
'/v1/admin/evacuation-status',
'/v1/admin/regional-rehome-control',
'/v1/admin/regional-rehome-preview',
'/v1/admin/runtime-status'
] as const
cloud/apps/relay/src/app.ts
+137 -51
View File
@@ -1,5 +1,9 @@
import {
AssignmentRequestSchema,
IdleRegionalRehomeRequestSchema,
type IdleRegionalRehomeRequest,
type IdleRegionalRehomeOutcome,
type RegionCorrectionResponse,
isRelayCellConnectionHardCap,
RELAY_ADMISSION_BUDGETS,
RELAY_DEFAULT_REGION,
@@ -39,7 +43,7 @@ import {
type AssignmentAdmissionRejection
} from './public-assignment-admission.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeCounts } from './relay-observability.js'
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
import {
isRegionalRehomeTrustProbe,
probeRegionalRehomeTrust
@@ -68,21 +72,28 @@ export function createRelayApp(
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
drainHost?: (input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceCellIncarnation: string
graceMs: number
}) => 'accepted' | 'already-accepted' | 'host-not-connected'
}) =>
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
regionalRehomeFetch?: typeof fetch
regionalRehomeTrustProbeHostExists?: (input: {
userId: string
relayHostId: string
}) => boolean
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
cellIncarnation?: string
isDraining?: () => boolean
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
recordAssignmentAdmission?: (
@@ -226,7 +237,8 @@ export function createRelayApp(
return context.json({ error: 'host_identity_mismatch' }, 403)
}
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
const requestedRegion = body.data.preferredRegion
const requestedRegion =
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
const targetRegion =
config.regionalPlacementEnabled !== false && requestedRegion
? requestedRegion
@@ -295,10 +307,30 @@ export function createRelayApp(
}
}
let assignment: RelayAssignment
let regionCorrection: RegionCorrectionResponse | undefined
try {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
if (body.data.regionCorrection?.action === 'report') {
const current = await operations.assignments.resolve(identity)
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
assignment = current
} else {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
}
if (body.data.regionCorrection) {
try {
regionCorrection = await operations.assignments.exchangeRegionCorrection(
identity,
body.data.regionCorrection,
assignment.assignmentEpoch
)
} catch (error) {
if (body.data.regionCorrection.action === 'report') throw error
// Optional measurement setup must not discard an otherwise valid placement.
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
@@ -353,13 +385,16 @@ export function createRelayApp(
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
lease
lease,
...(regionCorrection ? { regionCorrection } : {})
})
})
app.post('/v1/resolve', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
if (
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
@@ -433,6 +468,34 @@ export function createRelayApp(
operations.drain(body.data.graceMs)
return context.json({ ok: true })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = IdleRegionalRehomeCommandSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
if (config.role !== 'cell' || !operations.drainHost) {
return context.json({ error: 'cell_only' }, 404)
@@ -474,7 +537,7 @@ export function createRelayApp(
}
sharedRuntimeIdentityRejected = true
}
const outcome = operations.drainHost(body.data)
const outcome = await operations.drainHost(body.data)
return context.json({
v: 1,
outcome,
@@ -502,8 +565,7 @@ export function createRelayApp(
region: config.region ?? RELAY_DEFAULT_REGION,
imageDigest: config.imageDigest ?? null,
draining: operations.isDraining?.() ?? false,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
connectionCapacity:
config.connectionHardCap === undefined
? null
@@ -559,6 +621,18 @@ export function createRelayApp(
return context.json({ error: operationError(error) }, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
const preview = await operations.assignments.previewRegionalRehomeEligibility(
operations.regionalRehomeSafetySnapshot?.()
)
const outcomes = await operations.assignments.regionCorrectionOutcomes()
return context.json({ v: 1, preview, outcomes })
})
app.post('/v1/admin/regional-rehome-control', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
@@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z
})
.strict()
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
cohortPercent: z.number().int().min(0).max(100),
directorSafety: RegionalRehomeSafetySchema
})
const CellHeartbeatSchema = z
.object({
v: z.literal(1),
@@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
regionalRehomeProtocol: z.number().int().min(0).max(1),
regionalRehomeProtocol: z.number().int().min(0).max(3),
safety: RegionalRehomeSafetySchema
})
.strict()
const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
'DISABLE_REGIONAL_REHOMING'
])
}).strict()
]).superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled
? 'ENABLE_REGIONAL_REHOMING'
: 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeControlSchema = z
.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z
.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z
.number()
.int()
.min(60_000)
.max(60 * 60_000),
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
})
.strict()
])
.superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeTrustProbeSchema = z
.object({
@@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: z.number().int().positive(),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000)
})
.strict()
cloud/apps/relay/src/assignment-store.ts
+573 -799
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/cell-heartbeat-client.test.ts
+27 -21
View File
@@ -92,7 +92,7 @@ describe('cell heartbeat client', () => {
client.stop()
expect(JSON.parse(String(requests[1]!.body))).toMatchObject({
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 3,
safety: {
observedAt: 120,
sqlFailures: 0,
@@ -149,28 +149,34 @@ describe('cell heartbeat client', () => {
it('does not start outside an explicitly configured cell role', () => {
expect(
startCellHeartbeat({ ...CONFIG, role: 'director' }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, role: 'director' },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
expect(
startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, directorUrl: undefined },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
})
})
cloud/apps/relay/src/cell-heartbeat-client.ts
+6 -4
View File
@@ -70,8 +70,7 @@ export function startCellHeartbeat(
inFlightConnections: connectionCounts!.inFlightConnections,
reservedConnectionUnits: connectionCounts!.reservedConnectionUnits,
enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits,
connectionInclusionWatermark:
connectionCounts!.inclusionWatermark,
connectionInclusionWatermark: connectionCounts!.inclusionWatermark,
connectionHardCap: config.connectionHardCap,
connectionUnobservedBound: config.connectionUnobservedBound
})
@@ -94,7 +93,7 @@ export function startCellHeartbeat(
cellId: config.cellId,
cellIncarnation,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
safety: options.regionalRehomeSafety()
}),
signal: AbortSignal.timeout(10_000)
@@ -106,7 +105,10 @@ export function startCellHeartbeat(
}
} catch (error) {
// A heartbeat must fail closed without ever logging its bearer token.
console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '')
console.warn(
'[orca-relay] cell heartbeat failed',
error instanceof Error ? error.message : ''
)
} finally {
inFlight = false
}
cloud/apps/relay/src/cell-inventory-hold-samples.test.ts
+42
View File
@@ -67,4 +67,46 @@ describe('cell inventory hold samples', () => {
expect(samples.consumeCounts().cellInventoryHolds).toBe(2)
expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts())
})
// Why: this is the case the hold metrics alone cannot see. A NOWAIT grab that
// fails has no duration, so a retry storm used to leave every hold field at
// zero while the lock was saturated.
it('counts failed acquisitions in a window that recorded no holds', () => {
const samples = new CellInventoryHoldSamples()
for (let attempt = 0; attempt < 65; attempt++) samples.recordUnavailable()
const counts = samples.readCounts()
expect(counts.cellInventoryLockUnavailable).toBe(65)
expect(counts.cellInventoryHolds).toBe(0)
expect(counts.cellInventoryHoldMsMax).toBe(0)
})
it('reports failed acquisitions alongside the holds that did succeed', () => {
const samples = samplesOf([12, 34])
samples.recordUnavailable(3)
expect(samples.readCounts()).toMatchObject({
cellInventoryHolds: 2,
cellInventoryHoldMsMax: 34,
cellInventoryLockUnavailable: 3
})
})
it('ignores a failure count that is not a positive number', () => {
const samples = new CellInventoryHoldSamples()
samples.recordUnavailable(0)
samples.recordUnavailable(-2)
samples.recordUnavailable(Number.NaN)
expect(samples.readCounts()).toEqual(emptyCellInventoryHoldCounts())
})
it('resets failed acquisitions on consume', () => {
const samples = new CellInventoryHoldSamples()
samples.recordUnavailable(4)
expect(samples.consumeCounts().cellInventoryLockUnavailable).toBe(4)
expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts())
})
})
cloud/apps/relay/src/cell-inventory-hold-samples.ts
+37 -3
View File
@@ -5,6 +5,14 @@ export type CellInventoryHoldCounts = {
cellInventoryHoldMsMax: number
cellInventoryHoldMsP95: number
cellInventoryHolds: number
// Why: a failed acquisition produces no hold sample, so the hold fields alone
// read healthy while the lock is saturated. Split by wait policy, not by
// caller: fail-fast covers background sweeps that step aside by design AND
// request-path first attempts that retry, so it reads as contention pressure,
// not user-visible failure. An expired bounded wait has already spent its
// budget, so that lane is the one that tracks stalls.
cellInventoryLockUnavailable: number
cellInventoryLockTimeouts: number
}
// Bounded so a flush interval with heavy assignment traffic cannot grow the array
@@ -12,11 +20,19 @@ export type CellInventoryHoldCounts = {
const MAX_SAMPLES = 2_048
export function emptyCellInventoryHoldCounts(): CellInventoryHoldCounts {
return { cellInventoryHoldMsMax: 0, cellInventoryHoldMsP95: 0, cellInventoryHolds: 0 }
return {
cellInventoryHoldMsMax: 0,
cellInventoryHoldMsP95: 0,
cellInventoryHolds: 0,
cellInventoryLockUnavailable: 0,
cellInventoryLockTimeouts: 0
}
}
export class CellInventoryHoldSamples {
private samples: number[] = []
private unavailable = 0
private timeouts = 0
record(holdMs: number): void {
if (!Number.isFinite(holdMs) || holdMs < 0) return
@@ -24,19 +40,37 @@ export class CellInventoryHoldSamples {
this.samples.push(holdMs)
}
// Counted, not sampled: a failed acquisition has no duration to record.
recordUnavailable(count = 1): void {
if (!Number.isFinite(count) || count <= 0) return
this.unavailable += count
}
recordLockTimeout(count = 1): void {
if (!Number.isFinite(count) || count <= 0) return
this.timeouts += count
}
consumeCounts(): CellInventoryHoldCounts {
const counts = this.readCounts()
this.samples = []
this.unavailable = 0
this.timeouts = 0
return counts
}
readCounts(): CellInventoryHoldCounts {
if (this.samples.length === 0) return emptyCellInventoryHoldCounts()
const failures = {
cellInventoryLockUnavailable: this.unavailable,
cellInventoryLockTimeouts: this.timeouts
}
if (this.samples.length === 0) return { ...emptyCellInventoryHoldCounts(), ...failures }
const sorted = [...this.samples].sort((left, right) => left - right)
return {
cellInventoryHoldMsMax: round(sorted[sorted.length - 1]!),
cellInventoryHoldMsP95: round(sorted[Math.ceil(0.95 * sorted.length) - 1] ?? 0),
cellInventoryHolds: sorted.length
cellInventoryHolds: sorted.length,
...failures
}
}
}
cloud/apps/relay/src/cell-inventory-lock-census.test.ts
+7 -9
View File
@@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [
{ method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
{ method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
@@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map<string, Set<string>> {
bounds.forEach((method, index) => {
const end = bounds[index + 1]?.start ?? lines.length
const names = callees.get(method.name) ?? new Set<string>()
for (const call of lines.slice(method.start, end).join('\n').matchAll(
/this\.([A-Za-z_][\w]*)\s*\(/g
)) {
for (const call of lines
.slice(method.start, end)
.join('\n')
.matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) {
names.add(call[1]!)
}
callees.set(method.name, names)
@@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
describe('cell inventory lock call-site census', () => {
it('classifies every call site exactly as recorded', () => {
expect(readCallSites()).toEqual(
CENSUS.map(({ method, mode }) => ({ method, mode }))
)
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
})
// Why: the census only sees lockCellInventory calls, so a hand-written
cloud/apps/relay/src/cell-inventory-lock-contention.test.ts
+58
View File
@@ -260,6 +260,64 @@ describe('bounded cell-inventory lock wait', () => {
await database.close()
})
// Why: the 55P03 rolls the transaction back, so a drain on the commit path
// alone would report zero for exactly the windows that were contended.
it('reports a NOWAIT deferral that rolled its transaction back', async () => {
const database = await openFakePostgres()
fakes.query.mockImplementation(async (sql: string) => {
if (sql.includes('FOR UPDATE NOWAIT')) {
throw Object.assign(new Error('could not obtain lock'), { code: '55P03' })
}
return { rows: [], rowCount: 0 }
})
await expect(
database.transaction(async (transaction) => {
await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
failIfUnavailable: true,
measureHoldMs: true
})
})
).rejects.toThrow('database_lock_unavailable')
const counts = consumeRelayCellInventoryHold(database)
expect(counts.cellInventoryLockUnavailable).toBe(1)
expect(counts.cellInventoryLockTimeouts).toBe(0)
await database.close()
})
// Why: a bounded request-path wait raises the same 55P03 without NOWAIT. Folding
// it into the deferral counter would hide user-visible stalls among by-design
// sweep skips, which outnumber them by roughly an order of magnitude.
it('counts an expired bounded wait apart from a NOWAIT deferral', async () => {
const database = await openFakePostgres()
fakes.query.mockImplementation(async (sql: string) => {
if (sql.includes('FOR UPDATE') && !sql.includes('NOWAIT')) {
throw Object.assign(new Error('canceling statement due to lock timeout'), {
code: '55P03'
})
}
return { rows: [], rowCount: 0 }
})
await expect(
database.transaction(async (transaction) => {
await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
lockTimeoutMs: 500,
measureHoldMs: true
})
})
).rejects.toThrow()
const counts = consumeRelayCellInventoryHold(database)
// One per attempt, not per request: 55P03 is retryable, so an exhausted
// request contributes POSTGRES_TRANSACTION_ATTEMPTS timeouts. Reading the
// metric as affected-requests would overstate it threefold.
expect(counts.cellInventoryLockTimeouts).toBe(3)
expect(counts.cellInventoryLockUnavailable).toBe(0)
await database.close()
})
it('records no hold for a PostgreSQL transaction that took no measured lock', async () => {
const database = await openFakePostgres()
cloud/apps/relay/src/config.test.ts
+11
View File
@@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
}
describe('GCE relay capacity configuration', () => {
it('defaults optional region correction off and bounds the cohort', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0)
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5'
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5)
for (const invalid of ['-1', '101', '1.5', 'not-a-number']) {
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid
expect(() => loadRelayConfig(env)).toThrow()
}
})
it('requires distinct dedicated admin identities and accepts omitted values', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
cloud/apps/relay/src/config.ts
+7 -1
View File
@@ -75,11 +75,15 @@ const EnvSchema = z.object({
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(),
ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(),
ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(),
ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(),
ORCA_RELAY_IMAGE_DIGEST: z
.string()
.regex(/^sha256:[a-f0-9]{64}$/)
.optional(),
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2),
ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1),
ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64),
@@ -185,6 +189,7 @@ export type RelayConfig = {
databasePoolMax: number
publicAssignmentsEnabled: boolean
regionalPlacementEnabled?: boolean
regionCorrectionCohortPercent?: number
publicAssignmentConcurrency: number
publicAssignmentQueueMax: number
publicAssignmentWaitMs: number
@@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
databasePoolMax,
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY,
publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX,
publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS,
cloud/apps/relay/src/database-postgres-timeout.test.ts
+7 -1
View File
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
const fakes = vi.hoisted(() => ({
configs: [] as Array<Record<string, unknown>>,
@@ -119,11 +120,16 @@ describe('PostgreSQL relay deadlines', () => {
})
expect(ddl.length).toBeGreaterThan(0)
expect(ddl).toContain(POSTGRES_STATEMENT_STATS_MIGRATION)
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
ddl.every(
(statement) =>
statement === POSTGRES_STATEMENT_STATS_MIGRATION ||
/^(?:CREATE|ALTER TABLE)\b/i.test(body(statement))
)
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
cloud/apps/relay/src/database.test.ts
+32 -6
View File
@@ -18,6 +18,34 @@ afterEach(() => {
})
describe('relay database', () => {
it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-'))
temporaryDirectories.push(dataDir)
const legacy = await openRelayDatabase({ dataDir })
await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome')
await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation')
await legacy.query(
`INSERT INTO relay_control_capabilities
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing)
VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)`
)
await legacy.close()
const upgraded = await openRelayDatabase({ dataDir })
try {
expect(
await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities')
).toEqual([{ idle_regional_rehome: 0 }])
const columns = await upgraded.query(
"SELECT * FROM pragma_table_info('relay_region_rehome_attempts')"
)
expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({
dflt_value: '0'
})
} finally {
await upgraded.close()
}
})
it('creates every durable relay state table', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
@@ -54,6 +82,7 @@ describe('relay database', () => {
'relay_confirm_results',
'relay_confirmable_splices',
'relay_connection_bases',
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_devices',
'relay_direct_authorizations',
@@ -62,6 +91,7 @@ describe('relay database', () => {
'relay_migration_leases',
'relay_post_drain_migration_pins',
'relay_rate_windows',
'relay_region_decisions',
'relay_region_rehome_attempts',
'relay_region_rehome_control',
'relay_region_rehome_worker_state'
@@ -140,9 +170,7 @@ describe('relay database', () => {
const second = await openRelayDatabase({ dataDir })
expect(
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
'legacy-cell'
])
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell'])
).toEqual([{ region: 'us-central1' }])
await second.close()
})
@@ -165,9 +193,7 @@ describe('relay database', () => {
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true)
await database.close()
})
cloud/apps/relay/src/database.ts
+78 -3
View File
@@ -10,6 +10,8 @@ import {
type PostgresPoolPressureCounts
} from './postgres-pool-pressure.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
import { reportPostgresQueryFailure } from './postgres-query-failure.js'
import {
CellInventoryHoldSamples,
emptyCellInventoryHoldCounts,
@@ -197,6 +199,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
ON relay_assignment_region_preferences(observed_at);
CREATE TABLE IF NOT EXISTS relay_region_decisions (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
cohort_bucket BIGINT NOT NULL DEFAULT 0,
last_considered_at BIGINT NOT NULL DEFAULT 0,
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
PRIMARY KEY (user_id, relay_host_id)
);
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
finish_existing BIGINT NOT NULL,
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
worker_id TEXT PRIMARY KEY,
next_dispatch_at BIGINT NOT NULL,
@@ -228,6 +248,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
source_generation BIGINT NOT NULL DEFAULT 0,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
@@ -600,6 +621,9 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
POSTGRES_STATEMENT_STATS_MIGRATION,
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
@@ -607,7 +631,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
]
function postgresSql(sql: string): string {
@@ -758,6 +784,8 @@ class SqliteDatabase extends SqliteTransaction {
class PostgresTransaction implements RelayDatabase {
readonly dialect = 'postgres' as const
private heldFromMs: number | undefined
private lockUnavailable = 0
private lockTimeouts = 0
constructor(protected readonly client: pg.PoolClient) {}
@@ -768,6 +796,20 @@ class PostgresTransaction implements RelayDatabase {
return holdMs
}
// Drained by the owning database on both the commit and the rollback path: a
// 55P03 rolls the transaction back, so counting only on success would drop it.
consumeLockUnavailable(): number {
const count = this.lockUnavailable
this.lockUnavailable = 0
return count
}
consumeLockTimeouts(): number {
const count = this.lockTimeouts
this.lockTimeouts = 0
return count
}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
try {
const result = await this.client.query(postgresSql(sql), params)
@@ -803,8 +845,14 @@ class PostgresTransaction implements RelayDatabase {
options.failIfUnavailable &&
String((error as { code?: unknown }).code) === '55P03'
) {
if (options.measureHoldMs) this.lockUnavailable += 1
throw new Error('database_lock_unavailable')
}
// A bounded wait that expires raises the same 55P03 without NOWAIT. This is
// the request path, so it is counted apart from by-design sweep deferrals.
if (bounded && options.measureHoldMs && String((error as { code?: unknown }).code) === '55P03') {
this.lockTimeouts += 1
}
throw error
} finally {
// Restore on the error path too: the transaction may still be retried or
@@ -885,12 +933,25 @@ class PostgresDatabase implements RelayDatabase {
}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
const client = await this.pressure.connect()
const startedAt = performance.now()
let phase: 'acquire' | 'execute' = 'acquire'
let client: pg.PoolClient | undefined
try {
client = await this.pressure.connect()
phase = 'execute'
const result = await client.query(postgresSql(sql), params)
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
} catch (error) {
reportPostgresQueryFailure({
error,
phase,
sql,
elapsedMs: performance.now() - startedAt,
pool: this.pool
})
throw error
} finally {
client.release()
client?.release()
}
}
@@ -911,6 +972,7 @@ class PostgresDatabase implements RelayDatabase {
options.failIfUnavailable &&
String((error as { code?: unknown }).code) === '55P03'
) {
if (options.measureHoldMs) this.holds.recordUnavailable()
throw new Error('database_lock_unavailable')
}
throw error
@@ -929,9 +991,13 @@ class PostgresDatabase implements RelayDatabase {
const result = await operation(transaction)
await client.query('COMMIT')
this.holds.record(measuredHoldMs(transaction) ?? Number.NaN)
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
return result
} catch (error) {
await client.query('ROLLBACK').catch(() => undefined)
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
if (!retryablePostgresTransactionError(error) || attempt === POSTGRES_TRANSACTION_ATTEMPTS) {
if (retryablePostgresTransactionError(error) && options.reportRetries !== false) {
console.warn(
@@ -1013,6 +1079,15 @@ async function applySchema(database: RelayDatabase): Promise<void> {
for (const statement of SCHEMA.split(';')) {
if (statement.trim()) await database.query(statement)
}
for (const [table, column] of [
['relay_control_capabilities', 'idle_regional_rehome'],
['relay_region_rehome_attempts', 'source_generation']
]) {
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
if (!columns.some((existing) => existing.name === column)) {
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`)
}
}
}
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
cloud/apps/relay/src/drain-migration-row-lookup.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { expect, it } from 'vitest'
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import type { SqlRow } from './database.js'
function text(row: SqlRow, field: string): string {
const value = row[field]
if (typeof value !== 'string') {
throw new Error(`invalid_${field}`)
}
return value
}
it('keeps first assignment matches, lease order, row identity, and separate identity components', () => {
const rows = [
{ user_id: 'a:b', relay_host_id: 'c', value: 1 },
{ user_id: 'a', relay_host_id: 'b:c', value: 2 },
{ user_id: 'a:b', relay_host_id: 'c', value: 3 },
{ user_id: '', relay_host_id: '', value: 4 }
]
const lookup = createDrainMigrationRowLookup(rows, text)
for (const identity of [
{ userId: 'a:b', relayHostId: 'c' },
{ userId: 'a', relayHostId: 'b:c' },
{ userId: '', relayHostId: '' },
{ userId: 'missing', relayHostId: 'c' }
]) {
const expected = rows.filter(
(row) =>
row.user_id === identity.userId && row.relay_host_id === identity.relayHostId
)
expect(lookup.first(identity)).toBe(expected[0])
expect(lookup.all(identity)).toEqual(expected)
lookup.all(identity).forEach((row, index) => expect(row).toBe(expected[index]))
}
})
it('retains lazy validation and short circuiting when an inventory is malformed', () => {
const first = { user_id: 'user', relay_host_id: 'host' }
const identity = { userId: 'user', relayHostId: 'host' }
const lookup = createDrainMigrationRowLookup(
[first, { user_id: null, relay_host_id: 'bad' }],
text
)
expect(lookup.first(identity)).toBe(first)
expect(() => lookup.all(identity)).toThrow('invalid_user_id')
const unrelated = createDrainMigrationRowLookup(
[first, { user_id: 'other', relay_host_id: null }],
text
)
expect(unrelated.all(identity)).toEqual([first])
expect(() => unrelated.first({ userId: 'other', relayHostId: 'host' })).toThrow(
'invalid_relay_host_id'
)
})
it('does not share an index between refreshed inventories', () => {
const identity = { userId: 'user', relayHostId: 'host' }
const oldRow = { user_id: 'user', relay_host_id: 'host', version: 1 }
const newRow = { ...oldRow, version: 2 }
expect(createDrainMigrationRowLookup([oldRow], text).first(identity)).toBe(oldRow)
expect(createDrainMigrationRowLookup([newRow], text).first(identity)).toBe(newRow)
})
cloud/apps/relay/src/drain-migration-row-lookup.ts
+58
View File
@@ -0,0 +1,58 @@
import type { SqlRow } from './database.js'
type Identity = { userId: string; relayHostId: string }
type RowIndex = Map<string, Map<string, SqlRow[]>>
/** A single locked inventory, never retained across transactions or refreshed queries. */
export function createDrainMigrationRowLookup(
rows: SqlRow[],
readText: (row: SqlRow, field: string) => string
): {
first: (identity: Identity) => SqlRow | undefined
all: (identity: Identity) => SqlRow[]
} {
let index: RowIndex | null | undefined
const indexed = (identity: Identity): SqlRow[] | undefined => {
if (index === undefined) {
index = indexRows(rows)
}
return index?.get(identity.userId)?.get(identity.relayHostId)
}
const matches = (row: SqlRow, identity: Identity): boolean =>
readText(row, 'user_id') === identity.userId &&
readText(row, 'relay_host_id') === identity.relayHostId
return {
first(identity) {
const group = indexed(identity)
return index === null ? rows.find((row) => matches(row, identity)) : group?.[0]
},
all(identity) {
const group = indexed(identity)
return index === null ? rows.filter((row) => matches(row, identity)) : (group ?? [])
}
}
}
function indexRows(rows: SqlRow[]): RowIndex | null {
const index: RowIndex = new Map()
for (const row of rows) {
const userId = row.user_id
const hostId = row.relay_host_id
// Preserve the original lazy validation and refusal order for malformed database rows.
if (typeof userId !== 'string' || typeof hostId !== 'string') {
return null
}
let hosts = index.get(userId)
if (!hosts) {
hosts = new Map()
index.set(userId, hosts)
}
const group = hosts.get(hostId)
if (group) {
group.push(row)
} else {
hosts.set(hostId, [row])
}
}
return index
}
cloud/apps/relay/src/drain-migration-row-scaling.test.ts
+121
View File
@@ -0,0 +1,121 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { afterEach, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js'
let database: RelayDatabase | undefined
afterEach(async () => await database?.close())
it.each([false, true])(
'looks up a whole drain inventory with linear identity reads (expired: %s)',
async (expired) => {
database = await openInMemoryRelayDatabase()
let measuring = false
let identityReads = 0
let indexedRows = 0
const instrument = (delegate: RelayDatabase): RelayDatabase => ({
query: (sql, params) => delegate.query(sql, params),
queryLocked: async (sql, params, options) => {
const rows = await delegate.queryLocked(sql, params, options)
if (
!measuring ||
!sql.includes('WHERE EXISTS') ||
!(sql.includes('SELECT assignment.*') || sql.includes('SELECT lease.*'))
) {
return rows
}
indexedRows += rows.length
return rows.map(
(row): SqlRow =>
new Proxy(row, {
get(target, key) {
if (key === 'user_id' || key === 'relay_host_id') {
identityReads++
}
return Reflect.get(target, key)
}
})
)
},
transaction: (operation, options) =>
delegate.transaction((tx) => operation(instrument(tx)), options),
close: () => delegate.close()
})
let now = 100
const store = new RelayAssignmentStore(instrument(database), () => now, {
requireLiveCells: true
})
const cells = ['a', 'b'].map((id) => ({
id: `cell-${id}`,
url: `https://relay-${id}.example.com`,
capacityRequests: 500
}))
await store.reconcileCells(cells)
const incarnation = '11111111-1111-4111-8111-111111111111'
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
await store.setCellEnabled('cell-b', false)
const identities = Array.from({ length: 50 }, (_, index) => ({
userId: `user-${index % 5}`,
relayHostId: `host${String(index).padStart(12, '0')}`
}))
for (const identity of identities) {
await store.assign(identity)
}
await store.setCellEnabled('cell-b', true)
await store.setCellEnabled('cell-a', false)
for (const identity of identities) {
const migration = await store.startEvacuation(identity, 'cell-b')
await store.markMigrationTargetRegistered(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch
})
}
const attempt = {
attemptId: '55555555-5555-4555-8555-555555555555',
cellId: 'cell-a',
cellIncarnation: incarnation,
traceValue: '66666666-6666-4666-8666-666666666666',
plannedGraceMs: 120_000
}
await store.prepareCellDrainAttempt(attempt)
if (expired) {
now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
await store.releaseExpiredActivityLeases()
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
}
measuring = true
await expect(store.beginCellDrainSend(attempt)).resolves.toMatchObject({
state: 'send-may-have-started',
shouldSend: true
})
expect(indexedRows).toBeGreaterThanOrEqual(100)
expect(identityReads).toBeLessThanOrEqual(indexedRows * 2)
const migrations = await database.query(
'SELECT expires_at FROM relay_assignment_migrations'
)
expect(migrations).toHaveLength(50)
expect(
migrations.every(
(row) => row.expires_at === now + ASSIGNMENT_LIMITS.migrationLeaseMs
)
).toBe(true)
}
)
cloud/apps/relay/src/host-session-client-accept.test.ts
+141
View File
@@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => {
vi.useRealTimers()
})
it('does not admit new source work after a drain crosses activity acquisition', async () => {
const h = harness()
const control = await activeHost(h)
const slow = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slow.promise)
const client = new FakeSocket()
const capacity = { bind: vi.fn(), release: vi.fn() }
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
slow.resolve()
await accepting
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it('does not splice an attachment whose generation retired during basis persistence', async () => {
const h = harness()
await activeHost(h)
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const pending = [...session.pendingConns.values()][0]!
const slow = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
slow.resolve()
expect(await attaching).toBe(false)
expect(session.activeSplices.size).toBe(0)
expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId)
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true'))
expect(host.close).toHaveBeenCalled()
})
it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
@@ -364,6 +424,8 @@ describe('successful client accept timing', () => {
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const ownerProbe = vi.spyOn(session.pendingConns, 'has')
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
@@ -372,6 +434,7 @@ describe('successful client accept timing', () => {
)
expect(accepted).toBe(true)
expect(ownerProbe).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
@@ -390,6 +453,7 @@ describe('successful client accept timing', () => {
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
@@ -424,6 +488,80 @@ async function advanceToPing(control: FakeSocket, clock: { now: number }): Promi
return (JSON.parse(String(ping[0])) as { t: number }).t
}
// The attach resolves its owning session once and hands it to the unfenced leg;
// these hold the session it must be and the order the client hears about it.
describe('host data attach ownership', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const bystander = { ...identity, sub: 'user-2', relayHostId: 'qponmlkjihgfedcb' }
async function pendingAttach(h: ReturnType<typeof harness>) {
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
return { client, session, pending: [...session.pendingConns.values()][0]! }
}
it('attaches the session that owns the connection, not the first one registered', async () => {
const h = harness()
const idle = new FakeSocket()
await h.activate(idle as unknown as WebSocket, bystander, null, 1, false, 1, '1.4.197')
await activeHost(h)
const { client, session, pending } = await pendingAttach(h)
const idleSession = h.registry.get({
userId: bystander.sub,
relayHostId: bystander.relayHostId
})!
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
).toBe(true)
expect(client.send).toHaveBeenCalledWith(expect.stringContaining('"type":"relay-hello"'))
expect(session.activeSplices.has(pending.connId)).toBe(true)
expect(idleSession.activeSplices.size).toBe(0)
expect(idleSession.activeConnIds.size).toBe(0)
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
it('acknowledges the client only after the connection basis is persisted', async () => {
const h = harness()
await activeHost(h)
const { client, session, pending } = await pendingAttach(h)
const basis = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(basis.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
expect(h.store.recordConnectionBasis).toHaveBeenCalledOnce()
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('relay-hello'))
basis.resolve()
expect(await attaching).toBe(true)
expect(client.send).toHaveBeenCalledWith(expect.stringContaining('"type":"relay-hello"'))
expect(session.activeSplices.has(pending.connId)).toBe(true)
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -460,6 +598,9 @@ describe('control round-trip sampling', () => {
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
assignmentEpoch: 1,
controlGeneration: 1,
drainMode: 'none',
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
cloud/apps/relay/src/host-session-registry.test.ts
+620 -42
View File
@@ -4,6 +4,7 @@ import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -110,6 +111,7 @@ function createRegistry(
renewControlActivity: ReturnType<typeof vi.fn>
releaseActivity: ReturnType<typeof vi.fn>
observer: {
recordAuth: ReturnType<typeof vi.fn>
recordControlClose: ReturnType<typeof vi.fn>
recordSpliceClose: ReturnType<typeof vi.fn>
}
@@ -140,7 +142,10 @@ function createRegistry(
store as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
observer,
Date.now,
Math.random,
'incarnation-1'
)
// Mirrors the production signature exactly so a future positional shift fails to compile.
const bound = (
@@ -166,7 +171,14 @@ function createRegistry(
assignmentEpoch,
appVersion = '1.4.173'
) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion)
return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer }
return {
registry,
activate,
acquireActivity,
renewControlActivity,
releaseActivity,
observer
}
}
describe('host session cleanup races', () => {
@@ -398,26 +410,20 @@ describe('host session cleanup races', () => {
attemptId: '22222222-2222-4222-8222-222222222222'
})
).toThrow('regional_rehome_attempt_conflict')
expect(() =>
registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })
).toThrow('regional_rehome_assignment_epoch_mismatch')
expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow(
'regional_rehome_assignment_epoch_mismatch'
)
const rebound = new FakeSocket()
await activate(
rebound as unknown as WebSocket,
identity,
registry.get(request),
1,
true,
7
)
await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7)
expect(registry.get(request)?.state).toBe('drain-only')
expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
await vi.advanceTimersByTimeAsync(30_000)
expect(registry.get(request)).toBeNull()
expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }))
.not.toBeNull()
expect(
registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })
).not.toBeNull()
expect(secondSocket.close).not.toHaveBeenCalled()
})
@@ -513,14 +519,7 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
const rebindSocket = new FakeSocket()
const rebinding = activate(
rebindSocket as unknown as WebSocket,
identity,
original,
1,
true,
1
)
const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1)
rebindSocket.close()
blocked.resolve('control:production-gce-c3:1')
await rebinding
@@ -659,14 +658,7 @@ describe('host session cleanup races', () => {
originalSocket.close()
const replacementSocket = new FakeSocket()
await activate(
replacementSocket as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1)
const replacement = registry.get({
userId: identity.sub,
relayHostId: identity.relayHostId
@@ -696,14 +688,7 @@ describe('host session cleanup races', () => {
})
expect(original).not.toBeNull()
await activate(
new FakeSocket() as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
expect(renewControlActivity).toHaveBeenCalledOnce()
@@ -718,6 +703,53 @@ describe('host session cleanup races', () => {
vi.advanceTimersByTime(0)
})
it('ignores a denial belonging to the socket before a same-generation rebind', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const oldSocket = new FakeSocket()
await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
let reject!: (error: Error) => void
h.renewControlActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(replacement.close).not.toHaveBeenCalled()
expect(session.socket).toBe(replacement)
expect(session.generation).toBe(1)
})
it('ignores missing-activity recovery denial after an authority transition', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const socket = new FakeSocket()
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found'))
let reject!: (error: Error) => void
h.acquireActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).not.toHaveBeenCalled()
expect(session.state).toBe('drain-only')
})
it('keeps 15s pings while halving steady-state control renewals', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
@@ -732,9 +764,7 @@ describe('host session cleanup races', () => {
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
}
const pings = socket.send.mock.calls.filter((call) =>
String(call[0]).includes('"ping"')
)
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
expect(pings).toHaveLength(4)
expect(renewControlActivity).toHaveBeenCalledTimes(2)
const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt)
@@ -1118,3 +1148,551 @@ describe('host hello ack pending connections', () => {
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
describe('source-owned idle cutover', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const request = {
attemptId: 'idle-1',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
sourceCellIncarnation: 'incarnation-1',
targetCellId: 'target'
}
async function source(store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store)
const socket = new FakeSocket()
h.registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME])
)
socket.removeAllListeners('message')
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
return { ...h, socket, session: h.registry.get(request)! }
}
it('keeps either established client busy until both actually leave', async () => {
const h = await source()
h.session.activeConnIds.add('phone')
h.session.activeConnIds.add('ipad')
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
h.session.activeConnIds.delete('ipad')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
h.session.activeConnIds.delete('phone')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'committed' })
expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it.each([
{ userId: 'other-user' },
{ sourceAssignmentEpoch: 2 },
{ sourceGeneration: 2 },
{ sourceCellIncarnation: 'other-incarnation' },
{ targetCellId: 'other-target' }
])('rejects a reused operation ID with changed authority %j', async (change) => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const reconcile = vi.fn().mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(request, commit, reconcile)
const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile)
result.resolve({ outcome: 'deferred' })
expect(await conflicting).toEqual({ outcome: 'stale' })
expect(await moving).toEqual({ outcome: 'deferred' })
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
})
it('accounts for accepts before credential identity resolves', async () => {
const lookup = deferred<null>()
const h = await source({
resolveResume: vi.fn().mockReturnValue(lookup.promise),
resolveInviteForMove: vi.fn().mockResolvedValue(null)
})
const client = new FakeSocket()
const accept = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
lookup.resolve(null)
await accept
expect(h.socket.close).not.toHaveBeenCalled()
})
it('rejects new accepts and replacements synchronously while a commit awaits', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const moving = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const duplicate = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const client = new FakeSocket()
const release = vi.fn()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
{ release } as never
)
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(release).toHaveBeenCalledOnce()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1)
expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
result.resolve({ outcome: 'deferred' })
await moving
await duplicate
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'next' },
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'committed' })
})
it.each(['ambiguous', 'deferred'])(
'keeps %s outcomes fenced until locked reconciliation succeeds',
async (claim) => {
const h = await source()
const reconcile = vi
.fn()
.mockRejectedValueOnce(new Error('database unavailable'))
.mockRejectedValueOnce(new Error('database unavailable'))
.mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(
request,
claim === 'ambiguous'
? vi.fn().mockRejectedValue(new Error('lost commit reply'))
: vi.fn().mockResolvedValue({ outcome: 'deferred' }),
reconcile
)
await vi.advanceTimersByTimeAsync(50)
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'other' },
vi.fn(),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'busy' })
expect(h.socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(300)
expect(await moving).toEqual({ outcome: 'deferred' })
expect(reconcile).toHaveBeenCalledTimes(3)
expect(h.socket.close).not.toHaveBeenCalled()
}
)
it('owns accepted control mutations before the handler first awaits', async () => {
const mutation = deferred<RelayTokenClaims | null>()
const h = await source()
;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi
.fn()
.mockReturnValue(mutation.promise)
h.socket.emit(
'message',
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })),
false
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
mutation.resolve(identity)
await vi.advanceTimersByTimeAsync(0)
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'deferred' })
})
it('owns queued replacement activation before its first persistence await', async () => {
const h = await source()
const activation = deferred<string>()
const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } })
.assignments
assignments.activateControl = vi.fn().mockReturnValue(activation.promise)
const replacement = new FakeSocket()
const activating = h.activate(
replacement as unknown as WebSocket,
identity,
h.session,
2,
false,
1
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
activation.resolve('control:2')
await activating
})
it('retires changed authority even when the claim definitively deferred', async () => {
const h = await source()
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'deferred' }),
vi.fn().mockResolvedValue('stale')
)
).toEqual({ outcome: 'stale' })
expect(h.session.state).toBe('closed')
expect(h.releaseActivity).toHaveBeenCalled()
})
it('holds attach ownership through basis failure reservation cleanup', async () => {
const basis = deferred<void>()
const cleanup = deferred<void>()
const h = await source({
recordConnectionBasis: vi.fn().mockImplementation(async () => {
await basis.promise
throw new Error('basis failed')
}),
failReservation: vi.fn().mockReturnValue(cleanup.promise)
})
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const attached = h.registry.acceptHostData(
new FakeSocket() as unknown as WebSocket,
'conn',
'ticket',
1
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
basis.resolve()
await vi.advanceTimersByTimeAsync(0)
expect(h.session.activeConnIds.size).toBe(0)
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
cleanup.resolve()
await attached
})
it('rejects an attach mid-cutover before its ticket is ever examined', async () => {
const h = await source({ failReservation: vi.fn().mockResolvedValue(undefined) })
const result = deferred<{ outcome: 'deferred' }>()
// The cutover must already be in flight: an idle host is what it claims.
const moving = h.registry.idleRehome(request, () => result.promise, vi.fn())
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const host = new FakeSocket()
// The ticket below is the live one: only the cutover fence may reject it.
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, 'conn', 'ticket', 1)
).toBe(false)
expect(host.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.observer.recordAuth).not.toHaveBeenCalled()
expect(h.session.pendingConns.has('conn')).toBe(true)
expect(h.session.activeConnIds.size).toBe(0)
result.resolve({ outcome: 'deferred' })
await moving
})
it('holds no attach ownership when no session owns the connection', async () => {
const h = await source()
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, 'stranger', 'ticket', 1)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
expect.any(String)
)
// A leaked idle-work hold from the unowned attach would report `busy` here.
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn()
)
).toEqual({ outcome: 'committed' })
})
it('returns the durable operation outcome after source retirement', async () => {
const h = await source()
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
await h.registry.idleRehome(request, commit, vi.fn())
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed'))
).toEqual({ outcome: 'committed' })
expect(commit).toHaveBeenCalledOnce()
})
it('does not reopen a source overtaken by emergency drain', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const moving = h.registry.idleRehome(
request,
() => result.promise,
vi.fn().mockResolvedValue('not-committed')
)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
result.resolve({ outcome: 'deferred' })
await moving
expect(h.session.state).toBe('closed')
expect(h.registry.get(request)).toBeNull()
})
})
// The host data leg's owner lookup is the registry's only whole-inventory scan on
// an attach. These count what that scan touches, not how long it takes.
describe('host data attach owner lookup', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const SESSION_COUNT = 1000
const CONN_ID = 'conn-owned'
const OWNER_INDEX = { first: 0, middle: 499, last: 999 } as const
type Placement = keyof typeof OWNER_INDEX | 'absent'
type LookupCounts = { visits: number; membership: number }
function bindOwn<K, V>(map: Map<K, V>, property: string | symbol): unknown {
const value: unknown = Reflect.get(map, property, map)
return typeof value === 'function' ? value.bind(map) : value
}
// One visit per session the scan pulls off the map iterator; answers unchanged.
function countingValues<K, V>(map: Map<K, V>, counts: LookupCounts): Map<K, V> {
return new Proxy(map, {
get(target, property) {
if (property !== 'values') return bindOwn(target, property)
return function* (): Generator<V> {
for (const value of target.values()) {
counts.visits += 1
yield value
}
}
}
})
}
// One membership check per `pendingConns.has`; answers unchanged.
function countingHas<K, V>(map: Map<K, V>, counts: LookupCounts): Map<K, V> {
return new Proxy(map, {
get(target, property) {
if (property !== 'has') return bindOwn(target, property)
return (key: K) => {
counts.membership += 1
return target.has(key)
}
}
})
}
// The pre-change implementation, kept inline as the oracle the new counts are
// differenced against: two inventory arrays, two independent finds.
function legacyOwnerLookup(
sessions: Map<string, HostSession>,
connId: string
): { owner: HostSession | undefined; session: HostSession | undefined } {
const owner = [...sessions.values()].find((candidate) => candidate.pendingConns.has(connId))
const session = [...sessions.values()].find((candidate) => candidate.pendingConns.has(connId))
return { owner, session }
}
function pendingConn(client: FakeSocket, connTicket: string) {
return {
connId: CONN_ID,
connTicket,
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never
}
// Every decoy holds a pending conn of its own, so each membership check the
// scan makes is real work rather than a lookup in an empty map.
function decoySession(index: number, counts: LookupCounts): HostSession {
const pendingConns = new Map<string, unknown>([[`conn-decoy-${index}`, { connId: 'decoy' }]])
return {
relayHostId: `decoy-host-${index}`,
generation: 1,
state: 'active',
activeConnIds: new Set<string>(),
pendingConns: countingHas(pendingConns, counts)
} as unknown as HostSession
}
async function attachRegistry(placement: Placement, store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), {
failReservation: vi.fn().mockResolvedValue(undefined),
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
deactivateBasis: vi.fn().mockResolvedValue(undefined),
...store
})
const control = new FakeSocket()
await h.activate(control as unknown as WebSocket, identity, null, 1, false, 1)
const internals = h.registry as unknown as { sessions: Map<string, HostSession> }
const [ownerKey, owner] = [...internals.sessions.entries()][0]!
const counts: LookupCounts = { visits: 0, membership: 0 }
const client = new FakeSocket()
if (placement !== 'absent') owner.pendingConns.set(CONN_ID, pendingConn(client, 'ticket'))
owner.pendingConns = countingHas(owner.pendingConns, counts)
const ordered: HostSession[] = []
const sessions = new Map<string, HostSession>()
const ownerIndex = placement === 'absent' ? 0 : OWNER_INDEX[placement]
for (let index = 0; index < SESSION_COUNT; index += 1) {
const session = index === ownerIndex ? owner : decoySession(index, counts)
ordered.push(session)
sessions.set(index === ownerIndex ? ownerKey : `decoy-${index}`, session)
}
internals.sessions = countingValues(sessions, counts)
return { ...h, owner, ordered, counts, client, control, sessions: internals.sessions }
}
it.each([
{
placement: 'first',
before: { visits: 2000, membership: 2 },
after: { visits: 1, membership: 1 }
},
{
placement: 'middle',
before: { visits: 2000, membership: 1000 },
after: { visits: 500, membership: 500 }
},
{
placement: 'last',
before: { visits: 2000, membership: 2000 },
after: { visits: 1000, membership: 1000 }
},
{
placement: 'absent',
before: { visits: 2000, membership: 2000 },
after: { visits: 1000, membership: 1000 }
}
] as const)(
'visits the inventory once, not twice, for a $placement owner',
async ({ placement, before, after }) => {
const h = await attachRegistry(placement)
expect(h.sessions.size).toBe(SESSION_COUNT)
const oracle = legacyOwnerLookup(h.sessions, CONN_ID)
const legacy = { ...h.counts }
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
// An unusable ticket stops the attach immediately after the lookup, so the
// counts below belong to the lookup alone.
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'wrong', 1)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host data ticket'
)
expect(legacy).toEqual(before)
expect({ ...h.counts }).toEqual(after)
expect(oracle.owner).toBe(placement === 'absent' ? undefined : h.owner)
expect(oracle.owner).toBe(oracle.session)
}
)
it.each([
{ reason: 'ticket', ticket: 'wrong', generation: 1, state: 'active' },
{ reason: 'generation', ticket: 'ticket', generation: 2, state: 'active' },
{ reason: 'state', ticket: 'ticket', generation: 1, state: 'orphaned' }
] as const)('fails an attach whose $reason does not match the owner', async (input) => {
const h = await attachRegistry('middle')
h.owner.state = input.state
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(
host as unknown as WebSocket,
CONN_ID,
input.ticket,
input.generation
)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host data ticket'
)
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
expect(h.owner.activeConnIds.size).toBe(0)
})
it('rejects on the earlier duplicate owner rather than the later live one', async () => {
const h = await attachRegistry('middle')
h.ordered[0]!.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'stale-ticket') as never)
expect(legacyOwnerLookup(h.sessions, CONN_ID).owner).toBe(h.ordered[0])
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
})
it('splices the earlier duplicate owner and leaves the later one untouched', async () => {
const basis = vi.fn().mockRejectedValue(new Error('basis failed'))
const h = await attachRegistry('first', { recordConnectionBasis: basis })
const duplicate = h.ordered[3]!
duplicate.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'ticket') as never)
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.observer.recordAuth).toHaveBeenCalledWith(true)
expect(basis).toHaveBeenCalledOnce()
// The first owner's entry was consumed; the later duplicate never was.
expect(h.owner.pendingConns.has(CONN_ID)).toBe(false)
expect(duplicate.pendingConns.has(CONN_ID)).toBe(true)
expect(h.owner.activeConnIds.size).toBe(0)
})
})
cloud/apps/relay/src/host-session-registry.ts
+338 -56
View File
@@ -15,6 +15,7 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason,
@@ -25,10 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
@@ -78,7 +76,7 @@ export type HostSession = {
identity: RelayTokenClaims
readonly relayHostId: string
readonly generation: number
readonly assignmentEpoch: number
assignmentEpoch: number
readonly controlActivityId: string | null
readonly controlResumeSecret: string
// Why: reconnect churn is only actionable once it can be pinned to a client build.
@@ -94,6 +92,7 @@ export type HostSession = {
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
authorityRevision: number
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -108,10 +107,7 @@ export type HostSession = {
regionalDrainExpiresAt: number | null
}
export type RegionalHostDrainOutcome =
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
type PendingConnection = {
connId: string
@@ -184,6 +180,113 @@ export class HostSessionRegistry {
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
string,
{
attemptId: string
authorityKey: string
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
}
>()
async idleRehome(
input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceGeneration: number
sourceCellIncarnation: string
targetCellId: string
},
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
const authorityKey = JSON.stringify([
input.userId,
input.sourceAssignmentEpoch,
input.sourceGeneration,
input.sourceCellIncarnation,
input.targetCellId
])
const prior = this.idleAttempts.get(input.relayHostId)
if (prior) {
if (prior.attemptId !== input.attemptId) return { outcome: 'busy' }
return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' }
}
const session = this.get(input)
if (
this.draining ||
!session ||
session.state !== 'active' ||
session.generation !== input.sourceGeneration ||
session.assignmentEpoch !== input.sourceAssignmentEpoch ||
this.cellIncarnation !== input.sourceCellIncarnation
) {
const durable = await reconcile()
return { outcome: durable === 'committed' ? 'committed' : 'stale' }
}
if (
!session.socket ||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
)
return { outcome: 'deferred' }
if (
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
session.activeConnIds.size !== 0 ||
session.activeSplices.size !== 0 ||
session.pendingConns.size !== 0
)
return { outcome: 'busy' }
const revision = session.authorityRevision
const promise = Promise.resolve().then(async () => {
let outcome: 'committed' | 'deferred' | 'stale'
try {
outcome = (await commit()).outcome
if (outcome === 'deferred') {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
}
} catch {
let delay = 100
for (;;) {
try {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
break
} catch {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, delay)
timer.unref?.()
})
delay = Math.min(delay * 2, 5000)
}
}
}
if (this.get(input) === session) {
if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) {
this.closeDrainedSession(session)
}
}
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
this.idleAttempts.delete(input.relayHostId)
return { outcome }
})
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
return promise
}
private beginIdleWork(hostId: string): (() => void) | null {
if (this.idleAttempts.has(hostId)) return null
this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1)
return () => {
const remaining = (this.idleWork.get(hostId) ?? 1) - 1
if (remaining === 0) this.idleWork.delete(hostId)
else this.idleWork.set(hostId, remaining)
}
}
constructor(
private readonly config: RelayConfig,
private readonly verifyRelayToken: VerifyRelayToken,
@@ -192,7 +295,8 @@ export class HostSessionRegistry {
private readonly queuedByteBudget: ProcessQueuedByteBudget,
private readonly observer: RelayRuntimeObserver,
private readonly now: () => number = Date.now,
private readonly random: () => number = Math.random
private readonly random: () => number = Math.random,
private readonly cellIncarnation?: string
) {}
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
@@ -206,6 +310,25 @@ export class HostSessionRegistry {
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
const release = this.beginIdleWork(hostId)
if (!release) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
try {
await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation)
} finally {
release()
}
}
private async acceptClientUnfenced(
socket: WebSocket,
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
capacityReservation?.release()
@@ -295,6 +418,7 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED)
return
}
const admittingSocket = session.socket
const connId = randomUUID()
const connTicket = randomBytes(32).toString('base64url')
const identity = { userId: reservation.userId, relayHostId: hostId }
@@ -324,6 +448,20 @@ export class HostSessionRegistry {
) {
return
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
admittingSocket.readyState !== admittingSocket.OPEN
) {
capacityReservation?.release()
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
@@ -373,9 +511,34 @@ export class HostSessionRegistry {
connTicket: string,
generation: number
): Promise<boolean> {
const session = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
)
// First insertion-order owner, and the only scan the attach makes: the
// unfenced leg reuses this result instead of repeating the search.
let owner: HostSession | undefined
for (const candidate of this.sessions.values()) {
if (candidate.pendingConns.has(connId)) {
owner = candidate
break
}
}
const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {}
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return false
}
try {
return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation, owner)
} finally {
release()
}
}
private async acceptHostDataUnfenced(
socket: WebSocket,
connId: string,
connTicket: string,
generation: number,
session: HostSession | undefined
): Promise<boolean> {
const pending = session?.pendingConns.get(connId)
if (
!session ||
@@ -427,6 +590,27 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed')
return false
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
socket.readyState !== socket.OPEN ||
pending.client.readyState !== pending.client.OPEN
) {
session.activeConnIds.delete(connId)
pending.capacityReservation?.release()
this.deactivateBasisBestEffort(connId)
this.failReservationBestEffort(pending.reservation)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
}
this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING)
socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment')
return false
}
const close = wireSplice({
client: pending.client,
host: socket,
@@ -505,6 +689,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
@@ -513,6 +698,14 @@ export class HostSessionRegistry {
)
}
private sessionPlacementLogFields(session: HostSession) {
return {
assignmentEpoch: session.assignmentEpoch,
controlGeneration: session.generation,
drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none'
}
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
@@ -549,6 +742,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
@@ -562,6 +756,10 @@ export class HostSessionRegistry {
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
if (this.idleAttempts.has(identity.relayHostId)) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return
}
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
@@ -602,8 +800,7 @@ export class HostSessionRegistry {
socket: WebSocket | null,
context: string
): void {
void Promise.resolve()
.then(task)
void (async () => task())()
.catch((error: unknown) => {
const message = (error instanceof Error ? error.message : 'unknown')
// Untruncated, unlike peer-supplied close reasons: this is the
@@ -653,6 +850,7 @@ export class HostSessionRegistry {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
@@ -665,7 +863,8 @@ export class HostSessionRegistry {
relayHostId: string
sourceAssignmentEpoch: number
graceMs: number
}): RegionalHostDrainOutcome {
sourceCellIncarnation?: string
}): RegionalHostDrainOutcome | Promise<RegionalHostDrainOutcome> {
const session = this.get(input)
if (!session || session.state === 'closed') return 'host-not-connected'
if (session.assignmentEpoch !== input.sourceAssignmentEpoch) {
@@ -678,13 +877,11 @@ export class HostSessionRegistry {
this.reassertRegionalDrain(session)
return 'already-accepted'
}
session.authorityRevision += 1
session.regionalDrainAttemptId = input.attemptId
session.regionalDrainExpiresAt = this.now() + input.graceMs
this.reassertRegionalDrain(session)
session.regionalDrainTimer = setTimeout(
() => this.closeDrainedSession(session),
input.graceMs
)
session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs)
return 'accepted'
}
@@ -740,9 +937,9 @@ export class HostSessionRegistry {
const existing = this.sessions.get(key)
const rebind = Boolean(
existing &&
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
)
const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1
const ephemeral = nacl.box.keyPair()
@@ -784,7 +981,9 @@ export class HostSessionRegistry {
}, 10_000)
socket.once('message', (raw, isBinary) => {
clearTimeout(proofTimer)
const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const ack = isBinary
? null
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
if (
!ack?.success ||
@@ -826,6 +1025,11 @@ export class HostSessionRegistry {
appVersion: string,
connectionInclusionWatermark?: number
): Promise<void> {
const release = this.beginIdleWork(identity.relayHostId)
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return Promise.resolve()
}
const key = this.key(identity.sub, identity.relayHostId)
const previous = this.activationQueues.get(key) ?? Promise.resolve()
// The timeout only fails this waiting socket; the queue entry still chains
@@ -836,26 +1040,29 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
}, ACTIVATION_QUEUE_WAIT_MS)
queueWaitTimer.unref?.()
const activation = previous.catch(() => undefined).then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
const activation = previous
.catch(() => undefined)
.then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
this.activationQueues.set(key, activation)
const cleanup = (): void => {
release()
if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key)
}
void activation.then(cleanup, cleanup)
@@ -881,7 +1088,11 @@ export class HostSessionRegistry {
cellId: this.config.cellId,
assignmentEpoch,
generation,
connectionInclusionWatermark
connectionInclusionWatermark,
idleRegionalRehome:
this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ??
false,
cellIncarnation: this.cellIncarnation
}
)
await this.assignments.markMigrationTargetRegistered(
@@ -918,14 +1129,15 @@ export class HostSessionRegistry {
const previousSocket = existing.socket
if (existing.orphanTimer) clearTimeout(existing.orphanTimer)
existing.orphanTimer = null
existing.authorityRevision += 1
existing.assignmentEpoch = assignmentEpoch
existing.socket = socket
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
this.sendHelloAck(existing)
if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing)
@@ -980,6 +1192,7 @@ export class HostSessionRegistry {
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
authorityRevision: 0,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -1028,7 +1241,9 @@ export class HostSessionRegistry {
` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` +
` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` +
` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` +
(socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`)
(socketError === null
? ''
: ` error=${JSON.stringify(printableCloseReason(socketError))}`)
)
})
socket.on('message', (raw, isBinary) => {
@@ -1077,6 +1292,19 @@ export class HostSessionRegistry {
}
private async acceptRefresh(session: HostSession, raw: RawData): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptRefreshUnfenced(session, raw)
} finally {
release()
}
}
private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise<void> {
const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh'))
if (!parsed.success) return
const refreshed = await this.verifyRelayToken(parsed.data.relayJwt)
@@ -1107,6 +1335,16 @@ export class HostSessionRegistry {
if (controlActivityId && now >= session.activityRenewalDueAt) {
const attempt = ++session.activityRenewalAttempt
const startedAt = now
const socket = session.socket
const authorityRevision = session.authorityRevision
const current = (): boolean =>
this.sessions.get(key) === session &&
session.state !== 'closed' &&
session.socket === socket &&
socket.readyState === socket.OPEN &&
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
@@ -1117,11 +1355,16 @@ export class HostSessionRegistry {
}
)
.then(() => {
if (attempt <= session.activityRenewalCompletedAttempt) return
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS
})
.catch(async (error: unknown) => {
if (!current()) return
if (error instanceof Error && error.message === 'assignment_not_found') {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing')
return
}
if (error instanceof Error && error.message === 'activity_cell_not_authoritative') {
// Completion fences a late drain-only heartbeat after all source work is gone.
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed')
@@ -1144,8 +1387,22 @@ export class HostSessionRegistry {
cellId: this.config.cellId
}
)
if (!current()) {
// A replaced activity must not remain leased after its owner disappears.
if (
!this.sessions.get(key) ||
this.sessions.get(key)?.controlActivityId !== controlActivityId
) {
this.releaseActivityBestEffort(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
controlActivityId
)
}
return
}
this.observer.recordControlActivityRecovery?.(true)
} catch (acquireError: unknown) {
if (!current()) return
this.observer.recordControlActivityRecovery?.(false)
if (
acquireError instanceof Error &&
@@ -1218,6 +1475,21 @@ export class HostSessionRegistry {
private closeDrainedSession(session: HostSession): void {
if (session.state === 'closed') return
const forcedConnections = session.activeConnIds.size + session.pendingConns.size
if (forcedConnections > 0) {
console.warn(
JSON.stringify({
event: 'orca_relay_host_drain_forced_close',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
reason: this.draining ? 'emergency' : 'regional-deadline',
forcedConnections,
splices: session.activeSplices.size,
pending: session.pendingConns.size
})
)
}
if (session.heartbeatTimer) clearInterval(session.heartbeatTimer)
if (session.orphanTimer) clearTimeout(session.orphanTimer)
if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer)
@@ -1250,11 +1522,7 @@ export class HostSessionRegistry {
session.pendingConns.clear()
session.state = 'closed'
if (session.socket) {
closeRelayWebSocket(
session.socket,
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director')
}
const key = this.key(session.identity.sub, session.relayHostId)
if (this.sessions.get(key) === session) this.sessions.delete(key)
@@ -1278,6 +1546,23 @@ export class HostSessionRegistry {
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptControlCommandUnfenced(session, type, raw)
} finally {
release()
}
}
private async acceptControlCommandUnfenced(
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
if (typeof type !== 'string' || !session.socket) return
try {
@@ -1315,10 +1600,7 @@ export class HostSessionRegistry {
}
if (type === 'device-credential-install') {
const request = DeviceCredentialInstallSchema.parse(payload(raw, type))
if (
session.state !== 'active' &&
request.authorization.mode === 'authenticated-direct'
) {
if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') {
throw new Error('authorization_expired')
}
const installActivityId = `install:${request.reqId}`
cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const request = {
v: 1 as const,
attemptId: '33333333-3333-4333-8333-333333333333',
userId: 'idle-reconciliation-test',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 7,
targetCellId: 'target'
}
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
const store = new RelayAssignmentStore(database, () => 100_000_000)
await store.reconcileCells([
{ id: 'source', url: 'https://source.example.test', capacityRequests: 100 },
{ id: 'target', url: 'https://target.example.test', capacityRequests: 100 }
])
await store.assign(request)
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration,
cellIncarnation: request.sourceCellIncarnation
})
return { database, store }
}
describe('idle cutover durable reconciliation', () => {
it('only permits reopening when the exact source still owns the assignment', async () => {
const { store } = await setup()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed')
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration + 1,
cellIncarnation: request.sourceCellIncarnation
})
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('does not reopen an obsolete source after an assignment change', async () => {
const { store } = await setup()
await store.startEvacuation(request, request.targetCellId)
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('propagates unavailable durable state instead of declaring rollback', async () => {
const { database, store } = await setup()
vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable'))
await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable')
})
it('waits for an outstanding assignment transaction before deciding authority', async () => {
const { database, store } = await setup()
let release!: () => void
let entered!: () => void
const locked = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
const commit = database.transaction(async (transaction) => {
await transaction.queryLocked(
'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
entered()
await gate
await transaction.query(
'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
})
await locked
let settled = false
const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => {
settled = true
})
try {
await new Promise<void>((resolve) => setImmediate(resolve))
expect(settled).toBe(false)
} finally {
release()
await commit
await reconciliation
}
expect(await reconciliation).toBe('stale')
})
})
cloud/apps/relay/src/idle-regional-rehome-selection.ts
+117
View File
@@ -0,0 +1,117 @@
import { createHash } from 'node:crypto'
import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
[
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
}
cloud/apps/relay/src/idle-regional-rehome-store.test.ts
+396
View File
@@ -0,0 +1,396 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase, RelayLockOptions } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' }
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const cells = [
{
id: 'source',
url: 'https://source.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'target',
url: 'https://target.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
const request = {
v: 1 as const,
...identity,
attemptId: '33333333-3333-4333-8333-333333333333',
sourceCellId: cells[0]!.id,
sourceCellIncarnation: incarnations[0]!,
sourceAssignmentEpoch: assignment.assignmentEpoch,
sourceGeneration: 7,
targetCellId: cells[1]!.id
}
return { store, database, safety, request }
}
describe('constrained idle regional assignment transaction', () => {
it.each(['missing', 'disabled', 'future'] as const)(
'does only one read per tick with %s durable control and sees later enablement',
async (state) => {
const { store, database, safety } = await setup()
const control = (await database.query(
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
))[0]!
if (state === 'missing') {
await database.query('DELETE FROM relay_region_rehome_control')
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
)
}
const query = vi.spyOn(database, 'query')
const transaction = vi.spyOn(database, 'transaction')
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
expect(transaction).not.toHaveBeenCalled()
}
if (state === 'missing') {
const columns = Object.keys(control)
await database.query(
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
Object.values(control)
)
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
[safety.observedAt]
)
}
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(1)
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
}
)
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
const { store, database, safety, request } = await setup()
// Model three source activity units and seven units already reserved at the target.
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?',
[identity.userId, identity.relayHostId]
)
await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'")
await database.query(
"UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'",
[capacity]
)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
outcome: capacity === 11 ? 'committed' : 'deferred'
})
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
expect(await store.resolve(identity)).toMatchObject({
cellId: capacity === 11 ? 'target' : 'source',
assignmentEpoch: capacity === 11 ? 2 : 1
})
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const template = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
identity.userId,
identity.relayHostId
])
)[0]!
const columns = Object.keys(template)
for (let index = 0; index < 100; index++) {
const values = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`idle-store-test-${String(index).padStart(3, '0')}`,
`pagehost${String(index).padStart(8, '0')}`,
identity.userId,
identity.relayHostId
]
)
}
}
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first).toHaveLength(100)
expect(next).toHaveLength(1)
expect(next[0]!.relayHostId).toBe('pagehost00000099')
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
regionalRehomeCohortPercent: 100
})
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
})
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
'rechecks generation when replacement wins after the initial authority lookup',
async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
try {
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
} finally {
held.release()
}
expect(await commit).toEqual({ outcome: 'deferred' })
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
}
)
it('rejects source replacement when the cutover already holds assignment authority', async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
const replacement = store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const rejected = expect(replacement).rejects.toThrow('wrong_assignment')
held.release()
expect(await commit).toEqual({ outcome: 'committed' })
await rejected
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
})
it('finds the committed attempt after its database reply is lost', async () => {
const { store, safety, request, database } = await setup()
const transaction = database.transaction.bind(database)
const intercepted = vi
.spyOn(database, 'transaction')
.mockImplementation(async (operation, options) => {
let changed = false
const result = await transaction(
async (tx) =>
operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query')
return async (sql: string, params?: unknown[]) => {
if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true
return target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
),
options
)
if (changed) throw new Error('simulated_commit_reply_lost')
return result
})
await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow(
'simulated_commit_reply_lost'
)
intercepted.mockRestore()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed')
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1)
})
it('commits the requested move once and records its outcome without source retention', async () => {
const { store, database, safety, request } = await setup()
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts')
expect(attempts).toHaveLength(1)
expect(attempts[0]!.attempt_id).toBe(request.attemptId)
expect(Number(attempts[0]!.source_generation)).toBe(7)
})
it('rejects a replaced control and never substitutes a different target', async () => {
const { store, safety, request } = await setup()
expect(
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
).toEqual({ outcome: 'deferred' })
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 })
})
it('does not commit without process safety or cohort authorization', async () => {
const { store, safety, request, database } = await setup()
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
outcome: 'deferred'
})
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
const { store, safety, database } = await setup()
const before = await database.query('SELECT * FROM relay_assignments')
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(1)
expect(candidates[0]).toMatchObject({
sourceGeneration: 7,
sourceCellId: 'source',
targetCellId: 'target'
})
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
})
function holdStatement(database: RelayDatabase, fragment: string) {
let entered!: () => void
let release!: () => void
const arrival = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
let held = false
const transaction = database.transaction.bind(database)
vi.spyOn(database, 'transaction').mockImplementation((operation, options) =>
transaction(async (tx) => {
return operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query' || key === 'queryLocked')
return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => {
if (!held && sql.includes(fragment)) {
held = true
entered()
await gate
}
return key === 'queryLocked'
? target.queryLocked(sql, params, lockOptions)
: target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
)
}, options)
)
return { entered: arrival, release }
}
cloud/apps/relay/src/idle-regional-rehome-test-database.ts
+40
View File
@@ -0,0 +1,40 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
export async function openIdleRehomeTestDatabase(): Promise<RelayDatabase> {
const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL
if (!configured) return openInMemoryRelayDatabase()
const url = new URL(configured)
if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
throw new Error('idle_rehome_tests_require_local_postgres_55440')
}
const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}`
const admin = new pg.Client({ connectionString: configured })
await admin.connect()
try {
await admin.query(`CREATE SCHEMA ${schema}`)
url.searchParams.set('options', `-c search_path=${schema}`)
const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
const close = database.close.bind(database)
database.close = async () => {
try {
await close()
} finally {
try {
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
} finally {
await admin.end()
}
}
}
return database
} catch (error) {
try {
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await admin.end()
}
throw error
}
}
cloud/apps/relay/src/idle-regional-rehome-worker.test.ts
+105
View File
@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
const candidate = {
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
sourceGeneration: 3,
targetCellId: 'target'
}
const config = {
role: 'director',
regionCorrectionCohortPercent: 100,
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
rehomeDirectorServiceAccount: 'director@example.test'
} as RelayConfig
function setup(fetch: typeof globalThis.fetch) {
const selectIdleRegionalRehomeCandidates = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValue([candidate])
const claimRegionalRehome = vi.fn()
const recordRegionalRehomeDispatchFailure = vi.fn()
const worker = startRegionalRehomeWorker(
config,
{
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => ({ observedAt: 100 }) as never,
intervalMs: 60_000,
identityToken: async () => 'private-token',
fetch
}
)!
return {
worker,
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
}
}
describe('idle regional worker dispatch', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an idle request without claiming an assignment first', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({ v: 1, outcome: 'committed' })
)
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.claimRegionalRehome).not.toHaveBeenCalled()
expect(fetch).toHaveBeenCalledOnce()
const [url, init] = fetch.mock.calls[0]!
expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome')
const { sourceCellUrl: _, ...request } = candidate
expect(JSON.parse(String(init?.body))).toEqual({
...request,
cohortPercent: 100,
directorSafety: { observedAt: 100 }
})
})
it('progresses past busy hosts without charging a dispatch failure', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' }))
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' }))
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
c.selectIdleRegionalRehomeCandidates.mockResolvedValue([
candidate,
{
...candidate,
relayHostId: 'ponmlkjihgfedcba',
attemptId: '33333333-3333-4333-8333-333333333333'
}
])
await c.worker.run()
c.worker.stop()
expect(fetch).toHaveBeenCalledTimes(2)
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
it('does not charge a lost response as a claimed migration failure', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () => {
throw new Error('response lost')
})
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
})
cloud/apps/relay/src/index.ts
+8
View File
@@ -2,6 +2,7 @@ import {
formatAssignmentInventorySnapshot,
readAssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import { RelayAssignmentStore } from './assignment-store.js'
import { loadRelayConfig } from './config.js'
import { startCellHeartbeat } from './cell-heartbeat-client.js'
@@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
void runRelayBackgroundOperation(async () => {
const inventory = await readRegisteredMigrationInventory(database, Date.now())
for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line)
console.log(
JSON.stringify({
event: 'orca_relay_region_correction_outcomes',
observedAt: Date.now(),
outcomes: await readRegionCorrectionOutcomes(database, Date.now())
})
)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cloud/apps/relay/src/postgres-query-failure-postgres.test.ts
+70
View File
@@ -0,0 +1,70 @@
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
describePostgres('real PostgreSQL query failure phases', () => {
let database: RelayDatabase
beforeAll(async () => {
database = await openRelayDatabase({
databaseUrl,
dataDir: '',
poolMax: 1,
statementTimeoutMs: 50
})
})
afterAll(async () => {
await database.close()
})
afterEach(() => {
vi.restoreAllMocks()
})
it('distinguishes a server statement timeout and leaves the pool usable', async () => {
const log = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
await expect(database.query('SELECT pg_sleep(0.2)')).rejects.toMatchObject({ code: '57014' })
expect(JSON.parse(log.mock.calls[0]![0] as string)).toMatchObject({
event: 'orca_relay_postgres_query_failed',
phase: 'execute',
code: '57014',
connectionTimeout: false
})
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
})
it('distinguishes queue acquisition timeout without running the statement', async () => {
const log = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
let acquired!: () => void
const ready = new Promise<void>((resolve) => {
acquired = resolve
})
let release!: () => void
const wait = new Promise<void>((resolve) => {
release = resolve
})
const holder = database.transaction(async () => {
acquired()
await wait
})
await ready
try {
await expect(database.query('SELECT pg_sleep(0.2)')).rejects.toThrow(
'timeout exceeded when trying to connect'
)
expect(JSON.parse(log.mock.calls[0]![0] as string)).toMatchObject({
event: 'orca_relay_postgres_query_failed',
phase: 'acquire',
code: 'unknown',
connectionTimeout: true,
poolTotal: 1,
poolIdle: 0
})
} finally {
release()
await holder
}
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
})
})
cloud/apps/relay/src/postgres-query-failure.test.ts
+109
View File
@@ -0,0 +1,109 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const fakes = vi.hoisted(() => ({
connectError: undefined as unknown,
query: vi.fn(async (_sql: string, _params?: unknown[]) => ({ rows: [], rowCount: 0 })),
release: vi.fn()
}))
vi.mock('pg', () => ({
default: {
Pool: class {
totalCount = 10
idleCount = 0
waitingCount = 7
on = vi.fn()
async connect() {
if (fakes.connectError) throw fakes.connectError
return { query: fakes.query, release: fakes.release }
}
async end() {}
}
}
}))
import { openRelayDatabase, type RelayDatabase } from './database.js'
describe('PostgreSQL query failure diagnostics', () => {
let database: RelayDatabase
const sql = 'WITH assignment_state AS MATERIALIZED (SELECT $1) SELECT * FROM assignment_state'
beforeEach(async () => {
fakes.connectError = undefined
fakes.query.mockReset().mockResolvedValue({ rows: [], rowCount: 0 })
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
database = await openRelayDatabase({ databaseUrl: 'postgres://unused', dataDir: '' })
fakes.query.mockClear()
fakes.release.mockClear()
vi.mocked(console.warn).mockClear()
})
afterEach(async () => {
await database.close()
vi.restoreAllMocks()
})
it('identifies acquisition failure without issuing SQL or changing the error', async () => {
const error = new Error('timeout exceeded when trying to connect: private detail')
fakes.connectError = error
await expect(database.query(sql, ['private-token'])).rejects.toBe(error)
expect(fakes.query).not.toHaveBeenCalled()
expect(fakes.release).not.toHaveBeenCalled()
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toEqual({
event: 'orca_relay_postgres_query_failed',
phase: 'acquire',
operation: 'control-renewal',
code: 'unknown',
connectionTimeout: true,
elapsedMs: expect.any(Number),
poolTotal: 10,
poolIdle: 0,
poolWaiting: 7
})
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private')
})
it.each(['57014', '55P03', 'ECONNRESET'])(
'identifies execute failure %s and releases its client',
async (code) => {
const error = Object.assign(new Error('private-token'), { code, detail: sql })
fakes.query.mockRejectedValueOnce(error)
await expect(database.query(sql, ['private-token'])).rejects.toBe(error)
expect(fakes.query).toHaveBeenCalledOnce()
expect(fakes.release).toHaveBeenCalledOnce()
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'execute',
operation: 'control-renewal',
code,
connectionTimeout: false
})
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private-token')
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain(sql)
}
)
it('does not emit an arbitrary error code, message, query, or parameter', async () => {
const error = { code: 'private-code', message: 'private-message' }
fakes.query.mockRejectedValueOnce(error)
await expect(database.query('SELECT private_column', ['private-param'])).rejects.toBe(error)
const log = vi.mocked(console.warn).mock.calls[0]![0] as string
expect(JSON.parse(log)).toMatchObject({ operation: 'other', code: 'unknown' })
expect(log).not.toContain('private')
})
it('keeps the original error and releases the client if logging fails', async () => {
const error = new Error('database failure')
fakes.query.mockRejectedValueOnce(error)
vi.mocked(console.warn).mockImplementationOnce(() => {
throw new Error('logger failure')
})
await expect(database.query(sql)).rejects.toBe(error)
expect(fakes.release).toHaveBeenCalledOnce()
})
it('does not log successful queries', async () => {
await database.query(sql)
expect(console.warn).not.toHaveBeenCalled()
expect(fakes.release).toHaveBeenCalledOnce()
})
})
cloud/apps/relay/src/postgres-query-failure.ts
+55
View File
@@ -0,0 +1,55 @@
type QueryFailurePhase = 'acquire' | 'execute'
const ERROR_CODES = new Set([
'57014',
'55P03',
'40P01',
'40001',
'53300',
'57P01',
'57P02',
'57P03',
'08000',
'08001',
'08003',
'08006',
'ECONNRESET',
'ECONNREFUSED',
'ETIMEDOUT',
'EPIPE'
])
export function reportPostgresQueryFailure(input: {
error: unknown
phase: QueryFailurePhase
sql: string
elapsedMs: number
pool: { totalCount: number; idleCount: number; waitingCount: number }
}): void {
// Emit only bounded categories: error messages and SQL can contain credentials or identities.
try {
const error = input.error as { code?: unknown; message?: unknown } | null
const code =
typeof error?.code === 'string' && ERROR_CODES.has(error.code) ? error.code : 'unknown'
const connectionTimeout =
typeof error?.message === 'string' &&
error.message.includes('timeout exceeded when trying to connect')
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_query_failed',
phase: input.phase,
operation: /^\s*WITH\s+assignment_state\s+AS\s+MATERIALIZED\b/i.test(input.sql)
? 'control-renewal'
: 'other',
code,
connectionTimeout,
elapsedMs: Math.max(0, Math.round(input.elapsedMs)),
poolTotal: input.pool.totalCount,
poolIdle: input.pool.idleCount,
poolWaiting: input.pool.waitingCount
})
)
} catch {
// Diagnostics must not replace the original database failure.
}
}
cloud/apps/relay/src/postgres-statement-stats-postgres.test.ts
+121
View File
@@ -0,0 +1,121 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { openRelayDatabase } from './database.js'
import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
describePostgres('optional PostgreSQL statement statistics', () => {
let admin: pg.Client
let preloaded: boolean
const databases: string[] = []
const roles: string[] = []
beforeAll(async () => {
admin = new pg.Client({ connectionString: databaseUrl })
await admin.connect()
const result = await admin.query<{ loaded: boolean }>(
`SELECT 'pg_stat_statements' = ANY(string_to_array(
replace(current_setting('shared_preload_libraries'), ' ', ''), ','
)) AS loaded`
)
preloaded = result.rows[0]!.loaded
})
afterAll(async () => {
for (const database of databases) await admin.query(`DROP DATABASE IF EXISTS ${database}`)
for (const role of roles) await admin.query(`DROP ROLE IF EXISTS ${role}`)
await admin.end()
})
async function freshDatabase(): Promise<string> {
const name = `relay_stats_${randomUUID().replaceAll('-', '')}`
await admin.query(`CREATE DATABASE ${name}`)
databases.push(name)
const url = new URL(databaseUrl!)
url.pathname = `/${name}`
return url.toString()
}
async function connect(url: string): Promise<pg.Client> {
const client = new pg.Client({ connectionString: url, statement_timeout: 2_000 })
await client.connect()
return client
}
async function installed(client: pg.Client): Promise<boolean> {
const result = await client.query<{ present: boolean }>(
`SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') AS present`
)
return result.rows[0]!.present
}
it('exposes an existing collector idempotently, and skips servers without one', async () => {
const url = await freshDatabase()
const database = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
await database.close()
const client = await connect(url)
try {
expect(await installed(client)).toBe(preloaded)
if (preloaded) {
const before = await client.query('SELECT stats_reset FROM public.pg_stat_statements_info')
await client.query(POSTGRES_STATEMENT_STATS_MIGRATION)
const after = await client.query('SELECT stats_reset FROM public.pg_stat_statements_info')
expect(after.rows).toEqual(before.rows)
await client.query('SELECT calls, wal_bytes, shared_blks_dirtied FROM public.pg_stat_statements LIMIT 1')
} else {
await client.query(POSTGRES_STATEMENT_STATS_MIGRATION)
expect(await installed(client)).toBe(false)
}
} finally {
await client.end()
}
})
it.each([false, true])('tolerates missing extension privileges (read settings: %s)', async (readSettings) => {
const client = await connect(await freshDatabase())
const role = `relay_stats_role_${randomUUID().replaceAll('-', '')}`
await admin.query(`CREATE ROLE ${role}`)
roles.push(role)
if (readSettings) await admin.query(`GRANT pg_read_all_settings TO ${role}`)
try {
await client.query(`SET ROLE ${role}`)
await client.query(POSTGRES_STATEMENT_STATS_MIGRATION)
expect(await installed(client)).toBe(false)
expect((await client.query<{ value: number }>('SELECT 42 AS value')).rows[0]!.value).toBe(42)
} finally {
await client.end()
}
})
it('serializes concurrent catalog creation across directors', async () => {
const url = await freshDatabase()
const clients = await Promise.all(Array.from({ length: 5 }, async () => await connect(url)))
try {
await Promise.all(clients.map(async (client) => await client.query(POSTGRES_STATEMENT_STATS_MIGRATION)))
expect(await installed(clients[0]!)).toBe(preloaded)
} finally {
await Promise.all(clients.map(async (client) => await client.end()))
}
})
it('yields to an in-progress installer instead of blocking startup', async () => {
const url = await freshDatabase()
const owner = await connect(url)
const contender = await connect(url)
try {
await owner.query('BEGIN')
await owner.query(`SELECT pg_advisory_xact_lock(hashtext('orca-relay'), hashtext('statement-stats'))`)
await contender.query(POSTGRES_STATEMENT_STATS_MIGRATION)
expect(await installed(contender)).toBe(false)
await owner.query('COMMIT')
await contender.query(POSTGRES_STATEMENT_STATS_MIGRATION)
expect(await installed(contender)).toBe(preloaded)
} finally {
await owner.end()
await contender.end()
}
})
})
cloud/apps/relay/src/postgres-statement-stats.ts
+28
View File
@@ -0,0 +1,28 @@
// Expose an already-running collector; never preload a module or require elevated runtime privileges.
export const POSTGRES_STATEMENT_STATS_MIGRATION = `
DO $relay_statement_stats$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_catalog.pg_settings
WHERE name = 'shared_preload_libraries'
AND 'pg_stat_statements' = ANY(string_to_array(replace(setting, ' ', ''), ','))
) OR EXISTS (
SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_stat_statements'
) OR NOT EXISTS (
SELECT 1 FROM pg_catalog.pg_available_extensions WHERE name = 'pg_stat_statements'
) THEN
RETURN;
END IF;
IF NOT pg_try_advisory_xact_lock(hashtext('orca-relay'), hashtext('statement-stats')) THEN
RETURN;
END IF;
BEGIN
CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public;
EXCEPTION WHEN insufficient_privilege THEN
RAISE WARNING 'orca_relay_statement_stats_unavailable: insufficient privilege';
END;
END
$relay_statement_stats$;
`
cloud/apps/relay/src/region-correction-outcomes.ts
+29
View File
@@ -0,0 +1,29 @@
import type { RelayDatabase } from './database.js'
export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) {
const rows = await database.query(
`SELECT attempt.source_cell_id, attempt.target_cell_id,
CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted'
WHEN attempt.completed_at IS NOT NULL THEN 'completed'
WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state,
COUNT(*) AS count,
COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms,
COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units
FROM relay_region_rehome_attempts attempt
JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch
GROUP BY attempt.source_cell_id, attempt.target_cell_id, state
ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`,
[now]
)
return rows.map((row) => ({
sourceCellId: String(row.source_cell_id),
targetCellId: String(row.target_cell_id),
state: String(row.state),
count: Number(row.count),
oldestOpenMs: Number(row.oldest_open_ms),
targetReservedUnits: Number(row.target_reserved_units)
}))
}
cloud/apps/relay/src/region-correction-preview.ts
+157
View File
@@ -0,0 +1,157 @@
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
import {
REGIONAL_REHOME_CONCURRENT_LIMIT,
REGION_DECISION_TTL_MS
} from './region-correction-state.js'
export type RegionCorrectionPreview = {
observedAt: number
newClaimsEnabled: boolean
cohortPercent: number
openMigrations: number
availableMigrationSlots: number
globalSafetyFailure: string | null
counts: Record<string, number>
}
export async function previewRegionalRehomeEligibility(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
globalSafetyFailure: string | null
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<RegionCorrectionPreview> {
const { database, now } = input
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
await Promise.all([
database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at,
decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version,
decision.observed_at, decision.cohort_bucket,
(SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at,
(SELECT COUNT(*) FROM relay_assignment_migrations migration
WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations,
(SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease
WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id
AND lease.cell_id = assignment.cell_id) AS source_units,
(SELECT COUNT(*) FROM relay_control_capabilities host_capability
JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id
AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id
AND runtime.cell_incarnation = host_capability.cell_incarnation
WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id
AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch
AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ?
AND lease.updated_at >= runtime.started_at) AS capable_controls
FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision
ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`,
[now]
),
database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`),
database.query(`SELECT * FROM relay_cell_runtime`),
database.query(`SELECT * FROM relay_cell_capabilities`),
database.query(`SELECT * FROM relay_cell_rehome_safety`),
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
)
])
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
const runtimes = byCell(runtimeRows)
const capabilities = byCell(capabilityRows)
const safety = byCell(safetyRows)
const inventory = byCell(cells)
const control = controls[0]
const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS)
const openMigrations = Number(migrations[0]?.count ?? 0)
const counts: Record<string, number> = {}
const count = (reason: string) => {
counts[reason] = (counts[reason] ?? 0) + 1
}
const available = (cell: SqlRow): boolean => {
const id = String(cell.cell_id)
const runtime = runtimes.get(id)
const capability = capabilities.get(id)
return (
Number(cell.enabled) === 1 &&
cell.admission_state === 'general' &&
cell.region != null &&
runtime !== undefined &&
Number(runtime.ready) === 1 &&
Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs &&
capability !== undefined &&
capability.cell_incarnation === runtime.cell_incarnation &&
Number(capability.regional_rehome_protocol) >= 3
)
}
for (const host of hosts) {
let reason: string | null = null
const source = inventory.get(String(host.cell_id))
if (host.generation == null) reason = 'no-verified-decision'
else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge)
reason = 'expired'
else if (
Number(host.decision_epoch) !== Number(host.assignment_epoch) ||
host.incumbent_region !== source?.region
)
reason = 'basis-changed'
else if (
host.outcome !== 'conclusive' ||
Number(host.policy_version) !== 1 ||
host.preferred_region == null
)
reason = 'inconclusive-or-insufficient-improvement'
else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort'
else if (Number(host.open_migrations) > 0) reason = 'migration-open'
else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown)
reason = 'host-cooldown'
else if (!source || !available(source)) reason = 'source-ineligible'
else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive'
else if (
!input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now)
)
reason = 'source-unclean'
if (reason) {
count(reason)
continue
}
const targets = cells.filter(
(cell) =>
cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell)
)
const clean = targets.filter((cell) =>
input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now)
)
const capacity = clean.filter(
(cell) =>
input.connectionHeadroom.get(String(cell.cell_id)) !== false &&
Number(cell.reserved_requests) + Number(host.source_units) + 1 <=
Number(cell.capacity_requests)
)
if (targets.length === 0) count('no-eligible-target')
else if (clean.length === 0) count('target-unclean')
else if (capacity.length === 0) count('no-target-headroom')
else if (input.globalSafetyFailure) count('global-safety-blocked')
else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap')
else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`)
}
return {
observedAt: now,
newClaimsEnabled: Number(control?.enabled ?? 0) === 1,
cohortPercent: input.cohortPercent,
openMigrations,
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
globalSafetyFailure: input.globalSafetyFailure,
counts
}
}
cloud/apps/relay/src/region-correction-restart.test.ts
+119
View File
@@ -0,0 +1,119 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' }
const paths: string[] = []
const databases = new Set<RelayDatabase>()
afterEach(async () => {
for (const database of databases) await database.close()
databases.clear()
for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true })
})
async function setup() {
const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-'))
paths.push(dataDir)
let now = 1_000_000_000
const open = async () => {
const database = await openRelayDatabase({ dataDir })
databases.add(database)
return { database, store: new RelayAssignmentStore(database, () => now) }
}
const first = await open()
const cell = {
id: 'restart-us',
url: 'https://restart-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
}
await first.store.reconcileCells([cell])
await first.store.setCellEnabled(cell.id, true)
await first.store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
region: cell.region,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
const assignment = await first.store.assign(identity)
const issue = (store: RelayAssignmentStore) =>
store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
const window = (await issue(first.store)).window!
const report = {
v: 1 as const,
action: 'report' as const,
generation: window.generation,
assignmentEpoch: window.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
const restart = async () => {
await first.database.close()
databases.delete(first.database)
return open()
}
return {
...first, window, report, issue, restart,
setNow: (value: number) => { now = value }
}
}
describe('persisted region decisions across director restart', () => {
it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => {
const context = await setup()
const epoch = context.window.assignmentEpoch
await context.store.exchangeRegionCorrection(identity, {
v: 1, action: 'report', generation: context.window.generation,
assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter'
}, epoch)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'duplicate' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('inconclusive')
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'stale' })
})
it('uses server expiry after a restart regardless of an old client report', async () => {
const context = await setup()
context.setNow(context.window.expiresAt)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'expired' })
expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 })
})
it('does not interpret a persisted future-policy window using the old policy after rollback', async () => {
const context = await setup()
await context.database.query('UPDATE relay_region_decisions SET policy_version = 2')
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('pending')
expect(row.preferred_region).toBeNull()
expect(row.report_json).toBeNull()
})
it('keeps generation ordering when the server clock moves backwards across restart', async () => {
const context = await setup()
context.setNow(1_000_000_000 - 60_000)
const restarted = await context.restart()
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
})
})
cloud/apps/relay/src/region-correction-state.ts
+158
View File
@@ -0,0 +1,158 @@
import { relayHostLogDigest } from './relay-host-log-digest.js'
import { createHash } from 'node:crypto'
import type {
RegionCorrectionRequest,
RegionCorrectionResponse,
RelayRegion
} from '@orca-cloud/relay-contract'
import type { RelayDatabase } from './database.js'
type Identity = { userId: string; relayHostId: string }
export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000
export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8
export async function exchangeRegionCorrection(
database: RelayDatabase,
identity: Identity,
request: RegionCorrectionRequest,
assignmentEpoch: number,
now: number
): Promise<RegionCorrectionResponse> {
const result: RegionCorrectionResponse = await database.transaction(async (transaction) => {
const assignment = (
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
const region =
assignment &&
(
await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
assignment.cell_id
])
)[0]
if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) {
return { v: 1, reportStatus: 'basis-changed' }
}
const prior = (
await transaction.queryLocked(
`SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (request.action === 'issue-window') {
const generation = Number(prior?.generation ?? 0) + 1
if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted')
const expiresAt = now + REGION_DECISION_TTL_MS
const cohortBucket =
createHash('sha256')
.update(JSON.stringify([identity.userId, identity.relayHostId]))
.digest()
.readUInt32BE(0) % 100
await transaction.query(
`INSERT INTO relay_region_decisions
(user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region,
policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket)
VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
generation = excluded.generation, expires_at = excluded.expires_at,
assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region,
policy_version = 1, outcome = 'pending', preferred_region = NULL,
observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`,
[
identity.userId,
identity.relayHostId,
generation,
expiresAt,
assignmentEpoch,
region.region,
now,
cohortBucket
]
)
return {
v: 1,
window: {
generation,
expiresAt,
assignmentEpoch,
incumbentRegion: region.region as RelayRegion,
policyVersion: 1
}
}
}
if (!prior || Number(prior.generation) !== request.generation)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.policy_version) !== request.policyVersion)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' }
if (
request.assignmentEpoch !== assignmentEpoch ||
Number(prior.assignment_epoch) !== assignmentEpoch ||
prior.incumbent_region !== region.region
) {
return { v: 1, reportStatus: 'basis-changed' }
}
// The first report wins, including an inconclusive tombstone.
if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' }
let preferredRegion: RelayRegion | null = null
if (request.outcome === 'conclusive') {
const incumbent = request.measurements[region.region as RelayRegion]
const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1'
const targetRtt = request.measurements[target]
if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target
}
await transaction.query(
`UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ?
WHERE user_id = ? AND relay_host_id = ? AND generation = ?`,
[
request.outcome,
preferredRegion,
JSON.stringify(request),
identity.userId,
identity.relayHostId,
request.generation
]
)
return { v: 1, reportStatus: 'accepted' }
})
if (request.action === 'report' && result.reportStatus === 'accepted') {
const digest = relayHostLogDigest(identity.relayHostId)
// Stable sampling includes unchanged hosts for before/after comparisons.
if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) {
console.log(
JSON.stringify({
event: 'orca_relay_region_comparison',
relayHostIdDigest: digest,
assignmentEpoch,
generation: request.generation,
policyVersion: request.policyVersion,
outcome: request.outcome,
...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {})
})
)
}
}
return result
}
export async function previewRegionCorrection(
database: RelayDatabase,
now: number
): Promise<Record<string, number>> {
const rows = await database.query(
`SELECT CASE WHEN decision.expires_at <= ? THEN 'expired'
WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed'
WHEN decision.outcome = 'pending' THEN 'pending'
WHEN decision.preferred_region IS NULL THEN 'ineligible'
ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason,
COUNT(*) AS count
FROM relay_region_decisions decision
JOIN relay_assignments assignment ON assignment.user_id = decision.user_id
AND assignment.relay_host_id = decision.relay_host_id
GROUP BY reason`,
[now]
)
return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)]))
}
cloud/apps/relay/src/region-correction-store.test.ts
+359
View File
@@ -0,0 +1,359 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' }
const cells = [
{
id: 'decision-us',
url: 'https://decision-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'decision-asia',
url: 'https://decision-asia.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const opened: RelayDatabase[] = []
afterEach(async () => {
for (const database of opened.splice(0)) {
if (database.dialect === 'postgres') await cleanupPostgres(database)
await database.close()
}
})
async function cleanupPostgres(database: RelayDatabase) {
for (const table of [
'relay_control_connection_reservations',
'relay_region_decisions',
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_migration_incarnations',
'relay_assignment_region_preferences',
'relay_region_rehome_attempts',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
}
for (const table of [
'relay_cell_rehome_safety',
'relay_cell_capabilities',
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_runtime',
'relay_cell_connection_limits',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await database.query(
`DELETE FROM ${table} WHERE cell_id IN (?, ?)`,
cells.map((cell) => cell.id)
)
}
}
async function setup() {
const database =
process.env.ORCA_REGION_CORRECTION_POSTGRES === '1'
? await openRelayDatabase({
databaseUrl: requiredPostgresUrl(),
dataDir: '/tmp/orca-region-correction-unused'
})
: await openInMemoryRelayDatabase()
opened.push(database)
if (database.dialect === 'postgres') await cleanupPostgres(database)
let clock = 100_000_000
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100
})
await store.reconcileCells(cells)
for (const cell of cells) await store.setCellEnabled(cell.id, true)
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: clock - 1_000,
ready: true,
observedRequests: 0
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const activityId = await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
return {
database,
store,
assignment,
activityId,
now: () => clock,
advance: (ms: number) => {
clock += ms
}
}
}
function requiredPostgresUrl(): string {
const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL
if (!url || new URL(url).port !== '55440')
throw new Error('PostgreSQL tests require configured port 55440')
return url
}
async function window(context: Awaited<ReturnType<typeof setup>>) {
const result = await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
context.assignment.assignmentEpoch
)
return result.window!
}
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
const attemptId = '33333333-3333-4333-8333-333333333333'
await context.database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
[
attemptId,
identity.userId,
identity.relayHostId,
cells[0]!.id,
incarnations[0],
cells[1]!.id,
incarnations[1],
migration.previousEpoch,
migration.assignmentEpoch,
context.now(),
context.now()
]
)
return { migration }
}
describe('ordered region decisions and migration outcomes', () => {
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
const context = await setup()
const { migration } = await regionalMigration(context)
context.advance(1_000)
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
const outcomes = await context.store.regionCorrectionOutcomes()
expect(outcomes).toEqual([
expect.objectContaining({
sourceCellId: cells[0]!.id,
targetCellId: cells[1]!.id,
state: 'registering',
count: 1,
oldestOpenMs: 1_000
})
])
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
before
)
await context.store.activateControl(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch
})
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({ state: 'registered' })
])
await context.store.releaseActivity(identity, context.activityId)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({
state: 'completed',
targetReservedUnits: 0,
oldestOpenMs: 0
})
])
})
it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => {
const context = await setup()
const first = await window(context)
const second = await window(context)
expect(second.generation).toBe(first.generation + 1)
const report = {
v: 1 as const,
action: 'report' as const,
assignmentEpoch: first.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: first.generation },
first.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'accepted' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'duplicate' })
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
})
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
const context = await setup()
const query = context.database.query.bind(context.database)
const transaction = context.database.transaction.bind(context.database)
const queryLocked = context.database.queryLocked.bind(context.database)
context.database.query = async (sql, params) => {
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
return query(sql, params)
}
context.database.transaction = async () => {
throw new Error('preview_must_not_open_mutating_transaction')
}
context.database.queryLocked = async () => {
throw new Error('preview_must_not_lock')
}
try {
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
expect(JSON.stringify(preview)).not.toContain(identity.userId)
} finally {
context.database.query = query
context.database.transaction = transaction
context.database.queryLocked = queryLocked
}
})
it('allocates distinct ordered generations for concurrent window issuers', async () => {
const context = await setup()
const replies = await Promise.all([window(context), window(context), window(context)])
expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3])
const older = replies.find((reply) => reply.generation === 2)!
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: older.generation,
assignmentEpoch: older.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'delayed'
},
older.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
})
it('compares with assigned region, preserves hints, and never extends a window on report', async () => {
const context = await setup()
await context.store.assign(identity, 'asia-east2')
const issued = await window(context)
expect(issued.incumbentRegion).toBe('us-central1')
context.advance(50)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 110, 'asia-east2': 90 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]!
expect(Number(row.expires_at)).toBe(issued.expiresAt)
const hint = (
await context.database.query(
`SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`,
[identity.userId]
)
)[0]
expect(hint?.preferred_region).toBe('asia-east2')
context.advance(24 * 60 * 60_000)
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'late'
},
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'expired' })
})
it('rejects stale assignment basis and requires both thresholds', async () => {
const context = await setup()
const issued = await window(context)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 150, 'asia-east2': 100 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({
'us-central1-to-asia-east2': 1
})
await context.store.startEvacuation(identity, cells[1]!.id)
expect(
await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'basis-changed' })
})
})
cloud/apps/relay/src/regional-host-drain-app.test.ts
+207 -94
View File
@@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string, route?: string) =>
token === 'deploy-token' ||
(token === 'monitor-token' &&
(!route || route === '/v1/admin/regional-rehome-control')),
(!route ||
route === '/v1/admin/regional-rehome-control' ||
route === '/v1/admin/regional-rehome-preview')),
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) =>
token === 'deploy-token',
@@ -41,7 +43,83 @@ const request = {
graceMs: 60_000
}
describe('idle regional cutover endpoint', () => {
it('authenticates and fences the source before invoking a cutover', async () => {
const idleRehome = vi.fn(async () => ({ outcome: 'busy' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
idleRehome,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
const input = {
v: 1,
attemptId: request.attemptId,
userId: request.userId,
relayHostId: request.relayHostId,
sourceCellId: request.sourceCellId,
sourceCellIncarnation: cellIncarnation,
sourceAssignmentEpoch: 7,
sourceGeneration: 1,
targetCellId: 'target-cell',
cohortPercent: 100,
directorSafety: {
observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0
}
}
const path = '/v1/admin/host-idle-rehome'
expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401)
expect(
(
await postPath(app, path, 'rehome-token', {
...input,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(idleRehome).not.toHaveBeenCalled()
const response = await postPath(app, path, 'rehome-token', input)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'busy' })
expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input)
})
})
describe('regional host drain endpoint', () => {
it('exposes aggregate preview to monitors without a mutation path', async () => {
const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } }
const safety = { observedAt: 100 }
const previewRegionalRehomeEligibility = vi.fn(async () => preview)
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: {
previewRegionalRehomeEligibility,
regionCorrectionOutcomes: async () => []
} as never,
regionalRehomeSafetySnapshot: () => safety as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const path = '/v1/admin/regional-rehome-preview'
expect((await app.request(path)).status).toBe(401)
expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled()
const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] })
expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety)
expect(
(
await app.request(path, {
method: 'POST',
headers: { authorization: 'Bearer deploy-token' }
})
).status
).toBe(404)
})
it('accepts only the dedicated identity and exact cell generation', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => {
expect((await post(app, 'deploy-token', request)).status).toBe(401)
expect(
(await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})).status
(
await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(drainHost).toHaveBeenCalledOnce()
})
it('waits for an asynchronous drain operation before acknowledging', async () => {
let grant!: (value: 'accepted') => void
let entered!: () => void
const started = new Promise<void>((resolve) => {
entered = resolve
})
const drainHost = vi.fn(() => {
entered()
return new Promise<'accepted'>((resolve) => {
grant = resolve
})
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost,
cellIncarnation,
ready: vi.fn(async () => true)
})
const pending = post(app, 'rehome-token', request)
let acknowledged = false
void pending.then(() => {
acknowledged = true
})
await started
await Promise.resolve()
expect(acknowledged).toBe(false)
grant('accepted')
const response = await pending
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' })
})
it('rejects a failed asynchronous drain instead of acknowledging it', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost: async () => {
throw new Error('activity_cell_not_authoritative')
},
cellIncarnation,
ready: vi.fn(async () => true)
})
const response = await post(app, 'rehome-token', request)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' })
})
it('rejects malformed identities before touching the session registry', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -224,7 +354,7 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
observedAt: 100,
sqlFailures: 0,
@@ -235,12 +365,7 @@ describe('regional rehome director controls', () => {
databasePoolWaitMsMax: 0
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -266,18 +391,13 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
...observability.regionalRehomeRuntimeSafety(),
...emptyPostgresPoolPressureCounts()
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -301,12 +421,14 @@ describe('regional rehome director controls', () => {
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
const apply = {
v: 1,
action: 'apply',
@@ -319,39 +441,35 @@ describe('regional rehome director controls', () => {
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
apply
)).status).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status
).toBe(200)
expect(applyRegionalRehomeControl).toHaveBeenCalledOnce()
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
apply
)).status).toBe(403)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status
).toBe(403)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
...apply,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
).status
).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown))
.status
).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -382,12 +500,11 @@ describe('regional rehome director controls', () => {
}) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
const responseBody = await response.json()
@@ -448,12 +565,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
@@ -481,12 +597,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
@@ -504,18 +619,17 @@ describe('regional rehome director controls', () => {
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'monitor-token',
body
)).status).toBe(401)
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ ...body, unexpected: true }
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status
).toBe(401)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
...body,
unexpected: true
})
).status
).toBe(400)
})
it('fails closed when the source rejects the dedicated identity', async () => {
@@ -529,9 +643,9 @@ describe('regional rehome director controls', () => {
regionalRehomeProtocol: 1
}
})
const sourceFetch = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({ error: 'invalid_token' }, { status: 401 })
)
const sourceFetch = vi
.fn<typeof fetch>()
.mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 }))
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
@@ -540,12 +654,11 @@ describe('regional rehome director controls', () => {
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).toHaveBeenCalledOnce()
cloud/apps/relay/src/regional-rehome-postgres.test.ts
+260 -233
View File
@@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
async function cleanup(): Promise<void> {
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
}
await primary.query(
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
)
@@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
it('defaults to a closed correction cohort even with enabled durable control', async () => {
const context = await fixture()
const closed = new RelayAssignmentStore(primary, context.now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
expect(await cutover(closed, context.now())).toBeNull()
const preview = await closed.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.cohortPercent).toBe(0)
expect(preview.counts['outside-cohort']).toBe(1)
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('counts existing generic migrations against the optimization cap and preview', async () => {
const context = await fixture()
const safety = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
const before = await context.store.previewRegionalRehomeEligibility(safety)
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
for (let index = 0; index < 8; index++) {
const identity = {
userId: `pg-rehome-user-budget-${sequence}-${index}`,
relayHostId: `budgethost${String(index).padStart(6, '0')}`
}
await context.store.assign(identity, undefined, 'us-central1')
await context.store.startEvacuation(identity, context.target.id)
}
const preview = await context.store.previewRegionalRehomeEligibility(safety)
expect(preview.openMigrations).toBe(8)
expect(preview.availableMigrationSlots).toBe(0)
expect(preview.counts['concurrent-migration-cap']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('preview excludes request capacity exhaustion before a claim', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
[context.target.id]
)
const preview = await context.store.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.counts['no-target-headroom']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
})
it('claims through ambient per-cell sql retry noise', async () => {
const context = await fixture()
await primary.query(
@@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.source.id, context.target.id]
)
expect(await context.store.claimRegionalRehome()).not.toBeNull()
expect(await cutover(context.store, context.now())).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
@@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => {
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
expect(
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
context.identity.userId
])
).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
`UPDATE relay_region_decisions SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
@@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
@@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.target.id]
)
expect(await context.store.claimRegionalRehome()).toBeNull()
expect(await cutover(context.store, context.now())).toBeNull()
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await primary.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }])
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('lets only one director claim a host', async () => {
const context = await fixture()
const claims = await Promise.all([
context.store.claimRegionalRehome(),
context.competingStore.claimRegionalRehome()
cutover(context.store, context.now()),
cutover(context.competingStore, context.now())
])
expect(claims.filter(Boolean)).toHaveLength(1)
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
[context.identity.userId]
)
).toEqual([{ count: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
})
it('serializes an enable with a budget-exhausting failure without retries', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
const locked = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const primaryTransaction = primary.transaction.bind(primary)
const secondaryTransaction = secondary.transaction.bind(secondary)
let enableTransactions = 0
let failureTransactions = 0
let enablePid = 0
let failurePid = 0
const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) =>
primaryTransaction(async (transaction) => {
enableTransactions++
enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation({
dialect: 'postgres',
query: transaction.query.bind(transaction),
queryLocked: async (sql, params, lockOptions) => {
const rows = await transaction.queryLocked(sql, params, lockOptions)
if (sql.includes('FROM relay_region_rehome_control')) {
locked.resolve()
await release.promise
}
return rows
},
transaction: transaction.transaction.bind(transaction),
close: transaction.close.bind(transaction)
})
}, options)
)
const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) =>
secondaryTransaction(async (transaction) => {
failureTransactions++
failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation(transaction)
}, options)
)
const enable = context.store.applyRegionalRehomeControl({
expectedGeneration: 1,
enabled: true,
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
let failure: Promise<void> | undefined
let outcomes: PromiseSettledResult<unknown>[] = []
try {
await Promise.race([
locked.promise,
enable.then(() => {
throw new Error('enable completed before the control lock')
})
])
failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
// Observe the actual PostgreSQL wait before letting enable acquire the worker row.
await vi.waitFor(async () => {
expect(failurePid).not.toBe(0)
const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid])
expect(rows[0]!.blockers).toContain(enablePid)
}, { interval: 10, timeout: 800 })
} finally {
release.resolve()
outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])])
enableSpy.mockRestore()
failureSpy.mockRestore()
}
expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled'])
expect({ enableTransactions, failureTransactions }).toEqual({
enableTransactions: 1,
failureTransactions: 1
})
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: true
})
expect(await primary.query(
`SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state`
)).toEqual([{ consecutive_failures: '1', paused_until: '0' }])
[context.identity.userId]
)
).toEqual([{ count: '1' }])
})
it('increments the disable generation once across competing directors', async () => {
@@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
it('records one receipt across competing directors', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const receipts = await Promise.all([
context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'),
context.competingStore.recordRegionalRehomeDrainReceipt(
attempt!.attemptId,
'accepted'
)
])
expect(receipts.sort()).toEqual([false, true])
expect(await primary.query(
`SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ drain_outcome: 'accepted' }])
})
it('rechecks a preference changed while the assignment row is locked', async () => {
const context = await fixture()
let unlock!: () => void
@@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = cutover(context.store, context.now())
await primary.query(
`UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1',
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
[context.now(), context.identity.userId, context.identity.relayHostId]
)
@@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
await held
await expect(claim).resolves.toBeNull()
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('rechecks fleet safety under locks before mutating a candidate', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
let unlock!: () => void
let locked!: () => void
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
const held = secondary.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`,
[context.target.id]
)
await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [
context.target.id
])
await transaction.query(
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
[context.target.id]
@@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
unlock()
await held
await expect(claim).resolves.toBeNull()
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('pauses when one required cell exceeds the reconnect limit', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`,
[REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id]
)
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
context.source.id
])
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
})
it('does not retry a drain against a replacement source incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
context.advance(31_000)
await heartbeat(
context.store,
context.source,
'33333333-3333-4333-8333-333333333333',
1,
context.now()
)
await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull()
expect(await primary.query(
`SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ send_attempts: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('makes concurrent completion and expiry cleanup idempotent', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
const targetControl = await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.competingStore.abortExpiredRegionalRehomes()
])
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true, aborted: false }])
[context.identity.userId]
)
).toEqual([{ completed: true, aborted: false }])
})
it('will not complete against a replacement target incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
expect(await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed_at: null, aborted_at: null }])
expect(
await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
})
it('does not roll an unregistered target back to a stale regional source', async () => {
const context = await fixture()
await context.store.claimRegionalRehome()
await cutover(context.store, context.now())
context.advance(6 * 60_000)
await heartbeat(
context.store,
@@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
expect(
await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
})
it('completes after the drained host re-resolves through the director', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
// The drain recovery lands while both controls are still live.
await context.store.assign(context.identity, 'asia-east2')
expect(await controlAccounting(context.identity)).toEqual({
@@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => {
await context.store.releaseActivity(context.identity, context.sourceControl)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true }])
[context.identity.userId]
)
).toEqual([{ completed: true }])
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
@@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('repairs a skewed control counter before completing the rehome', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
await context.store.releaseActivity(context.identity, context.sourceControl)
// Damage already written by a pre-fix sticky grant.
await primary.query(
`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`,
[context.identity.userId]
)
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
context.identity.userId
])
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await controlAccounting(context.identity)).toEqual({
@@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function cutover(store: RelayAssignmentStore, now: number) {
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
if (!request) return null
const result = await store.commitIdleRegionalRehome(request, safety(now))
if (result.outcome !== 'committed') return null
const [attempt] = await primary.query(
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
)
return {
...request,
preferredRegion: String(attempt!.preferred_region),
assignmentEpoch: Number(attempt!.assignment_epoch)
}
}
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
@@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
await heartbeat(
store,
source,
'11111111-1111-4111-8111-111111111111',
1,
900_000
)
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
await heartbeat(
store,
target,
'22222222-2222-4222-8222-222222222222',
options.targetProtocol ?? 1,
options.targetProtocol ?? 3,
900_000
)
const identity = {
@@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => {
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: '11111111-1111-4111-8111-111111111111'
})
await store.assign(identity, preferredRegion)
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: {
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
}
},
assignment.assignmentEpoch
)
return {
preferredRegion,
store,
@@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
const storeOptions = {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
}
@@ -816,3 +831,15 @@ async function heartbeat(
}
})
}
function safety(now: number) {
return {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
}
cloud/apps/relay/src/regional-rehome-store.test.ts
+392 -609
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/regional-rehome-target-selection.test.ts
+53 -15
View File
@@ -25,6 +25,7 @@ async function setup() {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
@@ -83,11 +84,40 @@ async function setup() {
await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: incarnation(1)
})
await store.assign(identity, 'asia-east2')
const { window } = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
expect(window).toBeDefined()
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
return { database, store, beat, activatePreferredSource }
const safety = () => ({
observedAt: clock,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
return { database, store, beat, activatePreferredSource, safety }
}
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
@@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
describe('regional rehome target selection', () => {
it('never selects a target without connection headroom, even at lowest load', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(lowLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
it('falls to the next clean target when the load winner goes unclean', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(highLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
})
cloud/apps/relay/src/regional-rehome-worker.test.ts
+52 -165
View File
@@ -11,148 +11,37 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
preferredRegion: 'asia-east2',
sourceCellId: 'production-gce-c7',
sourceCellUrl: 'https://c7.relay.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'production-gce-c27',
targetCellIncarnation: '33333333-3333-4333-8333-333333333333',
previousEpoch: 7,
assignmentEpoch: 8,
drainGraceMs: 60_000,
sendAttempts: 1
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
vi.useFakeTimers()
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const worker = startRegionalRehomeWorker(config(), {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore, {
safetySnapshot: () => safety(Date.now()),
random: () => 0
})!
try {
await vi.advanceTimersByTimeAsync(0)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5_999)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(1)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
worker.stop()
await vi.advanceTimersByTimeAsync(60_000)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
} finally {
worker.stop()
vi.useRealTimers()
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDrainReceipt
} as unknown as RelayAssignmentStore
const requests: Array<{ url: string; init?: RequestInit }> = []
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async (audience) => {
expect(audience).toBe('https://relay.example.test/v1/admin/host-drain')
return 'secret-token'
},
fetch: (async (url, init) => {
requests.push({ url: String(url), init })
return Response.json({ v: 1, outcome: 'accepted' })
}) as typeof fetch
})!
await settleWorker()
now = 1_000
await worker.run()
worker.stop()
expect(requests).toHaveLength(1)
expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain')
expect(requests[0]!.url).not.toContain('secret-token')
expect(requests[0]!.init?.headers).toMatchObject({
authorization: 'Bearer secret-token'
})
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
graceMs: 60_000
})
expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111',
'accepted'
)
const logs = warn.mock.calls.map((call) => String(call[0])).join('\n')
expect(logs).not.toContain('private-user')
expect(logs).not.toContain('abcdefghijklmnop')
})
it('fails closed before the observation gate and records bounded dispatch failures', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'target',
previousEpoch: 1,
assignmentEpoch: 2,
drainGraceMs: 60_000,
sendAttempts: 1
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async () => {
throw new Error('token unavailable')
}
})!
await settleWorker()
claimRegionalRehome.mockClear()
now = 100
await worker.run()
worker.stop()
expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111'
)
})
it('keeps a failed poll out of the durable dispatch-failure budget', async () => {
let now = 0
const claimRegionalRehome = vi
.fn()
.mockResolvedValueOnce(null)
.mockRejectedValue(new Error('Connection terminated due to connection timeout'))
const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})!
await settleWorker()
now = 1_000
await expect(worker.run()).resolves.toBeUndefined()
worker.stop()
// The poll never claimed an attempt, so nothing was drained and nothing may
// be charged to the budget that latches the durable control off.
expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([
'orca_relay_regional_rehome_poll_failed'
])
})
it('passes unsafe process telemetry to the durable claim gate', async () => {
let now = 0
let sqlFailures = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
@@ -160,46 +49,40 @@ describe('regional rehome worker', () => {
intervalMs: 60_000
})!
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
sqlFailures = 1
await worker.run()
worker.stop()
expect(claimRegionalRehome).toHaveBeenCalledWith(
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
)
})
it('starts inert on directors so durable control can enable without a restart', async () => {
let now = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(
config(),
assignments,
{
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
}
)
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})
expect(worker).not.toBeNull()
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
await worker!.run()
worker!.stop()
expect(claimRegionalRehome).toHaveBeenCalledOnce()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
expect(
startRegionalRehomeWorker(
config({ role: 'cell' }),
{} as RelayAssignmentStore,
{ safetySnapshot: () => safety(1) }
)
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
safetySnapshot: () => safety(1)
})
).toBeNull()
})
@@ -208,16 +91,20 @@ describe('regional rehome worker', () => {
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
const processSafety = { ...safety(100), reconnects: limit * 10 }
const fleetSafety = { ...safety(100), reconnects: limit }
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)).toBeNull()
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)).toBe('elevated_reconnects')
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)
).toBeNull()
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)
).toBe('elevated_reconnects')
})
})
cloud/apps/relay/src/regional-rehome-worker.ts
+41 -62
View File
@@ -1,4 +1,4 @@
import { z } from 'zod'
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
@@ -20,13 +20,6 @@ export type RegionalRehomeWorker = {
stop: () => void
}
const RegionalHostDrainResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected'])
})
.strict()
export function startRegionalRehomeWorker(
config: RelayConfig,
assignments: RelayAssignmentStore,
@@ -42,7 +35,6 @@ export function startRegionalRehomeWorker(
}
const audience = config.rehomeAudience
const safetySnapshot = options.safetySnapshot
const now = options.now ?? Date.now
const fetchImpl = options.fetch ?? fetch
const tokenProvider =
options.identityToken ??
@@ -52,64 +44,50 @@ export function startRegionalRehomeWorker(
const run = async (): Promise<void> => {
if (stopped || inFlight) return
inFlight = true
let attemptId: string | null = null
try {
const processSafety = safetySnapshot()
const attempt = await assignments.claimRegionalRehome(processSafety)
if (!attempt) return
attemptId = attempt.attemptId
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
if (candidates.length === 0) return
const token = await tokenProvider(audience)
const response = await fetchImpl(
new URL('/v1/admin/host-drain', attempt.sourceCellUrl),
{
method: 'POST',
headers: {
authorization: `Bearer ${token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
attemptId: attempt.attemptId,
userId: attempt.userId,
relayHostId: attempt.relayHostId,
sourceCellId: attempt.sourceCellId,
sourceCellIncarnation: attempt.sourceCellIncarnation,
sourceAssignmentEpoch: attempt.previousEpoch,
graceMs: attempt.drainGraceMs
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
for (const candidate of candidates) {
if (stopped) return
const { sourceCellUrl, ...request } = candidate
try {
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
...request,
cohortPercent: config.regionCorrectionCohortPercent ?? 0,
directorSafety: safetySnapshot()
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
})
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
if (body.outcome === 'committed') {
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_committed',
sourceCellId: candidate.sourceCellId,
targetCellId: candidate.targetCellId
})
)
return
}
} catch (error) {
// The source may have committed; its durable outcome owns recovery.
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_request_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
)
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = RegionalHostDrainResponseSchema.safeParse(await response.json())
if (!body.success) throw new Error('regional_rehome_source_invalid_response')
await assignments.recordRegionalRehomeDrainReceipt(
attempt.attemptId,
body.data.outcome
)
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_dispatched',
sourceCellId: attempt.sourceCellId,
targetCellId: attempt.targetCellId,
outcome: body.data.outcome,
sendAttempts: attempt.sendAttempts
})
)
} catch (error) {
// Only a claimed attempt was drained. A poll that failed before the claim
// - a pool timeout on the once-a-second control read - dispatched nothing,
// so it must not spend the budget that latches the durable control off.
if (attemptId) {
await assignments
.recordRegionalRehomeDispatchFailure(attemptId)
.catch(() => undefined)
}
} catch (error) {
console.warn(
JSON.stringify({
event: attemptId
? 'orca_relay_regional_rehome_dispatch_failed'
: 'orca_relay_regional_rehome_poll_failed',
event: 'orca_relay_regional_rehome_poll_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
@@ -119,7 +97,8 @@ export function startRegionalRehomeWorker(
}
const timer = setInterval(
() => void run(),
options.intervalMs ?? jitteredSweepIntervalMs(1_000, options.random)
// Match the initial ten-moves/minute budget without replanning the join every second.
options.intervalMs ?? jitteredSweepIntervalMs(6_000, options.random)
)
timer.unref()
void run()
cloud/apps/relay/src/relay-observability.test.ts
+179
View File
@@ -5,6 +5,7 @@ import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
percentile,
RelayObservability,
type RelayProcessCounts
} from './relay-observability.js'
@@ -412,3 +413,181 @@ describe('relay observability', () => {
expect(recordSql.mock.calls.map((call) => call[1])).toEqual([true, false, true, false])
})
})
// The pre-change implementation, kept verbatim as the differential oracle. Both
// ranks sorted their own copy and the maximum was a zero-seeded fold.
function legacyPercentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
function legacyLatencySummary(samples: number[]): { p50: number; p95: number; max: number } {
const round = (value: number): number => Number(value.toFixed(3))
return {
p50: round(legacyPercentile(samples, 0.5)),
p95: round(legacyPercentile(samples, 0.95)),
max: round(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
// `-0` and `NaN` both survive a string round trip, unlike a bare equality check.
function describeNumber(value: number): string {
return Object.is(value, -0) ? '-0' : String(value)
}
function expectSameNumber(actual: number, expected: number, label: string): void {
expect(`${label} = ${describeNumber(actual)}`).toBe(`${label} = ${describeNumber(expected)}`)
}
function sparseWindow(size: number, filled: Record<number, number>): number[] {
const values: number[] = new Array<number>(size)
for (const [index, value] of Object.entries(filled)) values[Number(index)] = value
return values
}
// Lehmer generator: stays inside the safe-integer range so the window is
// byte-identical on every engine the relay runs on.
function deterministicWindow(size: number): number[] {
let seed = 20_260_912
return Array.from({ length: size }, () => {
seed = (seed * 48_271) % 2_147_483_647
return (seed % 4_000_000) / 1_000
})
}
const DENSE_WINDOWS: Array<{ name: string; values: number[] }> = [
{ name: 'empty', values: [] },
{ name: 'single', values: [7.5] },
{ name: 'single negative', values: [-7.5] },
{ name: 'ascending', values: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] },
{ name: 'descending', values: [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] },
{ name: 'duplicates', values: [4, 4, 4, 4, 4] },
// The trap: a sorted last element reads -1 here, the zero-seeded fold reads 0.
{ name: 'all negative', values: [-5, -1, -9, -3, -2] },
{ name: 'mixed signs', values: [-2, 3, -7, 0, 11, -0.5] },
{ name: 'signed zero', values: [0, -0, -0, 0] },
{ name: 'negative then signed zero', values: [-3, -0, -1] },
{ name: 'nan leading', values: [NaN, 5, 1, 9] },
{ name: 'nan trailing', values: [5, 1, 9, NaN] },
{ name: 'nan interleaved', values: [5, NaN, 1, NaN, 9] },
{ name: 'all nan', values: [NaN, NaN, NaN] },
{ name: 'positive infinity', values: [Infinity, 3, 1] },
{ name: 'negative infinity', values: [-Infinity, -3, -1] },
{ name: 'both infinities', values: [Infinity, -Infinity, 3, -Infinity] },
{ name: 'infinities and nan', values: [Infinity, NaN, -Infinity, 0] },
{ name: 'sub-millisecond rounding', values: [0.00049, 0.0005, 0.00051, 0.9995] },
{ name: 'reservoir sized', values: deterministicWindow(CONTROL_RTT_RESERVOIR_LIMIT) }
]
// Holes cannot reach the recorders, so they are exercised through `percentile`
// alone — the surface `host-session-registry` also calls.
const SPARSE_WINDOWS: Array<{ name: string; values: number[] }> = [
{ name: 'all holes', values: sparseWindow(4, {}) },
{ name: 'leading hole', values: sparseWindow(5, { 3: 8, 4: 2 }) },
{ name: 'trailing hole', values: sparseWindow(5, { 0: 8, 1: 2 }) },
{ name: 'interleaved holes', values: sparseWindow(6, { 0: 3, 2: -4, 5: 1 }) },
{ name: 'holes with nan', values: sparseWindow(5, { 1: NaN, 3: 6 }) }
]
const PERCENTILE_RANKS = [0, 0.05, 0.5, 0.9, 0.95, 0.99, 1]
type SortWork = { sorts: number; comparisons: number; copiedElements: number }
// Every sorted array here is a fresh spread copy, so its length is the number of
// elements copied to produce it.
function countSortWork(run: () => void): SortWork {
const work: SortWork = { sorts: 0, comparisons: 0, copiedElements: 0 }
const original = Array.prototype.sort
const patched = Array.prototype as { sort: unknown }
patched.sort = function <T>(this: T[], compare?: (left: T, right: T) => number): T[] {
work.sorts++
work.copiedElements += this.length
return original.call(this, (left: T, right: T) => {
work.comparisons++
return compare ? compare(left, right) : String(left) < String(right) ? -1 : 1
})
}
try {
run()
} finally {
patched.sort = original
}
return work
}
function summaryThroughFlush(samples: number[]): { p50: number; p95: number; max: number } {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (const sample of samples) observability.recordControlRenewal(sample, 'renewed')
observability.flush(counts)
const entry = entries[0]!
return {
p50: entry.controlRenewalLatencyMsP50 as number,
p95: entry.controlRenewalLatencyMsP95 as number,
max: entry.controlRenewalLatencyMsMax as number
}
}
describe('latency window summarisation', () => {
it('matches the pre-change percentile on every edge-case window', () => {
let compared = 0
for (const { name, values } of [...DENSE_WINDOWS, ...SPARSE_WINDOWS]) {
for (const rank of PERCENTILE_RANKS) {
expectSameNumber(
percentile(values, rank),
legacyPercentile(values, rank),
`${name} @ p${rank}`
)
compared++
}
}
expect(compared).toBe((DENSE_WINDOWS.length + SPARSE_WINDOWS.length) * PERCENTILE_RANKS.length)
})
it('matches the pre-change p50, p95 and maximum through a flush', () => {
let compared = 0
for (const { name, values } of DENSE_WINDOWS) {
const actual = summaryThroughFlush(values)
const expected = legacyLatencySummary(values)
expectSameNumber(actual.p50, expected.p50, `${name} p50`)
expectSameNumber(actual.p95, expected.p95, `${name} p95`)
// The zero-seeded fold, not the sorted last element: all-negative and NaN
// windows disagree between the two.
expectSameNumber(actual.max, expected.max, `${name} max`)
compared += 3
}
expect(compared).toBe(DENSE_WINDOWS.length * 3)
// The trap, spelled out: the sorted window ends at -1 but the fold reports 0.
expect(summaryThroughFlush([-5, -1, -9, -3, -2]).max).toBe(0)
expect(Number.isNaN(summaryThroughFlush([5, NaN, 1]).max)).toBe(true)
})
it('sorts each latency window once instead of once per rank', () => {
const samples = deterministicWindow(CONTROL_RTT_RESERVOIR_LIMIT)
const before = countSortWork(() => legacyLatencySummary(samples))
const after = countSortWork(() => summaryThroughFlush(samples))
expect(before.sorts).toBe(2)
expect(after.sorts).toBe(1)
expect(before.copiedElements).toBe(2 * CONTROL_RTT_RESERVOIR_LIMIT)
expect(after.copiedElements).toBe(CONTROL_RTT_RESERVOIR_LIMIT)
// Identical input and comparator, so the dropped sort is exactly half the
// comparator calls rather than an engine-specific constant.
expect(before.comparisons).toBeGreaterThan(CONTROL_RTT_RESERVOIR_LIMIT)
expect(after.comparisons).toBe(before.comparisons / 2)
})
it('never sorts an empty window and leaves the caller window untouched', () => {
const samples = [5, -1, NaN, 3, -0]
const before = samples.map(describeNumber)
expect(countSortWork(() => summaryThroughFlush([])).sorts).toBe(0)
expect(countSortWork(() => percentile([], 0.95)).sorts).toBe(0)
countSortWork(() => summaryThroughFlush(samples))
percentile(samples, 0.5)
expect(samples.map(describeNumber)).toEqual(before)
})
})
cloud/apps/relay/src/relay-observability.ts
+17 -5
View File
@@ -168,10 +168,18 @@ const emptyDeltas = (): RelayMetricDeltas => ({
controlActivityRecoveryFailures: 0
})
function ascending(values: number[]): number[] {
return [...values].sort((left, right) => left - right)
}
// Holes and NaN land past the requested rank, so the fallback still applies.
function nearestRank(sorted: number[], percentileRank: number): number {
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
export function percentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
return nearestRank(ascending(values), percentileRank)
}
function roundMs(value: number): number {
@@ -179,11 +187,15 @@ function roundMs(value: number): number {
}
// Spreading a window into Math.max blows the stack once a busy cell samples
// enough of it, so the maximum is folded instead.
// enough of it, so the maximum is folded instead. The fold is also not
// interchangeable with the sorted last element: it is seeded with zero, so an
// all-negative or NaN window reads differently.
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
// One sorted copy serves both ranks.
const sorted = samples.length === 0 ? samples : ascending(samples)
return {
p50: roundMs(percentile(samples, 0.5)),
p95: roundMs(percentile(samples, 0.95)),
p50: roundMs(nearestRank(sorted, 0.5)),
p95: roundMs(nearestRank(sorted, 0.95)),
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
cloud/apps/relay/src/relay-region-app.test.ts
+155
View File
@@ -40,6 +40,7 @@ describe('Relay region API', () => {
)
expect(response.status).toBe(200)
expect(await response.clone().json()).not.toHaveProperty('regionCorrection')
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'asiahost00000001' },
'asia-east2',
@@ -83,6 +84,160 @@ describe('Relay region API', () => {
)
})
it('preserves the cold-start hint and binds a negotiated window after placement', async () => {
const assignment = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn(async () => assignment)
const window = {
generation: 2,
expiresAt: Date.now() + 86_400_000,
assignmentEpoch: 7,
incumbentRegion: 'asia-east2',
policyVersion: 1
}
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = { v: 1, action: 'issue-window' }
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
'asia-east2',
'asia-east2'
)
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({
v: 1,
window
})
})
it('returns successful placement when optional window storage is unavailable', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: async () => ({
cellId: 'asia-c1',
region: 'asia-east2',
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
}),
exchangeRegionCorrection: async () => {
throw new Error('database unavailable')
}
} as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection: { v: 1, action: 'issue-window' }
})
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
})
})
it('does not place or write a legacy hint when reporting migration evidence', async () => {
const current = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn()
const resolve = vi.fn(async () => current)
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'us-central1',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7)
})
it('does not manufacture an assignment for a report whose assignment disappeared', async () => {
const assign = vi.fn()
const exchangeRegionCorrection = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
regionCorrection: {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'timeout'
}
})
)
expect(response.status).toBe(409)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).not.toHaveBeenCalled()
})
it('exposes only the store-provided healthy catalog from directors', async () => {
const regionCatalog = vi.fn(async () => [
{ region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] }
cloud/apps/relay/src/relay-server.ts
+25 -8
View File
@@ -20,14 +20,12 @@ import { createRelayApp } from './app.js'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { RelayCredentialStore } from './credential-store.js'
import type { RelayDatabase } from './database.js'
import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js'
import { HostSessionRegistry } from './host-session-registry.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import { RelayObservability } from './relay-observability.js'
import {
RelayConnectionLedger,
type RelayConnectionUpgrade
} from './relay-connection-ledger.js'
import { combineRegionalRehomeSafety } from './regional-rehome-safety.js'
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js'
import { createRelayReadiness } from './relay-readiness.js'
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
@@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void {
function admissionSource(request: IncomingMessage): string {
const forwarded = request.headers['x-forwarded-for']
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '')
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
.split(',')
.map((entry) => entry.trim())
.filter(Boolean)
@@ -112,6 +110,7 @@ export function createRelayServer(
const store = new RelayCredentialStore(observedDatabase, options.now)
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
requireLiveCells: config.role === 'director',
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
recordControlRenewal: (durationMs, outcome) =>
observability.recordControlRenewal?.(durationMs, outcome)
})
@@ -127,17 +126,35 @@ export function createRelayServer(
queuedBytes,
observability,
options.now,
options.random
options.random,
cellIncarnation
)
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
return Promise.resolve({ outcome: 'deferred' })
}
return sessions.idleRehome(input,
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
input.directorSafety,
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
), input.cohortPercent),
() => assignments.reconcileIdleRegionalRehome(input)
)
},
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
cellIncarnation,
isDraining: () => sessions.isDraining(),
runtimeCounts: () => runtimeCounts(),
regionalRehomeSafetySnapshot: () => ({
...observability.regionalRehomeRuntimeSafety(),
...readRelayDatabasePoolPressure(database)
}),
ready,
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
recordAssignmentRejectionReason: (lane, reason) =>
@@ -339,7 +356,7 @@ export function createRelayServer(
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
// Released combined-service invites gain their first durable cell assignment here.
const assignment = identity
? (await assignments.resolve(identity)) ?? (await assignments.assign(identity))
? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity)))
: null
if (!invite || !assignment) {
phoneAdmission?.hostData.release()
cloud/apps/relay/src/relay-sweep-schedule.test.ts
+3 -3
View File
@@ -19,7 +19,7 @@ describe('sweep schedule jitter', () => {
expect(SWEEP_JITTER_FRACTION).toBeGreaterThan(0)
})
it('jitters the regional rehome dispatch tick, which every director runs each second', () => {
it('jitters the six-second regional rehome dispatch tick across directors', () => {
const timers: number[] = []
const setIntervalSpy = vi
.spyOn(globalThis, 'setInterval')
@@ -35,14 +35,14 @@ describe('sweep schedule jitter', () => {
rehomeAudience: 'https://rehome.example.test',
rehomeDirectorServiceAccount: 'rehome@example.test'
} as never,
{ claimRegionalRehome: async () => null } as never,
{ selectIdleRegionalRehomeCandidates: async () => [] } as never,
{ random: () => 0.5, safetySnapshot: () => ({}) as never }
)
} finally {
setIntervalSpy.mockRestore()
}
expect(timers).toEqual([1_100])
expect(timers).toEqual([6_600])
})
// Why: index.ts boots a server on import, so its wiring can only be read.
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md
+8
View File
@@ -0,0 +1,8 @@
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
```text
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
```
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts
+146
View File
@@ -0,0 +1,146 @@
import { z } from 'zod'
import {
Base6432ByteSchema,
Base64Raw24ByteSchema,
Base64Url32ByteSchema,
EpochMsSchema,
GenerationSchema,
OpaqueIdSchema,
PositiveDurationMsSchema,
RelayHostIdSchema
} from './wire-scalars.js'
const AppVersionSchema = z.string().min(1).max(128)
const BoundedCiphertextSchema = z
.string()
.min(1)
.max(16 * 1024)
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
const ConnectionKindSchema = z.enum(['invite', 'resume'])
export const HostHelloSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
assignmentEpoch: GenerationSchema,
hostPublicKeyB64: Base6432ByteSchema,
appVersion: AppVersionSchema,
previousGeneration: GenerationSchema.optional(),
controlResumeSecret: Base64Url32ByteSchema.optional()
})
.strict()
export const HostChallengeSchema = z
.object({
challengeId: OpaqueIdSchema,
relayEphemeralPublicKeyB64: Base6432ByteSchema,
nonceB64: Base64Raw24ByteSchema,
ciphertextB64: BoundedCiphertextSchema,
expiresAt: EpochMsSchema
})
.strict()
export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
.object({
v: z.literal(1),
generation: GenerationSchema,
controlResumeSecret: Base64Url32ByteSchema,
leaseExpiresAt: EpochMsSchema,
activeConnIds: z.array(OpaqueIdSchema).max(8),
pendingConns: z.array(PendingConnectionSchema).max(8)
})
.strict()
export const ConnectionOpenSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema,
relayDeviceId: OpaqueIdSchema,
attachDeadlineMs: PositiveDurationMsSchema
})
.strict()
export const HostDataAuthSchema = z
.object({
v: z.literal(1),
connTicket: Base64Url32ByteSchema,
generation: GenerationSchema
})
.strict()
export const InviteCreateSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const InviteCreatedSchema = z
.object({
reqId: OpaqueIdSchema,
inviteToken: Base64Url32ByteSchema,
expiresAt: EpochMsSchema,
maxAttempts: z.number().int().positive().max(16)
})
.strict()
export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
export type HostHello = z.infer<typeof HostHelloSchema>
export type HostChallenge = z.infer<typeof HostChallengeSchema>
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
export type InviteCreate = z.infer<typeof InviteCreateSchema>
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
export type Drain = z.infer<typeof DrainSchema>
export type Heartbeat = z.infer<typeof HeartbeatSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts
+75
View File
@@ -0,0 +1,75 @@
import { z } from 'zod'
import {
Base64Url32ByteSchema,
CanonicalHttpsOriginSchema,
EpochMsSchema,
GenerationSchema,
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
})
.strict()
export const AssignmentResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
})
.strict()
export const ResolveRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
resumeToken: Base64Url32ByteSchema
})
.strict()
export const ResolveResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
leaseExpiresAt: EpochMsSchema
})
.strict()
export const RelayMovedSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema
})
.strict()
export function isTrustedNewerMove(input: {
sourceOrigin: string
configuredDirectorOrigin: string
currentAssignmentEpoch: number
move: z.infer<typeof RelayMovedSchema>
}): boolean {
// Why: cells and stale director responses must never redirect a credential-bearing client.
return (
input.sourceOrigin === input.configuredDirectorOrigin &&
input.move.assignmentEpoch > input.currentAssignmentEpoch
)
}
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
export type RelayMoved = z.infer<typeof RelayMovedSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts
+71
View File
@@ -0,0 +1,71 @@
import { z } from 'zod'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
function isCanonicalHttpsOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts
+20
View File
@@ -0,0 +1,20 @@
import { z } from 'zod'
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
export const OpaqueIdSchema = z.string().min(1).max(128)
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
} catch {
return false
}
}, 'must be a canonical HTTPS origin')
cloud/apps/relay/tsconfig.build.json
+1 -1
View File
@@ -6,5 +6,5 @@
"outDir": "dist",
"rootDir": "src"
},
"exclude": ["src/**/*.test.ts"]
"exclude": ["src/**/*.test.ts", "src/test-fixtures/**"]
}
cloud/dev/scripts/deploy-relay-blue-green.mjs
+22 -1
View File
@@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
'orca-cloud-relay-regional-placement-enabled'
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
export const DIRECTOR_REHOME_IDENTITY_ENV =
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
@@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) {
return { changed: additions.length > 0, value: JSON.stringify(desired) }
}
export function correctionCohortPercent(value) {
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
throw new Error('region correction cohort must be an integer from 0 to 100')
}
return String(value)
}
export function directorDeploymentEnvironment(config) {
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
if (config.image !== undefined && imageDigest === undefined) {
@@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) {
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
}
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve') {
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
}
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
const rehomeDirectorServiceAccount = projectServiceAccount(
@@ -302,7 +314,8 @@ export function parseArguments(argv) {
values['rehome-director-service-account'] !== undefined ||
values['rehome-audience'] !== undefined ||
values['expected-rehome-generation'] !== undefined ||
values['rehome-control-origin'] !== undefined
values['rehome-control-origin'] !== undefined ||
values['region-correction-cohort-percent'] !== undefined
) {
throw new Error('director configuration arguments require --role director')
}
@@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) {
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
const currentEnvironment = revisionEnvironment(servingRevision)
const deploymentEnvironment = directorDeploymentEnvironment(config)
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
)
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve' &&
config['expected-rehome-generation'] === undefined) {
throw new Error('cohort changes require an exact disabled regional-rehome generation')
}
const mutableEnvironment = {
...deploymentEnvironment,
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
cloud/dev/scripts/deploy-relay-blue-green.test.mjs
+42
View File
@@ -4,6 +4,8 @@ import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import {
activeRevision,
correctionCohortPercent,
DIRECTOR_CORRECTION_COHORT_ENV,
cloudRunTrafficTag,
DIRECTOR_ADMISSION_ENVIRONMENT,
DIRECTOR_REGIONAL_PLACEMENT_ENV,
@@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err
/forbidden/
)
})
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
}
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
const harness = directorHarness()
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
await deployDirector({}, 'candidate-new', harness.operations)
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
}
})
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
const harness = directorHarness()
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
assert.equal(harness.state.nextRevision, 2)
await deployDirector({}, 'candidate-new', harness.operations)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
})
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
const harness = directorHarness()
let verified = 0
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
await deployDirector(config, 'candidate-new', { ...harness.operations,
assertRegionalRehomeDisabled: async () => { verified++ } })
assert.ok(verified >= 2)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
cloud/dev/scripts/probe-relay-rehome-trust.mjs
+20 -3
View File
@@ -38,7 +38,7 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env)
export async function probeRehomeTrust(config, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const response = await fetchAdminOnceMore(
const request = () => fetchAdminOnceMore(
fetchImpl,
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
{
@@ -55,9 +55,26 @@ export async function probeRehomeTrust(config, dependencies = {}) {
},
{ wait: dependencies.wait }
)
const body = await response.json().catch(() => ({}))
let response = await request()
let body = await response.json().catch(() => ({}))
// The director wraps source HTTP failures in 409; retry only explicit transient statuses.
if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) {
await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000)
response = await request()
body = await response.json().catch(() => ({}))
}
if (!response.ok) {
throw new Error(`application-mediated rehome trust probe returned ${response.status}`)
const safeReasons = new Set([
'invalid_token', 'director_only', 'invalid_request',
'regional_rehome_trust_not_configured',
'regional_rehome_trust_probe_source_unavailable',
'regional_rehome_trust_probe_source_invalid_response',
'regional_rehome_trust_probe_not_proven',
...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504]
.map((status) => `regional_rehome_trust_probe_source_${status}`)
])
const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error'
throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`)
}
if (
body.v !== 1 ||
cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
+37
View File
@@ -131,3 +131,40 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells',
)
}
})
test('retries one director-wrapped source 503 without relaxing the proof', async () => {
let calls = 0
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => ++calls === 1
? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
: Response.json(provenProbe)
})
assert.equal(calls, 2)
assert.equal(result.proven, true)
})
test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => {
for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => { throw new Error('must not retry') },
fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) }
}), error => {
assert.match(error.message, /returned 409/)
assert.ok(!error.message.includes('secret-token-example'))
if (reason.endsWith('_403')) assert.match(error.message, /source_403/)
return true
})
assert.equal(calls, 1)
}
})
test('stops after the second wrapped transient failure', async () => {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
}), /returned 409.*source_503/)
assert.equal(calls, 2)
})
cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs
+13 -3
View File
@@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
throw error
}
const serving = (service.status?.traffic ?? []).filter(
@@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) =>
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
)
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
(typeof cohortSettings[0].value !== 'string' ||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
throw new Error('serving region correction cohort is invalid')
}
const cohort_percent = cohortSettings[0]?.value ?? '0'
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version }
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
@@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version }
return { version: reference.version, cohort_percent }
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs
+32 -4
View File
@@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
}
})
assert.deepEqual(result, { version: '11' })
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
assert.equal(calls[1][3], 'relay-serving')
})
@@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
]) {
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
}), { version: '1' })
}), { version: '1', cohort_percent: '0' })
}
})
@@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => {
notFound.code = 'NOT_FOUND'
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: () => { throw notFound }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: { spec: { containers: [{ env: [] }] } }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
})
test('classifies real absent-service stderr without weakening revision failures', () => {
@@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () =>
run: () => { throw denied }
}), denied)
})
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
for (const value of ['0', '1', '17', '100']) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), { version: '11', cohort_percent: value })
}
})
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
const cases = [
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
[{ name, value: '1' }, { name, value: '2' }]
]
for (const settings of cases) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push(...settings)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), /cohort is invalid/)
}
})
cloud/dev/scripts/relay-production-same-cap-wave.mjs
+6 -3
View File
@@ -87,18 +87,21 @@ export function canaryAuthority(input) {
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
authority.selectorGeneration !== Number(expected.selectorGeneration) ||
!Number.isSafeInteger(authority.selectorGeneration) ||
authority.selectorGeneration < 0 ||
!Number.isSafeInteger(selectorGeneration) ||
selectorGeneration < authority.selectorGeneration ||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
// same SHA; every field above still pins this batch to that exact canary.
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
currentSha: expected.commitSha,
cloud/dev/scripts/relay-production-same-cap-wave.test.mjs
+36 -1
View File
@@ -109,6 +109,41 @@ test('seals and verifies canary authority for later batches', () => {
}), /does not match/)
})
test('reuses a canary across selector advances only within the same control epoch', () => {
const authority = canaryAuthority({
cellIds: 'production-gce-c7', targetDigest, rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), authority)
}
for (const generation of ['12', '-1', 'NaN', 'Infinity', '13.5', '9007199254740992']) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), /does not match/)
}
for (const generation of [-1, NaN, Infinity, 13.5, '13', Number.MAX_SAFE_INTEGER + 1]) {
assert.throws(() => verifyCanaryAuthority({
...authority, selectorGeneration: generation
}, expected), /does not match/)
}
for (const mismatch of [
{ rehomeGeneration: '3' }, { rehomeGeneration: '5' },
{ targetDigest: rollbackDigest }, { rollbackDigest: targetDigest }, { runId: '43' }
]) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, ...mismatch
}), /does not match/)
}
})
function gitIn(root, ...args) {
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
}
@@ -158,7 +193,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
selectorGeneration: '21',
rehomeGeneration: '4'
}, repositoryRoot)
assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+13 -7
View File
@@ -77,14 +77,14 @@ function rollPlan({ cellId, cap, protocol }) {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
}),
self_link: null
},
@@ -178,11 +178,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -204,7 +202,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.throws(
() => validateCapacityPlan(plan, {
...config,
regionalRehomeProtocol: String(1 - protocol)
regionalRehomeProtocol: '0'
}),
/reviewed image and capacity/,
cellId
@@ -239,3 +237,11 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
})
// Both trusted versions must prove the same authenticated drain boundary.
it('proves rehome trust for protocol 3 on forward and rollback rolls', () => {
const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0]
assert.match(step, /inputs\.rollback-rehome-protocol != '0'/)
assert.match(step, /inputs\.target-rehome-protocol != '0'/)
assert.match(step, /probe-relay-rehome-trust\.mjs/)
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+3 -3
View File
@@ -9,7 +9,7 @@ const REHOME_CONFIG =
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) {
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
}
return Number(regionalRehomeProtocol)
@@ -43,7 +43,7 @@ export function parseCapacityPlanArguments(argv) {
(!values['rollback-image'] ||
!values['rehome-director-service-account'] ||
!values['rehome-audience'] ||
!['0', '1'].includes(values['regional-rehome-protocol']))
!['0', '1', '3'].includes(values['regional-rehome-protocol']))
) throw new Error('same-cap validation requires rollback image and rehome trust config')
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
@@ -227,7 +227,7 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
])
}
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1
if (rehomeTrusted) {
expected.push(
[

Some files were not shown because too many files have changed in this diff Show More