starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-26 16:02:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into brennanb2025/orphan-chat-tab-cleanup

Browse Source
This commit is contained in:
Merge Sim
2026-09-13 13:31:34 -07:00
parent e192b623c4 c6548b98f4
commit 77058d656d
1203 changed files with 74278 additions and 14306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+4
View File
@@ -31,6 +31,10 @@
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
.github/CONTRIBUTING.md
+5
View File
@@ -21,6 +21,11 @@ pnpm install
pnpm dev
```
Ordinary installs include native optional dependencies for the current OS and CPU only.
Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and
arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants.
See [the install policy](../docs/reference/pnpm-install-policy.md).
## Branch Naming
Use a clear, descriptive branch name that reflects the change.
.github/actions/install-node-dependencies/action.yml
+13 -15
View File
@@ -10,6 +10,10 @@ inputs:
description: Node.js version override; defaults to the version declared in package.json.
required: false
default: ''
cache-dependency-path:
description: Lockfiles for the pnpm download store; include mobile/pnpm-lock.yaml only when the job installs mobile dependencies.
required: false
default: pnpm-lock.yaml
persist-native-cache:
description: Save restored native modules at job end. Set false when a later step overwrites the same path with a different ABI.
required: false
@@ -39,9 +43,7 @@ runs:
with:
install: false
# Why both lockfiles: setup-node keys the pnpm store on the root lockfile alone, so
# jobs that also install mobile restored a store with none of the React Native tree
# in it and re-downloaded the lot on every run.
# Desktop-only jobs should not miss their download cache when mobile dependencies change.
- name: Setup Node.js
id: default-node
if: inputs.node-version == ''
@@ -49,9 +51,7 @@ runs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Setup requested Node.js
id: requested-node
@@ -60,9 +60,7 @@ runs:
with:
node-version: ${{ inputs.node-version }}
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Validate native runtime
shell: bash
@@ -152,9 +150,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Restore compiled native modules without saving
id: native-cache-restore-only
@@ -163,9 +161,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
.github/workflows/adhoc-mac-build.yml
+5 -5
View File
@@ -160,16 +160,14 @@ jobs:
- name: Checkout the requested ref
uses: actions/checkout@v6
env:
# Full-history checkout must also preserve case-twin branch and tag names.
GIT_DEFAULT_REF_FORMAT: reftable
with:
# Why an input at all rather than just github.ref: the whole point is to
# build code that has not landed, and the workflow definition itself
# always comes from the dispatch ref — naming the branch here instead
# applies main's current copy of this file to an arbitrary branch.
ref: ${{ steps.vetted.outputs.sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# This job only reads stablyai/orca and never pushes; every write goes
# to the adhoc repo through a minted App token passed by env. Not
# persisting the checkout credential shrinks the blast radius if a build
@@ -197,13 +195,15 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
.github/workflows/cloud-deploy-relay-production-director.yml
+12 -2
View File
@@ -13,6 +13,11 @@ on:
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
@@ -62,6 +67,7 @@ jobs:
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
@@ -106,8 +112,11 @@ jobs:
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
@@ -218,7 +227,8 @@ jobs:
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}"
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+3 -3
View File
@@ -67,8 +67,8 @@ jobs:
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
@@ -599,7 +599,7 @@ jobs:
| jq -e '.control.enabled == false' >/dev/null
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+3 -3
View File
@@ -26,13 +26,13 @@ on:
required: true
default: '1'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
@@ -62,7 +62,7 @@ on:
required: false
type: string
canary-run-id:
description: Successful same-commit canary run required for batch-apply
description: Successful same-code canary in this rehome control generation; reusable across batches
required: false
type: string
confirmation:
.github/workflows/computer-e2e.yml
+9
View File
@@ -6,6 +6,9 @@ on:
- '.github/workflows/computer-e2e.yml'
- 'config/electron-builder.config.cjs'
- 'config/scripts/build-computer-macos.mjs'
- 'config/scripts/build-native-for-platform.mjs'
- 'config/scripts/build-native-for-platform.test.mjs'
- 'config/scripts/pnpm-cli-invocation.mjs'
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/build-windows-cli-launcher.test.mjs'
- 'config/scripts/computer-e2e-workflow.test.mjs'
@@ -157,6 +160,12 @@ jobs:
pnpm vitest run
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
# Why: the parallel launcher's cancellation tests are darwin-only and no
# other PR job runs on macOS.
- name: Parallel native build launcher cancellation
run: >-
pnpm vitest run --config config/vitest.config.ts
config/scripts/build-native-for-platform.test.mjs
- name: Authenticated helper owner-loss smoke
run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1
- name: Swift tests and signed universal helper verification
.github/workflows/daily-mac-build.yml
+5 -2
View File
@@ -90,7 +90,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the daily repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -167,6 +168,8 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
@@ -174,7 +177,7 @@ jobs:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
.github/workflows/dev-channel-win-build.yml
+2
View File
@@ -219,6 +219,8 @@ jobs:
# Why retried: pnpm install triggers electron's postinstall, which pulls the
# Electron binary from GitHub release assets, and that CDN returns transient
# 504s often enough to lose a build to it.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
.github/workflows/e2e.yml
+32 -1
View File
@@ -170,8 +170,39 @@ jobs:
# artifact instead of starting five concurrent electron-vite builds.
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
# launches but never creates a BrowserWindow.
- name: Balance E2E shard from timing evidence
env:
ORCA_BACKGROUND_LAUNCH: '1'
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
ORCA_E2E_WEB_CLIENT: '1'
ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay
run: |
mkdir -p ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --list --reporter=json > ci-shards/discovery.json
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
node config/scripts/ci-e2e-shard-plan.mjs ci-shards/discovery.json '${{ matrix.shard }}' ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --test-list=ci-shards/selected.txt --list --reporter=json > ci-shards/selected-discovery.json
node config/scripts/ci-e2e-shard-plan.mjs --verify ci-shards/assignment.json ci-shards/selected-discovery.json
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --test-list=ci-shards/selected.txt
- name: Upload E2E shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: e2e-shard-${{ matrix.shard_name }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
# The frame benchmark needs a mapped window, which the headless shards exclude.
- name: Run worktree first-paint benchmark
if: matrix.shard == '1/14'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
.github/workflows/hourly-mac-build.yml
+5 -2
View File
@@ -137,7 +137,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: ${{ needs.preflight.outputs.head_sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -174,13 +175,15 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
.github/workflows/mobile-ios-release.yml
+2
View File
@@ -94,6 +94,8 @@ jobs:
run: node -e 'const fs = require("node:fs"); const { expo } = require("./app.json"); fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${expo.version}\nbuild_number=${expo.ios.buildNumber}\n`)'
- name: Expo prebuild
env:
ORCA_IOS_APS_ENVIRONMENT: production
run: npx expo prebuild --platform ios --no-install
- name: Install CocoaPods
.github/workflows/mobile.yml
+4
View File
@@ -41,6 +41,10 @@ jobs:
uses: actions/checkout@v6
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
.github/workflows/pi-owner-runtime.yml
+29
View File
@@ -0,0 +1,29 @@
name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/agent-status-handler-source.ts'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
.github/workflows/pi-provider-runtime.yml
+28
View File
@@ -0,0 +1,28 @@
name: Pi extension provider verification
on:
pull_request:
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
.github/workflows/pr.yml
+21 -16
View File
@@ -126,6 +126,9 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Lint
run: pnpm exec oxlint --format github
@@ -273,6 +276,12 @@ jobs:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node .github/scripts/check-root-directory-entries.mjs "$BASE_SHA" "$HEAD_SHA"
# Why here: the READMEs embed media owned by docs/site and resources/onboarding,
# and the classifier skips static_analysis for docs-only diffs. This job runs
# on every PR and needs no install.
- name: Check README local links
run: node config/scripts/check-readme-local-links.mjs
typecheck:
needs: [code_paths]
if: needs.code_paths.outputs.typecheck == 'true'
@@ -775,18 +784,9 @@ jobs:
[[ "$rpm_marker" == rpm ]] || { echo "Expected rpm marker, got: $rpm_marker"; exit 1; }
- name: Verify headless serve signal shutdown
run: node config/scripts/run-headless-serve-shutdown-docker.mjs --appimage dist/orca-linux.AppImage
- name: Verify extracted launcher serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint launcher
- name: Verify AppImage CLI registration and serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint appimage
--signal-target serving-electron --int-delivery pid
--appimage dist/orca-linux.AppImage --all-entrypoints
# A default container reproduces the hostile AppImage launch environment.
- name: Verify Linux CLI launch contract
@@ -835,9 +835,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# vitest runs here directly rather than through `pnpm test`, so the addon
# assertions only hold once install-node-dependencies has rebuilt natives.
@@ -846,6 +846,11 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
src/main/windows-registry-addon.test.ts
config/scripts/windows-process-tree-gyp-path.test.mjs
config/scripts/windows-process-tree-gyp-rebuild.test.mjs
config/scripts/package-electron-runtime-contract.test.mjs
config/scripts/electron-builder-runtime-resources.test.mjs
src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts
src/main/browser/browser-route-tcp-egress.electron.test.ts
src/main/browser/browser-route-webrtc-egress.electron.test.ts
@@ -903,9 +908,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Prepare Electron native runtime
run: node config/scripts/ensure-native-runtime.mjs --runtime=electron
.github/workflows/release-cut.yml
+2
View File
@@ -1262,6 +1262,8 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
.github/workflows/release-mac-build.yml
+3 -1
View File
@@ -64,13 +64,15 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
- name: Verify macOS signing environment
run: node config/scripts/verify-macos-release-env.mjs
.github/workflows/unit-tests.yml
+23 -1
View File
@@ -37,8 +37,19 @@ jobs:
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
- name: Test shard
# The real two-cell transport test imports cloud relay source and its contracts.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test shard
env:
ORCA_BALANCE_UNIT_SHARDS: '1'
ORCA_BACKGROUND_LAUNCH: '1'
run: |
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
pnpm exec vitest run --config config/vitest.config.ts \
--exclude=src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts \
--exclude=src/main/daemon/shell-ready.test.ts \
@@ -58,3 +69,14 @@ jobs:
--exclude=src/shared/posix-command-path-lookup.test.ts \
--exclude=tests/e2e/cross-version-wire/** \
--shard=${{ matrix.shard }}/${{ matrix.shard_total }}
- name: Upload unit shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: unit-shard-node-${{ matrix.node }}-${{ matrix.shard }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
.github/workflows/windows-signing-rehearsal.yml
+2
View File
@@ -68,6 +68,8 @@ jobs:
restore-keys: |
electron-builder-win-
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
.gitignore
+7
View File
@@ -28,6 +28,9 @@ out/
/build/
release/
native/**/.build/
# node-gyp output for the vendored Windows registry addon; generated per host and ABI.
native/windows-registry/build/
native/windows-registry/bin/
# pnpm
.pnpm-store/
@@ -103,13 +106,17 @@ docs/**
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-pty-transcript-capture.md
!docs/reference/agent-session-search-query-tuning.md
!docs/reference/agent-status-store.md
!docs/reference/antigravity-readiness-evidence.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
!docs/reference/ime-regression-checklist.md
!docs/reference/linux-glibc-compatibility.md
!docs/reference/macos-press-and-hold.md
!docs/reference/orcad-operations.md
!docs/reference/pnpm-install-policy.md
!docs/reference/relay-grace-time-reconfiguration.md
!docs/reference/windows-cmd-shim-resolution.md
!docs/reference/windows-daemon-host-relocation.md
AGENTS.md
+16
View File
@@ -33,6 +33,14 @@ Never use vague names like `helpers`, `utils`, `common`, `misc`, or `shared-stuf
## Type Declarations: Prefer `.ts` Over `.d.ts`
## Type Assertions: Prefer Checked Types
Avoid type assertions except `as const`. Unavoidable casts need a line-specific `SAFETY:` explanation:
```ts
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: Explain the verified invariant here.
```
# Verifying Changes
- **Typecheck**: `pnpm tc` (or `tc:node` / `tc:cli` / `tc:web`)
@@ -60,6 +68,10 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
- **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.
## Native Dependency Installs
Ordinary `pnpm install` covers the host OS and CPU only. Before packaging for another architecture — including `pnpm build:mac`, which builds x64 and arm64 by default — run `pnpm install:release`. electron-builder only warns on a missing `extraResources` source, so the `beforePack` guard is what turns a thin install into a build failure instead of a silently broken artifact; see [`docs/reference/pnpm-install-policy.md`](./docs/reference/pnpm-install-policy.md).
## SSH Use Case
All changes must consider the SSH use case. Don't assume local-only execution. Before changing anything that reports on, stops, or lists remote work, follow [`docs/reference/ssh-execution-boundary.md`](./docs/reference/ssh-execution-boundary.md): the execution host owns everything that touches execution, and loss of contact is never evidence of process death — the verdict vocabulary is `live` / `unverifiable` / `exited`, with no synonyms.
@@ -72,6 +84,10 @@ All changes must consider folder workspaces as well as git worktrees. Don't assu
The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy.
## Agent Terminal Screens
A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md).
## Remote Wire Compatibility
Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change.
README.md
+8 -8
View File
@@ -54,7 +54,7 @@ Fan one prompt across five agents, each in its own isolated git worktree — com
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/assets/feature-wall/parallel-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/parallel-worktrees.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/site/public/docs/tab-split.gif" type="image/gif"><img src="docs/site/public/docs/posters/tab-split.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -68,7 +68,7 @@ Ghostty-class terminals with WebGL rendering, infinite splits, and scrollback th
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="docs/assets/feature-wall/terminal-splits.gif" type="image/gif"><img src="docs/assets/feature-wall/terminal-splits.jpg" alt="Terminal splits" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="resources/onboarding/feature-wall/tile-02.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-02.poster.jpg" alt="Terminal splits" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -82,7 +82,7 @@ Click any UI element in a real Chromium window to send its HTML, CSS, and a crop
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/assets/feature-wall/design-mode.gif" type="image/gif"><img src="docs/assets/feature-wall/design-mode.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/site/public/docs/orca-design-mode.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-05.poster.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -96,7 +96,7 @@ Browse PRs, issues, and project boards in-app — open a worktree from any task
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="docs/assets/feature-wall/github-linear.gif" type="image/gif"><img src="docs/assets/feature-wall/github-linear.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="resources/onboarding/feature-wall/tile-03.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-03.poster.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -110,7 +110,7 @@ Run agents on a beefy remote box with full file editing, git, and terminals —
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="docs/assets/feature-wall/ssh-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/ssh-worktrees.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="resources/onboarding/feature-wall/tile-06.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-06.poster.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -124,7 +124,7 @@ Drop comments on any diff line and ship them back to the agent — review, edit,
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/assets/feature-wall/annotate-diff.gif" type="image/gif"><img src="docs/assets/feature-wall/annotate-diff.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/site/public/docs/annotate-ai-diff.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-08.poster.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -138,7 +138,7 @@ VS Code's editor with autosave everywhere — drag files or images straight into
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="docs/assets/feature-wall/file-drag.gif" type="image/gif"><img src="docs/assets/feature-wall/file-drag.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="resources/onboarding/feature-wall/tile-07.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-07.poster.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -152,7 +152,7 @@ Agents drive Orca too — script every workflow with `orca worktree create`, `sn
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="docs/assets/feature-wall/orca-cli.gif" type="image/gif"><img src="docs/assets/feature-wall/orca-cli.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="resources/onboarding/feature-wall/tile-09.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-09.poster.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
</td>
</tr>
</table>
cloud/apps/push/src/fcm-client.test.ts
+13 -24
View File
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
const NOW = 1_700_000_000_000
@@ -20,6 +20,7 @@ function delivery(agentState: 'needs-input' | null = 'needs-input') {
agentState,
title: 'Agent needs input',
body: 'Waiting on your answer',
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1'
}
})
@@ -59,27 +60,14 @@ describe('fcm client', () => {
expect(JSON.parse(request.body)).toEqual({
message: {
token: TOKEN,
notification: { title: 'Agent needs input', body: 'Waiting on your answer' },
android: {
priority: 'HIGH',
ttl: '300s',
collapse_key: createHash('sha256')
.update(
createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
)
.digest('hex')
.slice(0, 32),
notification: {
channel_id: 'orca-desktop',
tag: createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
}
},
android: { priority: 'HIGH', ttl: '300s' },
data: {
title: 'Agent needs input',
message: 'Waiting on your answer',
tag: delivery().collapseId,
channelId: 'orca-desktop',
hostFingerprint: HOST,
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: '7',
@@ -96,7 +84,7 @@ describe('fcm client', () => {
await fcm.send(delivery(null), { token: TOKEN })
const message = JSON.parse(fake.requests[0]!.body) as {
message: {
android: { collapse_key: string; notification: { tag: string } }
android: Record<string, unknown>
data: Record<string, string>
}
}
@@ -108,9 +96,10 @@ describe('fcm client', () => {
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
expect(message.message.data.coalescedCount).toBeUndefined()
expect(message.message.android.notification.tag).toBe(tag)
expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag))
expect(message.message.android.collapse_key).toHaveLength(32)
expect(message.message.data.tag).toBe(tag)
expect(message.message.android).not.toHaveProperty('collapse_key')
expect(message.message).not.toHaveProperty('notification')
expect(message.message.data).not.toHaveProperty('body')
})
it('marks an unregistered token dead from the status or the error detail', async () => {
cloud/apps/push/src/fcm-client.ts
+11 -19
View File
@@ -1,5 +1,4 @@
import { providerRetryAfter } from './provider-retry-delay.js'
import { createHash } from 'node:crypto'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
@@ -22,12 +21,6 @@ type FcmErrorBody = {
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
}
// FCM collapse_key is a short opaque string, so the collapse id is hashed
// rather than truncated: truncation would merge unrelated notifications.
export function fcmCollapseKey(collapseId: string): string {
return createHash('sha256').update(collapseId).digest('hex').slice(0, 32)
}
export function fcmMessageBody(input: {
delivery: PushDelivery
token: string
@@ -39,24 +32,23 @@ export function fcmMessageBody(input: {
return JSON.stringify({
message: {
token: input.token,
...(delivery.orca.kind === 'dismiss'
? {}
: { notification: { title: delivery.title, body: delivery.body } }),
android: {
priority: 'HIGH',
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`,
collapse_key: fcmCollapseKey(delivery.collapseId),
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`
},
// Notification payloads collapse offline; Expo renders these data messages natively.
data: {
...orcaDataStrings(delivery.orca),
...(delivery.orca.kind === 'dismiss'
? {}
: {
notification: {
channel_id:
delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
tag: delivery.collapseId
}
title: delivery.title,
message: delivery.body,
tag: delivery.collapseId,
channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
...(delivery.sound === false ? { sound: '' } : {})
})
},
data: orcaDataStrings(delivery.orca)
}
}
})
}
cloud/apps/push/src/push-delivery-message.ts
+2
View File
@@ -5,6 +5,7 @@ export type PushOrcaData = {
kind?: 'alert' | 'dismiss'
hostFingerprint: string
worktreeId?: string
paneKey?: string
notificationId?: string
notificationSeq: number
notificationEpoch: string
@@ -51,6 +52,7 @@ export function buildPushDelivery(input: {
orca: {
...(notification.kind ? { kind: notification.kind } : {}),
hostFingerprint,
...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }),
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
...(notification.notificationId === undefined
? {}
cloud/apps/push/src/push-dismissal-provider.test.ts
+4
View File
@@ -23,5 +23,9 @@ it('dismissal provider payloads cannot display a new alert or play a sound', ()
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
expect(android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('collapse_key')
expect(android.data).not.toHaveProperty('title')
expect(android.data).not.toHaveProperty('message')
expect(android.data).not.toHaveProperty('sound')
expect(android.data.kind).toBe('dismiss')
})
cloud/apps/push/src/push-notification-sound.test.ts
+5 -1
View File
@@ -23,7 +23,11 @@ it('carries a silent preference through validation to APNs and Android payloads'
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.android.notification.channel_id
.data.channelId
).toBe('orca-desktop-silent')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.sound
).toBe('')
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
})
cloud/apps/push/src/push-pane-routing.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { expect, it } from 'vitest'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js'
it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => {
const base = {
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
worktreeId: 'folder:/work'
}
const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111'
for (const extra of [{}, { paneKey }]) {
const notification = PushNotificationSchema.parse({ ...base, ...extra })
const delivery = buildPushDelivery({
notification,
hostFingerprint: 'host',
registrationId: 'phone',
expiresAt: Date.now() + 300000
})
expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined)
expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined)
}
})
cloud/apps/push/src/push-server-send.test.ts
+2 -2
View File
@@ -56,7 +56,7 @@ describe('push gateway send route', () => {
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } }
message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } }
})
const afterDeath = await harness.post(
@@ -179,7 +179,7 @@ describe('push gateway send route', () => {
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
message: { android: { notification: { tag: string } }; data: Record<string, string> }
}
expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.coalescedCount).toBeUndefined()
})
cloud/apps/relay-ops/src/incident-monitor.test.ts
+20
View File
@@ -273,6 +273,26 @@ describe('incident monitor evaluator', () => {
)
})
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
for (const errors of [1, 2, 3]) {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
}
const excess = healthySample()
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
)
const auth = healthySample()
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
expect(evaluateIncidentSample(auth, startedAt).status).toBe('freeze')
const pressure = healthySample()
pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1)
pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze')
})
it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
cloud/apps/relay-ops/src/incident-monitor.ts
+2 -1
View File
@@ -101,7 +101,8 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
directorCpuUtilization: 0.8,
directorMemoryUtilization: 0.8,
directorConcurrency: 64,
directorErrors: 0,
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
directorErrors: 3,
authErrors: 0,
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
cloud/apps/relay/src/admin-token-verifier.ts
+1
View File
@@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
'/v1/admin/cell-status',
'/v1/admin/evacuation-status',
'/v1/admin/regional-rehome-control',
'/v1/admin/regional-rehome-preview',
'/v1/admin/runtime-status'
] as const
cloud/apps/relay/src/app.ts
+137 -51
View File
@@ -1,5 +1,9 @@
import {
AssignmentRequestSchema,
IdleRegionalRehomeRequestSchema,
type IdleRegionalRehomeRequest,
type IdleRegionalRehomeOutcome,
type RegionCorrectionResponse,
isRelayCellConnectionHardCap,
RELAY_ADMISSION_BUDGETS,
RELAY_DEFAULT_REGION,
@@ -39,7 +43,7 @@ import {
type AssignmentAdmissionRejection
} from './public-assignment-admission.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeCounts } from './relay-observability.js'
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
import {
isRegionalRehomeTrustProbe,
probeRegionalRehomeTrust
@@ -68,21 +72,28 @@ export function createRelayApp(
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
drainHost?: (input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceCellIncarnation: string
graceMs: number
}) => 'accepted' | 'already-accepted' | 'host-not-connected'
}) =>
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
regionalRehomeFetch?: typeof fetch
regionalRehomeTrustProbeHostExists?: (input: {
userId: string
relayHostId: string
}) => boolean
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
cellIncarnation?: string
isDraining?: () => boolean
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
recordAssignmentAdmission?: (
@@ -226,7 +237,8 @@ export function createRelayApp(
return context.json({ error: 'host_identity_mismatch' }, 403)
}
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
const requestedRegion = body.data.preferredRegion
const requestedRegion =
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
const targetRegion =
config.regionalPlacementEnabled !== false && requestedRegion
? requestedRegion
@@ -295,10 +307,30 @@ export function createRelayApp(
}
}
let assignment: RelayAssignment
let regionCorrection: RegionCorrectionResponse | undefined
try {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
if (body.data.regionCorrection?.action === 'report') {
const current = await operations.assignments.resolve(identity)
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
assignment = current
} else {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
}
if (body.data.regionCorrection) {
try {
regionCorrection = await operations.assignments.exchangeRegionCorrection(
identity,
body.data.regionCorrection,
assignment.assignmentEpoch
)
} catch (error) {
if (body.data.regionCorrection.action === 'report') throw error
// Optional measurement setup must not discard an otherwise valid placement.
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
@@ -353,13 +385,16 @@ export function createRelayApp(
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
lease
lease,
...(regionCorrection ? { regionCorrection } : {})
})
})
app.post('/v1/resolve', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
if (
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
@@ -433,6 +468,34 @@ export function createRelayApp(
operations.drain(body.data.graceMs)
return context.json({ ok: true })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = IdleRegionalRehomeCommandSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
if (config.role !== 'cell' || !operations.drainHost) {
return context.json({ error: 'cell_only' }, 404)
@@ -474,7 +537,7 @@ export function createRelayApp(
}
sharedRuntimeIdentityRejected = true
}
const outcome = operations.drainHost(body.data)
const outcome = await operations.drainHost(body.data)
return context.json({
v: 1,
outcome,
@@ -502,8 +565,7 @@ export function createRelayApp(
region: config.region ?? RELAY_DEFAULT_REGION,
imageDigest: config.imageDigest ?? null,
draining: operations.isDraining?.() ?? false,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
connectionCapacity:
config.connectionHardCap === undefined
? null
@@ -559,6 +621,18 @@ export function createRelayApp(
return context.json({ error: operationError(error) }, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
const preview = await operations.assignments.previewRegionalRehomeEligibility(
operations.regionalRehomeSafetySnapshot?.()
)
const outcomes = await operations.assignments.regionCorrectionOutcomes()
return context.json({ v: 1, preview, outcomes })
})
app.post('/v1/admin/regional-rehome-control', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
@@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z
})
.strict()
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
cohortPercent: z.number().int().min(0).max(100),
directorSafety: RegionalRehomeSafetySchema
})
const CellHeartbeatSchema = z
.object({
v: z.literal(1),
@@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
regionalRehomeProtocol: z.number().int().min(0).max(1),
regionalRehomeProtocol: z.number().int().min(0).max(3),
safety: RegionalRehomeSafetySchema
})
.strict()
const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
'DISABLE_REGIONAL_REHOMING'
])
}).strict()
]).superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled
? 'ENABLE_REGIONAL_REHOMING'
: 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeControlSchema = z
.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z
.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z
.number()
.int()
.min(60_000)
.max(60 * 60_000),
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
})
.strict()
])
.superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeTrustProbeSchema = z
.object({
@@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: z.number().int().positive(),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000)
})
.strict()
cloud/apps/relay/src/assignment-store.ts
+573 -799
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/cell-heartbeat-client.test.ts
+27 -21
View File
@@ -92,7 +92,7 @@ describe('cell heartbeat client', () => {
client.stop()
expect(JSON.parse(String(requests[1]!.body))).toMatchObject({
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 3,
safety: {
observedAt: 120,
sqlFailures: 0,
@@ -149,28 +149,34 @@ describe('cell heartbeat client', () => {
it('does not start outside an explicitly configured cell role', () => {
expect(
startCellHeartbeat({ ...CONFIG, role: 'director' }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, role: 'director' },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
expect(
startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, directorUrl: undefined },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
})
})
cloud/apps/relay/src/cell-heartbeat-client.ts
+6 -4
View File
@@ -70,8 +70,7 @@ export function startCellHeartbeat(
inFlightConnections: connectionCounts!.inFlightConnections,
reservedConnectionUnits: connectionCounts!.reservedConnectionUnits,
enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits,
connectionInclusionWatermark:
connectionCounts!.inclusionWatermark,
connectionInclusionWatermark: connectionCounts!.inclusionWatermark,
connectionHardCap: config.connectionHardCap,
connectionUnobservedBound: config.connectionUnobservedBound
})
@@ -94,7 +93,7 @@ export function startCellHeartbeat(
cellId: config.cellId,
cellIncarnation,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
safety: options.regionalRehomeSafety()
}),
signal: AbortSignal.timeout(10_000)
@@ -106,7 +105,10 @@ export function startCellHeartbeat(
}
} catch (error) {
// A heartbeat must fail closed without ever logging its bearer token.
console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '')
console.warn(
'[orca-relay] cell heartbeat failed',
error instanceof Error ? error.message : ''
)
} finally {
inFlight = false
}
cloud/apps/relay/src/cell-inventory-lock-census.test.ts
+7 -9
View File
@@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [
{ method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
{ method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
@@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map<string, Set<string>> {
bounds.forEach((method, index) => {
const end = bounds[index + 1]?.start ?? lines.length
const names = callees.get(method.name) ?? new Set<string>()
for (const call of lines.slice(method.start, end).join('\n').matchAll(
/this\.([A-Za-z_][\w]*)\s*\(/g
)) {
for (const call of lines
.slice(method.start, end)
.join('\n')
.matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) {
names.add(call[1]!)
}
callees.set(method.name, names)
@@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
describe('cell inventory lock call-site census', () => {
it('classifies every call site exactly as recorded', () => {
expect(readCallSites()).toEqual(
CENSUS.map(({ method, mode }) => ({ method, mode }))
)
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
})
// Why: the census only sees lockCellInventory calls, so a hand-written
cloud/apps/relay/src/config.test.ts
+11
View File
@@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
}
describe('GCE relay capacity configuration', () => {
it('defaults optional region correction off and bounds the cohort', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0)
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5'
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5)
for (const invalid of ['-1', '101', '1.5', 'not-a-number']) {
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid
expect(() => loadRelayConfig(env)).toThrow()
}
})
it('requires distinct dedicated admin identities and accepts omitted values', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
cloud/apps/relay/src/config.ts
+7 -1
View File
@@ -75,11 +75,15 @@ const EnvSchema = z.object({
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(),
ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(),
ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(),
ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(),
ORCA_RELAY_IMAGE_DIGEST: z
.string()
.regex(/^sha256:[a-f0-9]{64}$/)
.optional(),
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2),
ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1),
ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64),
@@ -185,6 +189,7 @@ export type RelayConfig = {
databasePoolMax: number
publicAssignmentsEnabled: boolean
regionalPlacementEnabled?: boolean
regionCorrectionCohortPercent?: number
publicAssignmentConcurrency: number
publicAssignmentQueueMax: number
publicAssignmentWaitMs: number
@@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
databasePoolMax,
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY,
publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX,
publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS,
cloud/apps/relay/src/database.test.ts
+32 -6
View File
@@ -18,6 +18,34 @@ afterEach(() => {
})
describe('relay database', () => {
it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-'))
temporaryDirectories.push(dataDir)
const legacy = await openRelayDatabase({ dataDir })
await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome')
await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation')
await legacy.query(
`INSERT INTO relay_control_capabilities
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing)
VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)`
)
await legacy.close()
const upgraded = await openRelayDatabase({ dataDir })
try {
expect(
await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities')
).toEqual([{ idle_regional_rehome: 0 }])
const columns = await upgraded.query(
"SELECT * FROM pragma_table_info('relay_region_rehome_attempts')"
)
expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({
dflt_value: '0'
})
} finally {
await upgraded.close()
}
})
it('creates every durable relay state table', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
@@ -54,6 +82,7 @@ describe('relay database', () => {
'relay_confirm_results',
'relay_confirmable_splices',
'relay_connection_bases',
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_devices',
'relay_direct_authorizations',
@@ -62,6 +91,7 @@ describe('relay database', () => {
'relay_migration_leases',
'relay_post_drain_migration_pins',
'relay_rate_windows',
'relay_region_decisions',
'relay_region_rehome_attempts',
'relay_region_rehome_control',
'relay_region_rehome_worker_state'
@@ -140,9 +170,7 @@ describe('relay database', () => {
const second = await openRelayDatabase({ dataDir })
expect(
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
'legacy-cell'
])
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell'])
).toEqual([{ region: 'us-central1' }])
await second.close()
})
@@ -165,9 +193,7 @@ describe('relay database', () => {
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true)
await database.close()
})
cloud/apps/relay/src/database.ts
+33 -1
View File
@@ -197,6 +197,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
ON relay_assignment_region_preferences(observed_at);
CREATE TABLE IF NOT EXISTS relay_region_decisions (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
cohort_bucket BIGINT NOT NULL DEFAULT 0,
last_considered_at BIGINT NOT NULL DEFAULT 0,
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
PRIMARY KEY (user_id, relay_host_id)
);
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
finish_existing BIGINT NOT NULL,
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
worker_id TEXT PRIMARY KEY,
next_dispatch_at BIGINT NOT NULL,
@@ -228,6 +246,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
source_generation BIGINT NOT NULL DEFAULT 0,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
@@ -600,6 +619,8 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
@@ -607,7 +628,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
]
function postgresSql(sql: string): string {
@@ -1013,6 +1036,15 @@ async function applySchema(database: RelayDatabase): Promise<void> {
for (const statement of SCHEMA.split(';')) {
if (statement.trim()) await database.query(statement)
}
for (const [table, column] of [
['relay_control_capabilities', 'idle_regional_rehome'],
['relay_region_rehome_attempts', 'source_generation']
]) {
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
if (!columns.some((existing) => existing.name === column)) {
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`)
}
}
}
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
cloud/apps/relay/src/drain-migration-row-lookup.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { expect, it } from 'vitest'
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import type { SqlRow } from './database.js'
function text(row: SqlRow, field: string): string {
const value = row[field]
if (typeof value !== 'string') {
throw new Error(`invalid_${field}`)
}
return value
}
it('keeps first assignment matches, lease order, row identity, and separate identity components', () => {
const rows = [
{ user_id: 'a:b', relay_host_id: 'c', value: 1 },
{ user_id: 'a', relay_host_id: 'b:c', value: 2 },
{ user_id: 'a:b', relay_host_id: 'c', value: 3 },
{ user_id: '', relay_host_id: '', value: 4 }
]
const lookup = createDrainMigrationRowLookup(rows, text)
for (const identity of [
{ userId: 'a:b', relayHostId: 'c' },
{ userId: 'a', relayHostId: 'b:c' },
{ userId: '', relayHostId: '' },
{ userId: 'missing', relayHostId: 'c' }
]) {
const expected = rows.filter(
(row) =>
row.user_id === identity.userId && row.relay_host_id === identity.relayHostId
)
expect(lookup.first(identity)).toBe(expected[0])
expect(lookup.all(identity)).toEqual(expected)
lookup.all(identity).forEach((row, index) => expect(row).toBe(expected[index]))
}
})
it('retains lazy validation and short circuiting when an inventory is malformed', () => {
const first = { user_id: 'user', relay_host_id: 'host' }
const identity = { userId: 'user', relayHostId: 'host' }
const lookup = createDrainMigrationRowLookup(
[first, { user_id: null, relay_host_id: 'bad' }],
text
)
expect(lookup.first(identity)).toBe(first)
expect(() => lookup.all(identity)).toThrow('invalid_user_id')
const unrelated = createDrainMigrationRowLookup(
[first, { user_id: 'other', relay_host_id: null }],
text
)
expect(unrelated.all(identity)).toEqual([first])
expect(() => unrelated.first({ userId: 'other', relayHostId: 'host' })).toThrow(
'invalid_relay_host_id'
)
})
it('does not share an index between refreshed inventories', () => {
const identity = { userId: 'user', relayHostId: 'host' }
const oldRow = { user_id: 'user', relay_host_id: 'host', version: 1 }
const newRow = { ...oldRow, version: 2 }
expect(createDrainMigrationRowLookup([oldRow], text).first(identity)).toBe(oldRow)
expect(createDrainMigrationRowLookup([newRow], text).first(identity)).toBe(newRow)
})
cloud/apps/relay/src/drain-migration-row-lookup.ts
+58
View File
@@ -0,0 +1,58 @@
import type { SqlRow } from './database.js'
type Identity = { userId: string; relayHostId: string }
type RowIndex = Map<string, Map<string, SqlRow[]>>
/** A single locked inventory, never retained across transactions or refreshed queries. */
export function createDrainMigrationRowLookup(
rows: SqlRow[],
readText: (row: SqlRow, field: string) => string
): {
first: (identity: Identity) => SqlRow | undefined
all: (identity: Identity) => SqlRow[]
} {
let index: RowIndex | null | undefined
const indexed = (identity: Identity): SqlRow[] | undefined => {
if (index === undefined) {
index = indexRows(rows)
}
return index?.get(identity.userId)?.get(identity.relayHostId)
}
const matches = (row: SqlRow, identity: Identity): boolean =>
readText(row, 'user_id') === identity.userId &&
readText(row, 'relay_host_id') === identity.relayHostId
return {
first(identity) {
const group = indexed(identity)
return index === null ? rows.find((row) => matches(row, identity)) : group?.[0]
},
all(identity) {
const group = indexed(identity)
return index === null ? rows.filter((row) => matches(row, identity)) : (group ?? [])
}
}
}
function indexRows(rows: SqlRow[]): RowIndex | null {
const index: RowIndex = new Map()
for (const row of rows) {
const userId = row.user_id
const hostId = row.relay_host_id
// Preserve the original lazy validation and refusal order for malformed database rows.
if (typeof userId !== 'string' || typeof hostId !== 'string') {
return null
}
let hosts = index.get(userId)
if (!hosts) {
hosts = new Map()
index.set(userId, hosts)
}
const group = hosts.get(hostId)
if (group) {
group.push(row)
} else {
hosts.set(hostId, [row])
}
}
return index
}
cloud/apps/relay/src/drain-migration-row-scaling.test.ts
+121
View File
@@ -0,0 +1,121 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { afterEach, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js'
let database: RelayDatabase | undefined
afterEach(async () => await database?.close())
it.each([false, true])(
'looks up a whole drain inventory with linear identity reads (expired: %s)',
async (expired) => {
database = await openInMemoryRelayDatabase()
let measuring = false
let identityReads = 0
let indexedRows = 0
const instrument = (delegate: RelayDatabase): RelayDatabase => ({
query: (sql, params) => delegate.query(sql, params),
queryLocked: async (sql, params, options) => {
const rows = await delegate.queryLocked(sql, params, options)
if (
!measuring ||
!sql.includes('WHERE EXISTS') ||
!(sql.includes('SELECT assignment.*') || sql.includes('SELECT lease.*'))
) {
return rows
}
indexedRows += rows.length
return rows.map(
(row): SqlRow =>
new Proxy(row, {
get(target, key) {
if (key === 'user_id' || key === 'relay_host_id') {
identityReads++
}
return Reflect.get(target, key)
}
})
)
},
transaction: (operation, options) =>
delegate.transaction((tx) => operation(instrument(tx)), options),
close: () => delegate.close()
})
let now = 100
const store = new RelayAssignmentStore(instrument(database), () => now, {
requireLiveCells: true
})
const cells = ['a', 'b'].map((id) => ({
id: `cell-${id}`,
url: `https://relay-${id}.example.com`,
capacityRequests: 500
}))
await store.reconcileCells(cells)
const incarnation = '11111111-1111-4111-8111-111111111111'
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
await store.setCellEnabled('cell-b', false)
const identities = Array.from({ length: 50 }, (_, index) => ({
userId: `user-${index % 5}`,
relayHostId: `host${String(index).padStart(12, '0')}`
}))
for (const identity of identities) {
await store.assign(identity)
}
await store.setCellEnabled('cell-b', true)
await store.setCellEnabled('cell-a', false)
for (const identity of identities) {
const migration = await store.startEvacuation(identity, 'cell-b')
await store.markMigrationTargetRegistered(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch
})
}
const attempt = {
attemptId: '55555555-5555-4555-8555-555555555555',
cellId: 'cell-a',
cellIncarnation: incarnation,
traceValue: '66666666-6666-4666-8666-666666666666',
plannedGraceMs: 120_000
}
await store.prepareCellDrainAttempt(attempt)
if (expired) {
now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
await store.releaseExpiredActivityLeases()
for (const cell of cells) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: incarnation,
startedAt: 50,
ready: true,
observedRequests: 0
})
}
}
measuring = true
await expect(store.beginCellDrainSend(attempt)).resolves.toMatchObject({
state: 'send-may-have-started',
shouldSend: true
})
expect(indexedRows).toBeGreaterThanOrEqual(100)
expect(identityReads).toBeLessThanOrEqual(indexedRows * 2)
const migrations = await database.query(
'SELECT expires_at FROM relay_assignment_migrations'
)
expect(migrations).toHaveLength(50)
expect(
migrations.every(
(row) => row.expires_at === now + ASSIGNMENT_LIMITS.migrationLeaseMs
)
).toBe(true)
}
)
cloud/apps/relay/src/host-session-client-accept.test.ts
+141
View File
@@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => {
vi.useRealTimers()
})
it('does not admit new source work after a drain crosses activity acquisition', async () => {
const h = harness()
const control = await activeHost(h)
const slow = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slow.promise)
const client = new FakeSocket()
const capacity = { bind: vi.fn(), release: vi.fn() }
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
slow.resolve()
await accepting
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it('does not splice an attachment whose generation retired during basis persistence', async () => {
const h = harness()
await activeHost(h)
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const pending = [...session.pendingConns.values()][0]!
const slow = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
slow.resolve()
expect(await attaching).toBe(false)
expect(session.activeSplices.size).toBe(0)
expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId)
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true'))
expect(host.close).toHaveBeenCalled()
})
it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
@@ -364,6 +424,8 @@ describe('successful client accept timing', () => {
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const ownerProbe = vi.spyOn(session.pendingConns, 'has')
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
@@ -372,6 +434,7 @@ describe('successful client accept timing', () => {
)
expect(accepted).toBe(true)
expect(ownerProbe).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
@@ -390,6 +453,7 @@ describe('successful client accept timing', () => {
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
@@ -424,6 +488,80 @@ async function advanceToPing(control: FakeSocket, clock: { now: number }): Promi
return (JSON.parse(String(ping[0])) as { t: number }).t
}
// The attach resolves its owning session once and hands it to the unfenced leg;
// these hold the session it must be and the order the client hears about it.
describe('host data attach ownership', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const bystander = { ...identity, sub: 'user-2', relayHostId: 'qponmlkjihgfedcb' }
async function pendingAttach(h: ReturnType<typeof harness>) {
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
return { client, session, pending: [...session.pendingConns.values()][0]! }
}
it('attaches the session that owns the connection, not the first one registered', async () => {
const h = harness()
const idle = new FakeSocket()
await h.activate(idle as unknown as WebSocket, bystander, null, 1, false, 1, '1.4.197')
await activeHost(h)
const { client, session, pending } = await pendingAttach(h)
const idleSession = h.registry.get({
userId: bystander.sub,
relayHostId: bystander.relayHostId
})!
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
).toBe(true)
expect(client.send).toHaveBeenCalledWith(expect.stringContaining('"type":"relay-hello"'))
expect(session.activeSplices.has(pending.connId)).toBe(true)
expect(idleSession.activeSplices.size).toBe(0)
expect(idleSession.activeConnIds.size).toBe(0)
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
it('acknowledges the client only after the connection basis is persisted', async () => {
const h = harness()
await activeHost(h)
const { client, session, pending } = await pendingAttach(h)
const basis = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(basis.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
expect(h.store.recordConnectionBasis).toHaveBeenCalledOnce()
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('relay-hello'))
basis.resolve()
expect(await attaching).toBe(true)
expect(client.send).toHaveBeenCalledWith(expect.stringContaining('"type":"relay-hello"'))
expect(session.activeSplices.has(pending.connId)).toBe(true)
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -460,6 +598,9 @@ describe('control round-trip sampling', () => {
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
assignmentEpoch: 1,
controlGeneration: 1,
drainMode: 'none',
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
cloud/apps/relay/src/host-session-registry.test.ts
+620 -42
View File
@@ -4,6 +4,7 @@ import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -110,6 +111,7 @@ function createRegistry(
renewControlActivity: ReturnType<typeof vi.fn>
releaseActivity: ReturnType<typeof vi.fn>
observer: {
recordAuth: ReturnType<typeof vi.fn>
recordControlClose: ReturnType<typeof vi.fn>
recordSpliceClose: ReturnType<typeof vi.fn>
}
@@ -140,7 +142,10 @@ function createRegistry(
store as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
observer,
Date.now,
Math.random,
'incarnation-1'
)
// Mirrors the production signature exactly so a future positional shift fails to compile.
const bound = (
@@ -166,7 +171,14 @@ function createRegistry(
assignmentEpoch,
appVersion = '1.4.173'
) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion)
return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer }
return {
registry,
activate,
acquireActivity,
renewControlActivity,
releaseActivity,
observer
}
}
describe('host session cleanup races', () => {
@@ -398,26 +410,20 @@ describe('host session cleanup races', () => {
attemptId: '22222222-2222-4222-8222-222222222222'
})
).toThrow('regional_rehome_attempt_conflict')
expect(() =>
registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })
).toThrow('regional_rehome_assignment_epoch_mismatch')
expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow(
'regional_rehome_assignment_epoch_mismatch'
)
const rebound = new FakeSocket()
await activate(
rebound as unknown as WebSocket,
identity,
registry.get(request),
1,
true,
7
)
await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7)
expect(registry.get(request)?.state).toBe('drain-only')
expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
await vi.advanceTimersByTimeAsync(30_000)
expect(registry.get(request)).toBeNull()
expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }))
.not.toBeNull()
expect(
registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })
).not.toBeNull()
expect(secondSocket.close).not.toHaveBeenCalled()
})
@@ -513,14 +519,7 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
const rebindSocket = new FakeSocket()
const rebinding = activate(
rebindSocket as unknown as WebSocket,
identity,
original,
1,
true,
1
)
const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1)
rebindSocket.close()
blocked.resolve('control:production-gce-c3:1')
await rebinding
@@ -659,14 +658,7 @@ describe('host session cleanup races', () => {
originalSocket.close()
const replacementSocket = new FakeSocket()
await activate(
replacementSocket as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1)
const replacement = registry.get({
userId: identity.sub,
relayHostId: identity.relayHostId
@@ -696,14 +688,7 @@ describe('host session cleanup races', () => {
})
expect(original).not.toBeNull()
await activate(
new FakeSocket() as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
expect(renewControlActivity).toHaveBeenCalledOnce()
@@ -718,6 +703,53 @@ describe('host session cleanup races', () => {
vi.advanceTimersByTime(0)
})
it('ignores a denial belonging to the socket before a same-generation rebind', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const oldSocket = new FakeSocket()
await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
let reject!: (error: Error) => void
h.renewControlActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(replacement.close).not.toHaveBeenCalled()
expect(session.socket).toBe(replacement)
expect(session.generation).toBe(1)
})
it('ignores missing-activity recovery denial after an authority transition', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const socket = new FakeSocket()
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found'))
let reject!: (error: Error) => void
h.acquireActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).not.toHaveBeenCalled()
expect(session.state).toBe('drain-only')
})
it('keeps 15s pings while halving steady-state control renewals', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
@@ -732,9 +764,7 @@ describe('host session cleanup races', () => {
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
}
const pings = socket.send.mock.calls.filter((call) =>
String(call[0]).includes('"ping"')
)
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
expect(pings).toHaveLength(4)
expect(renewControlActivity).toHaveBeenCalledTimes(2)
const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt)
@@ -1118,3 +1148,551 @@ describe('host hello ack pending connections', () => {
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
describe('source-owned idle cutover', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const request = {
attemptId: 'idle-1',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
sourceCellIncarnation: 'incarnation-1',
targetCellId: 'target'
}
async function source(store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store)
const socket = new FakeSocket()
h.registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME])
)
socket.removeAllListeners('message')
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
return { ...h, socket, session: h.registry.get(request)! }
}
it('keeps either established client busy until both actually leave', async () => {
const h = await source()
h.session.activeConnIds.add('phone')
h.session.activeConnIds.add('ipad')
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
h.session.activeConnIds.delete('ipad')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
h.session.activeConnIds.delete('phone')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'committed' })
expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it.each([
{ userId: 'other-user' },
{ sourceAssignmentEpoch: 2 },
{ sourceGeneration: 2 },
{ sourceCellIncarnation: 'other-incarnation' },
{ targetCellId: 'other-target' }
])('rejects a reused operation ID with changed authority %j', async (change) => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const reconcile = vi.fn().mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(request, commit, reconcile)
const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile)
result.resolve({ outcome: 'deferred' })
expect(await conflicting).toEqual({ outcome: 'stale' })
expect(await moving).toEqual({ outcome: 'deferred' })
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
})
it('accounts for accepts before credential identity resolves', async () => {
const lookup = deferred<null>()
const h = await source({
resolveResume: vi.fn().mockReturnValue(lookup.promise),
resolveInviteForMove: vi.fn().mockResolvedValue(null)
})
const client = new FakeSocket()
const accept = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
lookup.resolve(null)
await accept
expect(h.socket.close).not.toHaveBeenCalled()
})
it('rejects new accepts and replacements synchronously while a commit awaits', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const moving = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const duplicate = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const client = new FakeSocket()
const release = vi.fn()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
{ release } as never
)
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(release).toHaveBeenCalledOnce()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1)
expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
result.resolve({ outcome: 'deferred' })
await moving
await duplicate
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'next' },
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'committed' })
})
it.each(['ambiguous', 'deferred'])(
'keeps %s outcomes fenced until locked reconciliation succeeds',
async (claim) => {
const h = await source()
const reconcile = vi
.fn()
.mockRejectedValueOnce(new Error('database unavailable'))
.mockRejectedValueOnce(new Error('database unavailable'))
.mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(
request,
claim === 'ambiguous'
? vi.fn().mockRejectedValue(new Error('lost commit reply'))
: vi.fn().mockResolvedValue({ outcome: 'deferred' }),
reconcile
)
await vi.advanceTimersByTimeAsync(50)
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'other' },
vi.fn(),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'busy' })
expect(h.socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(300)
expect(await moving).toEqual({ outcome: 'deferred' })
expect(reconcile).toHaveBeenCalledTimes(3)
expect(h.socket.close).not.toHaveBeenCalled()
}
)
it('owns accepted control mutations before the handler first awaits', async () => {
const mutation = deferred<RelayTokenClaims | null>()
const h = await source()
;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi
.fn()
.mockReturnValue(mutation.promise)
h.socket.emit(
'message',
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })),
false
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
mutation.resolve(identity)
await vi.advanceTimersByTimeAsync(0)
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'deferred' })
})
it('owns queued replacement activation before its first persistence await', async () => {
const h = await source()
const activation = deferred<string>()
const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } })
.assignments
assignments.activateControl = vi.fn().mockReturnValue(activation.promise)
const replacement = new FakeSocket()
const activating = h.activate(
replacement as unknown as WebSocket,
identity,
h.session,
2,
false,
1
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
activation.resolve('control:2')
await activating
})
it('retires changed authority even when the claim definitively deferred', async () => {
const h = await source()
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'deferred' }),
vi.fn().mockResolvedValue('stale')
)
).toEqual({ outcome: 'stale' })
expect(h.session.state).toBe('closed')
expect(h.releaseActivity).toHaveBeenCalled()
})
it('holds attach ownership through basis failure reservation cleanup', async () => {
const basis = deferred<void>()
const cleanup = deferred<void>()
const h = await source({
recordConnectionBasis: vi.fn().mockImplementation(async () => {
await basis.promise
throw new Error('basis failed')
}),
failReservation: vi.fn().mockReturnValue(cleanup.promise)
})
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const attached = h.registry.acceptHostData(
new FakeSocket() as unknown as WebSocket,
'conn',
'ticket',
1
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
basis.resolve()
await vi.advanceTimersByTimeAsync(0)
expect(h.session.activeConnIds.size).toBe(0)
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
cleanup.resolve()
await attached
})
it('rejects an attach mid-cutover before its ticket is ever examined', async () => {
const h = await source({ failReservation: vi.fn().mockResolvedValue(undefined) })
const result = deferred<{ outcome: 'deferred' }>()
// The cutover must already be in flight: an idle host is what it claims.
const moving = h.registry.idleRehome(request, () => result.promise, vi.fn())
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const host = new FakeSocket()
// The ticket below is the live one: only the cutover fence may reject it.
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, 'conn', 'ticket', 1)
).toBe(false)
expect(host.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.observer.recordAuth).not.toHaveBeenCalled()
expect(h.session.pendingConns.has('conn')).toBe(true)
expect(h.session.activeConnIds.size).toBe(0)
result.resolve({ outcome: 'deferred' })
await moving
})
it('holds no attach ownership when no session owns the connection', async () => {
const h = await source()
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, 'stranger', 'ticket', 1)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
expect.any(String)
)
// A leaked idle-work hold from the unowned attach would report `busy` here.
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn()
)
).toEqual({ outcome: 'committed' })
})
it('returns the durable operation outcome after source retirement', async () => {
const h = await source()
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
await h.registry.idleRehome(request, commit, vi.fn())
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed'))
).toEqual({ outcome: 'committed' })
expect(commit).toHaveBeenCalledOnce()
})
it('does not reopen a source overtaken by emergency drain', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const moving = h.registry.idleRehome(
request,
() => result.promise,
vi.fn().mockResolvedValue('not-committed')
)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
result.resolve({ outcome: 'deferred' })
await moving
expect(h.session.state).toBe('closed')
expect(h.registry.get(request)).toBeNull()
})
})
// The host data leg's owner lookup is the registry's only whole-inventory scan on
// an attach. These count what that scan touches, not how long it takes.
describe('host data attach owner lookup', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const SESSION_COUNT = 1000
const CONN_ID = 'conn-owned'
const OWNER_INDEX = { first: 0, middle: 499, last: 999 } as const
type Placement = keyof typeof OWNER_INDEX | 'absent'
type LookupCounts = { visits: number; membership: number }
function bindOwn<K, V>(map: Map<K, V>, property: string | symbol): unknown {
const value: unknown = Reflect.get(map, property, map)
return typeof value === 'function' ? value.bind(map) : value
}
// One visit per session the scan pulls off the map iterator; answers unchanged.
function countingValues<K, V>(map: Map<K, V>, counts: LookupCounts): Map<K, V> {
return new Proxy(map, {
get(target, property) {
if (property !== 'values') return bindOwn(target, property)
return function* (): Generator<V> {
for (const value of target.values()) {
counts.visits += 1
yield value
}
}
}
})
}
// One membership check per `pendingConns.has`; answers unchanged.
function countingHas<K, V>(map: Map<K, V>, counts: LookupCounts): Map<K, V> {
return new Proxy(map, {
get(target, property) {
if (property !== 'has') return bindOwn(target, property)
return (key: K) => {
counts.membership += 1
return target.has(key)
}
}
})
}
// The pre-change implementation, kept inline as the oracle the new counts are
// differenced against: two inventory arrays, two independent finds.
function legacyOwnerLookup(
sessions: Map<string, HostSession>,
connId: string
): { owner: HostSession | undefined; session: HostSession | undefined } {
const owner = [...sessions.values()].find((candidate) => candidate.pendingConns.has(connId))
const session = [...sessions.values()].find((candidate) => candidate.pendingConns.has(connId))
return { owner, session }
}
function pendingConn(client: FakeSocket, connTicket: string) {
return {
connId: CONN_ID,
connTicket,
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never
}
// Every decoy holds a pending conn of its own, so each membership check the
// scan makes is real work rather than a lookup in an empty map.
function decoySession(index: number, counts: LookupCounts): HostSession {
const pendingConns = new Map<string, unknown>([[`conn-decoy-${index}`, { connId: 'decoy' }]])
return {
relayHostId: `decoy-host-${index}`,
generation: 1,
state: 'active',
activeConnIds: new Set<string>(),
pendingConns: countingHas(pendingConns, counts)
} as unknown as HostSession
}
async function attachRegistry(placement: Placement, store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), {
failReservation: vi.fn().mockResolvedValue(undefined),
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
deactivateBasis: vi.fn().mockResolvedValue(undefined),
...store
})
const control = new FakeSocket()
await h.activate(control as unknown as WebSocket, identity, null, 1, false, 1)
const internals = h.registry as unknown as { sessions: Map<string, HostSession> }
const [ownerKey, owner] = [...internals.sessions.entries()][0]!
const counts: LookupCounts = { visits: 0, membership: 0 }
const client = new FakeSocket()
if (placement !== 'absent') owner.pendingConns.set(CONN_ID, pendingConn(client, 'ticket'))
owner.pendingConns = countingHas(owner.pendingConns, counts)
const ordered: HostSession[] = []
const sessions = new Map<string, HostSession>()
const ownerIndex = placement === 'absent' ? 0 : OWNER_INDEX[placement]
for (let index = 0; index < SESSION_COUNT; index += 1) {
const session = index === ownerIndex ? owner : decoySession(index, counts)
ordered.push(session)
sessions.set(index === ownerIndex ? ownerKey : `decoy-${index}`, session)
}
internals.sessions = countingValues(sessions, counts)
return { ...h, owner, ordered, counts, client, control, sessions: internals.sessions }
}
it.each([
{
placement: 'first',
before: { visits: 2000, membership: 2 },
after: { visits: 1, membership: 1 }
},
{
placement: 'middle',
before: { visits: 2000, membership: 1000 },
after: { visits: 500, membership: 500 }
},
{
placement: 'last',
before: { visits: 2000, membership: 2000 },
after: { visits: 1000, membership: 1000 }
},
{
placement: 'absent',
before: { visits: 2000, membership: 2000 },
after: { visits: 1000, membership: 1000 }
}
] as const)(
'visits the inventory once, not twice, for a $placement owner',
async ({ placement, before, after }) => {
const h = await attachRegistry(placement)
expect(h.sessions.size).toBe(SESSION_COUNT)
const oracle = legacyOwnerLookup(h.sessions, CONN_ID)
const legacy = { ...h.counts }
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
// An unusable ticket stops the attach immediately after the lookup, so the
// counts below belong to the lookup alone.
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'wrong', 1)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host data ticket'
)
expect(legacy).toEqual(before)
expect({ ...h.counts }).toEqual(after)
expect(oracle.owner).toBe(placement === 'absent' ? undefined : h.owner)
expect(oracle.owner).toBe(oracle.session)
}
)
it.each([
{ reason: 'ticket', ticket: 'wrong', generation: 1, state: 'active' },
{ reason: 'generation', ticket: 'ticket', generation: 2, state: 'active' },
{ reason: 'state', ticket: 'ticket', generation: 1, state: 'orphaned' }
] as const)('fails an attach whose $reason does not match the owner', async (input) => {
const h = await attachRegistry('middle')
h.owner.state = input.state
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(
host as unknown as WebSocket,
CONN_ID,
input.ticket,
input.generation
)
).toBe(false)
expect(h.observer.recordAuth).toHaveBeenCalledExactlyOnceWith(false)
expect(host.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
'invalid host data ticket'
)
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
expect(h.owner.activeConnIds.size).toBe(0)
})
it('rejects on the earlier duplicate owner rather than the later live one', async () => {
const h = await attachRegistry('middle')
h.ordered[0]!.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'stale-ticket') as never)
expect(legacyOwnerLookup(h.sessions, CONN_ID).owner).toBe(h.ordered[0])
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.owner.pendingConns.has(CONN_ID)).toBe(true)
})
it('splices the earlier duplicate owner and leaves the later one untouched', async () => {
const basis = vi.fn().mockRejectedValue(new Error('basis failed'))
const h = await attachRegistry('first', { recordConnectionBasis: basis })
const duplicate = h.ordered[3]!
duplicate.pendingConns.set(CONN_ID, pendingConn(new FakeSocket(), 'ticket') as never)
h.counts.visits = 0
h.counts.membership = 0
const host = new FakeSocket()
expect(
await h.registry.acceptHostData(host as unknown as WebSocket, CONN_ID, 'ticket', 1)
).toBe(false)
expect({ ...h.counts }).toEqual({ visits: 1, membership: 1 })
expect(h.observer.recordAuth).toHaveBeenCalledWith(true)
expect(basis).toHaveBeenCalledOnce()
// The first owner's entry was consumed; the later duplicate never was.
expect(h.owner.pendingConns.has(CONN_ID)).toBe(false)
expect(duplicate.pendingConns.has(CONN_ID)).toBe(true)
expect(h.owner.activeConnIds.size).toBe(0)
})
})
cloud/apps/relay/src/host-session-registry.ts
+338 -56
View File
@@ -15,6 +15,7 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason,
@@ -25,10 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
@@ -78,7 +76,7 @@ export type HostSession = {
identity: RelayTokenClaims
readonly relayHostId: string
readonly generation: number
readonly assignmentEpoch: number
assignmentEpoch: number
readonly controlActivityId: string | null
readonly controlResumeSecret: string
// Why: reconnect churn is only actionable once it can be pinned to a client build.
@@ -94,6 +92,7 @@ export type HostSession = {
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
authorityRevision: number
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -108,10 +107,7 @@ export type HostSession = {
regionalDrainExpiresAt: number | null
}
export type RegionalHostDrainOutcome =
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
type PendingConnection = {
connId: string
@@ -184,6 +180,113 @@ export class HostSessionRegistry {
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
string,
{
attemptId: string
authorityKey: string
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
}
>()
async idleRehome(
input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceGeneration: number
sourceCellIncarnation: string
targetCellId: string
},
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
const authorityKey = JSON.stringify([
input.userId,
input.sourceAssignmentEpoch,
input.sourceGeneration,
input.sourceCellIncarnation,
input.targetCellId
])
const prior = this.idleAttempts.get(input.relayHostId)
if (prior) {
if (prior.attemptId !== input.attemptId) return { outcome: 'busy' }
return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' }
}
const session = this.get(input)
if (
this.draining ||
!session ||
session.state !== 'active' ||
session.generation !== input.sourceGeneration ||
session.assignmentEpoch !== input.sourceAssignmentEpoch ||
this.cellIncarnation !== input.sourceCellIncarnation
) {
const durable = await reconcile()
return { outcome: durable === 'committed' ? 'committed' : 'stale' }
}
if (
!session.socket ||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
)
return { outcome: 'deferred' }
if (
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
session.activeConnIds.size !== 0 ||
session.activeSplices.size !== 0 ||
session.pendingConns.size !== 0
)
return { outcome: 'busy' }
const revision = session.authorityRevision
const promise = Promise.resolve().then(async () => {
let outcome: 'committed' | 'deferred' | 'stale'
try {
outcome = (await commit()).outcome
if (outcome === 'deferred') {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
}
} catch {
let delay = 100
for (;;) {
try {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
break
} catch {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, delay)
timer.unref?.()
})
delay = Math.min(delay * 2, 5000)
}
}
}
if (this.get(input) === session) {
if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) {
this.closeDrainedSession(session)
}
}
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
this.idleAttempts.delete(input.relayHostId)
return { outcome }
})
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
return promise
}
private beginIdleWork(hostId: string): (() => void) | null {
if (this.idleAttempts.has(hostId)) return null
this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1)
return () => {
const remaining = (this.idleWork.get(hostId) ?? 1) - 1
if (remaining === 0) this.idleWork.delete(hostId)
else this.idleWork.set(hostId, remaining)
}
}
constructor(
private readonly config: RelayConfig,
private readonly verifyRelayToken: VerifyRelayToken,
@@ -192,7 +295,8 @@ export class HostSessionRegistry {
private readonly queuedByteBudget: ProcessQueuedByteBudget,
private readonly observer: RelayRuntimeObserver,
private readonly now: () => number = Date.now,
private readonly random: () => number = Math.random
private readonly random: () => number = Math.random,
private readonly cellIncarnation?: string
) {}
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
@@ -206,6 +310,25 @@ export class HostSessionRegistry {
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
const release = this.beginIdleWork(hostId)
if (!release) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
try {
await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation)
} finally {
release()
}
}
private async acceptClientUnfenced(
socket: WebSocket,
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
capacityReservation?.release()
@@ -295,6 +418,7 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED)
return
}
const admittingSocket = session.socket
const connId = randomUUID()
const connTicket = randomBytes(32).toString('base64url')
const identity = { userId: reservation.userId, relayHostId: hostId }
@@ -324,6 +448,20 @@ export class HostSessionRegistry {
) {
return
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
admittingSocket.readyState !== admittingSocket.OPEN
) {
capacityReservation?.release()
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
@@ -373,9 +511,34 @@ export class HostSessionRegistry {
connTicket: string,
generation: number
): Promise<boolean> {
const session = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
)
// First insertion-order owner, and the only scan the attach makes: the
// unfenced leg reuses this result instead of repeating the search.
let owner: HostSession | undefined
for (const candidate of this.sessions.values()) {
if (candidate.pendingConns.has(connId)) {
owner = candidate
break
}
}
const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {}
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return false
}
try {
return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation, owner)
} finally {
release()
}
}
private async acceptHostDataUnfenced(
socket: WebSocket,
connId: string,
connTicket: string,
generation: number,
session: HostSession | undefined
): Promise<boolean> {
const pending = session?.pendingConns.get(connId)
if (
!session ||
@@ -427,6 +590,27 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed')
return false
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
socket.readyState !== socket.OPEN ||
pending.client.readyState !== pending.client.OPEN
) {
session.activeConnIds.delete(connId)
pending.capacityReservation?.release()
this.deactivateBasisBestEffort(connId)
this.failReservationBestEffort(pending.reservation)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
}
this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING)
socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment')
return false
}
const close = wireSplice({
client: pending.client,
host: socket,
@@ -505,6 +689,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
@@ -513,6 +698,14 @@ export class HostSessionRegistry {
)
}
private sessionPlacementLogFields(session: HostSession) {
return {
assignmentEpoch: session.assignmentEpoch,
controlGeneration: session.generation,
drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none'
}
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
@@ -549,6 +742,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
@@ -562,6 +756,10 @@ export class HostSessionRegistry {
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
if (this.idleAttempts.has(identity.relayHostId)) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return
}
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
@@ -602,8 +800,7 @@ export class HostSessionRegistry {
socket: WebSocket | null,
context: string
): void {
void Promise.resolve()
.then(task)
void (async () => task())()
.catch((error: unknown) => {
const message = (error instanceof Error ? error.message : 'unknown')
// Untruncated, unlike peer-supplied close reasons: this is the
@@ -653,6 +850,7 @@ export class HostSessionRegistry {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
@@ -665,7 +863,8 @@ export class HostSessionRegistry {
relayHostId: string
sourceAssignmentEpoch: number
graceMs: number
}): RegionalHostDrainOutcome {
sourceCellIncarnation?: string
}): RegionalHostDrainOutcome | Promise<RegionalHostDrainOutcome> {
const session = this.get(input)
if (!session || session.state === 'closed') return 'host-not-connected'
if (session.assignmentEpoch !== input.sourceAssignmentEpoch) {
@@ -678,13 +877,11 @@ export class HostSessionRegistry {
this.reassertRegionalDrain(session)
return 'already-accepted'
}
session.authorityRevision += 1
session.regionalDrainAttemptId = input.attemptId
session.regionalDrainExpiresAt = this.now() + input.graceMs
this.reassertRegionalDrain(session)
session.regionalDrainTimer = setTimeout(
() => this.closeDrainedSession(session),
input.graceMs
)
session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs)
return 'accepted'
}
@@ -740,9 +937,9 @@ export class HostSessionRegistry {
const existing = this.sessions.get(key)
const rebind = Boolean(
existing &&
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
)
const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1
const ephemeral = nacl.box.keyPair()
@@ -784,7 +981,9 @@ export class HostSessionRegistry {
}, 10_000)
socket.once('message', (raw, isBinary) => {
clearTimeout(proofTimer)
const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const ack = isBinary
? null
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
if (
!ack?.success ||
@@ -826,6 +1025,11 @@ export class HostSessionRegistry {
appVersion: string,
connectionInclusionWatermark?: number
): Promise<void> {
const release = this.beginIdleWork(identity.relayHostId)
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return Promise.resolve()
}
const key = this.key(identity.sub, identity.relayHostId)
const previous = this.activationQueues.get(key) ?? Promise.resolve()
// The timeout only fails this waiting socket; the queue entry still chains
@@ -836,26 +1040,29 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
}, ACTIVATION_QUEUE_WAIT_MS)
queueWaitTimer.unref?.()
const activation = previous.catch(() => undefined).then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
const activation = previous
.catch(() => undefined)
.then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
this.activationQueues.set(key, activation)
const cleanup = (): void => {
release()
if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key)
}
void activation.then(cleanup, cleanup)
@@ -881,7 +1088,11 @@ export class HostSessionRegistry {
cellId: this.config.cellId,
assignmentEpoch,
generation,
connectionInclusionWatermark
connectionInclusionWatermark,
idleRegionalRehome:
this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ??
false,
cellIncarnation: this.cellIncarnation
}
)
await this.assignments.markMigrationTargetRegistered(
@@ -918,14 +1129,15 @@ export class HostSessionRegistry {
const previousSocket = existing.socket
if (existing.orphanTimer) clearTimeout(existing.orphanTimer)
existing.orphanTimer = null
existing.authorityRevision += 1
existing.assignmentEpoch = assignmentEpoch
existing.socket = socket
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
this.sendHelloAck(existing)
if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing)
@@ -980,6 +1192,7 @@ export class HostSessionRegistry {
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
authorityRevision: 0,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -1028,7 +1241,9 @@ export class HostSessionRegistry {
` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` +
` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` +
` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` +
(socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`)
(socketError === null
? ''
: ` error=${JSON.stringify(printableCloseReason(socketError))}`)
)
})
socket.on('message', (raw, isBinary) => {
@@ -1077,6 +1292,19 @@ export class HostSessionRegistry {
}
private async acceptRefresh(session: HostSession, raw: RawData): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptRefreshUnfenced(session, raw)
} finally {
release()
}
}
private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise<void> {
const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh'))
if (!parsed.success) return
const refreshed = await this.verifyRelayToken(parsed.data.relayJwt)
@@ -1107,6 +1335,16 @@ export class HostSessionRegistry {
if (controlActivityId && now >= session.activityRenewalDueAt) {
const attempt = ++session.activityRenewalAttempt
const startedAt = now
const socket = session.socket
const authorityRevision = session.authorityRevision
const current = (): boolean =>
this.sessions.get(key) === session &&
session.state !== 'closed' &&
session.socket === socket &&
socket.readyState === socket.OPEN &&
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
@@ -1117,11 +1355,16 @@ export class HostSessionRegistry {
}
)
.then(() => {
if (attempt <= session.activityRenewalCompletedAttempt) return
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS
})
.catch(async (error: unknown) => {
if (!current()) return
if (error instanceof Error && error.message === 'assignment_not_found') {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing')
return
}
if (error instanceof Error && error.message === 'activity_cell_not_authoritative') {
// Completion fences a late drain-only heartbeat after all source work is gone.
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed')
@@ -1144,8 +1387,22 @@ export class HostSessionRegistry {
cellId: this.config.cellId
}
)
if (!current()) {
// A replaced activity must not remain leased after its owner disappears.
if (
!this.sessions.get(key) ||
this.sessions.get(key)?.controlActivityId !== controlActivityId
) {
this.releaseActivityBestEffort(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
controlActivityId
)
}
return
}
this.observer.recordControlActivityRecovery?.(true)
} catch (acquireError: unknown) {
if (!current()) return
this.observer.recordControlActivityRecovery?.(false)
if (
acquireError instanceof Error &&
@@ -1218,6 +1475,21 @@ export class HostSessionRegistry {
private closeDrainedSession(session: HostSession): void {
if (session.state === 'closed') return
const forcedConnections = session.activeConnIds.size + session.pendingConns.size
if (forcedConnections > 0) {
console.warn(
JSON.stringify({
event: 'orca_relay_host_drain_forced_close',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
reason: this.draining ? 'emergency' : 'regional-deadline',
forcedConnections,
splices: session.activeSplices.size,
pending: session.pendingConns.size
})
)
}
if (session.heartbeatTimer) clearInterval(session.heartbeatTimer)
if (session.orphanTimer) clearTimeout(session.orphanTimer)
if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer)
@@ -1250,11 +1522,7 @@ export class HostSessionRegistry {
session.pendingConns.clear()
session.state = 'closed'
if (session.socket) {
closeRelayWebSocket(
session.socket,
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director')
}
const key = this.key(session.identity.sub, session.relayHostId)
if (this.sessions.get(key) === session) this.sessions.delete(key)
@@ -1278,6 +1546,23 @@ export class HostSessionRegistry {
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptControlCommandUnfenced(session, type, raw)
} finally {
release()
}
}
private async acceptControlCommandUnfenced(
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
if (typeof type !== 'string' || !session.socket) return
try {
@@ -1315,10 +1600,7 @@ export class HostSessionRegistry {
}
if (type === 'device-credential-install') {
const request = DeviceCredentialInstallSchema.parse(payload(raw, type))
if (
session.state !== 'active' &&
request.authorization.mode === 'authenticated-direct'
) {
if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') {
throw new Error('authorization_expired')
}
const installActivityId = `install:${request.reqId}`
cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const request = {
v: 1 as const,
attemptId: '33333333-3333-4333-8333-333333333333',
userId: 'idle-reconciliation-test',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 7,
targetCellId: 'target'
}
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
const store = new RelayAssignmentStore(database, () => 100_000_000)
await store.reconcileCells([
{ id: 'source', url: 'https://source.example.test', capacityRequests: 100 },
{ id: 'target', url: 'https://target.example.test', capacityRequests: 100 }
])
await store.assign(request)
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration,
cellIncarnation: request.sourceCellIncarnation
})
return { database, store }
}
describe('idle cutover durable reconciliation', () => {
it('only permits reopening when the exact source still owns the assignment', async () => {
const { store } = await setup()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed')
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration + 1,
cellIncarnation: request.sourceCellIncarnation
})
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('does not reopen an obsolete source after an assignment change', async () => {
const { store } = await setup()
await store.startEvacuation(request, request.targetCellId)
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('propagates unavailable durable state instead of declaring rollback', async () => {
const { database, store } = await setup()
vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable'))
await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable')
})
it('waits for an outstanding assignment transaction before deciding authority', async () => {
const { database, store } = await setup()
let release!: () => void
let entered!: () => void
const locked = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
const commit = database.transaction(async (transaction) => {
await transaction.queryLocked(
'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
entered()
await gate
await transaction.query(
'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
})
await locked
let settled = false
const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => {
settled = true
})
try {
await new Promise<void>((resolve) => setImmediate(resolve))
expect(settled).toBe(false)
} finally {
release()
await commit
await reconciliation
}
expect(await reconciliation).toBe('stale')
})
})
cloud/apps/relay/src/idle-regional-rehome-selection.ts
+117
View File
@@ -0,0 +1,117 @@
import { createHash } from 'node:crypto'
import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
[
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
}
cloud/apps/relay/src/idle-regional-rehome-store.test.ts
+396
View File
@@ -0,0 +1,396 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase, RelayLockOptions } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' }
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const cells = [
{
id: 'source',
url: 'https://source.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'target',
url: 'https://target.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
const request = {
v: 1 as const,
...identity,
attemptId: '33333333-3333-4333-8333-333333333333',
sourceCellId: cells[0]!.id,
sourceCellIncarnation: incarnations[0]!,
sourceAssignmentEpoch: assignment.assignmentEpoch,
sourceGeneration: 7,
targetCellId: cells[1]!.id
}
return { store, database, safety, request }
}
describe('constrained idle regional assignment transaction', () => {
it.each(['missing', 'disabled', 'future'] as const)(
'does only one read per tick with %s durable control and sees later enablement',
async (state) => {
const { store, database, safety } = await setup()
const control = (await database.query(
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
))[0]!
if (state === 'missing') {
await database.query('DELETE FROM relay_region_rehome_control')
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
)
}
const query = vi.spyOn(database, 'query')
const transaction = vi.spyOn(database, 'transaction')
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
expect(transaction).not.toHaveBeenCalled()
}
if (state === 'missing') {
const columns = Object.keys(control)
await database.query(
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
Object.values(control)
)
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
[safety.observedAt]
)
}
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(1)
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
}
)
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
const { store, database, safety, request } = await setup()
// Model three source activity units and seven units already reserved at the target.
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?',
[identity.userId, identity.relayHostId]
)
await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'")
await database.query(
"UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'",
[capacity]
)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
outcome: capacity === 11 ? 'committed' : 'deferred'
})
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
expect(await store.resolve(identity)).toMatchObject({
cellId: capacity === 11 ? 'target' : 'source',
assignmentEpoch: capacity === 11 ? 2 : 1
})
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const template = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
identity.userId,
identity.relayHostId
])
)[0]!
const columns = Object.keys(template)
for (let index = 0; index < 100; index++) {
const values = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`idle-store-test-${String(index).padStart(3, '0')}`,
`pagehost${String(index).padStart(8, '0')}`,
identity.userId,
identity.relayHostId
]
)
}
}
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first).toHaveLength(100)
expect(next).toHaveLength(1)
expect(next[0]!.relayHostId).toBe('pagehost00000099')
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
regionalRehomeCohortPercent: 100
})
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
})
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
'rechecks generation when replacement wins after the initial authority lookup',
async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
try {
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
} finally {
held.release()
}
expect(await commit).toEqual({ outcome: 'deferred' })
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
}
)
it('rejects source replacement when the cutover already holds assignment authority', async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
const replacement = store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const rejected = expect(replacement).rejects.toThrow('wrong_assignment')
held.release()
expect(await commit).toEqual({ outcome: 'committed' })
await rejected
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
})
it('finds the committed attempt after its database reply is lost', async () => {
const { store, safety, request, database } = await setup()
const transaction = database.transaction.bind(database)
const intercepted = vi
.spyOn(database, 'transaction')
.mockImplementation(async (operation, options) => {
let changed = false
const result = await transaction(
async (tx) =>
operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query')
return async (sql: string, params?: unknown[]) => {
if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true
return target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
),
options
)
if (changed) throw new Error('simulated_commit_reply_lost')
return result
})
await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow(
'simulated_commit_reply_lost'
)
intercepted.mockRestore()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed')
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1)
})
it('commits the requested move once and records its outcome without source retention', async () => {
const { store, database, safety, request } = await setup()
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts')
expect(attempts).toHaveLength(1)
expect(attempts[0]!.attempt_id).toBe(request.attemptId)
expect(Number(attempts[0]!.source_generation)).toBe(7)
})
it('rejects a replaced control and never substitutes a different target', async () => {
const { store, safety, request } = await setup()
expect(
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
).toEqual({ outcome: 'deferred' })
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 })
})
it('does not commit without process safety or cohort authorization', async () => {
const { store, safety, request, database } = await setup()
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
outcome: 'deferred'
})
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
const { store, safety, database } = await setup()
const before = await database.query('SELECT * FROM relay_assignments')
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(1)
expect(candidates[0]).toMatchObject({
sourceGeneration: 7,
sourceCellId: 'source',
targetCellId: 'target'
})
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
})
function holdStatement(database: RelayDatabase, fragment: string) {
let entered!: () => void
let release!: () => void
const arrival = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
let held = false
const transaction = database.transaction.bind(database)
vi.spyOn(database, 'transaction').mockImplementation((operation, options) =>
transaction(async (tx) => {
return operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query' || key === 'queryLocked')
return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => {
if (!held && sql.includes(fragment)) {
held = true
entered()
await gate
}
return key === 'queryLocked'
? target.queryLocked(sql, params, lockOptions)
: target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
)
}, options)
)
return { entered: arrival, release }
}
cloud/apps/relay/src/idle-regional-rehome-test-database.ts
+40
View File
@@ -0,0 +1,40 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
export async function openIdleRehomeTestDatabase(): Promise<RelayDatabase> {
const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL
if (!configured) return openInMemoryRelayDatabase()
const url = new URL(configured)
if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
throw new Error('idle_rehome_tests_require_local_postgres_55440')
}
const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}`
const admin = new pg.Client({ connectionString: configured })
await admin.connect()
try {
await admin.query(`CREATE SCHEMA ${schema}`)
url.searchParams.set('options', `-c search_path=${schema}`)
const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
const close = database.close.bind(database)
database.close = async () => {
try {
await close()
} finally {
try {
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
} finally {
await admin.end()
}
}
}
return database
} catch (error) {
try {
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await admin.end()
}
throw error
}
}
cloud/apps/relay/src/idle-regional-rehome-worker.test.ts
+105
View File
@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
const candidate = {
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
sourceGeneration: 3,
targetCellId: 'target'
}
const config = {
role: 'director',
regionCorrectionCohortPercent: 100,
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
rehomeDirectorServiceAccount: 'director@example.test'
} as RelayConfig
function setup(fetch: typeof globalThis.fetch) {
const selectIdleRegionalRehomeCandidates = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValue([candidate])
const claimRegionalRehome = vi.fn()
const recordRegionalRehomeDispatchFailure = vi.fn()
const worker = startRegionalRehomeWorker(
config,
{
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => ({ observedAt: 100 }) as never,
intervalMs: 60_000,
identityToken: async () => 'private-token',
fetch
}
)!
return {
worker,
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
}
}
describe('idle regional worker dispatch', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an idle request without claiming an assignment first', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({ v: 1, outcome: 'committed' })
)
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.claimRegionalRehome).not.toHaveBeenCalled()
expect(fetch).toHaveBeenCalledOnce()
const [url, init] = fetch.mock.calls[0]!
expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome')
const { sourceCellUrl: _, ...request } = candidate
expect(JSON.parse(String(init?.body))).toEqual({
...request,
cohortPercent: 100,
directorSafety: { observedAt: 100 }
})
})
it('progresses past busy hosts without charging a dispatch failure', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' }))
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' }))
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
c.selectIdleRegionalRehomeCandidates.mockResolvedValue([
candidate,
{
...candidate,
relayHostId: 'ponmlkjihgfedcba',
attemptId: '33333333-3333-4333-8333-333333333333'
}
])
await c.worker.run()
c.worker.stop()
expect(fetch).toHaveBeenCalledTimes(2)
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
it('does not charge a lost response as a claimed migration failure', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () => {
throw new Error('response lost')
})
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
})
cloud/apps/relay/src/index.ts
+8
View File
@@ -2,6 +2,7 @@ import {
formatAssignmentInventorySnapshot,
readAssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import { RelayAssignmentStore } from './assignment-store.js'
import { loadRelayConfig } from './config.js'
import { startCellHeartbeat } from './cell-heartbeat-client.js'
@@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
void runRelayBackgroundOperation(async () => {
const inventory = await readRegisteredMigrationInventory(database, Date.now())
for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line)
console.log(
JSON.stringify({
event: 'orca_relay_region_correction_outcomes',
observedAt: Date.now(),
outcomes: await readRegionCorrectionOutcomes(database, Date.now())
})
)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cloud/apps/relay/src/region-correction-outcomes.ts
+29
View File
@@ -0,0 +1,29 @@
import type { RelayDatabase } from './database.js'
export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) {
const rows = await database.query(
`SELECT attempt.source_cell_id, attempt.target_cell_id,
CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted'
WHEN attempt.completed_at IS NOT NULL THEN 'completed'
WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state,
COUNT(*) AS count,
COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms,
COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units
FROM relay_region_rehome_attempts attempt
JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch
GROUP BY attempt.source_cell_id, attempt.target_cell_id, state
ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`,
[now]
)
return rows.map((row) => ({
sourceCellId: String(row.source_cell_id),
targetCellId: String(row.target_cell_id),
state: String(row.state),
count: Number(row.count),
oldestOpenMs: Number(row.oldest_open_ms),
targetReservedUnits: Number(row.target_reserved_units)
}))
}
cloud/apps/relay/src/region-correction-preview.ts
+157
View File
@@ -0,0 +1,157 @@
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
import {
REGIONAL_REHOME_CONCURRENT_LIMIT,
REGION_DECISION_TTL_MS
} from './region-correction-state.js'
export type RegionCorrectionPreview = {
observedAt: number
newClaimsEnabled: boolean
cohortPercent: number
openMigrations: number
availableMigrationSlots: number
globalSafetyFailure: string | null
counts: Record<string, number>
}
export async function previewRegionalRehomeEligibility(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
globalSafetyFailure: string | null
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<RegionCorrectionPreview> {
const { database, now } = input
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
await Promise.all([
database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at,
decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version,
decision.observed_at, decision.cohort_bucket,
(SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at,
(SELECT COUNT(*) FROM relay_assignment_migrations migration
WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations,
(SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease
WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id
AND lease.cell_id = assignment.cell_id) AS source_units,
(SELECT COUNT(*) FROM relay_control_capabilities host_capability
JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id
AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id
AND runtime.cell_incarnation = host_capability.cell_incarnation
WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id
AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch
AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ?
AND lease.updated_at >= runtime.started_at) AS capable_controls
FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision
ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`,
[now]
),
database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`),
database.query(`SELECT * FROM relay_cell_runtime`),
database.query(`SELECT * FROM relay_cell_capabilities`),
database.query(`SELECT * FROM relay_cell_rehome_safety`),
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
)
])
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
const runtimes = byCell(runtimeRows)
const capabilities = byCell(capabilityRows)
const safety = byCell(safetyRows)
const inventory = byCell(cells)
const control = controls[0]
const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS)
const openMigrations = Number(migrations[0]?.count ?? 0)
const counts: Record<string, number> = {}
const count = (reason: string) => {
counts[reason] = (counts[reason] ?? 0) + 1
}
const available = (cell: SqlRow): boolean => {
const id = String(cell.cell_id)
const runtime = runtimes.get(id)
const capability = capabilities.get(id)
return (
Number(cell.enabled) === 1 &&
cell.admission_state === 'general' &&
cell.region != null &&
runtime !== undefined &&
Number(runtime.ready) === 1 &&
Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs &&
capability !== undefined &&
capability.cell_incarnation === runtime.cell_incarnation &&
Number(capability.regional_rehome_protocol) >= 3
)
}
for (const host of hosts) {
let reason: string | null = null
const source = inventory.get(String(host.cell_id))
if (host.generation == null) reason = 'no-verified-decision'
else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge)
reason = 'expired'
else if (
Number(host.decision_epoch) !== Number(host.assignment_epoch) ||
host.incumbent_region !== source?.region
)
reason = 'basis-changed'
else if (
host.outcome !== 'conclusive' ||
Number(host.policy_version) !== 1 ||
host.preferred_region == null
)
reason = 'inconclusive-or-insufficient-improvement'
else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort'
else if (Number(host.open_migrations) > 0) reason = 'migration-open'
else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown)
reason = 'host-cooldown'
else if (!source || !available(source)) reason = 'source-ineligible'
else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive'
else if (
!input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now)
)
reason = 'source-unclean'
if (reason) {
count(reason)
continue
}
const targets = cells.filter(
(cell) =>
cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell)
)
const clean = targets.filter((cell) =>
input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now)
)
const capacity = clean.filter(
(cell) =>
input.connectionHeadroom.get(String(cell.cell_id)) !== false &&
Number(cell.reserved_requests) + Number(host.source_units) + 1 <=
Number(cell.capacity_requests)
)
if (targets.length === 0) count('no-eligible-target')
else if (clean.length === 0) count('target-unclean')
else if (capacity.length === 0) count('no-target-headroom')
else if (input.globalSafetyFailure) count('global-safety-blocked')
else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap')
else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`)
}
return {
observedAt: now,
newClaimsEnabled: Number(control?.enabled ?? 0) === 1,
cohortPercent: input.cohortPercent,
openMigrations,
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
globalSafetyFailure: input.globalSafetyFailure,
counts
}
}
cloud/apps/relay/src/region-correction-restart.test.ts
+119
View File
@@ -0,0 +1,119 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' }
const paths: string[] = []
const databases = new Set<RelayDatabase>()
afterEach(async () => {
for (const database of databases) await database.close()
databases.clear()
for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true })
})
async function setup() {
const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-'))
paths.push(dataDir)
let now = 1_000_000_000
const open = async () => {
const database = await openRelayDatabase({ dataDir })
databases.add(database)
return { database, store: new RelayAssignmentStore(database, () => now) }
}
const first = await open()
const cell = {
id: 'restart-us',
url: 'https://restart-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
}
await first.store.reconcileCells([cell])
await first.store.setCellEnabled(cell.id, true)
await first.store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
region: cell.region,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
const assignment = await first.store.assign(identity)
const issue = (store: RelayAssignmentStore) =>
store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
const window = (await issue(first.store)).window!
const report = {
v: 1 as const,
action: 'report' as const,
generation: window.generation,
assignmentEpoch: window.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
const restart = async () => {
await first.database.close()
databases.delete(first.database)
return open()
}
return {
...first, window, report, issue, restart,
setNow: (value: number) => { now = value }
}
}
describe('persisted region decisions across director restart', () => {
it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => {
const context = await setup()
const epoch = context.window.assignmentEpoch
await context.store.exchangeRegionCorrection(identity, {
v: 1, action: 'report', generation: context.window.generation,
assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter'
}, epoch)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'duplicate' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('inconclusive')
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'stale' })
})
it('uses server expiry after a restart regardless of an old client report', async () => {
const context = await setup()
context.setNow(context.window.expiresAt)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'expired' })
expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 })
})
it('does not interpret a persisted future-policy window using the old policy after rollback', async () => {
const context = await setup()
await context.database.query('UPDATE relay_region_decisions SET policy_version = 2')
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('pending')
expect(row.preferred_region).toBeNull()
expect(row.report_json).toBeNull()
})
it('keeps generation ordering when the server clock moves backwards across restart', async () => {
const context = await setup()
context.setNow(1_000_000_000 - 60_000)
const restarted = await context.restart()
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
})
})
cloud/apps/relay/src/region-correction-state.ts
+158
View File
@@ -0,0 +1,158 @@
import { relayHostLogDigest } from './relay-host-log-digest.js'
import { createHash } from 'node:crypto'
import type {
RegionCorrectionRequest,
RegionCorrectionResponse,
RelayRegion
} from '@orca-cloud/relay-contract'
import type { RelayDatabase } from './database.js'
type Identity = { userId: string; relayHostId: string }
export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000
export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8
export async function exchangeRegionCorrection(
database: RelayDatabase,
identity: Identity,
request: RegionCorrectionRequest,
assignmentEpoch: number,
now: number
): Promise<RegionCorrectionResponse> {
const result: RegionCorrectionResponse = await database.transaction(async (transaction) => {
const assignment = (
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
const region =
assignment &&
(
await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
assignment.cell_id
])
)[0]
if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) {
return { v: 1, reportStatus: 'basis-changed' }
}
const prior = (
await transaction.queryLocked(
`SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (request.action === 'issue-window') {
const generation = Number(prior?.generation ?? 0) + 1
if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted')
const expiresAt = now + REGION_DECISION_TTL_MS
const cohortBucket =
createHash('sha256')
.update(JSON.stringify([identity.userId, identity.relayHostId]))
.digest()
.readUInt32BE(0) % 100
await transaction.query(
`INSERT INTO relay_region_decisions
(user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region,
policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket)
VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
generation = excluded.generation, expires_at = excluded.expires_at,
assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region,
policy_version = 1, outcome = 'pending', preferred_region = NULL,
observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`,
[
identity.userId,
identity.relayHostId,
generation,
expiresAt,
assignmentEpoch,
region.region,
now,
cohortBucket
]
)
return {
v: 1,
window: {
generation,
expiresAt,
assignmentEpoch,
incumbentRegion: region.region as RelayRegion,
policyVersion: 1
}
}
}
if (!prior || Number(prior.generation) !== request.generation)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.policy_version) !== request.policyVersion)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' }
if (
request.assignmentEpoch !== assignmentEpoch ||
Number(prior.assignment_epoch) !== assignmentEpoch ||
prior.incumbent_region !== region.region
) {
return { v: 1, reportStatus: 'basis-changed' }
}
// The first report wins, including an inconclusive tombstone.
if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' }
let preferredRegion: RelayRegion | null = null
if (request.outcome === 'conclusive') {
const incumbent = request.measurements[region.region as RelayRegion]
const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1'
const targetRtt = request.measurements[target]
if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target
}
await transaction.query(
`UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ?
WHERE user_id = ? AND relay_host_id = ? AND generation = ?`,
[
request.outcome,
preferredRegion,
JSON.stringify(request),
identity.userId,
identity.relayHostId,
request.generation
]
)
return { v: 1, reportStatus: 'accepted' }
})
if (request.action === 'report' && result.reportStatus === 'accepted') {
const digest = relayHostLogDigest(identity.relayHostId)
// Stable sampling includes unchanged hosts for before/after comparisons.
if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) {
console.log(
JSON.stringify({
event: 'orca_relay_region_comparison',
relayHostIdDigest: digest,
assignmentEpoch,
generation: request.generation,
policyVersion: request.policyVersion,
outcome: request.outcome,
...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {})
})
)
}
}
return result
}
export async function previewRegionCorrection(
database: RelayDatabase,
now: number
): Promise<Record<string, number>> {
const rows = await database.query(
`SELECT CASE WHEN decision.expires_at <= ? THEN 'expired'
WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed'
WHEN decision.outcome = 'pending' THEN 'pending'
WHEN decision.preferred_region IS NULL THEN 'ineligible'
ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason,
COUNT(*) AS count
FROM relay_region_decisions decision
JOIN relay_assignments assignment ON assignment.user_id = decision.user_id
AND assignment.relay_host_id = decision.relay_host_id
GROUP BY reason`,
[now]
)
return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)]))
}
cloud/apps/relay/src/region-correction-store.test.ts
+359
View File
@@ -0,0 +1,359 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' }
const cells = [
{
id: 'decision-us',
url: 'https://decision-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'decision-asia',
url: 'https://decision-asia.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const opened: RelayDatabase[] = []
afterEach(async () => {
for (const database of opened.splice(0)) {
if (database.dialect === 'postgres') await cleanupPostgres(database)
await database.close()
}
})
async function cleanupPostgres(database: RelayDatabase) {
for (const table of [
'relay_control_connection_reservations',
'relay_region_decisions',
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_migration_incarnations',
'relay_assignment_region_preferences',
'relay_region_rehome_attempts',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
}
for (const table of [
'relay_cell_rehome_safety',
'relay_cell_capabilities',
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_runtime',
'relay_cell_connection_limits',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await database.query(
`DELETE FROM ${table} WHERE cell_id IN (?, ?)`,
cells.map((cell) => cell.id)
)
}
}
async function setup() {
const database =
process.env.ORCA_REGION_CORRECTION_POSTGRES === '1'
? await openRelayDatabase({
databaseUrl: requiredPostgresUrl(),
dataDir: '/tmp/orca-region-correction-unused'
})
: await openInMemoryRelayDatabase()
opened.push(database)
if (database.dialect === 'postgres') await cleanupPostgres(database)
let clock = 100_000_000
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100
})
await store.reconcileCells(cells)
for (const cell of cells) await store.setCellEnabled(cell.id, true)
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: clock - 1_000,
ready: true,
observedRequests: 0
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const activityId = await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
return {
database,
store,
assignment,
activityId,
now: () => clock,
advance: (ms: number) => {
clock += ms
}
}
}
function requiredPostgresUrl(): string {
const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL
if (!url || new URL(url).port !== '55440')
throw new Error('PostgreSQL tests require configured port 55440')
return url
}
async function window(context: Awaited<ReturnType<typeof setup>>) {
const result = await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
context.assignment.assignmentEpoch
)
return result.window!
}
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
const attemptId = '33333333-3333-4333-8333-333333333333'
await context.database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
[
attemptId,
identity.userId,
identity.relayHostId,
cells[0]!.id,
incarnations[0],
cells[1]!.id,
incarnations[1],
migration.previousEpoch,
migration.assignmentEpoch,
context.now(),
context.now()
]
)
return { migration }
}
describe('ordered region decisions and migration outcomes', () => {
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
const context = await setup()
const { migration } = await regionalMigration(context)
context.advance(1_000)
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
const outcomes = await context.store.regionCorrectionOutcomes()
expect(outcomes).toEqual([
expect.objectContaining({
sourceCellId: cells[0]!.id,
targetCellId: cells[1]!.id,
state: 'registering',
count: 1,
oldestOpenMs: 1_000
})
])
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
before
)
await context.store.activateControl(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch
})
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({ state: 'registered' })
])
await context.store.releaseActivity(identity, context.activityId)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({
state: 'completed',
targetReservedUnits: 0,
oldestOpenMs: 0
})
])
})
it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => {
const context = await setup()
const first = await window(context)
const second = await window(context)
expect(second.generation).toBe(first.generation + 1)
const report = {
v: 1 as const,
action: 'report' as const,
assignmentEpoch: first.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: first.generation },
first.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'accepted' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'duplicate' })
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
})
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
const context = await setup()
const query = context.database.query.bind(context.database)
const transaction = context.database.transaction.bind(context.database)
const queryLocked = context.database.queryLocked.bind(context.database)
context.database.query = async (sql, params) => {
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
return query(sql, params)
}
context.database.transaction = async () => {
throw new Error('preview_must_not_open_mutating_transaction')
}
context.database.queryLocked = async () => {
throw new Error('preview_must_not_lock')
}
try {
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
expect(JSON.stringify(preview)).not.toContain(identity.userId)
} finally {
context.database.query = query
context.database.transaction = transaction
context.database.queryLocked = queryLocked
}
})
it('allocates distinct ordered generations for concurrent window issuers', async () => {
const context = await setup()
const replies = await Promise.all([window(context), window(context), window(context)])
expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3])
const older = replies.find((reply) => reply.generation === 2)!
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: older.generation,
assignmentEpoch: older.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'delayed'
},
older.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
})
it('compares with assigned region, preserves hints, and never extends a window on report', async () => {
const context = await setup()
await context.store.assign(identity, 'asia-east2')
const issued = await window(context)
expect(issued.incumbentRegion).toBe('us-central1')
context.advance(50)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 110, 'asia-east2': 90 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]!
expect(Number(row.expires_at)).toBe(issued.expiresAt)
const hint = (
await context.database.query(
`SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`,
[identity.userId]
)
)[0]
expect(hint?.preferred_region).toBe('asia-east2')
context.advance(24 * 60 * 60_000)
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'late'
},
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'expired' })
})
it('rejects stale assignment basis and requires both thresholds', async () => {
const context = await setup()
const issued = await window(context)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 150, 'asia-east2': 100 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({
'us-central1-to-asia-east2': 1
})
await context.store.startEvacuation(identity, cells[1]!.id)
expect(
await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'basis-changed' })
})
})
cloud/apps/relay/src/regional-host-drain-app.test.ts
+207 -94
View File
@@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string, route?: string) =>
token === 'deploy-token' ||
(token === 'monitor-token' &&
(!route || route === '/v1/admin/regional-rehome-control')),
(!route ||
route === '/v1/admin/regional-rehome-control' ||
route === '/v1/admin/regional-rehome-preview')),
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) =>
token === 'deploy-token',
@@ -41,7 +43,83 @@ const request = {
graceMs: 60_000
}
describe('idle regional cutover endpoint', () => {
it('authenticates and fences the source before invoking a cutover', async () => {
const idleRehome = vi.fn(async () => ({ outcome: 'busy' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
idleRehome,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
const input = {
v: 1,
attemptId: request.attemptId,
userId: request.userId,
relayHostId: request.relayHostId,
sourceCellId: request.sourceCellId,
sourceCellIncarnation: cellIncarnation,
sourceAssignmentEpoch: 7,
sourceGeneration: 1,
targetCellId: 'target-cell',
cohortPercent: 100,
directorSafety: {
observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0
}
}
const path = '/v1/admin/host-idle-rehome'
expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401)
expect(
(
await postPath(app, path, 'rehome-token', {
...input,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(idleRehome).not.toHaveBeenCalled()
const response = await postPath(app, path, 'rehome-token', input)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'busy' })
expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input)
})
})
describe('regional host drain endpoint', () => {
it('exposes aggregate preview to monitors without a mutation path', async () => {
const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } }
const safety = { observedAt: 100 }
const previewRegionalRehomeEligibility = vi.fn(async () => preview)
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: {
previewRegionalRehomeEligibility,
regionCorrectionOutcomes: async () => []
} as never,
regionalRehomeSafetySnapshot: () => safety as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const path = '/v1/admin/regional-rehome-preview'
expect((await app.request(path)).status).toBe(401)
expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled()
const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] })
expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety)
expect(
(
await app.request(path, {
method: 'POST',
headers: { authorization: 'Bearer deploy-token' }
})
).status
).toBe(404)
})
it('accepts only the dedicated identity and exact cell generation', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => {
expect((await post(app, 'deploy-token', request)).status).toBe(401)
expect(
(await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})).status
(
await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(drainHost).toHaveBeenCalledOnce()
})
it('waits for an asynchronous drain operation before acknowledging', async () => {
let grant!: (value: 'accepted') => void
let entered!: () => void
const started = new Promise<void>((resolve) => {
entered = resolve
})
const drainHost = vi.fn(() => {
entered()
return new Promise<'accepted'>((resolve) => {
grant = resolve
})
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost,
cellIncarnation,
ready: vi.fn(async () => true)
})
const pending = post(app, 'rehome-token', request)
let acknowledged = false
void pending.then(() => {
acknowledged = true
})
await started
await Promise.resolve()
expect(acknowledged).toBe(false)
grant('accepted')
const response = await pending
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' })
})
it('rejects a failed asynchronous drain instead of acknowledging it', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost: async () => {
throw new Error('activity_cell_not_authoritative')
},
cellIncarnation,
ready: vi.fn(async () => true)
})
const response = await post(app, 'rehome-token', request)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' })
})
it('rejects malformed identities before touching the session registry', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -224,7 +354,7 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
observedAt: 100,
sqlFailures: 0,
@@ -235,12 +365,7 @@ describe('regional rehome director controls', () => {
databasePoolWaitMsMax: 0
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -266,18 +391,13 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
...observability.regionalRehomeRuntimeSafety(),
...emptyPostgresPoolPressureCounts()
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -301,12 +421,14 @@ describe('regional rehome director controls', () => {
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
const apply = {
v: 1,
action: 'apply',
@@ -319,39 +441,35 @@ describe('regional rehome director controls', () => {
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
apply
)).status).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status
).toBe(200)
expect(applyRegionalRehomeControl).toHaveBeenCalledOnce()
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
apply
)).status).toBe(403)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status
).toBe(403)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
...apply,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
).status
).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown))
.status
).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -382,12 +500,11 @@ describe('regional rehome director controls', () => {
}) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
const responseBody = await response.json()
@@ -448,12 +565,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
@@ -481,12 +597,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
@@ -504,18 +619,17 @@ describe('regional rehome director controls', () => {
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'monitor-token',
body
)).status).toBe(401)
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ ...body, unexpected: true }
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status
).toBe(401)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
...body,
unexpected: true
})
).status
).toBe(400)
})
it('fails closed when the source rejects the dedicated identity', async () => {
@@ -529,9 +643,9 @@ describe('regional rehome director controls', () => {
regionalRehomeProtocol: 1
}
})
const sourceFetch = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({ error: 'invalid_token' }, { status: 401 })
)
const sourceFetch = vi
.fn<typeof fetch>()
.mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 }))
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
@@ -540,12 +654,11 @@ describe('regional rehome director controls', () => {
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).toHaveBeenCalledOnce()
cloud/apps/relay/src/regional-rehome-postgres.test.ts
+260 -233
View File
@@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
async function cleanup(): Promise<void> {
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
}
await primary.query(
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
)
@@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
it('defaults to a closed correction cohort even with enabled durable control', async () => {
const context = await fixture()
const closed = new RelayAssignmentStore(primary, context.now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
expect(await cutover(closed, context.now())).toBeNull()
const preview = await closed.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.cohortPercent).toBe(0)
expect(preview.counts['outside-cohort']).toBe(1)
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('counts existing generic migrations against the optimization cap and preview', async () => {
const context = await fixture()
const safety = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
const before = await context.store.previewRegionalRehomeEligibility(safety)
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
for (let index = 0; index < 8; index++) {
const identity = {
userId: `pg-rehome-user-budget-${sequence}-${index}`,
relayHostId: `budgethost${String(index).padStart(6, '0')}`
}
await context.store.assign(identity, undefined, 'us-central1')
await context.store.startEvacuation(identity, context.target.id)
}
const preview = await context.store.previewRegionalRehomeEligibility(safety)
expect(preview.openMigrations).toBe(8)
expect(preview.availableMigrationSlots).toBe(0)
expect(preview.counts['concurrent-migration-cap']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('preview excludes request capacity exhaustion before a claim', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
[context.target.id]
)
const preview = await context.store.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.counts['no-target-headroom']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
})
it('claims through ambient per-cell sql retry noise', async () => {
const context = await fixture()
await primary.query(
@@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.source.id, context.target.id]
)
expect(await context.store.claimRegionalRehome()).not.toBeNull()
expect(await cutover(context.store, context.now())).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
@@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => {
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
expect(
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
context.identity.userId
])
).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
`UPDATE relay_region_decisions SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
@@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
@@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.target.id]
)
expect(await context.store.claimRegionalRehome()).toBeNull()
expect(await cutover(context.store, context.now())).toBeNull()
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await primary.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }])
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('lets only one director claim a host', async () => {
const context = await fixture()
const claims = await Promise.all([
context.store.claimRegionalRehome(),
context.competingStore.claimRegionalRehome()
cutover(context.store, context.now()),
cutover(context.competingStore, context.now())
])
expect(claims.filter(Boolean)).toHaveLength(1)
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
[context.identity.userId]
)
).toEqual([{ count: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
})
it('serializes an enable with a budget-exhausting failure without retries', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
const locked = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const primaryTransaction = primary.transaction.bind(primary)
const secondaryTransaction = secondary.transaction.bind(secondary)
let enableTransactions = 0
let failureTransactions = 0
let enablePid = 0
let failurePid = 0
const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) =>
primaryTransaction(async (transaction) => {
enableTransactions++
enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation({
dialect: 'postgres',
query: transaction.query.bind(transaction),
queryLocked: async (sql, params, lockOptions) => {
const rows = await transaction.queryLocked(sql, params, lockOptions)
if (sql.includes('FROM relay_region_rehome_control')) {
locked.resolve()
await release.promise
}
return rows
},
transaction: transaction.transaction.bind(transaction),
close: transaction.close.bind(transaction)
})
}, options)
)
const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) =>
secondaryTransaction(async (transaction) => {
failureTransactions++
failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation(transaction)
}, options)
)
const enable = context.store.applyRegionalRehomeControl({
expectedGeneration: 1,
enabled: true,
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
let failure: Promise<void> | undefined
let outcomes: PromiseSettledResult<unknown>[] = []
try {
await Promise.race([
locked.promise,
enable.then(() => {
throw new Error('enable completed before the control lock')
})
])
failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
// Observe the actual PostgreSQL wait before letting enable acquire the worker row.
await vi.waitFor(async () => {
expect(failurePid).not.toBe(0)
const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid])
expect(rows[0]!.blockers).toContain(enablePid)
}, { interval: 10, timeout: 800 })
} finally {
release.resolve()
outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])])
enableSpy.mockRestore()
failureSpy.mockRestore()
}
expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled'])
expect({ enableTransactions, failureTransactions }).toEqual({
enableTransactions: 1,
failureTransactions: 1
})
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: true
})
expect(await primary.query(
`SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state`
)).toEqual([{ consecutive_failures: '1', paused_until: '0' }])
[context.identity.userId]
)
).toEqual([{ count: '1' }])
})
it('increments the disable generation once across competing directors', async () => {
@@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
it('records one receipt across competing directors', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const receipts = await Promise.all([
context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'),
context.competingStore.recordRegionalRehomeDrainReceipt(
attempt!.attemptId,
'accepted'
)
])
expect(receipts.sort()).toEqual([false, true])
expect(await primary.query(
`SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ drain_outcome: 'accepted' }])
})
it('rechecks a preference changed while the assignment row is locked', async () => {
const context = await fixture()
let unlock!: () => void
@@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = cutover(context.store, context.now())
await primary.query(
`UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1',
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
[context.now(), context.identity.userId, context.identity.relayHostId]
)
@@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
await held
await expect(claim).resolves.toBeNull()
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('rechecks fleet safety under locks before mutating a candidate', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
let unlock!: () => void
let locked!: () => void
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
const held = secondary.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`,
[context.target.id]
)
await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [
context.target.id
])
await transaction.query(
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
[context.target.id]
@@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
unlock()
await held
await expect(claim).resolves.toBeNull()
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('pauses when one required cell exceeds the reconnect limit', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`,
[REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id]
)
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
context.source.id
])
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
})
it('does not retry a drain against a replacement source incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
context.advance(31_000)
await heartbeat(
context.store,
context.source,
'33333333-3333-4333-8333-333333333333',
1,
context.now()
)
await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull()
expect(await primary.query(
`SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ send_attempts: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('makes concurrent completion and expiry cleanup idempotent', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
const targetControl = await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.competingStore.abortExpiredRegionalRehomes()
])
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true, aborted: false }])
[context.identity.userId]
)
).toEqual([{ completed: true, aborted: false }])
})
it('will not complete against a replacement target incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
expect(await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed_at: null, aborted_at: null }])
expect(
await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
})
it('does not roll an unregistered target back to a stale regional source', async () => {
const context = await fixture()
await context.store.claimRegionalRehome()
await cutover(context.store, context.now())
context.advance(6 * 60_000)
await heartbeat(
context.store,
@@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
expect(
await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
})
it('completes after the drained host re-resolves through the director', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
// The drain recovery lands while both controls are still live.
await context.store.assign(context.identity, 'asia-east2')
expect(await controlAccounting(context.identity)).toEqual({
@@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => {
await context.store.releaseActivity(context.identity, context.sourceControl)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true }])
[context.identity.userId]
)
).toEqual([{ completed: true }])
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
@@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('repairs a skewed control counter before completing the rehome', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
await context.store.releaseActivity(context.identity, context.sourceControl)
// Damage already written by a pre-fix sticky grant.
await primary.query(
`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`,
[context.identity.userId]
)
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
context.identity.userId
])
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await controlAccounting(context.identity)).toEqual({
@@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function cutover(store: RelayAssignmentStore, now: number) {
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
if (!request) return null
const result = await store.commitIdleRegionalRehome(request, safety(now))
if (result.outcome !== 'committed') return null
const [attempt] = await primary.query(
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
)
return {
...request,
preferredRegion: String(attempt!.preferred_region),
assignmentEpoch: Number(attempt!.assignment_epoch)
}
}
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
@@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
await heartbeat(
store,
source,
'11111111-1111-4111-8111-111111111111',
1,
900_000
)
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
await heartbeat(
store,
target,
'22222222-2222-4222-8222-222222222222',
options.targetProtocol ?? 1,
options.targetProtocol ?? 3,
900_000
)
const identity = {
@@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => {
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: '11111111-1111-4111-8111-111111111111'
})
await store.assign(identity, preferredRegion)
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: {
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
}
},
assignment.assignmentEpoch
)
return {
preferredRegion,
store,
@@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
const storeOptions = {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
}
@@ -816,3 +831,15 @@ async function heartbeat(
}
})
}
function safety(now: number) {
return {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
}
cloud/apps/relay/src/regional-rehome-store.test.ts
+392 -609
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/regional-rehome-target-selection.test.ts
+53 -15
View File
@@ -25,6 +25,7 @@ async function setup() {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
@@ -83,11 +84,40 @@ async function setup() {
await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: incarnation(1)
})
await store.assign(identity, 'asia-east2')
const { window } = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
expect(window).toBeDefined()
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
return { database, store, beat, activatePreferredSource }
const safety = () => ({
observedAt: clock,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
return { database, store, beat, activatePreferredSource, safety }
}
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
@@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
describe('regional rehome target selection', () => {
it('never selects a target without connection headroom, even at lowest load', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(lowLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
it('falls to the next clean target when the load winner goes unclean', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(highLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
})
cloud/apps/relay/src/regional-rehome-worker.test.ts
+52 -165
View File
@@ -11,148 +11,37 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
preferredRegion: 'asia-east2',
sourceCellId: 'production-gce-c7',
sourceCellUrl: 'https://c7.relay.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'production-gce-c27',
targetCellIncarnation: '33333333-3333-4333-8333-333333333333',
previousEpoch: 7,
assignmentEpoch: 8,
drainGraceMs: 60_000,
sendAttempts: 1
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
vi.useFakeTimers()
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const worker = startRegionalRehomeWorker(config(), {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore, {
safetySnapshot: () => safety(Date.now()),
random: () => 0
})!
try {
await vi.advanceTimersByTimeAsync(0)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5_999)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(1)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
worker.stop()
await vi.advanceTimersByTimeAsync(60_000)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
} finally {
worker.stop()
vi.useRealTimers()
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDrainReceipt
} as unknown as RelayAssignmentStore
const requests: Array<{ url: string; init?: RequestInit }> = []
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async (audience) => {
expect(audience).toBe('https://relay.example.test/v1/admin/host-drain')
return 'secret-token'
},
fetch: (async (url, init) => {
requests.push({ url: String(url), init })
return Response.json({ v: 1, outcome: 'accepted' })
}) as typeof fetch
})!
await settleWorker()
now = 1_000
await worker.run()
worker.stop()
expect(requests).toHaveLength(1)
expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain')
expect(requests[0]!.url).not.toContain('secret-token')
expect(requests[0]!.init?.headers).toMatchObject({
authorization: 'Bearer secret-token'
})
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
graceMs: 60_000
})
expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111',
'accepted'
)
const logs = warn.mock.calls.map((call) => String(call[0])).join('\n')
expect(logs).not.toContain('private-user')
expect(logs).not.toContain('abcdefghijklmnop')
})
it('fails closed before the observation gate and records bounded dispatch failures', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'target',
previousEpoch: 1,
assignmentEpoch: 2,
drainGraceMs: 60_000,
sendAttempts: 1
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async () => {
throw new Error('token unavailable')
}
})!
await settleWorker()
claimRegionalRehome.mockClear()
now = 100
await worker.run()
worker.stop()
expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111'
)
})
it('keeps a failed poll out of the durable dispatch-failure budget', async () => {
let now = 0
const claimRegionalRehome = vi
.fn()
.mockResolvedValueOnce(null)
.mockRejectedValue(new Error('Connection terminated due to connection timeout'))
const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})!
await settleWorker()
now = 1_000
await expect(worker.run()).resolves.toBeUndefined()
worker.stop()
// The poll never claimed an attempt, so nothing was drained and nothing may
// be charged to the budget that latches the durable control off.
expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([
'orca_relay_regional_rehome_poll_failed'
])
})
it('passes unsafe process telemetry to the durable claim gate', async () => {
let now = 0
let sqlFailures = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
@@ -160,46 +49,40 @@ describe('regional rehome worker', () => {
intervalMs: 60_000
})!
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
sqlFailures = 1
await worker.run()
worker.stop()
expect(claimRegionalRehome).toHaveBeenCalledWith(
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
)
})
it('starts inert on directors so durable control can enable without a restart', async () => {
let now = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(
config(),
assignments,
{
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
}
)
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})
expect(worker).not.toBeNull()
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
await worker!.run()
worker!.stop()
expect(claimRegionalRehome).toHaveBeenCalledOnce()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
expect(
startRegionalRehomeWorker(
config({ role: 'cell' }),
{} as RelayAssignmentStore,
{ safetySnapshot: () => safety(1) }
)
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
safetySnapshot: () => safety(1)
})
).toBeNull()
})
@@ -208,16 +91,20 @@ describe('regional rehome worker', () => {
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
const processSafety = { ...safety(100), reconnects: limit * 10 }
const fleetSafety = { ...safety(100), reconnects: limit }
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)).toBeNull()
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)).toBe('elevated_reconnects')
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)
).toBeNull()
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)
).toBe('elevated_reconnects')
})
})
cloud/apps/relay/src/regional-rehome-worker.ts
+41 -62
View File
@@ -1,4 +1,4 @@
import { z } from 'zod'
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
@@ -20,13 +20,6 @@ export type RegionalRehomeWorker = {
stop: () => void
}
const RegionalHostDrainResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected'])
})
.strict()
export function startRegionalRehomeWorker(
config: RelayConfig,
assignments: RelayAssignmentStore,
@@ -42,7 +35,6 @@ export function startRegionalRehomeWorker(
}
const audience = config.rehomeAudience
const safetySnapshot = options.safetySnapshot
const now = options.now ?? Date.now
const fetchImpl = options.fetch ?? fetch
const tokenProvider =
options.identityToken ??
@@ -52,64 +44,50 @@ export function startRegionalRehomeWorker(
const run = async (): Promise<void> => {
if (stopped || inFlight) return
inFlight = true
let attemptId: string | null = null
try {
const processSafety = safetySnapshot()
const attempt = await assignments.claimRegionalRehome(processSafety)
if (!attempt) return
attemptId = attempt.attemptId
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
if (candidates.length === 0) return
const token = await tokenProvider(audience)
const response = await fetchImpl(
new URL('/v1/admin/host-drain', attempt.sourceCellUrl),
{
method: 'POST',
headers: {
authorization: `Bearer ${token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
attemptId: attempt.attemptId,
userId: attempt.userId,
relayHostId: attempt.relayHostId,
sourceCellId: attempt.sourceCellId,
sourceCellIncarnation: attempt.sourceCellIncarnation,
sourceAssignmentEpoch: attempt.previousEpoch,
graceMs: attempt.drainGraceMs
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
for (const candidate of candidates) {
if (stopped) return
const { sourceCellUrl, ...request } = candidate
try {
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
...request,
cohortPercent: config.regionCorrectionCohortPercent ?? 0,
directorSafety: safetySnapshot()
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
})
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
if (body.outcome === 'committed') {
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_committed',
sourceCellId: candidate.sourceCellId,
targetCellId: candidate.targetCellId
})
)
return
}
} catch (error) {
// The source may have committed; its durable outcome owns recovery.
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_request_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
)
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = RegionalHostDrainResponseSchema.safeParse(await response.json())
if (!body.success) throw new Error('regional_rehome_source_invalid_response')
await assignments.recordRegionalRehomeDrainReceipt(
attempt.attemptId,
body.data.outcome
)
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_dispatched',
sourceCellId: attempt.sourceCellId,
targetCellId: attempt.targetCellId,
outcome: body.data.outcome,
sendAttempts: attempt.sendAttempts
})
)
} catch (error) {
// Only a claimed attempt was drained. A poll that failed before the claim
// - a pool timeout on the once-a-second control read - dispatched nothing,
// so it must not spend the budget that latches the durable control off.
if (attemptId) {
await assignments
.recordRegionalRehomeDispatchFailure(attemptId)
.catch(() => undefined)
}
} catch (error) {
console.warn(
JSON.stringify({
event: attemptId
? 'orca_relay_regional_rehome_dispatch_failed'
: 'orca_relay_regional_rehome_poll_failed',
event: 'orca_relay_regional_rehome_poll_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
@@ -119,7 +97,8 @@ export function startRegionalRehomeWorker(
}
const timer = setInterval(
() => void run(),
options.intervalMs ?? jitteredSweepIntervalMs(1_000, options.random)
// Match the initial ten-moves/minute budget without replanning the join every second.
options.intervalMs ?? jitteredSweepIntervalMs(6_000, options.random)
)
timer.unref()
void run()
cloud/apps/relay/src/relay-observability.test.ts
+179
View File
@@ -5,6 +5,7 @@ import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
percentile,
RelayObservability,
type RelayProcessCounts
} from './relay-observability.js'
@@ -412,3 +413,181 @@ describe('relay observability', () => {
expect(recordSql.mock.calls.map((call) => call[1])).toEqual([true, false, true, false])
})
})
// The pre-change implementation, kept verbatim as the differential oracle. Both
// ranks sorted their own copy and the maximum was a zero-seeded fold.
function legacyPercentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
function legacyLatencySummary(samples: number[]): { p50: number; p95: number; max: number } {
const round = (value: number): number => Number(value.toFixed(3))
return {
p50: round(legacyPercentile(samples, 0.5)),
p95: round(legacyPercentile(samples, 0.95)),
max: round(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
// `-0` and `NaN` both survive a string round trip, unlike a bare equality check.
function describeNumber(value: number): string {
return Object.is(value, -0) ? '-0' : String(value)
}
function expectSameNumber(actual: number, expected: number, label: string): void {
expect(`${label} = ${describeNumber(actual)}`).toBe(`${label} = ${describeNumber(expected)}`)
}
function sparseWindow(size: number, filled: Record<number, number>): number[] {
const values: number[] = new Array<number>(size)
for (const [index, value] of Object.entries(filled)) values[Number(index)] = value
return values
}
// Lehmer generator: stays inside the safe-integer range so the window is
// byte-identical on every engine the relay runs on.
function deterministicWindow(size: number): number[] {
let seed = 20_260_912
return Array.from({ length: size }, () => {
seed = (seed * 48_271) % 2_147_483_647
return (seed % 4_000_000) / 1_000
})
}
const DENSE_WINDOWS: Array<{ name: string; values: number[] }> = [
{ name: 'empty', values: [] },
{ name: 'single', values: [7.5] },
{ name: 'single negative', values: [-7.5] },
{ name: 'ascending', values: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] },
{ name: 'descending', values: [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] },
{ name: 'duplicates', values: [4, 4, 4, 4, 4] },
// The trap: a sorted last element reads -1 here, the zero-seeded fold reads 0.
{ name: 'all negative', values: [-5, -1, -9, -3, -2] },
{ name: 'mixed signs', values: [-2, 3, -7, 0, 11, -0.5] },
{ name: 'signed zero', values: [0, -0, -0, 0] },
{ name: 'negative then signed zero', values: [-3, -0, -1] },
{ name: 'nan leading', values: [NaN, 5, 1, 9] },
{ name: 'nan trailing', values: [5, 1, 9, NaN] },
{ name: 'nan interleaved', values: [5, NaN, 1, NaN, 9] },
{ name: 'all nan', values: [NaN, NaN, NaN] },
{ name: 'positive infinity', values: [Infinity, 3, 1] },
{ name: 'negative infinity', values: [-Infinity, -3, -1] },
{ name: 'both infinities', values: [Infinity, -Infinity, 3, -Infinity] },
{ name: 'infinities and nan', values: [Infinity, NaN, -Infinity, 0] },
{ name: 'sub-millisecond rounding', values: [0.00049, 0.0005, 0.00051, 0.9995] },
{ name: 'reservoir sized', values: deterministicWindow(CONTROL_RTT_RESERVOIR_LIMIT) }
]
// Holes cannot reach the recorders, so they are exercised through `percentile`
// alone — the surface `host-session-registry` also calls.
const SPARSE_WINDOWS: Array<{ name: string; values: number[] }> = [
{ name: 'all holes', values: sparseWindow(4, {}) },
{ name: 'leading hole', values: sparseWindow(5, { 3: 8, 4: 2 }) },
{ name: 'trailing hole', values: sparseWindow(5, { 0: 8, 1: 2 }) },
{ name: 'interleaved holes', values: sparseWindow(6, { 0: 3, 2: -4, 5: 1 }) },
{ name: 'holes with nan', values: sparseWindow(5, { 1: NaN, 3: 6 }) }
]
const PERCENTILE_RANKS = [0, 0.05, 0.5, 0.9, 0.95, 0.99, 1]
type SortWork = { sorts: number; comparisons: number; copiedElements: number }
// Every sorted array here is a fresh spread copy, so its length is the number of
// elements copied to produce it.
function countSortWork(run: () => void): SortWork {
const work: SortWork = { sorts: 0, comparisons: 0, copiedElements: 0 }
const original = Array.prototype.sort
const patched = Array.prototype as { sort: unknown }
patched.sort = function <T>(this: T[], compare?: (left: T, right: T) => number): T[] {
work.sorts++
work.copiedElements += this.length
return original.call(this, (left: T, right: T) => {
work.comparisons++
return compare ? compare(left, right) : String(left) < String(right) ? -1 : 1
})
}
try {
run()
} finally {
patched.sort = original
}
return work
}
function summaryThroughFlush(samples: number[]): { p50: number; p95: number; max: number } {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (const sample of samples) observability.recordControlRenewal(sample, 'renewed')
observability.flush(counts)
const entry = entries[0]!
return {
p50: entry.controlRenewalLatencyMsP50 as number,
p95: entry.controlRenewalLatencyMsP95 as number,
max: entry.controlRenewalLatencyMsMax as number
}
}
describe('latency window summarisation', () => {
it('matches the pre-change percentile on every edge-case window', () => {
let compared = 0
for (const { name, values } of [...DENSE_WINDOWS, ...SPARSE_WINDOWS]) {
for (const rank of PERCENTILE_RANKS) {
expectSameNumber(
percentile(values, rank),
legacyPercentile(values, rank),
`${name} @ p${rank}`
)
compared++
}
}
expect(compared).toBe((DENSE_WINDOWS.length + SPARSE_WINDOWS.length) * PERCENTILE_RANKS.length)
})
it('matches the pre-change p50, p95 and maximum through a flush', () => {
let compared = 0
for (const { name, values } of DENSE_WINDOWS) {
const actual = summaryThroughFlush(values)
const expected = legacyLatencySummary(values)
expectSameNumber(actual.p50, expected.p50, `${name} p50`)
expectSameNumber(actual.p95, expected.p95, `${name} p95`)
// The zero-seeded fold, not the sorted last element: all-negative and NaN
// windows disagree between the two.
expectSameNumber(actual.max, expected.max, `${name} max`)
compared += 3
}
expect(compared).toBe(DENSE_WINDOWS.length * 3)
// The trap, spelled out: the sorted window ends at -1 but the fold reports 0.
expect(summaryThroughFlush([-5, -1, -9, -3, -2]).max).toBe(0)
expect(Number.isNaN(summaryThroughFlush([5, NaN, 1]).max)).toBe(true)
})
it('sorts each latency window once instead of once per rank', () => {
const samples = deterministicWindow(CONTROL_RTT_RESERVOIR_LIMIT)
const before = countSortWork(() => legacyLatencySummary(samples))
const after = countSortWork(() => summaryThroughFlush(samples))
expect(before.sorts).toBe(2)
expect(after.sorts).toBe(1)
expect(before.copiedElements).toBe(2 * CONTROL_RTT_RESERVOIR_LIMIT)
expect(after.copiedElements).toBe(CONTROL_RTT_RESERVOIR_LIMIT)
// Identical input and comparator, so the dropped sort is exactly half the
// comparator calls rather than an engine-specific constant.
expect(before.comparisons).toBeGreaterThan(CONTROL_RTT_RESERVOIR_LIMIT)
expect(after.comparisons).toBe(before.comparisons / 2)
})
it('never sorts an empty window and leaves the caller window untouched', () => {
const samples = [5, -1, NaN, 3, -0]
const before = samples.map(describeNumber)
expect(countSortWork(() => summaryThroughFlush([])).sorts).toBe(0)
expect(countSortWork(() => percentile([], 0.95)).sorts).toBe(0)
countSortWork(() => summaryThroughFlush(samples))
percentile(samples, 0.5)
expect(samples.map(describeNumber)).toEqual(before)
})
})
cloud/apps/relay/src/relay-observability.ts
+17 -5
View File
@@ -168,10 +168,18 @@ const emptyDeltas = (): RelayMetricDeltas => ({
controlActivityRecoveryFailures: 0
})
function ascending(values: number[]): number[] {
return [...values].sort((left, right) => left - right)
}
// Holes and NaN land past the requested rank, so the fallback still applies.
function nearestRank(sorted: number[], percentileRank: number): number {
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
export function percentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
return nearestRank(ascending(values), percentileRank)
}
function roundMs(value: number): number {
@@ -179,11 +187,15 @@ function roundMs(value: number): number {
}
// Spreading a window into Math.max blows the stack once a busy cell samples
// enough of it, so the maximum is folded instead.
// enough of it, so the maximum is folded instead. The fold is also not
// interchangeable with the sorted last element: it is seeded with zero, so an
// all-negative or NaN window reads differently.
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
// One sorted copy serves both ranks.
const sorted = samples.length === 0 ? samples : ascending(samples)
return {
p50: roundMs(percentile(samples, 0.5)),
p95: roundMs(percentile(samples, 0.95)),
p50: roundMs(nearestRank(sorted, 0.5)),
p95: roundMs(nearestRank(sorted, 0.95)),
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
cloud/apps/relay/src/relay-region-app.test.ts
+155
View File
@@ -40,6 +40,7 @@ describe('Relay region API', () => {
)
expect(response.status).toBe(200)
expect(await response.clone().json()).not.toHaveProperty('regionCorrection')
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'asiahost00000001' },
'asia-east2',
@@ -83,6 +84,160 @@ describe('Relay region API', () => {
)
})
it('preserves the cold-start hint and binds a negotiated window after placement', async () => {
const assignment = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn(async () => assignment)
const window = {
generation: 2,
expiresAt: Date.now() + 86_400_000,
assignmentEpoch: 7,
incumbentRegion: 'asia-east2',
policyVersion: 1
}
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = { v: 1, action: 'issue-window' }
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
'asia-east2',
'asia-east2'
)
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({
v: 1,
window
})
})
it('returns successful placement when optional window storage is unavailable', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: async () => ({
cellId: 'asia-c1',
region: 'asia-east2',
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
}),
exchangeRegionCorrection: async () => {
throw new Error('database unavailable')
}
} as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection: { v: 1, action: 'issue-window' }
})
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
})
})
it('does not place or write a legacy hint when reporting migration evidence', async () => {
const current = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn()
const resolve = vi.fn(async () => current)
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'us-central1',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7)
})
it('does not manufacture an assignment for a report whose assignment disappeared', async () => {
const assign = vi.fn()
const exchangeRegionCorrection = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
regionCorrection: {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'timeout'
}
})
)
expect(response.status).toBe(409)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).not.toHaveBeenCalled()
})
it('exposes only the store-provided healthy catalog from directors', async () => {
const regionCatalog = vi.fn(async () => [
{ region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] }
cloud/apps/relay/src/relay-server.ts
+25 -8
View File
@@ -20,14 +20,12 @@ import { createRelayApp } from './app.js'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { RelayCredentialStore } from './credential-store.js'
import type { RelayDatabase } from './database.js'
import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js'
import { HostSessionRegistry } from './host-session-registry.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import { RelayObservability } from './relay-observability.js'
import {
RelayConnectionLedger,
type RelayConnectionUpgrade
} from './relay-connection-ledger.js'
import { combineRegionalRehomeSafety } from './regional-rehome-safety.js'
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js'
import { createRelayReadiness } from './relay-readiness.js'
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
@@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void {
function admissionSource(request: IncomingMessage): string {
const forwarded = request.headers['x-forwarded-for']
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '')
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
.split(',')
.map((entry) => entry.trim())
.filter(Boolean)
@@ -112,6 +110,7 @@ export function createRelayServer(
const store = new RelayCredentialStore(observedDatabase, options.now)
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
requireLiveCells: config.role === 'director',
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
recordControlRenewal: (durationMs, outcome) =>
observability.recordControlRenewal?.(durationMs, outcome)
})
@@ -127,17 +126,35 @@ export function createRelayServer(
queuedBytes,
observability,
options.now,
options.random
options.random,
cellIncarnation
)
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
return Promise.resolve({ outcome: 'deferred' })
}
return sessions.idleRehome(input,
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
input.directorSafety,
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
), input.cohortPercent),
() => assignments.reconcileIdleRegionalRehome(input)
)
},
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
cellIncarnation,
isDraining: () => sessions.isDraining(),
runtimeCounts: () => runtimeCounts(),
regionalRehomeSafetySnapshot: () => ({
...observability.regionalRehomeRuntimeSafety(),
...readRelayDatabasePoolPressure(database)
}),
ready,
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
recordAssignmentRejectionReason: (lane, reason) =>
@@ -339,7 +356,7 @@ export function createRelayServer(
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
// Released combined-service invites gain their first durable cell assignment here.
const assignment = identity
? (await assignments.resolve(identity)) ?? (await assignments.assign(identity))
? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity)))
: null
if (!invite || !assignment) {
phoneAdmission?.hostData.release()
cloud/apps/relay/src/relay-sweep-schedule.test.ts
+3 -3
View File
@@ -19,7 +19,7 @@ describe('sweep schedule jitter', () => {
expect(SWEEP_JITTER_FRACTION).toBeGreaterThan(0)
})
it('jitters the regional rehome dispatch tick, which every director runs each second', () => {
it('jitters the six-second regional rehome dispatch tick across directors', () => {
const timers: number[] = []
const setIntervalSpy = vi
.spyOn(globalThis, 'setInterval')
@@ -35,14 +35,14 @@ describe('sweep schedule jitter', () => {
rehomeAudience: 'https://rehome.example.test',
rehomeDirectorServiceAccount: 'rehome@example.test'
} as never,
{ claimRegionalRehome: async () => null } as never,
{ selectIdleRegionalRehomeCandidates: async () => [] } as never,
{ random: () => 0.5, safetySnapshot: () => ({}) as never }
)
} finally {
setIntervalSpy.mockRestore()
}
expect(timers).toEqual([1_100])
expect(timers).toEqual([6_600])
})
// Why: index.ts boots a server on import, so its wiring can only be read.
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md
+8
View File
@@ -0,0 +1,8 @@
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
```text
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
```
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts
+146
View File
@@ -0,0 +1,146 @@
import { z } from 'zod'
import {
Base6432ByteSchema,
Base64Raw24ByteSchema,
Base64Url32ByteSchema,
EpochMsSchema,
GenerationSchema,
OpaqueIdSchema,
PositiveDurationMsSchema,
RelayHostIdSchema
} from './wire-scalars.js'
const AppVersionSchema = z.string().min(1).max(128)
const BoundedCiphertextSchema = z
.string()
.min(1)
.max(16 * 1024)
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
const ConnectionKindSchema = z.enum(['invite', 'resume'])
export const HostHelloSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
assignmentEpoch: GenerationSchema,
hostPublicKeyB64: Base6432ByteSchema,
appVersion: AppVersionSchema,
previousGeneration: GenerationSchema.optional(),
controlResumeSecret: Base64Url32ByteSchema.optional()
})
.strict()
export const HostChallengeSchema = z
.object({
challengeId: OpaqueIdSchema,
relayEphemeralPublicKeyB64: Base6432ByteSchema,
nonceB64: Base64Raw24ByteSchema,
ciphertextB64: BoundedCiphertextSchema,
expiresAt: EpochMsSchema
})
.strict()
export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
.object({
v: z.literal(1),
generation: GenerationSchema,
controlResumeSecret: Base64Url32ByteSchema,
leaseExpiresAt: EpochMsSchema,
activeConnIds: z.array(OpaqueIdSchema).max(8),
pendingConns: z.array(PendingConnectionSchema).max(8)
})
.strict()
export const ConnectionOpenSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema,
relayDeviceId: OpaqueIdSchema,
attachDeadlineMs: PositiveDurationMsSchema
})
.strict()
export const HostDataAuthSchema = z
.object({
v: z.literal(1),
connTicket: Base64Url32ByteSchema,
generation: GenerationSchema
})
.strict()
export const InviteCreateSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const InviteCreatedSchema = z
.object({
reqId: OpaqueIdSchema,
inviteToken: Base64Url32ByteSchema,
expiresAt: EpochMsSchema,
maxAttempts: z.number().int().positive().max(16)
})
.strict()
export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
export type HostHello = z.infer<typeof HostHelloSchema>
export type HostChallenge = z.infer<typeof HostChallengeSchema>
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
export type InviteCreate = z.infer<typeof InviteCreateSchema>
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
export type Drain = z.infer<typeof DrainSchema>
export type Heartbeat = z.infer<typeof HeartbeatSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts
+75
View File
@@ -0,0 +1,75 @@
import { z } from 'zod'
import {
Base64Url32ByteSchema,
CanonicalHttpsOriginSchema,
EpochMsSchema,
GenerationSchema,
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
})
.strict()
export const AssignmentResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
})
.strict()
export const ResolveRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
resumeToken: Base64Url32ByteSchema
})
.strict()
export const ResolveResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
leaseExpiresAt: EpochMsSchema
})
.strict()
export const RelayMovedSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema
})
.strict()
export function isTrustedNewerMove(input: {
sourceOrigin: string
configuredDirectorOrigin: string
currentAssignmentEpoch: number
move: z.infer<typeof RelayMovedSchema>
}): boolean {
// Why: cells and stale director responses must never redirect a credential-bearing client.
return (
input.sourceOrigin === input.configuredDirectorOrigin &&
input.move.assignmentEpoch > input.currentAssignmentEpoch
)
}
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
export type RelayMoved = z.infer<typeof RelayMovedSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts
+71
View File
@@ -0,0 +1,71 @@
import { z } from 'zod'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
function isCanonicalHttpsOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts
+20
View File
@@ -0,0 +1,20 @@
import { z } from 'zod'
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
export const OpaqueIdSchema = z.string().min(1).max(128)
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
} catch {
return false
}
}, 'must be a canonical HTTPS origin')
cloud/apps/relay/tsconfig.build.json
+1 -1
View File
@@ -6,5 +6,5 @@
"outDir": "dist",
"rootDir": "src"
},
"exclude": ["src/**/*.test.ts"]
"exclude": ["src/**/*.test.ts", "src/test-fixtures/**"]
}
cloud/dev/scripts/deploy-relay-blue-green.mjs
+22 -1
View File
@@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
'orca-cloud-relay-regional-placement-enabled'
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
export const DIRECTOR_REHOME_IDENTITY_ENV =
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
@@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) {
return { changed: additions.length > 0, value: JSON.stringify(desired) }
}
export function correctionCohortPercent(value) {
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
throw new Error('region correction cohort must be an integer from 0 to 100')
}
return String(value)
}
export function directorDeploymentEnvironment(config) {
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
if (config.image !== undefined && imageDigest === undefined) {
@@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) {
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
}
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve') {
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
}
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
const rehomeDirectorServiceAccount = projectServiceAccount(
@@ -302,7 +314,8 @@ export function parseArguments(argv) {
values['rehome-director-service-account'] !== undefined ||
values['rehome-audience'] !== undefined ||
values['expected-rehome-generation'] !== undefined ||
values['rehome-control-origin'] !== undefined
values['rehome-control-origin'] !== undefined ||
values['region-correction-cohort-percent'] !== undefined
) {
throw new Error('director configuration arguments require --role director')
}
@@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) {
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
const currentEnvironment = revisionEnvironment(servingRevision)
const deploymentEnvironment = directorDeploymentEnvironment(config)
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
)
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve' &&
config['expected-rehome-generation'] === undefined) {
throw new Error('cohort changes require an exact disabled regional-rehome generation')
}
const mutableEnvironment = {
...deploymentEnvironment,
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
cloud/dev/scripts/deploy-relay-blue-green.test.mjs
+42
View File
@@ -4,6 +4,8 @@ import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import {
activeRevision,
correctionCohortPercent,
DIRECTOR_CORRECTION_COHORT_ENV,
cloudRunTrafficTag,
DIRECTOR_ADMISSION_ENVIRONMENT,
DIRECTOR_REGIONAL_PLACEMENT_ENV,
@@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err
/forbidden/
)
})
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
}
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
const harness = directorHarness()
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
await deployDirector({}, 'candidate-new', harness.operations)
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
}
})
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
const harness = directorHarness()
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
assert.equal(harness.state.nextRevision, 2)
await deployDirector({}, 'candidate-new', harness.operations)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
})
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
const harness = directorHarness()
let verified = 0
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
await deployDirector(config, 'candidate-new', { ...harness.operations,
assertRegionalRehomeDisabled: async () => { verified++ } })
assert.ok(verified >= 2)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
cloud/dev/scripts/probe-relay-rehome-trust.mjs
+20 -3
View File
@@ -38,7 +38,7 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env)
export async function probeRehomeTrust(config, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const response = await fetchAdminOnceMore(
const request = () => fetchAdminOnceMore(
fetchImpl,
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
{
@@ -55,9 +55,26 @@ export async function probeRehomeTrust(config, dependencies = {}) {
},
{ wait: dependencies.wait }
)
const body = await response.json().catch(() => ({}))
let response = await request()
let body = await response.json().catch(() => ({}))
// The director wraps source HTTP failures in 409; retry only explicit transient statuses.
if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) {
await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000)
response = await request()
body = await response.json().catch(() => ({}))
}
if (!response.ok) {
throw new Error(`application-mediated rehome trust probe returned ${response.status}`)
const safeReasons = new Set([
'invalid_token', 'director_only', 'invalid_request',
'regional_rehome_trust_not_configured',
'regional_rehome_trust_probe_source_unavailable',
'regional_rehome_trust_probe_source_invalid_response',
'regional_rehome_trust_probe_not_proven',
...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504]
.map((status) => `regional_rehome_trust_probe_source_${status}`)
])
const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error'
throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`)
}
if (
body.v !== 1 ||
cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
+37
View File
@@ -131,3 +131,40 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells',
)
}
})
test('retries one director-wrapped source 503 without relaxing the proof', async () => {
let calls = 0
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => ++calls === 1
? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
: Response.json(provenProbe)
})
assert.equal(calls, 2)
assert.equal(result.proven, true)
})
test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => {
for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => { throw new Error('must not retry') },
fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) }
}), error => {
assert.match(error.message, /returned 409/)
assert.ok(!error.message.includes('secret-token-example'))
if (reason.endsWith('_403')) assert.match(error.message, /source_403/)
return true
})
assert.equal(calls, 1)
}
})
test('stops after the second wrapped transient failure', async () => {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
}), /returned 409.*source_503/)
assert.equal(calls, 2)
})
cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs
+13 -3
View File
@@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
throw error
}
const serving = (service.status?.traffic ?? []).filter(
@@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) =>
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
)
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
(typeof cohortSettings[0].value !== 'string' ||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
throw new Error('serving region correction cohort is invalid')
}
const cohort_percent = cohortSettings[0]?.value ?? '0'
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version }
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
@@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version }
return { version: reference.version, cohort_percent }
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs
+32 -4
View File
@@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
}
})
assert.deepEqual(result, { version: '11' })
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
assert.equal(calls[1][3], 'relay-serving')
})
@@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
]) {
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
}), { version: '1' })
}), { version: '1', cohort_percent: '0' })
}
})
@@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => {
notFound.code = 'NOT_FOUND'
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: () => { throw notFound }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: { spec: { containers: [{ env: [] }] } }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
})
test('classifies real absent-service stderr without weakening revision failures', () => {
@@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () =>
run: () => { throw denied }
}), denied)
})
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
for (const value of ['0', '1', '17', '100']) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), { version: '11', cohort_percent: value })
}
})
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
const cases = [
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
[{ name, value: '1' }, { name, value: '2' }]
]
for (const settings of cases) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push(...settings)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), /cohort is invalid/)
}
})
cloud/dev/scripts/relay-production-same-cap-wave.mjs
+6 -3
View File
@@ -87,18 +87,21 @@ export function canaryAuthority(input) {
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
authority.selectorGeneration !== Number(expected.selectorGeneration) ||
!Number.isSafeInteger(authority.selectorGeneration) ||
authority.selectorGeneration < 0 ||
!Number.isSafeInteger(selectorGeneration) ||
selectorGeneration < authority.selectorGeneration ||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
// same SHA; every field above still pins this batch to that exact canary.
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
currentSha: expected.commitSha,
cloud/dev/scripts/relay-production-same-cap-wave.test.mjs
+36 -1
View File
@@ -109,6 +109,41 @@ test('seals and verifies canary authority for later batches', () => {
}), /does not match/)
})
test('reuses a canary across selector advances only within the same control epoch', () => {
const authority = canaryAuthority({
cellIds: 'production-gce-c7', targetDigest, rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), authority)
}
for (const generation of ['12', '-1', 'NaN', 'Infinity', '13.5', '9007199254740992']) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), /does not match/)
}
for (const generation of [-1, NaN, Infinity, 13.5, '13', Number.MAX_SAFE_INTEGER + 1]) {
assert.throws(() => verifyCanaryAuthority({
...authority, selectorGeneration: generation
}, expected), /does not match/)
}
for (const mismatch of [
{ rehomeGeneration: '3' }, { rehomeGeneration: '5' },
{ targetDigest: rollbackDigest }, { rollbackDigest: targetDigest }, { runId: '43' }
]) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, ...mismatch
}), /does not match/)
}
})
function gitIn(root, ...args) {
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
}
@@ -158,7 +193,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
selectorGeneration: '21',
rehomeGeneration: '4'
}, repositoryRoot)
assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+13 -7
View File
@@ -77,14 +77,14 @@ function rollPlan({ cellId, cap, protocol }) {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
}),
self_link: null
},
@@ -178,11 +178,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -204,7 +202,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.throws(
() => validateCapacityPlan(plan, {
...config,
regionalRehomeProtocol: String(1 - protocol)
regionalRehomeProtocol: '0'
}),
/reviewed image and capacity/,
cellId
@@ -239,3 +237,11 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
})
// Both trusted versions must prove the same authenticated drain boundary.
it('proves rehome trust for protocol 3 on forward and rollback rolls', () => {
const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0]
assert.match(step, /inputs\.rollback-rehome-protocol != '0'/)
assert.match(step, /inputs\.target-rehome-protocol != '0'/)
assert.match(step, /probe-relay-rehome-trust\.mjs/)
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+3 -3
View File
@@ -9,7 +9,7 @@ const REHOME_CONFIG =
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) {
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
}
return Number(regionalRehomeProtocol)
@@ -43,7 +43,7 @@ export function parseCapacityPlanArguments(argv) {
(!values['rollback-image'] ||
!values['rehome-director-service-account'] ||
!values['rehome-audience'] ||
!['0', '1'].includes(values['regional-rehome-protocol']))
!['0', '1', '3'].includes(values['regional-rehome-protocol']))
) throw new Error('same-cap validation requires rollback image and rehome trust config')
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
@@ -227,7 +227,7 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
])
}
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1
if (rehomeTrusted) {
expected.push(
[
cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
+4
View File
@@ -756,6 +756,10 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
cloud/dev/scripts/verify-relay-capacity-transition.mjs
+2 -2
View File
@@ -108,8 +108,8 @@ export function parseCapacityTransitionArguments(argv) {
const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined
? undefined
: integer(values['regional-rehome-protocol'], '--regional-rehome-protocol')
if (regionalRehomeProtocol !== undefined && ![0, 1].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0 or 1')
if (regionalRehomeProtocol !== undefined && ![0, 1, 3].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0, 1, or 3')
}
if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) {
throw new Error('unavailable runtime cannot prove the regional rehome protocol')
cloud/docs/orca-relay-operations.md
+71 -5
View File
@@ -466,11 +466,16 @@ After a deployment traffic shift, preserve the old revision/tag until metrics an
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
Idle regional correction requires both source and target cells to advertise
`regionalRehomeProtocol >= 3`. PR #20105 introduced this capability version with
the idle handoff implementation. With that runtime, both rehome trust environment
settings must be configured to advertise 3; otherwise the cell advertises 0.
An older trusted runtime can advertise 1: configuring trust alone does not upgrade
its implementation. The separate `connectionCapacityProtocol: 2` health field does
not establish regional-correction readiness. Verify the live runtime version and
image, not only instance-template configuration, before rollout or enablement.
Incompatible cells are excluded from correction selection; enabling the cohort
cannot override this check. Director and cell deployments are separate operations.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
@@ -491,3 +496,64 @@ Run and record each scenario in staging before launch:
- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback.
The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them.
## Optional measured region correction (deployment gated)
New optimization claims require both the durable regional-rehome control and
`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**).
Turning either gate off stops new optional moves; ordinary migration cleanup and
recovery continue. Legacy preferred-region hints do not certify a correction. Both
cells must advertise regional protocol3 and the authenticated desktop control must
advertise idle-regional-rehome-v1. The source must have no actual client sockets or
pending admission/control work; a live control socket alone does not prevent a move.
The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**.
It returns full-population eligibility/exclusion counts, open-migration capacity,
process-safety gating and aggregate migration outcomes; it never claims a
host or changes the failure budget. This is advisory, with separately read state:
concurrent assignments, capacity changes, rate pauses and control changes can make
the next claim differ. Inspect the durable control separately before enabling.
Do not treat an unavailable/failed preview as zero eligible hosts.
`orca_relay_region_correction_outcomes` reports attempts by source/target,
registration/completion/abort state, oldest open age and target
reservation units every five minutes. `orca_relay_region_comparison` samples a
stable 10% of accepted reports (including unchanged hosts), keyed by host digest,
assignment epoch and decision generation. Existing control RTT and client-accept
logs include assignment epoch, control generation and drain mode; join those for
matched before/after and unchanged-cohort comparisons. Client accept latency is
connection setup, not application command round trip. No application-latency
improvement has been demonstrated by probe differences alone.
Quiet live connections count as work and defer optional correction indefinitely.
A returning client may race with the short admission gate and retry normally. No
optimization timer may close an established client. Investigate failed registration,
ambiguous authority, stuck reservations and reconnect/failure rates against agreed
limits. A database outage can keep the source fenced until locked reconciliation
establishes its authority; timeout alone is not permission to reopen admissions.
All directors must run the reviewed idle worker before enabling. Record the tested
immutable source and rollback revisions, then verify the ordinary migration recovery
path before rollout. There is no retained-source table or renewal protocol. Deploying
supporting cells/desktops and enabling a cohort require separate rollout authorization
and explicit numerical stop criteria; this change enables neither.
### Setting the correction cohort during a reviewed director rollout
The existing **Deploy Relay Production Director** workflow accepts
`region-correction-cohort-percent`: `preserve` (default) or an integer0–100.
It carries the cohort onto both candidate and compatible rollback revisions and
verifies the environment before promotion. If the predecessor has no setting,
`preserve` stamps zero. An explicit change requires the exact disabled durable
rehome generation; configuring a nonzero cohort does not itself enable the sweep.
The usual image, identity, health and traffic checks remain in force. No workflow
was dispatched as part of implementation.
Terraform reads the cohort from the same traffic-serving revision used to preserve
regional placement. A later apply therefore preserves a workflow-set cohort,
including explicit zero; only an absent service/setting bootstraps to0. Malformed
or ambiguous live settings fail the plan instead of silently resetting the cohort.
The audited director workflow owns subsequent changes.
Before the first nonzero cohort, verify compatible protocol2 cells, updated
cleanup workers, preview eligibility, both serving/rollback images and the
explicitly approved observation/stop criteria.
cloud/docs/relay-incident-monitor.md
+6 -1
View File
@@ -112,7 +112,8 @@ durably marked consumed before mutation and cannot authorize another run.
| Director instances | outside 5–6 |
| Director CPU or memory | over 80% |
| Director concurrency | over 64 |
| Unexpected director 5xx or auth 5xx in five minutes | over 0 |
| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
| Auth 5xx in five minutes | over 0 |
| Connections per cell process | over 500 |
| Queued bytes per cell process | over 48 MiB |
| Blocked or expired/unregistered migration | over 0 |
@@ -276,3 +277,7 @@ without its segment is a compile error in relay-contract, not a silent gap.
load the director's three-connection database pool.
- Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence.
- Added the manual production workflow. It has not been dispatched.
### Director error allowance (2026-09-12)
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
cloud/infra/terraform/relay.tf
+5
View File
@@ -169,6 +169,11 @@ resource "google_cloud_run_v2_service" "relay" {
}
}
env {
name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT"
value = data.external.relay_serving_regional_placement_version.result.cohort_percent
}
ports {
container_port = 8080
}
cloud/packages/push-contract/src/send-messages.ts
+2 -1
View File
@@ -26,7 +26,8 @@ export const PushNotificationSchema = z
agentState: PushAgentStateSchema.nullable(),
title: z.string().min(1).max(PUSH_LIMITS.titleMaxChars),
body: z.string().max(PUSH_LIMITS.bodyMaxChars),
worktreeId: z.string().min(1).max(2048).optional()
worktreeId: z.string().min(1).max(2048).optional(),
paneKey: z.string().min(1).max(2048).optional()
})
.strict()
.refine(
cloud/packages/relay-contract/src/control-messages.ts
+14 -2
View File
@@ -50,6 +50,7 @@ export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
@@ -121,11 +122,22 @@ export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const AuthRefreshSchema = z
.object({
relayJwt: z
.string()
.min(1)
.max(8 * 1024)
})
.strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
cloud/packages/relay-contract/src/director-messages.ts
+12 -3
View File
@@ -7,8 +7,15 @@ import {
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
import {
RegionCorrectionRequestSchema,
RegionCorrectionResponseSchema
} from './region-correction.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
const SignedAssignmentLeaseSchema = z
.string()
.min(1)
.max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
@@ -17,7 +24,8 @@ export const AssignmentRequestSchema = z
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
preferredRegion: RelayRegionSchema.optional(),
regionCorrection: RegionCorrectionRequestSchema.optional()
})
.strict()
@@ -26,7 +34,8 @@ export const AssignmentResponseSchema = z
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
lease: SignedAssignmentLeaseSchema,
regionCorrection: RegionCorrectionResponseSchema.optional()
})
.strict()
cloud/packages/relay-contract/src/idle-regional-rehome.ts
+26
View File
@@ -0,0 +1,26 @@
import { z } from 'zod'
import { GenerationSchema, RelayHostIdSchema } from './wire-scalars.js'
export const IdleRegionalRehomeRequestSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
userId: z.string().min(1).max(256),
relayHostId: RelayHostIdSchema,
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: GenerationSchema.refine((value) => value > 0),
sourceGeneration: GenerationSchema.refine((value) => value > 0),
targetCellId: z.string().min(1).max(128)
})
.strict()
export const IdleRegionalRehomeResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['busy', 'committed', 'deferred', 'stale'])
})
.strict()
export type IdleRegionalRehomeRequest = z.infer<typeof IdleRegionalRehomeRequestSchema>
export type IdleRegionalRehomeOutcome = z.infer<typeof IdleRegionalRehomeResponseSchema>['outcome']
cloud/packages/relay-contract/src/index.ts
+2
View File
@@ -13,3 +13,5 @@ export * from './resume-confirmation-contract.js'
export * from './relay-regions.js'
export * from './splice-state-machine.js'
export * from './wire-scalars.js'
export * from './region-correction.js'
export * from './idle-regional-rehome.js'
cloud/packages/relay-contract/src/region-correction.test.ts
+76
View File
@@ -0,0 +1,76 @@
import { describe, expect, it } from 'vitest'
import { AssignmentRequestSchema, AssignmentResponseSchema } from './director-messages.js'
import { DrainSchema } from './control-messages.js'
import { RegionCorrectionRequestSchema } from './region-correction.js'
const report = {
v: 1,
action: 'report',
generation: 3,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const retention = {
mode: 'finish-existing',
attemptId: '11111111-1111-4111-8111-111111111111',
sourceGeneration: 3,
sourceAssignmentEpoch: 7
}
describe('region correction wire boundaries', () => {
it('keeps legacy assignment shapes readable without negotiated fields', () => {
expect(
AssignmentRequestSchema.parse({ v: 1, relayHostId: 'abcdefghijklmnop' })
).not.toHaveProperty('regionCorrection')
expect(
AssignmentResponseSchema.parse({
v: 1,
cellUrl: 'https://cell.example',
assignmentEpoch: 1,
lease: 'synthetic-lease'
})
).not.toHaveProperty('regionCorrection')
})
it('accepts complete comparison evidence and explicit inconclusive reports', () => {
expect(RegionCorrectionRequestSchema.safeParse(report).success).toBe(true)
const { measurements: _measurements, ...basis } = report
expect(
RegionCorrectionRequestSchema.safeParse({
...basis,
outcome: 'inconclusive',
reason: 'probe-unavailable'
}).success
).toBe(true)
})
it.each([
{ measurements: { 'us-central1': 40 } },
{ measurements: { 'us-central1': -1, 'asia-east2': 10 } },
{ measurements: { 'us-central1': Infinity, 'asia-east2': 10 } },
{ measurements: { 'us-central1': 120_001, 'asia-east2': 10 } },
{ generation: Number.MAX_SAFE_INTEGER + 1 },
{ assignmentEpoch: 1.2 },
{ policyVersion: 2 },
{ outcome: 'inconclusive', reason: 'timeout' }
])('rejects ambiguous or unbounded evidence: %j', (override) => {
expect(RegionCorrectionRequestSchema.safeParse({ ...report, ...override }).success).toBe(false)
})
it('rejects reporting and issuing a window in the same request', () => {
expect(
RegionCorrectionRequestSchema.safeParse({
...report,
action: 'issue-window'
}).success
).toBe(false)
})
it('uses ordinary drain and rejects the superseded retention extension', () => {
const ordinary = { graceMs: 0, recovery: 'resolve-director' }
expect(DrainSchema.parse(ordinary)).toEqual(ordinary)
expect(DrainSchema.safeParse({ ...ordinary, retention }).success).toBe(false)
})
})
cloud/packages/relay-contract/src/region-correction.ts
+60
View File
@@ -0,0 +1,60 @@
import { z } from 'zod'
import { EpochMsSchema, GenerationSchema } from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const RttSchema = z.number().finite().nonnegative().max(120_000)
export const RegionMeasurementsSchema = z
.object({
'us-central1': RttSchema,
'asia-east2': RttSchema
})
.strict()
export const RegionMeasurementWindowSchema = z
.object({
generation: GenerationSchema,
expiresAt: EpochMsSchema,
assignmentEpoch: GenerationSchema,
incumbentRegion: RelayRegionSchema,
policyVersion: z.literal(1)
})
.strict()
const ReportBasis = {
v: z.literal(1),
action: z.literal('report'),
generation: GenerationSchema,
assignmentEpoch: GenerationSchema,
policyVersion: z.literal(1)
}
export const RegionCorrectionRequestSchema = z.union([
z.object({ v: z.literal(1), action: z.literal('issue-window') }).strict(),
z
.object({
...ReportBasis,
outcome: z.literal('conclusive'),
measurements: RegionMeasurementsSchema
})
.strict(),
z
.object({
...ReportBasis,
outcome: z.literal('inconclusive'),
reason: z.string().min(1).max(64)
})
.strict()
])
export const RegionCorrectionResponseSchema = z
.object({
v: z.literal(1),
window: RegionMeasurementWindowSchema.optional(),
reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional()
})
.strict()
export type RegionMeasurements = z.infer<typeof RegionMeasurementsSchema>
export type RegionMeasurementWindow = z.infer<typeof RegionMeasurementWindowSchema>
export type RegionCorrectionRequest = z.infer<typeof RegionCorrectionRequestSchema>
export type RegionCorrectionResponse = z.infer<typeof RegionCorrectionResponseSchema>
config/electron-builder.config.cjs
+18 -4
View File
@@ -7,6 +7,7 @@ const {
verifyPackagedDaemonEntryBoots
} = require('./scripts/verify-packaged-daemon-entry.cjs')
const {
assertPackagedNativeVariantsInstalled,
createPackagedRuntimeNodeModuleResources,
prunePackagedRuntimeNodeModules,
verifyPackagedMainRuntimeDeps
@@ -147,6 +148,16 @@ const rpmElectronRuntimeDependencies = [
// config/nsis/orca-installer-hooks.nsh, which registers the same set on Windows.
const MARKDOWN_FILE_EXTENSIONS = ['md', 'markdown', 'mdx']
// Why: the config must load on a host-only install without resolving unused Windows addons.
// This is load-time tolerance only; beforePack enforces that the target's natives are installed.
// Why one package: @vscode/windows-process-tree is the only os: win32 npm addon;
// @orca/windows-registry is a workspace link present on every host, so its presence proves nothing.
const windowsRuntimeResources = existsSync(
join(__dirname, '..', 'node_modules', '@vscode', 'windows-process-tree', 'package.json')
)
? createPackagedRuntimeNodeModuleResources('win32')
: []
/** @type {import('electron-builder').Configuration} */
module.exports = {
appId,
@@ -278,6 +289,9 @@ module.exports = {
verifyStaticAppImagePackage(file, arch)
}
},
beforePack: (context) => {
assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch)
},
afterPack: async (context) => {
const resourcesDir =
context.electronPlatformName === 'darwin'
@@ -319,9 +333,9 @@ module.exports = {
// Why: a Linux runner-image glibc bump silently shipped a node-pty pty.node
// requiring GLIBC_2.34, crashing the app on startup on Ubuntu 20.04 (#9902).
// Fail packaging if any bundled native binary exceeds the supported floor.
// Why after the prune: cross-builds intentionally install every optional
// native variant, so an arm64 slice still carries the x64 @parcel/watcher
// until prunePackagedRuntimeNodeModules drops it.
// Why after the prune: `pnpm install:release` widens the CPU set for cross-builds,
// so an arm64 slice can still carry the x64 @parcel/watcher until
// prunePackagedRuntimeNodeModules drops it.
if (context.electronPlatformName === 'linux') {
// Why the arch is passed: symbol-version checks pass happily on a wrong-architecture binary,
// so a cross-built slice could ship the host's pty.node and only fail at runtime.
@@ -418,7 +432,7 @@ module.exports = {
...(isWinDevChannel ? { verifyUpdateCodeSignature: false } : {}),
extraResources: [
...commonExtraResources,
...createPackagedRuntimeNodeModuleResources('win32'),
...windowsRuntimeResources,
winSpeechNativeResource,
{
from: 'resources/win32/bin/orca.cmd',

Some files were not shown because too many files have changed in this diff Show More