mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 17:17:38 +00:00
Merge origin/main (3bfb1149da, Part 2's squash) into settle-split part 3
Part 2 landed on main as3bfb1149da, so files this part never changed take main's text. Where both sides changed a file, the result is this part's change laid onto main, checked against Part 2's final head674dbedd87: - the client coalescer keeps this part's merged `actionablePromptIds` and `working` beside main's `hasNewer` (#28333, the "New" line above unseen messages); - the transcript row keeps this part's actionable-prompt import beside main's new-rows marker imports (#28333).
This commit is contained in:
@@ -54,7 +54,7 @@ runs:
|
||||
env:
|
||||
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
|
||||
NODE_VERSION: ${{ inputs.node-version }}
|
||||
NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }}
|
||||
NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }}
|
||||
run: |
|
||||
scope="${ImageOS:-$RUNNER_OS}"
|
||||
if [ -r /etc/os-release ]; then
|
||||
@@ -120,4 +120,9 @@ runs:
|
||||
shell: bash
|
||||
env:
|
||||
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
|
||||
run: node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME"
|
||||
run: |
|
||||
if [ "$RUNNER_OS" = Windows ] && [ "$NATIVE_RUNTIME" = node ]; then
|
||||
node config/scripts/initialize-windows-native-toolchain.mjs
|
||||
else
|
||||
node config/scripts/ensure-native-runtime.mjs --runtime="$NATIVE_RUNTIME"
|
||||
fi
|
||||
|
||||
@@ -63,5 +63,5 @@ runs:
|
||||
fi
|
||||
pnpm build:orcad-prebuilds
|
||||
fi
|
||||
pnpm build:orcad-prebuilds --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
|
||||
pnpm build:orcad-prebuilds --smoke
|
||||
node config/scripts/build-orcad-prebuilds.mjs --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
|
||||
node config/scripts/build-orcad-prebuilds.mjs --smoke
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Prepare Wayland clipboard helper
|
||||
description: Reuse the source-matched Ubuntu 20.04 helper, validating its architecture and glibc requirements every time.
|
||||
|
||||
inputs:
|
||||
save-cache:
|
||||
description: Publish a validated helper from the default branch.
|
||||
default: 'false'
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Resolve Wayland clipboard build identity
|
||||
id: identity
|
||||
shell: bash
|
||||
run: |
|
||||
node --input-type=module <<'NODE'
|
||||
import { appendFileSync } from 'node:fs'
|
||||
import { waylandClipboardFingerprint } from './config/scripts/build-wayland-clipboard.mjs'
|
||||
const fingerprint = waylandClipboardFingerprint(process.cwd(), process.arch)
|
||||
// Immutable invalid hits need a version bump here before publishing their repair.
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `key=wayland-clipboard-v1-${process.arch}-${fingerprint}\n`)
|
||||
NODE
|
||||
|
||||
- name: Restore Wayland clipboard helper
|
||||
id: cache
|
||||
uses: actions/cache/restore@v5
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: native/wayland-clipboard/.build
|
||||
key: ${{ steps.identity.outputs.key }}
|
||||
|
||||
- name: Build or validate Wayland clipboard helper
|
||||
shell: bash
|
||||
env:
|
||||
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
|
||||
run: |
|
||||
if node config/scripts/build-wayland-clipboard.mjs; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$CACHE_HIT" != true ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
|
||||
rm -rf -- native/wayland-clipboard/.build
|
||||
node config/scripts/build-wayland-clipboard.mjs
|
||||
|
||||
- name: Save validated Wayland clipboard helper from main
|
||||
if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
|
||||
uses: actions/cache/save@v5
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: native/wayland-clipboard/.build
|
||||
key: ${{ steps.identity.outputs.key }}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
export function verifyRequiredPrContext(parseYaml) {
|
||||
const directory = '.github/workflows'
|
||||
const providers = readdirSync(directory)
|
||||
.filter((file) => /\.ya?ml$/.test(file))
|
||||
.flatMap((file) => {
|
||||
const workflow = parseYaml(readFileSync(join(directory, file), 'utf8'))
|
||||
return Object.entries(workflow.jobs)
|
||||
.filter(([id, job]) => (job.name ?? id) === 'PR checks passed')
|
||||
.map(([id]) => `${file}:${id}`)
|
||||
})
|
||||
if (providers.length !== 1 || providers[0] !== 'pr.yml:verify') {
|
||||
throw new Error(`Required PR context must belong only to pr.yml:verify; found ${providers}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const yaml = globalThis.Bun?.YAML
|
||||
if (!yaml) {
|
||||
throw new Error('Run workflow context validation with the pinned Bun runtime.')
|
||||
}
|
||||
verifyRequiredPrContext(yaml.parse)
|
||||
console.log('Required PR check context verified.')
|
||||
}
|
||||
@@ -8,6 +8,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
warm-linux-arm:
|
||||
if: github.event_name != 'pull_request'
|
||||
concurrency:
|
||||
group: ci-cache-warmup-linux-arm-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
@@ -53,7 +54,39 @@ jobs:
|
||||
- name: Verify native cache is usable
|
||||
run: node config/scripts/ensure-native-runtime.mjs --check-only
|
||||
|
||||
- name: Cache Windows CLI launcher
|
||||
if: runner.arch == 'X64'
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
native/windows-cli-launcher/.build
|
||||
native/windows-cli-launcher/target
|
||||
native/windows-agent-hook/.build
|
||||
native/windows-agent-hook/target
|
||||
~/.cargo/registry
|
||||
key: windows-cli-launcher-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('native/windows-cli-launcher/**', 'native/windows-agent-hook/**', 'config/scripts/build-windows-cli-launcher.mjs', 'config/scripts/build-windows-agent-hook.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }}
|
||||
|
||||
- name: Build the Windows native executables
|
||||
if: runner.arch == 'X64'
|
||||
shell: pwsh
|
||||
run: |
|
||||
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
|
||||
rustup toolchain install stable --profile minimal
|
||||
rustup default stable
|
||||
}
|
||||
node config/scripts/build-windows-agent-hook.mjs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
node config/scripts/build-windows-cli-launcher.mjs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
- name: Warm the Windows ARM server launcher
|
||||
if: runner.arch == 'ARM64'
|
||||
uses: ./.github/actions/prepare-windows-cli-launcher
|
||||
with:
|
||||
arches: arm64
|
||||
|
||||
warm-linux-package-fixtures:
|
||||
if: github.event_name != 'pull_request'
|
||||
concurrency:
|
||||
group: ci-cache-warmup-linux-package-fixtures-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
|
||||
@@ -10,6 +10,7 @@ on:
|
||||
- '.github/actions/install-node-dependencies/**'
|
||||
- '.github/actions/restore-pnpm-verification/**'
|
||||
- '.github/actions/prepare-native-runtime/**'
|
||||
- '.github/actions/prepare-windows-cli-launcher/**'
|
||||
- '.github/actions/prepare-git-compatibility/**'
|
||||
- '.github/actions/prepare-linux-package-fixture/**'
|
||||
- 'config/docker/headless-serve-shutdown/**'
|
||||
@@ -21,6 +22,8 @@ on:
|
||||
- '.npmrc'
|
||||
- '.pnpmfile.cjs'
|
||||
- 'config/scripts/ensure-native-runtime.mjs'
|
||||
- 'config/scripts/initialize-windows-native-toolchain.mjs'
|
||||
- 'config/scripts/capture-windows-native-environment.cmd'
|
||||
- 'config/scripts/rebuild-native-deps.mjs'
|
||||
- 'config/scripts/node-pty-job-ownership.cjs'
|
||||
- 'config/scripts/windows-pe-machine.cjs'
|
||||
@@ -34,9 +37,14 @@ on:
|
||||
- 'config/scripts/space-sharing-copy.mjs'
|
||||
- 'config/patches/**'
|
||||
- 'native/windows-registry/**'
|
||||
- 'native/windows-cli-launcher/**'
|
||||
- 'native/windows-agent-hook/**'
|
||||
- 'config/scripts/build-windows-cli-launcher.mjs'
|
||||
- 'config/scripts/build-windows-agent-hook.mjs'
|
||||
- 'config/scripts/windows-rust-executable-build.mjs'
|
||||
- 'resources/build/icon.ico'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/ci-cache-warmup.yml'
|
||||
- '.github/workflows/ci-cache-warmup-platforms.yml'
|
||||
- '.github/workflows/ci-cache-warmup-platform-jobs.yml'
|
||||
|
||||
|
||||
@@ -14,10 +14,14 @@ on:
|
||||
- '.github/actions/install-node-dependencies/**'
|
||||
- '.github/actions/restore-pnpm-verification/**'
|
||||
- '.github/actions/prepare-native-runtime/**'
|
||||
- '.github/actions/prepare-windows-cli-launcher/**'
|
||||
- '.github/actions/prepare-git-compatibility/**'
|
||||
- '.github/actions/prepare-headless-compiler/**'
|
||||
- 'config/scripts/headless-detector-compiler-cache.mjs'
|
||||
- '.github/actions/prepare-linux-package-fixture/**'
|
||||
- '.github/actions/prepare-wayland-clipboard/**'
|
||||
- 'native/wayland-clipboard/**'
|
||||
- 'config/scripts/build-wayland-clipboard.mjs'
|
||||
- 'config/docker/headless-serve-shutdown/**'
|
||||
- 'config/docker/cli-launch-contract/**'
|
||||
- 'config/docker/daemon-shutdown-descendants/**'
|
||||
@@ -27,6 +31,8 @@ on:
|
||||
- '.npmrc'
|
||||
- '.pnpmfile.cjs'
|
||||
- 'config/scripts/ensure-native-runtime.mjs'
|
||||
- 'config/scripts/initialize-windows-native-toolchain.mjs'
|
||||
- 'config/scripts/capture-windows-native-environment.cmd'
|
||||
- 'config/scripts/rebuild-native-deps.mjs'
|
||||
- 'config/scripts/node-pty-job-ownership.cjs'
|
||||
- 'config/scripts/windows-pe-machine.cjs'
|
||||
@@ -44,10 +50,9 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/ci-cache-warmup.yml'
|
||||
- '.github/workflows/ci-cache-warmup-platform-jobs.yml'
|
||||
- '.github/workflows/ci-cache-warmup-platforms.yml'
|
||||
- '.github/actions/prepare-headless-compiler/**'
|
||||
- 'config/scripts/headless-detector-compiler-cache.mjs'
|
||||
- '.github/actions/prepare-wayland-clipboard/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,6 +88,10 @@ jobs:
|
||||
|
||||
- uses: ./.github/actions/prepare-git-compatibility
|
||||
|
||||
- uses: ./.github/actions/prepare-wayland-clipboard
|
||||
with:
|
||||
save-cache: 'true'
|
||||
|
||||
warm-platforms:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
uses: ./.github/workflows/ci-cache-warmup-platform-jobs.yml
|
||||
|
||||
@@ -47,8 +47,8 @@ jobs:
|
||||
timeout-minutes: 75
|
||||
environment: production
|
||||
env:
|
||||
# These drains are unpaced, but restart-safe still needs a quiet runtime for the production
|
||||
# pace window, the same one the same-cap drain uses.
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
|
||||
|
||||
@@ -54,8 +54,7 @@ jobs:
|
||||
# /v1/assign at hosts / window and is re-placed through the director's drain-return
|
||||
# lane. The wave validator admits only its closed set, and anything below the 300000
|
||||
# default only on US general cells: Asia drains are bound by their targets' accept
|
||||
# rate. The transition wait below is the 15-min migration lease plus this window, plus
|
||||
# its excess over 300000, since restart-safe waits a window of quiet after the drain. A
|
||||
# rate. The transition wait below is the 15-min migration lease plus this window. A
|
||||
# cell on an older image drains at once, or refuses a window above 300000.
|
||||
DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
|
||||
steps:
|
||||
@@ -504,7 +503,7 @@ jobs:
|
||||
run: echo 'Exact same-cap rollback point verified.'
|
||||
|
||||
# The sample can age the first one-hour token by up to 20 min; the drain's restart-safe
|
||||
# wait (20 min at the default pace, 50 at the slowest) gets a fresh one.
|
||||
# wait (20 min at the default pace, 35 at the slowest) gets a fresh one.
|
||||
- id: drain-auth
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
@@ -555,7 +554,7 @@ jobs:
|
||||
# What the cell accepted, which is 0 when an older image fell back to an unpaced drain.
|
||||
echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \
|
||||
>> "${GITHUB_OUTPUT}"
|
||||
# Restart-safe means an empty runtime for a whole pace window after the drain; leases
|
||||
# Restart-safe means an empty runtime that lasts past the drain's pace window; leases
|
||||
# held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
@@ -564,7 +563,7 @@ jobs:
|
||||
--heartbeat either --admission migration-only --draining required \
|
||||
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
|
||||
--timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS + (DRAIN_PACE_WINDOW_MS > 300000 ? DRAIN_PACE_WINDOW_MS - 300000 : 0)))"
|
||||
--timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))"
|
||||
echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- id: capacity-auth
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
name: Operate Relay Production Reserve Watch
|
||||
|
||||
# Watches one cell already switched to reserve mode and flips it back to db on the first stop
|
||||
# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing.
|
||||
# An operator watches the run. A refused watch or a flip back that did not land raises an ::error::
|
||||
# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch
|
||||
# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: dry-run logs what a trip would write; write flips the cell back on a trip
|
||||
required: true
|
||||
default: dry-run
|
||||
type: choice
|
||||
options: [dry-run, write]
|
||||
cell-id:
|
||||
description: Exact cell id already in reserve mode, e.g. production-gce-c26
|
||||
required: true
|
||||
type: string
|
||||
duration-minutes:
|
||||
description: How long to watch, 5 to 300
|
||||
required: true
|
||||
default: '120'
|
||||
type: string
|
||||
flipped-at:
|
||||
description: >-
|
||||
When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503
|
||||
background is the median of the 10 minutes before it
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
background-503-per-minute:
|
||||
description: >-
|
||||
Director 503s per minute before the flip; overrides the one measured before flipped-at.
|
||||
One of the two is required
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
unverifiable-limit:
|
||||
description: Consecutive samples with any unreadable signal that count as a trip
|
||||
required: true
|
||||
default: '3'
|
||||
type: string
|
||||
confirmation:
|
||||
description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md.
|
||||
concurrency:
|
||||
group: relay-production-reserve-watch-${{ inputs.cell-id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# Explicit bash runs with pipefail, so the script's exit code survives the tee.
|
||||
shell: bash
|
||||
working-directory: cloud
|
||||
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_ID: ${{ inputs.cell-id }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
GCP_PROJECT_ID: onorca-cloud
|
||||
|
||||
jobs:
|
||||
watch:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Six 55-minute segments cover the longest watch; under the 360-minute job limit.
|
||||
timeout-minutes: 350
|
||||
environment: production
|
||||
outputs:
|
||||
state: ${{ steps.outcome.outputs.state }}
|
||||
verdict: ${{ steps.outcome.outputs.verdict }}
|
||||
env:
|
||||
DURATION_MINUTES: ${{ inputs.duration-minutes }}
|
||||
FLIPPED_AT: ${{ inputs.flipped-at }}
|
||||
BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }}
|
||||
UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }}
|
||||
# One segment, after the auth step before it: fixed text, the inputs arrive as env vars.
|
||||
WATCH_SEGMENT: |
|
||||
set -euo pipefail
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs watch \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \
|
||||
--duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \
|
||||
--flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \
|
||||
--unverifiable-limit "${UNVERIFIABLE_LIMIT}" \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \
|
||||
| tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- id: google-auth-1
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- id: watch-1
|
||||
name: Watch segment 1 of 6
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-2
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-2
|
||||
name: Watch segment 2 of 6
|
||||
if: ${{ steps.watch-1.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-3
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-3
|
||||
name: Watch segment 3 of 6
|
||||
if: ${{ steps.watch-2.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-4
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-4
|
||||
name: Watch segment 4 of 6
|
||||
if: ${{ steps.watch-3.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-5
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-5
|
||||
name: Watch segment 5 of 6
|
||||
if: ${{ steps.watch-4.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: google-auth-6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- id: watch-6
|
||||
name: Watch segment 6 of 6
|
||||
if: ${{ steps.watch-5.outputs.state == 'watching' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }}
|
||||
run: bash -c "${WATCH_SEGMENT}"
|
||||
|
||||
- id: outcome
|
||||
name: Record the watch outcome
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs outcome \
|
||||
--state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}"
|
||||
|
||||
# The documented flow writes reserve before dispatching, so a refusal can leave it unwatched.
|
||||
- name: Alarm on a refused watch
|
||||
if: ${{ always() && steps.outcome.outputs.state == 'refused' }}
|
||||
env:
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
WATCH_STATE: ${{ steps.outcome.outputs.state }}
|
||||
WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}"
|
||||
echo '```json'
|
||||
tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
flip-back:
|
||||
needs: watch
|
||||
# Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
|
||||
# operator too) flips back like a trip. In dry-run this only plans the write.
|
||||
if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
|
||||
runs-on: ubuntu-latest
|
||||
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
|
||||
timeout-minutes: 60
|
||||
environment: production
|
||||
# Only the run's per-cell group above, which a running run never yields (cancel-in-progress off).
|
||||
# Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel
|
||||
# can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the
|
||||
# lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
|
||||
env:
|
||||
VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- id: google-auth
|
||||
if: ${{ always() }}
|
||||
uses: google-github-actions/auth@v2
|
||||
with:
|
||||
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
|
||||
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
|
||||
token_format: id_token
|
||||
id_token_audience: https://relay.onorca.dev/v1/admin/drain
|
||||
id_token_include_email: true
|
||||
|
||||
- uses: google-github-actions/setup-gcloud@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
skip_install: true
|
||||
|
||||
- name: Flip the cell back to db, or preview it
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
|
||||
run: |
|
||||
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
|
||||
ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
|
||||
echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
|
||||
export ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs flip-back \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" \
|
||||
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
|
||||
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
|
||||
--confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \
|
||||
| tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
|
||||
|
||||
# The flip-back line is logged only once the write (or its dry-run plan) finished; without
|
||||
# it a missing token, a failed login or a silent cell left the cell in reserve, unwatched.
|
||||
- name: Alarm when the flip back did not land
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
|
||||
exit 0
|
||||
fi
|
||||
{
|
||||
echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode"
|
||||
echo "Flip it to db by hand with Operate Relay Production Cell Flags."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
|
||||
--mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}"
|
||||
|
||||
- name: Audit summary
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
{
|
||||
echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}"
|
||||
echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "Verdict: ${VERDICT}"
|
||||
echo '```json'
|
||||
cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)'
|
||||
echo '```'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
@@ -69,8 +69,8 @@ jobs:
|
||||
DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
|
||||
DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
|
||||
CELL_ORIGIN: https://c3.relay-staging.onorca.dev
|
||||
# These drains are unpaced, but restart-safe still needs a quiet runtime for the production
|
||||
# pace window, the same one the same-cap drain uses.
|
||||
# These drains are unpaced, but restart-safe still waits out the production pace window
|
||||
# from the drain, the same one the same-cap drain uses.
|
||||
RESTART_SAFE_PACE_WINDOW_MS: '300000'
|
||||
TARGET_CELL_ID: staging-gce-c3
|
||||
FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
|
||||
|
||||
@@ -4,12 +4,14 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- cloud/**
|
||||
- src/packages/process-host/**
|
||||
- .github/workflows/cloud-*.yml
|
||||
- .github/actions/cloud-sql-rollout-lease/**
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- cloud/**
|
||||
- src/packages/process-host/**
|
||||
- .github/workflows/cloud-*.yml
|
||||
- .github/actions/cloud-sql-rollout-lease/**
|
||||
|
||||
@@ -93,8 +95,8 @@ jobs:
|
||||
- run: pnpm build
|
||||
- run: pnpm typecheck
|
||||
|
||||
# Runs in parallel with build. `pnpm test` compiles the one workspace
|
||||
# package it needs through the relay pretest hook, so it does not depend on
|
||||
# Runs in parallel with build. `pnpm test` compiles the workspace
|
||||
# packages it needs through pretest hooks, so it does not depend on
|
||||
# `pnpm build` having run.
|
||||
test:
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -113,6 +115,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
|
||||
ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
|
||||
steps:
|
||||
|
||||
+65
-38
@@ -52,7 +52,9 @@ jobs:
|
||||
name: build e2e app
|
||||
if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -60,10 +62,44 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Install native build tools
|
||||
env:
|
||||
ORCA_E2E_APT_PACKAGES: build-essential python3
|
||||
run: &install_e2e_tools |
|
||||
if [ "$ORCA_E2E_APT_PACKAGES" = 'build-essential python3' ] &&
|
||||
command -v make >/dev/null && command -v g++ >/dev/null && command -v python3 >/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
read -r -a packages <<< "$ORCA_E2E_APT_PACKAGES"
|
||||
# Why apt-mirrors.txt: runner images resolve mirror+file: sources through it, so editing the lists alone leaves Azure in use.
|
||||
for source in /etc/apt/apt-mirrors.txt /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
|
||||
if [ -f "$source" ]; then
|
||||
sudo sed -i 's|https*://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source"
|
||||
fi
|
||||
done
|
||||
sudo tee /etc/apt/apt.conf.d/99-orca-e2e >/dev/null <<'APTCONF'
|
||||
Acquire::http::Timeout "15";
|
||||
Acquire::https::Timeout "15";
|
||||
Acquire::Retries "1";
|
||||
APTCONF
|
||||
timeout 120 sudo apt-get update
|
||||
timeout 300 sudo apt-get install -y "${packages[@]}" || timeout 300 sudo apt-get install -y --fix-missing "${packages[@]}"
|
||||
|
||||
# Why: the build's plain-Node daemon smoke load resolves node-pty.
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
id: deps
|
||||
with:
|
||||
native-runtime: node
|
||||
persist-native-cache: 'false'
|
||||
cache-electron-package: 'true'
|
||||
|
||||
# Save Node's ABI before the Electron preparation replaces the same build paths.
|
||||
- name: Save compiled Node native modules
|
||||
if: steps.deps.outputs.native-cache-key != '' && steps.deps.outputs.native-cache-hit != 'true'
|
||||
uses: actions/cache/save@v5
|
||||
with:
|
||||
path: ${{ steps.deps.outputs.native-cache-path }}
|
||||
key: ${{ steps.deps.outputs.native-cache-key }}
|
||||
|
||||
# Why: building here avoids parallel builds inside Playwright globalSetup;
|
||||
# paired-browser specs also need the standalone web bundle.
|
||||
@@ -88,7 +124,19 @@ jobs:
|
||||
|
||||
- wait: e2e-cli
|
||||
|
||||
- name: Check shared native runtime action
|
||||
id: native-runtime-action
|
||||
run: |
|
||||
if [ -f .github/actions/prepare-native-runtime/action.yml ]; then
|
||||
echo 'available=true' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'available=false' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Only the completed output upload overlaps the change from Node to Electron ABI.
|
||||
- name: Upload E2E build output
|
||||
id: e2e-build-upload
|
||||
background: true
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: e2e-build-out${{ inputs.artifact_suffix }}
|
||||
@@ -97,46 +145,25 @@ jobs:
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
|
||||
# Build Electron-native dependencies once per workflow. Consumer shards restore
|
||||
# this immutable cache instead of compiling the same ABI concurrently.
|
||||
prepare-native-cache:
|
||||
name: prepare Electron native cache
|
||||
if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Prepare shared Electron native runtime
|
||||
id: electron-native-runtime
|
||||
if: steps.native-runtime-action.outputs.available == 'true' && steps.deps.outputs.node-version != ''
|
||||
uses: ./.github/actions/prepare-native-runtime
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
native-runtime: electron
|
||||
node-version: ${{ steps.deps.outputs.node-version }}
|
||||
|
||||
- name: Install native build tools
|
||||
env:
|
||||
ORCA_E2E_APT_PACKAGES: build-essential python3
|
||||
run: &install_e2e_tools |
|
||||
read -r -a packages <<< "$ORCA_E2E_APT_PACKAGES"
|
||||
# Why apt-mirrors.txt: runner images resolve mirror+file: sources through it, so editing the lists alone leaves Azure in use.
|
||||
for source in /etc/apt/apt-mirrors.txt /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
|
||||
if [ -f "$source" ]; then
|
||||
sudo sed -i 's|https*://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source"
|
||||
fi
|
||||
done
|
||||
sudo tee /etc/apt/apt.conf.d/99-orca-e2e >/dev/null <<'APTCONF'
|
||||
Acquire::http::Timeout "15";
|
||||
Acquire::https::Timeout "15";
|
||||
Acquire::Retries "1";
|
||||
APTCONF
|
||||
timeout 120 sudo apt-get update
|
||||
timeout 300 sudo apt-get install -y "${packages[@]}" || timeout 300 sudo apt-get install -y --fix-missing "${packages[@]}"
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
- name: Prepare Electron natives for older refs
|
||||
if: steps.native-runtime-action.outputs.available != 'true' || steps.deps.outputs.node-version == ''
|
||||
uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
|
||||
- wait: e2e-build-upload
|
||||
|
||||
e2e:
|
||||
name: e2e ${{ matrix.shard_name }}
|
||||
needs: [build, prepare-native-cache]
|
||||
needs: [build]
|
||||
if: inputs.test_files == ''
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -188,7 +215,7 @@ jobs:
|
||||
# The Azure archive took 16 minutes for one font package; bound setup
|
||||
# separately so a slow mirror cannot consume the shard's test budget.
|
||||
env: &e2e_tool_packages
|
||||
ORCA_E2E_APT_PACKAGES: build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
|
||||
ORCA_E2E_APT_PACKAGES: build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils xdotool
|
||||
run: *install_e2e_tools
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
@@ -263,7 +290,7 @@ jobs:
|
||||
|
||||
changed-e2e:
|
||||
name: changed e2e specs
|
||||
needs: [build, prepare-native-cache]
|
||||
needs: [build]
|
||||
if: inputs.test_files != '' && (github.event_name != 'pull_request' || inputs.run_changed_e2e)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
@@ -351,7 +378,7 @@ jobs:
|
||||
|
||||
orcad-serve-mode-switch:
|
||||
name: orca serve Electron/orcad mode switch (D7)
|
||||
needs: [build, prepare-native-cache]
|
||||
needs: [build]
|
||||
if: >-
|
||||
inputs.test_files == '' ||
|
||||
contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') ||
|
||||
@@ -411,7 +438,7 @@ jobs:
|
||||
# orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds.
|
||||
orcad-ssh-docker:
|
||||
name: ssh on managed orcad (Docker ${{ matrix.shard }})
|
||||
needs: [build, prepare-native-cache]
|
||||
needs: [build]
|
||||
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
@@ -36,6 +36,7 @@ on:
|
||||
- 'config/scripts/mobile-release-check-scope*'
|
||||
- 'config/scripts/mobile-test-change-scope*'
|
||||
- 'config/scripts/pr-code-change-scope.mjs'
|
||||
- 'config/scripts/pr-package-test-paths.mjs'
|
||||
- 'config/scripts/pr-layout-oracle-route.mjs'
|
||||
# Why main too: two pull requests can each pass against their own base and disagree once both
|
||||
# land, and `verify` never runs on main. The same source paths as above, less the CI inputs only
|
||||
|
||||
@@ -21,34 +21,13 @@ on:
|
||||
- '.github/actions/prepare-native-runtime/**'
|
||||
- '.github/actions/prepare-orcad-prebuilds/**'
|
||||
- '.github/workflows/node-server-tests.yml'
|
||||
# Relevant main pushes qualify every platform after the dependency check.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'src/**'
|
||||
- 'config/**'
|
||||
- 'native/**'
|
||||
- 'tests/**'
|
||||
- 'resources/**'
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
- 'pnpm-workspace.yaml'
|
||||
- 'tsconfig.json'
|
||||
- '.npmrc'
|
||||
- '.pnpmfile.cjs'
|
||||
- '.github/actions/install-node-dependencies/**'
|
||||
- '.github/actions/restore-pnpm-verification/**'
|
||||
- '.github/actions/prepare-headless-compiler/**'
|
||||
- '.github/actions/prepare-native-runtime/**'
|
||||
- '.github/actions/prepare-orcad-prebuilds/**'
|
||||
- '.github/workflows/node-server-tests.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_template:
|
||||
description: Also merge every lane's slot into the desktop orcad template artifact
|
||||
type: boolean
|
||||
default: false
|
||||
# Release packaging calls this to build the orcad template it ships (design D2).
|
||||
# Release and dev builds qualify every platform for the template they ship.
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
@@ -59,13 +38,14 @@ on:
|
||||
description: Upload each lane's release slot and merge them into the orcad-template artifact
|
||||
type: boolean
|
||||
default: false
|
||||
# Requalify fresh builds daily; publish missing immutable Windows cache keys.
|
||||
schedule:
|
||||
- cron: '30 11 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Main pushes must finish detection before they can supersede relevant qualification.
|
||||
# Keep template callers independent while replacing superseded PR qualification.
|
||||
concurrency:
|
||||
group: node-server-${{ (inputs.build_template || github.event_name == 'push') && format('run-{0}', github.run_id) || github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ !inputs.build_template && github.event_name != 'push' }}
|
||||
@@ -140,9 +120,8 @@ jobs:
|
||||
concurrency:
|
||||
group: node-server-persistence-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
|
||||
# Missing/failed detection runs the full matrix; manual runs remain unconditional.
|
||||
# A draft carries no platform verdict; readiness re-triggers this workflow. Spelled against
|
||||
# the event name so the push and schedule paths do not rest on a null property comparison.
|
||||
# Missing/failed detection runs the full matrix; scheduled and manual runs stay unconditional.
|
||||
# Drafts re-trigger at readiness; inherited caller events must not depend on PR fields.
|
||||
if: >-
|
||||
${{ !cancelled() && needs.changes.outputs.should_run != 'false' &&
|
||||
(github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
|
||||
|
||||
+45
-25
@@ -74,8 +74,12 @@ jobs:
|
||||
/README.md
|
||||
/docs/readme/
|
||||
/.github/scripts/check-root-directory-entries.mjs
|
||||
/.github/scripts/check-required-pr-context.mjs
|
||||
/.github/workflows/
|
||||
/config/.bun-version
|
||||
/config/scripts/check-readme-local-links.mjs
|
||||
/config/scripts/pr-code-change-scope.mjs
|
||||
/config/scripts/pr-package-test-paths.mjs
|
||||
/config/scripts/pr-layout-oracle-route.mjs
|
||||
/config/scripts/pr-e2e-source-routing.mjs
|
||||
/config/scripts/ci-e2e-job-selection.mjs
|
||||
@@ -94,6 +98,14 @@ jobs:
|
||||
- name: Check README local links
|
||||
run: node config/scripts/check-readme-local-links.mjs
|
||||
|
||||
- name: Set up Bun for required workflow validation
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version-file: config/.bun-version
|
||||
|
||||
- name: Check required PR context ownership
|
||||
run: bun .github/scripts/check-required-pr-context.mjs
|
||||
|
||||
# Readiness changes eligibility for advisory tests, not the already-tested source.
|
||||
- name: Find identical successful required checks
|
||||
id: readiness
|
||||
@@ -208,6 +220,7 @@ jobs:
|
||||
with:
|
||||
bun-version-file: config/.bun-version
|
||||
|
||||
# Run gate scripts directly to avoid repeated package-manager startup.
|
||||
- name: Check process-host imports
|
||||
if: '!cancelled()'
|
||||
id: process-host-imports
|
||||
@@ -228,7 +241,7 @@ jobs:
|
||||
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
|
||||
run: |
|
||||
if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
|
||||
bun --bun run check:main-sync-spawn-ratchet
|
||||
bun config/scripts/check-main-sync-spawn-ratchet.mjs
|
||||
|
||||
# Keep each check in its own log while sharing this runner.
|
||||
- name: Lint
|
||||
@@ -240,7 +253,7 @@ jobs:
|
||||
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
|
||||
run: |
|
||||
if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
|
||||
pnpm exec oxlint --format github
|
||||
node node_modules/oxlint/bin/oxlint --format github
|
||||
|
||||
- name: Reject low-evidence patterns
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
@@ -259,11 +272,11 @@ jobs:
|
||||
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
|
||||
run: |
|
||||
if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
|
||||
pnpm run audit:code-quality:native
|
||||
node node_modules/oxlint/bin/oxlint --config config/oxlint-code-quality-native-plugins.json src config tests mobile --deny-warnings
|
||||
|
||||
- name: Enforce type-aware code-quality baseline
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run audit:code-quality:type-aware
|
||||
run: node node_modules/oxlint/bin/oxlint --type-aware --config config/oxlint-code-quality-type-aware.json src config tests --deny-warnings
|
||||
|
||||
# Mobile installation changes import resolution for the native cycle check.
|
||||
- wait: native-code-quality
|
||||
@@ -283,11 +296,11 @@ jobs:
|
||||
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
|
||||
run: |
|
||||
if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
|
||||
pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
|
||||
node config/scripts/check-changed-code-quality.mjs "${{ github.event.pull_request.base.sha }}"
|
||||
|
||||
- name: Enforce React Doctor on changed lines
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:react-doctor:changed -- "${{ github.event.pull_request.base.sha }}"
|
||||
run: node config/scripts/check-react-doctor-changed.mjs "${{ github.event.pull_request.base.sha }}"
|
||||
|
||||
- wait: changed-code-quality
|
||||
|
||||
@@ -305,15 +318,15 @@ jobs:
|
||||
|
||||
- name: Check Zustand selector fan-out budget
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:zustand-selector-fanout
|
||||
run: node config/scripts/zustand-selector-fanout-benchmark.mjs --check
|
||||
|
||||
- name: Check reliability gate manifest
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:reliability-gates
|
||||
run: node config/scripts/check-reliability-gates.mjs
|
||||
|
||||
- name: Enforce dead design-system classes
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:dead-classes
|
||||
run: node node_modules/oxlint/bin/oxlint --config config/oxlint-dead-classes.json src/renderer
|
||||
|
||||
- name: Check VM runtime rollback compatibility
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
@@ -339,19 +352,19 @@ jobs:
|
||||
|
||||
- name: Enforce max-lines ratchet
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:max-lines-ratchet
|
||||
run: node config/scripts/check-max-lines-ratchet.mjs
|
||||
|
||||
- name: Enforce ts-nocheck ratchet
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:ts-nocheck-ratchet
|
||||
run: node config/scripts/check-ts-nocheck-ratchet.mjs
|
||||
|
||||
- name: Enforce runtime Electron-import ratchet
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:runtime-electron-ratchet
|
||||
run: node config/scripts/check-runtime-electron-ratchet.mjs
|
||||
|
||||
- name: Enforce focus-read ratchet
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:focus-read-ratchet
|
||||
run: node config/scripts/check-focus-read-ratchet.mjs
|
||||
|
||||
- name: Enforce layout-writer ratchets
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
@@ -359,7 +372,7 @@ jobs:
|
||||
|
||||
- name: Check Node runtime pin
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run check:node-runtime-pin
|
||||
run: node config/scripts/check-node-runtime-pin.mjs
|
||||
|
||||
# Why: extraction writes sorted evidence to an isolated temporary path,
|
||||
# so feature PRs need one normalized AST pass rather than a three-OS matrix.
|
||||
@@ -403,19 +416,19 @@ jobs:
|
||||
|
||||
- name: Verify the generated RPC params catalog
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run verify:rpc-params-catalog
|
||||
run: node config/scripts/generate-rpc-params-catalog.mjs --check
|
||||
|
||||
- name: Verify the generated ACP protocol schema
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run verify:acp-protocol
|
||||
run: node config/scripts/acp/generate-protocol.mjs --check
|
||||
|
||||
- name: Verify bundled skill guides
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run verify:bundled-skill-guides
|
||||
run: node config/scripts/generate-bundled-skill-guides.mjs --check
|
||||
|
||||
- name: Verify skill freshness manifest
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm run verify:skill-bundle-manifest
|
||||
run: node config/scripts/generate-skill-bundle-manifest.mjs
|
||||
|
||||
- name: Verify localization catalogs
|
||||
if: '!cancelled()'
|
||||
@@ -426,7 +439,7 @@ jobs:
|
||||
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
|
||||
run: |
|
||||
if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
|
||||
pnpm run verify:localization-catalogs
|
||||
bun config/scripts/verify-localization-catalogs.mjs
|
||||
|
||||
- name: Verify localization coverage
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
@@ -454,11 +467,11 @@ jobs:
|
||||
|
||||
- name: Check feature wall asset budget
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm check:feature-wall-assets
|
||||
run: node config/scripts/check-feature-wall-assets.mjs
|
||||
|
||||
- name: Verify macOS entitlements
|
||||
if: needs.code_paths.outputs.static_analysis == 'true'
|
||||
run: pnpm verify:macos-entitlements
|
||||
run: node config/scripts/verify-macos-entitlements.mjs
|
||||
|
||||
# Planning shares setup and stays off the compiler's critical path.
|
||||
- name: Plan unit selection
|
||||
@@ -1088,6 +1101,15 @@ jobs:
|
||||
src/main/browser/browser-route-h3-egress.electron.test.ts
|
||||
src/main/browser/browser-route-dns-prefetch.electron.test.ts
|
||||
src/main/persistence/profile-state/profile-state-writer-stall.electron.test.ts
|
||||
src/main/browser/browser-popup-compatibility.electron.test.ts
|
||||
src/main/browser/browser-popup-native-compatibility.electron.test.ts
|
||||
src/main/browser/browser-popup-background-compatibility.electron.test.ts
|
||||
|
||||
# The isolated glibc-floor build writes only the Wayland helper, so JavaScript can overlap.
|
||||
- name: Build native components
|
||||
id: linux-native
|
||||
background: true
|
||||
uses: ./.github/actions/prepare-wayland-clipboard
|
||||
|
||||
# Restore independent fixtures during builds; keep native lifecycle probes serial.
|
||||
- uses: ./.github/actions/prepare-linux-package-fixture
|
||||
@@ -1143,10 +1165,7 @@ jobs:
|
||||
- name: Build mobile web bundle
|
||||
run: pnpm run build:mobile-web
|
||||
|
||||
- name: Build native components
|
||||
run: pnpm run build:native
|
||||
|
||||
- wait: [linux-package-tools, web-client]
|
||||
- wait: [linux-native, linux-package-tools, web-client]
|
||||
|
||||
- name: Package unpacked app
|
||||
env:
|
||||
@@ -1479,6 +1498,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
verify:
|
||||
name: PR checks passed
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- code_paths
|
||||
|
||||
@@ -2336,6 +2336,11 @@ jobs:
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
outputs:
|
||||
run_id: ${{ steps.dispatch.outputs.run_id }}
|
||||
workflow_sha: ${{ steps.dispatch.outputs.workflow_sha }}
|
||||
run_attempt: ${{ steps.dispatch.outputs.run_attempt }}
|
||||
wait_deadline_ms: ${{ steps.dispatch.outputs.wait_deadline_ms }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -2346,7 +2351,8 @@ jobs:
|
||||
node-version-file: package.json
|
||||
|
||||
- name: Run isolated macOS release build
|
||||
run: node config/scripts/run-release-mac-build-workflow.mjs
|
||||
id: dispatch
|
||||
run: node config/scripts/run-release-mac-build-workflow.mjs dispatch
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RELEASE_MAC_BUILD_REF: ${{ github.ref_name }}
|
||||
@@ -2355,11 +2361,50 @@ jobs:
|
||||
RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }}
|
||||
RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml
|
||||
|
||||
verify-mac-build:
|
||||
needs:
|
||||
- cut
|
||||
- build-mac
|
||||
- build
|
||||
# Qualify macOS even when a sibling build fails; publication still requires every build.
|
||||
if: >-
|
||||
!cancelled() &&
|
||||
needs.cut.result == 'success' &&
|
||||
needs.cut.outputs.should_release == 'true' &&
|
||||
needs.build-mac.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 95
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: package.json
|
||||
|
||||
- name: Require the exact isolated macOS release build to succeed
|
||||
run: node config/scripts/run-release-mac-build-workflow.mjs wait
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RELEASE_MAC_BUILD_REF: ${{ github.ref_name }}
|
||||
RELEASE_MAC_BUILD_RELEASE_RUN_ID: ${{ github.run_id }}
|
||||
RELEASE_MAC_BUILD_RUN_ID: ${{ needs.build-mac.outputs.run_id }}
|
||||
RELEASE_MAC_BUILD_WORKFLOW_SHA: ${{ needs.build-mac.outputs.workflow_sha }}
|
||||
RELEASE_MAC_BUILD_RUN_ATTEMPT: ${{ needs.build-mac.outputs.run_attempt }}
|
||||
RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: ${{ needs.build-mac.outputs.wait_deadline_ms }}
|
||||
RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }}
|
||||
RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml
|
||||
|
||||
publish-release:
|
||||
needs:
|
||||
- cut
|
||||
- build
|
||||
- build-mac
|
||||
- verify-mac-build
|
||||
- skill-sharing-linux-floor-release-gate
|
||||
- skill-sharing-release-gate
|
||||
- terminal-rendering-golden
|
||||
@@ -2369,6 +2414,7 @@ jobs:
|
||||
needs.cut.result == 'success' &&
|
||||
needs.build.result == 'success' &&
|
||||
needs.build-mac.result == 'success' &&
|
||||
needs.verify-mac-build.result == 'success' &&
|
||||
needs.skill-sharing-linux-floor-release-gate.result == 'success' &&
|
||||
needs.skill-sharing-release-gate.result == 'success' &&
|
||||
needs.terminal-rendering-golden.result == 'success'
|
||||
|
||||
@@ -62,12 +62,23 @@ on:
|
||||
description: Comma-separated cell ids from src/main/ssh/ssh-windows-host-cells.ts; empty runs the default set (orcad-cli-* cells run only when named).
|
||||
required: false
|
||||
default: ''
|
||||
preparation_mode:
|
||||
description: Compare inbox capability preparation overlapping builds or running after them.
|
||||
type: choice
|
||||
options: [overlap, serial]
|
||||
required: false
|
||||
default: overlap
|
||||
arm_inbox_only:
|
||||
description: Run only the native ARM inbox lane for a preparation timing comparison.
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ssh-windows-hosts-${{ github.event.pull_request.number || github.ref }}
|
||||
group: ssh-windows-hosts-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'workflow_dispatch' && inputs.arm_inbox_only && format('-{0}', inputs.preparation_mode || 'overlap') || '' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
@@ -77,21 +88,10 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: x64
|
||||
runner: windows-2022
|
||||
server: inbox
|
||||
- arch: x64
|
||||
runner: windows-2022
|
||||
server: preview
|
||||
archive: OpenSSH-Win64.zip
|
||||
- arch: arm64
|
||||
runner: windows-11-arm
|
||||
server: inbox
|
||||
- arch: arm64
|
||||
runner: windows-11-arm
|
||||
server: preview
|
||||
archive: OpenSSH-ARM64.zip
|
||||
include: >-
|
||||
${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.arm_inbox_only &&
|
||||
'[{"arch":"arm64","runner":"windows-11-arm","server":"inbox"}]' ||
|
||||
'[{"arch":"x64","runner":"windows-2022","server":"inbox"},{"arch":"x64","runner":"windows-2022","server":"preview","archive":"OpenSSH-Win64.zip"},{"arch":"arm64","runner":"windows-11-arm","server":"inbox"},{"arch":"arm64","runner":"windows-11-arm","server":"preview","archive":"OpenSSH-ARM64.zip"}]') }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
# Installing the inbox capability alone can take several minutes on a fresh image; the CLI matrix
|
||||
# cells launch the e2e app several times each.
|
||||
@@ -99,6 +99,7 @@ jobs:
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
ORCA_ISOLATED_SSH_CI: '1'
|
||||
INBOX_PREPARATION_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.preparation_mode == 'serial' && 'serial' || 'overlap' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -127,11 +128,11 @@ jobs:
|
||||
background: true
|
||||
shell: pwsh
|
||||
run: |
|
||||
if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){
|
||||
if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64' -and $env:INBOX_PREPARATION_MODE -eq 'overlap'){
|
||||
$receipts=Join-Path $pwd '.build/ssh-windows-host-receipts'
|
||||
New-Item -ItemType Directory -Force -Path $receipts | Out-Null
|
||||
. config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1
|
||||
Initialize-WindowsInboxSshCapability -Arch '${{ matrix.arch }}' -Receipt (Join-Path $receipts 'inbox-capability-preparation.json')
|
||||
Initialize-WindowsInboxSshCapability -Arch '${{ matrix.arch }}' -Receipt (Join-Path $receipts 'inbox-capability-preparation.json') -PreparationMode $env:INBOX_PREPARATION_MODE
|
||||
}
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
id: dependencies
|
||||
@@ -164,6 +165,14 @@ jobs:
|
||||
pnpm build:orcad-prebuilds --require-slots "win32-${{ matrix.arch }}"
|
||||
node config/scripts/build-orcad-template.mjs --targets "win32-${{ matrix.arch }}"
|
||||
- wait: inbox-capability
|
||||
- name: Prepare the Windows inbox SSH capability after the builds
|
||||
if: ${{ matrix.server == 'inbox' && matrix.arch == 'arm64' && env.INBOX_PREPARATION_MODE == 'serial' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
$receipts=Join-Path $pwd '.build/ssh-windows-host-receipts'
|
||||
New-Item -ItemType Directory -Force -Path $receipts | Out-Null
|
||||
. config/ci/windows-ssh-provider/preview-ssh/windows-ssh-capability.ps1
|
||||
Initialize-WindowsInboxSshCapability -Arch '${{ matrix.arch }}' -Receipt (Join-Path $receipts 'inbox-capability-preparation.json') -PreparationMode $env:INBOX_PREPARATION_MODE
|
||||
- name: Run the Windows host cells against a private ${{ matrix.server }} sshd
|
||||
shell: pwsh
|
||||
timeout-minutes: ${{ contains(github.event.inputs.cells || '', 'orcad-cli') && 130 || 50 }}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Unit transform cache comparison
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: unit-module-cache-comparison-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
baseline:
|
||||
uses: ./.github/workflows/unit-tests.yml
|
||||
with:
|
||||
node_versions: '["24.21.0"]'
|
||||
runner: ubuntu-24.04-arm
|
||||
shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
|
||||
artifact_prefix: baseline-unit
|
||||
phase_diagnostics: true
|
||||
run_relay_integration: false
|
||||
|
||||
producer:
|
||||
needs: baseline
|
||||
uses: ./.github/workflows/unit-tests.yml
|
||||
with:
|
||||
node_versions: '["24.21.0"]'
|
||||
runner: ubuntu-24.04-arm
|
||||
shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
|
||||
artifact_prefix: producer-unit
|
||||
module_cache: cold
|
||||
module_cache_namespace: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
module_cache_seed: true
|
||||
phase_diagnostics: true
|
||||
run_relay_integration: false
|
||||
|
||||
warm:
|
||||
needs: producer
|
||||
uses: ./.github/workflows/unit-tests.yml
|
||||
with:
|
||||
node_versions: '["24.21.0"]'
|
||||
runner: ubuntu-24.04-arm
|
||||
shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
|
||||
artifact_prefix: warm-unit
|
||||
module_cache: warm
|
||||
module_cache_namespace: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
phase_diagnostics: true
|
||||
run_relay_integration: false
|
||||
@@ -31,13 +31,43 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
module_cache:
|
||||
description: Transform cache policy (off, cold producer, or warm exact restore).
|
||||
default: 'off'
|
||||
type: string
|
||||
|
||||
module_cache_namespace:
|
||||
description: Isolate a same-source cache comparison from existing archives.
|
||||
default: ''
|
||||
type: string
|
||||
|
||||
module_cache_seed:
|
||||
description: Publish comparison caches from a manually dispatched workflow.
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
phase_diagnostics:
|
||||
description: Attach bounded per-file worker phase diagnostics.
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
artifact_prefix:
|
||||
description: Distinguish evidence from repeated same-source comparison lanes.
|
||||
default: unit
|
||||
type: string
|
||||
|
||||
run_relay_integration:
|
||||
description: Run relay contracts alongside normal unit checks.
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
if: inputs.run_shards
|
||||
name: tests node ${{ matrix.node }} ${{ matrix.shard.index }}/${{ matrix.shard.count }}
|
||||
name: ${{ inputs.artifact_prefix != 'unit' && format('{0} ', inputs.artifact_prefix) || '' }}tests node ${{ matrix.node }} ${{ matrix.shard.index }}/${{ matrix.shard.count }}
|
||||
runs-on: ${{ inputs.runner }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
@@ -65,17 +95,51 @@ jobs:
|
||||
with:
|
||||
bun-version-file: config/.bun-version
|
||||
|
||||
- name: Resolve unit module cache
|
||||
id: module-cache
|
||||
if: inputs.module_cache == 'cold' || inputs.module_cache == 'warm'
|
||||
env:
|
||||
ORCA_UNIT_SHARD_INDEX: ${{ matrix.shard.index }}
|
||||
ORCA_UNIT_SHARD_COUNT: ${{ matrix.shard.count }}
|
||||
ORCA_UNIT_CACHE_NAMESPACE: ${{ inputs.module_cache_namespace }}
|
||||
run: node config/scripts/ci-unit-module-cache.mjs identity
|
||||
|
||||
- name: Restore exact unit module cache
|
||||
id: module-cache-restore
|
||||
if: inputs.module_cache == 'warm' && steps.module-cache.outputs.key != ''
|
||||
continue-on-error: true
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
path: ${{ steps.module-cache.outputs.path }}
|
||||
key: ${{ steps.module-cache.outputs.key }}
|
||||
|
||||
- name: Validate restored unit module cache or use cold transforms
|
||||
if: steps.module-cache.outputs.key != ''
|
||||
env:
|
||||
ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-hit }}
|
||||
run: node config/scripts/ci-unit-module-cache.mjs prepare
|
||||
|
||||
- uses: actions/download-artifact@v8
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: unit-selection-attempt-${{ github.run_attempt }}
|
||||
path: ci-shards/
|
||||
|
||||
- name: Record full unit discovery
|
||||
if: inputs.phase_diagnostics
|
||||
continue-on-error: true
|
||||
run: >-
|
||||
node --input-type=module -e '
|
||||
import { discoverUnitFiles } from "./config/scripts/ci-unit-files.mjs";
|
||||
import { writeAssignment } from "./config/scripts/ci-shard-assignment.mjs";
|
||||
writeAssignment("ci-shards/unit-discovery.json", { files: discoverUnitFiles() });'
|
||||
|
||||
- name: Test shard
|
||||
env:
|
||||
NODE_COMPILE_CACHE: ${{ runner.temp }}/orca-node-compile-cache-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.node }}-${{ matrix.shard.index }}
|
||||
ORCA_BALANCE_UNIT_SHARDS: '1'
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
ORCA_UNIT_PHASE_REPORT: ${{ inputs.phase_diagnostics && 'ci-shards/unit-phases.json' || '' }}
|
||||
run: |
|
||||
ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
|
||||
export ORCA_SHARD_SOURCE_SHA
|
||||
@@ -83,18 +147,37 @@ jobs:
|
||||
--shard=${{ matrix.shard.index }}/${{ matrix.shard.count }} \
|
||||
${{ inputs.runner == 'ubuntu-24.04-arm' && '--maxWorkers=4' || '' }}
|
||||
|
||||
- name: Seal unit module cache
|
||||
id: module-cache-seal
|
||||
if: >-
|
||||
success() && steps.module-cache.outputs.key != '' &&
|
||||
steps.module-cache-restore.outputs.cache-hit != 'true' &&
|
||||
(github.event_name == 'pull_request' ||
|
||||
github.ref == 'refs/heads/main' && github.event_name != 'pull_request' ||
|
||||
inputs.module_cache_seed && github.event_name == 'workflow_dispatch')
|
||||
run: node config/scripts/ci-unit-module-cache.mjs seal
|
||||
|
||||
- name: Save unit module cache
|
||||
if: steps.module-cache-seal.outputs.ready == 'true'
|
||||
continue-on-error: true
|
||||
uses: actions/cache/save@v5
|
||||
with:
|
||||
path: ${{ steps.module-cache.outputs.path }}
|
||||
key: ${{ steps.module-cache.outputs.key }}
|
||||
|
||||
- name: Upload unit shard assignment
|
||||
if: always()
|
||||
# Diagnostic upload outages must not change the test verdict.
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unit-shard-node-${{ matrix.node }}-${{ matrix.shard.index }}-attempt-${{ github.run_attempt }}
|
||||
name: ${{ inputs.artifact_prefix }}-shard-node-${{ matrix.node }}-${{ matrix.shard.index }}-attempt-${{ github.run_attempt }}
|
||||
path: ci-shards/
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
relay_integration:
|
||||
if: inputs.run_relay_integration
|
||||
name: relay integration node ${{ matrix.node }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Windows native preparation comparison
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Exact commit containing this benchmark and the Windows compiler candidate.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: windows-native-preparation-comparison-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
|
||||
jobs:
|
||||
compare:
|
||||
name: Native preparation comparison (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
os: [windows-2022, windows-11-arm]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.source_sha }}
|
||||
persist-credentials: false
|
||||
- name: Record measurement start and validate source
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
run: |
|
||||
[[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
|
||||
mkdir -p .build/windows-native-benchmark
|
||||
node -e 'require("node:fs").writeFileSync(".build/windows-native-benchmark/job-start.json", JSON.stringify({startedAt:new Date().toISOString()}))'
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
node-version: '24.21.0'
|
||||
native-runtime: none
|
||||
cache-pnpm-store: 'false'
|
||||
cache-pnpm-verification: 'false'
|
||||
- name: Compare isolated cold and same-build warm preparation
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
BENCHMARK_RUNNER: ${{ matrix.os }}
|
||||
run: node config/scripts/benchmark-windows-native-preparation.mjs
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: windows-native-preparation-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: .build/windows-native-benchmark/
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
@@ -1,11 +1,5 @@
|
||||
name: ZCode read-only account contract
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/main/zcode/**'
|
||||
- 'src/shared/node-bounded-file-reader*'
|
||||
- 'src/main/rate-limits/**'
|
||||
- '.github/workflows/zcode-account-contract.yml'
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -13,24 +7,7 @@ concurrency:
|
||||
group: zcode-account-contract-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
credential-files:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-22.04, macos-15, windows-2022]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version-file: config/.bun-version
|
||||
- name: Verify read-only credentials and account changes
|
||||
run: pnpm test src/main/zcode/zcode-v2-plan-credentials.test.ts src/main/zcode/zcode-v2-personal-overlay-raw-json.test.ts src/main/rate-limits/service-zcode-usage.test.ts src/shared/node-bounded-file-reader-sync.test.ts
|
||||
account-contract:
|
||||
uses: ./.github/workflows/zcode-credential-contract.yml
|
||||
with:
|
||||
contract: account
|
||||
|
||||
@@ -3,18 +3,29 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/main/zcode/**'
|
||||
- 'src/shared/node-bounded-file-reader*'
|
||||
- 'src/main/rate-limits/**'
|
||||
- 'src/shared/secure-file*'
|
||||
- 'src/shared/secure-path-windows-acl*'
|
||||
- 'src/shared/windows-security-descriptor*'
|
||||
- '.github/workflows/zcode-account-contract.yml'
|
||||
- '.github/workflows/zcode-credential-contract.yml'
|
||||
- 'config/scripts/zcode-credential-workflow.test.mjs'
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
inputs:
|
||||
contract:
|
||||
description: Keep the account workflow's manual entry point on its original test group.
|
||||
type: string
|
||||
default: credentials
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: coding-plan-contract-${{ github.event.pull_request.number || github.ref }}
|
||||
group: coding-plan-contract-${{ github.event.pull_request.number || github.ref }}-${{ inputs.contract || 'credentials' }}
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
credential-files:
|
||||
name: ZCode credentials (${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -26,12 +37,57 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Select credential test groups
|
||||
id: scope
|
||||
shell: bash
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
MANUAL_ACCOUNT: ${{ inputs.contract == 'account' }}
|
||||
MANUAL_CREDENTIALS: ${{ inputs.contract != 'account' }}
|
||||
run: |
|
||||
if [ "$EVENT_NAME" != pull_request ]; then
|
||||
echo "account=$MANUAL_ACCOUNT" >> "$GITHUB_OUTPUT"
|
||||
echo "credentials=$MANUAL_CREDENTIALS" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
account=true
|
||||
credentials=true
|
||||
diff_status=2
|
||||
if git rev-parse --verify HEAD^1 >/dev/null 2>&1; then
|
||||
diff_status=0
|
||||
git diff --quiet --no-renames HEAD^1 HEAD || diff_status=$?
|
||||
fi
|
||||
if [ "$diff_status" = 1 ]; then
|
||||
if git diff --quiet --no-renames HEAD^1 HEAD -- \
|
||||
src/main/zcode/ ':(glob)src/shared/node-bounded-file-reader*' src/main/rate-limits/ \
|
||||
.github/workflows/zcode-account-contract.yml \
|
||||
.github/workflows/zcode-credential-contract.yml \
|
||||
config/scripts/zcode-credential-workflow.test.mjs; then
|
||||
account=false
|
||||
fi
|
||||
if git diff --quiet --no-renames HEAD^1 HEAD -- \
|
||||
src/main/zcode/ ':(glob)src/shared/secure-file*' ':(glob)src/shared/secure-path-windows-acl*' \
|
||||
':(glob)src/shared/windows-security-descriptor*' \
|
||||
.github/workflows/zcode-credential-contract.yml \
|
||||
config/scripts/zcode-credential-workflow.test.mjs; then
|
||||
credentials=false
|
||||
fi
|
||||
fi
|
||||
echo "account=$account" >> "$GITHUB_OUTPUT"
|
||||
echo "credentials=$credentials" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
id: dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
id: bun
|
||||
with:
|
||||
bun-version-file: config/.bun-version
|
||||
- name: Verify read-only credentials and account changes
|
||||
if: ${{ !cancelled() && steps.scope.outputs.account == 'true' && steps.dependencies.outcome == 'success' && steps.bun.outcome == 'success' }}
|
||||
run: pnpm test src/main/zcode/zcode-v2-plan-credentials.test.ts src/main/zcode/zcode-v2-personal-overlay-raw-json.test.ts src/main/rate-limits/service-zcode-usage.test.ts src/shared/node-bounded-file-reader-sync.test.ts
|
||||
- name: Verify rollback and native Windows permissions
|
||||
if: ${{ !cancelled() && steps.scope.outputs.credentials == 'true' && steps.dependencies.outcome == 'success' && steps.bun.outcome == 'success' }}
|
||||
run: pnpm test src/main/zcode/zcode-plan-api-key-store.test.ts src/main/zcode/zcode-plan-api-key-store-plaintext-rollback.test.ts src/main/zcode/zcode-plan-api-key-store-publication-race.test.ts src/main/zcode/zcode-plan-api-key-store-native-windows.test.ts src/shared/secure-file.test.ts
|
||||
|
||||
@@ -25,6 +25,10 @@
|
||||
{
|
||||
"name": "native-chat-design-tokens",
|
||||
"specifier": "./config/oxlint-plugins/native-chat-design-tokens.mjs"
|
||||
},
|
||||
{
|
||||
"name": "ime-enter-guard",
|
||||
"specifier": "./config/oxlint-plugins/ime-enter-guard.mjs"
|
||||
}
|
||||
],
|
||||
"categories": {
|
||||
@@ -176,6 +180,7 @@
|
||||
"files": ["src/renderer/src/**/*.{ts,tsx}"],
|
||||
"rules": {
|
||||
"renderer-scrollbar-style/require-styled-vertical-scrollbar": "error",
|
||||
"ime-enter-guard/require-ime-guard": "error",
|
||||
"no-restricted-properties": [
|
||||
"error",
|
||||
{
|
||||
@@ -211,6 +216,7 @@
|
||||
"react/globals": "off",
|
||||
"quadratic-buffer-concat/no-loop-carried-concat": "off",
|
||||
"renderer-scrollbar-style/require-styled-vertical-scrollbar": "off",
|
||||
"ime-enter-guard/require-ime-guard": "off",
|
||||
"native-chat-design-tokens/no-arbitrary-font-size": "off",
|
||||
"native-chat-design-tokens/no-softened-foreground": "off",
|
||||
"native-chat-design-tokens/no-weakened-focus-ring": "off"
|
||||
|
||||
@@ -60,6 +60,14 @@ Fill in [`.github/pull_request_template.md`](./.github/pull_request_template.md)
|
||||
|
||||
Cover all four concisely. Don't pad or walk the diff.
|
||||
|
||||
## Merging Pull Requests
|
||||
|
||||
Before merging, verify that the required checks and aggregate gate in the full PR workflow have passed for the exact current PR commit. For GitHub, identify the `.github/workflows/pr.yml` run and its aggregate `PR checks passed` (job ID `verify`); Mobile Checks is separate and never substitutes for it. Check workflow identity as well as the displayed job name. If the head changes, repeat the check for the new head.
|
||||
|
||||
Pending, failed, cancelled, or missing required checks block agent merges even when repository settings allow them. `gh pr merge --auto` is not proof that CI passed: GitHub may merge immediately when checks are optional. Never use `--admin` or disable protection to bypass a failed check. An explicit user instruction to make an emergency exception overrides this default; otherwise fix the failure or wait. Use `--match-head-commit` to prevent merging a different commit after validation.
|
||||
|
||||
CI changes must exercise the affected checks in the PR. Required check names must be unique across workflows, and workflow changes must preserve the full aggregate's failure handling and conditional job requirements.
|
||||
|
||||
# Considerations
|
||||
|
||||
## Worktree Safety
|
||||
|
||||
@@ -242,9 +242,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
|
||||
|
||||
- **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
|
||||
- **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
|
||||
- **WeChat:** Scan to join the Orca community WeChat group 11.
|
||||
- **WeChat:** Group 11 is full. Scan to join the Orca community WeChat group 12.
|
||||
|
||||
<img src="docs/assets/wechat-qr-group11.jpg" alt="WeChat group 11 QR code for the Orca community" width="160" />
|
||||
<img src="docs/assets/wechat-qr-group12.jpg" alt="WeChat group 12 QR code for the Orca community" width="160" />
|
||||
|
||||
- **Feedback & Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
|
||||
- **Privacy:** See the [privacy & telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { DurablePushStore } from './durable-push-store.js'
|
||||
import {
|
||||
cleanupDurablePushFixtures,
|
||||
DEVICE_HEAD_SQL,
|
||||
durablePushTestDatabaseUrl,
|
||||
fixture,
|
||||
notification,
|
||||
pauseAfter,
|
||||
within
|
||||
} from './durable-push-store.test-fixture.js'
|
||||
|
||||
afterEach(cleanupDurablePushFixtures)
|
||||
|
||||
it.skipIf(!durablePushTestDatabaseUrl)(
|
||||
'claims the same device in another schema while a fixture holds its device lock',
|
||||
async () => {
|
||||
const first = await fixture()
|
||||
const second = await fixture()
|
||||
const [firstScope] = await first.db.query(
|
||||
'SELECT current_database() AS database, current_schema() AS schema'
|
||||
)
|
||||
const [secondScope] = await second.db.query(
|
||||
'SELECT current_database() AS database, current_schema() AS schema'
|
||||
)
|
||||
expect(firstScope?.database).toBe(secondScope?.database)
|
||||
expect(firstScope?.schema).not.toBe(secondScope?.schema)
|
||||
await first.store.accept('host', 'phone', notification(1))
|
||||
await second.store.accept('host', 'phone', notification(2))
|
||||
const paused = pauseAfter(first.db, DEVICE_HEAD_SQL)
|
||||
const held = new DurablePushStore(paused.wrapped, first.clock).claim()
|
||||
await paused.atCandidate
|
||||
try {
|
||||
expect(await within(first.store.claim(), 2_000)).toBeNull()
|
||||
expect((await within(second.store.claim(), 2_000))?.notification.notificationSeq).toBe(2)
|
||||
} finally {
|
||||
paused.release()
|
||||
await held
|
||||
}
|
||||
expect((await held)?.notification.notificationSeq).toBe(1)
|
||||
}
|
||||
)
|
||||
@@ -11,7 +11,10 @@ const cleanups: (() => Promise<void>)[] = []
|
||||
export async function cleanupDurablePushFixtures(): Promise<void> {
|
||||
await Promise.all(cleanups.splice(0).map((cleanup) => cleanup()))
|
||||
}
|
||||
export const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'): PushNotification => ({
|
||||
export const notification = (
|
||||
seq: number,
|
||||
kind: 'alert' | 'dismiss' = 'alert'
|
||||
): PushNotification => ({
|
||||
notificationId: `notification-${seq}`,
|
||||
notificationEpoch: 'epoch',
|
||||
notificationSeq: seq,
|
||||
@@ -21,11 +24,26 @@ export const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'):
|
||||
body: '',
|
||||
kind
|
||||
})
|
||||
// ownDatabase: advisory locks are database-wide, so a test that holds a global key must not share one.
|
||||
|
||||
// Schemas isolate rows, but advisory locks need the same fixture boundary.
|
||||
function scopeFixtureLocks(database: PushDatabase, scope: string): PushDatabase {
|
||||
return {
|
||||
dialect: database.dialect,
|
||||
query: (sql, params) => database.query(sql, params),
|
||||
transaction: (run) => database.transaction((tx) => run(scopeFixtureLocks(tx, scope))),
|
||||
lockQuotaScope: (key) => database.lockQuotaScope(`${scope}:${key}`),
|
||||
tryLockScope: (key) => database.tryLockScope(`${scope}:${key}`),
|
||||
tryLockSharedScope: (key) => database.tryLockSharedScope(`${scope}:${key}`),
|
||||
close: () => database.close()
|
||||
}
|
||||
}
|
||||
|
||||
// ownDatabase keeps production lock keys for mixed-revision tests.
|
||||
export async function fixture({ ownDatabase = false } = {}) {
|
||||
const databaseUrl = durablePushTestDatabaseUrl
|
||||
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440')
|
||||
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') {
|
||||
throw new Error('isolated_postgres_port_required')
|
||||
}
|
||||
let db: PushDatabase
|
||||
if (databaseUrl) {
|
||||
const admin = new pg.Client({ connectionString: databaseUrl })
|
||||
@@ -38,7 +56,9 @@ export async function fixture({ ownDatabase = false } = {}) {
|
||||
} finally {
|
||||
try {
|
||||
await admin.query(
|
||||
ownDatabase ? `DROP DATABASE IF EXISTS ${name}` : `DROP SCHEMA IF EXISTS ${name} CASCADE`
|
||||
ownDatabase
|
||||
? `DROP DATABASE IF EXISTS ${name}`
|
||||
: `DROP SCHEMA IF EXISTS ${name} CASCADE`
|
||||
)
|
||||
} finally {
|
||||
await admin.end()
|
||||
@@ -53,7 +73,8 @@ export async function fixture({ ownDatabase = false } = {}) {
|
||||
await admin.query(`CREATE SCHEMA ${name}`)
|
||||
url.searchParams.set('options', `-c search_path=${name}`)
|
||||
}
|
||||
db = scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
|
||||
scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
|
||||
db = ownDatabase ? scoped : scopeFixtureLocks(scoped, name)
|
||||
} else {
|
||||
db = await openInMemoryPushDatabase()
|
||||
cleanups.push(() => db.close())
|
||||
@@ -117,7 +138,9 @@ export async function within<T>(operation: Promise<T>, ms: number): Promise<T> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation,
|
||||
new Promise<T>((_, reject) => (timer = setTimeout(() => reject(new Error('claim_blocked')), ms)))
|
||||
new Promise<T>(
|
||||
(_, reject) => (timer = setTimeout(() => reject(new Error('claim_blocked')), ms))
|
||||
)
|
||||
])
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@orca-cloud/relay",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
@@ -11,7 +11,7 @@
|
||||
"lint": "tsc -p tsconfig.json --noEmit",
|
||||
"pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build",
|
||||
"start": "node dist/index.js",
|
||||
"test": "vitest run",
|
||||
"test": "node --import tsx --conditions=orca-source ../../dev/scripts/run-relay-tests.mjs",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
+40
-19
@@ -81,6 +81,11 @@ import {
|
||||
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
|
||||
import type { RelayJWKSet } from './stale-if-error-jwks.js'
|
||||
import { stagingAsiaProofMembership } from './staging-asia-proof-admission.js'
|
||||
import {
|
||||
runInStickySlot,
|
||||
StickySlotStatements,
|
||||
type StickySlotSample
|
||||
} from './sticky-slot-statements.js'
|
||||
|
||||
const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
|
||||
isRelayCellConnectionHardCap
|
||||
@@ -159,6 +164,7 @@ export function createRelayApp(
|
||||
) => void
|
||||
recordDrainReturnRetryAfter?: (seconds: number) => void
|
||||
recordAdmissionServiceMs?: (lane: 'sticky' | 'drain-return', durationMs: number) => void
|
||||
recordStickySlot?: (sample: StickySlotSample) => void
|
||||
recordAssignmentUnavailable?: (cause: AssignmentUnavailableCause) => void
|
||||
recordRegionRequest?: (region: RelayRegion | undefined) => void
|
||||
// Step 3 shadow directory; absent unless the director polls seat feeds.
|
||||
@@ -269,16 +275,25 @@ export function createRelayApp(
|
||||
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
|
||||
}
|
||||
// The slot's hold, not the request's latency: herd recovery is slots / this.
|
||||
const timeStickySlot = (lease: { release(): void }): { release(): void } => {
|
||||
const timeStickySlot = (
|
||||
lease: { release(): void },
|
||||
queueWaitMs: number
|
||||
): { release(): void; measure<T>(work: () => Promise<T>): Promise<T> } => {
|
||||
const startedAt = performance.now()
|
||||
const statements = new StickySlotStatements()
|
||||
let released = false
|
||||
return {
|
||||
release: () => {
|
||||
if (released) return
|
||||
released = true
|
||||
operations.recordAdmissionServiceMs?.('sticky', performance.now() - startedAt)
|
||||
lease.release()
|
||||
}
|
||||
try {
|
||||
operations.recordAdmissionServiceMs?.('sticky', performance.now() - startedAt)
|
||||
operations.recordStickySlot?.({ queueWaitMs, ...statements.close() })
|
||||
} finally {
|
||||
lease.release()
|
||||
}
|
||||
},
|
||||
measure: (work) => runInStickySlot(statements, work)
|
||||
}
|
||||
}
|
||||
// Classified server-side from the host's own assignment row, never from a
|
||||
@@ -480,8 +495,11 @@ export function createRelayApp(
|
||||
}
|
||||
let admission: { release(): void } | null = null
|
||||
let lane: AssignmentAdmissionLane = 'placement'
|
||||
// Charges the sticky slot's SQL to its request; any other lane runs work as is.
|
||||
let measureHeld = <T>(work: () => Promise<T>): Promise<T> => work()
|
||||
if (body.data.reconnect) {
|
||||
let rejection: AssignmentAdmissionRejection | undefined
|
||||
const queuedAt = performance.now()
|
||||
const stickySlot = await stickyAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
|
||||
rejection = reason
|
||||
})
|
||||
@@ -497,12 +515,12 @@ export function createRelayApp(
|
||||
})
|
||||
return rejectStickyAssignment(context)
|
||||
}
|
||||
const fastLane = timeStickySlot(stickySlot)
|
||||
const fastLane = timeStickySlot(stickySlot, performance.now() - queuedAt)
|
||||
let verified: ResolvedRelayAssignment | null = null
|
||||
try {
|
||||
verified = await operations.assignments.resolve(identity, {
|
||||
classifyHomeRollIsolation: true
|
||||
})
|
||||
verified = await fastLane.measure(() =>
|
||||
operations.assignments.resolve(identity, { classifyHomeRollIsolation: true })
|
||||
)
|
||||
} catch (error) {
|
||||
fastLane.release()
|
||||
operations.recordAssignmentAdmission?.('sticky-rejected')
|
||||
@@ -546,6 +564,7 @@ export function createRelayApp(
|
||||
} else if (verified) {
|
||||
lane = 'sticky'
|
||||
admission = fastLane
|
||||
measureHeld = fastLane.measure
|
||||
operations.recordAssignmentAdmission?.('sticky')
|
||||
} else {
|
||||
// An unverified hint joins the placement lane with today's semantics.
|
||||
@@ -574,25 +593,27 @@ export function createRelayApp(
|
||||
let regionCorrection: RegionCorrectionResponse | undefined
|
||||
try {
|
||||
if (body.data.regionCorrection?.action === 'report') {
|
||||
const current = await operations.assignments.resolve(identity)
|
||||
const current = await measureHeld(() => operations.assignments.resolve(identity))
|
||||
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
|
||||
assignment = current
|
||||
} else {
|
||||
// The host asked: it may leave a reserve cell the director proves dead.
|
||||
assignment = await operations.assignments.assign(
|
||||
identity,
|
||||
requestedRegion,
|
||||
requestedRegion ? targetRegion : undefined,
|
||||
undefined,
|
||||
{ ...step5, hostRequested: true }
|
||||
assignment = await measureHeld(() =>
|
||||
operations.assignments.assign(
|
||||
identity,
|
||||
requestedRegion,
|
||||
requestedRegion ? targetRegion : undefined,
|
||||
undefined,
|
||||
{ ...step5, hostRequested: true }
|
||||
)
|
||||
)
|
||||
}
|
||||
if (body.data.regionCorrection) {
|
||||
try {
|
||||
regionCorrection = await operations.assignments.exchangeRegionCorrection(
|
||||
identity,
|
||||
body.data.regionCorrection,
|
||||
assignment.assignmentEpoch
|
||||
const correction = body.data.regionCorrection
|
||||
const epoch = assignment.assignmentEpoch
|
||||
regionCorrection = await measureHeld(() =>
|
||||
operations.assignments.exchangeRegionCorrection(identity, correction, epoch)
|
||||
)
|
||||
} catch (error) {
|
||||
if (body.data.regionCorrection.action === 'report') throw error
|
||||
|
||||
@@ -330,6 +330,19 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
|
||||
expect(cell.book(identity, 9)).toEqual([{ outcome: 'ok' }])
|
||||
})
|
||||
|
||||
it('leaves a drained seat to its drain grace instead of the shorter demotion close', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
cell.book(identity, 9)
|
||||
await cell.connect(identity, 9)
|
||||
const page = cell.relay.sessions.seatFeed(null)
|
||||
const joinedAt = 'full' in page ? page.full[0]!.joinedAt : -1
|
||||
cell.relay.sessions.drain(120_000)
|
||||
const seat = { v: 1 as const, userId: 'user-1', relayHostId: identity.hostId, epoch: 9, joinedAt }
|
||||
expect(cell.relay.sessions.demote(seat)).toBe('draining')
|
||||
expect(cell.relay.sessions.demote(seat)).toBe('draining')
|
||||
})
|
||||
|
||||
it('gives every booking unit back when the cell starts to drain', async () => {
|
||||
const cell = await startCell({ admitMode: 'reserve' })
|
||||
const identity = await cell.host()
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import { applyPostgresSchema } from './postgres-schema-startup.js'
|
||||
import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
|
||||
import { reportPostgresQueryFailure } from './postgres-query-failure.js'
|
||||
import { noteStickySlotPoolWait, noteStickySlotStatement } from './sticky-slot-statements.js'
|
||||
import {
|
||||
isPostgresReadTimeout,
|
||||
isRelayDatabaseLayerError,
|
||||
@@ -910,19 +911,25 @@ type PostgresQueryTimeout = () => number | undefined
|
||||
|
||||
async function timedQuery(
|
||||
client: pg.PoolClient,
|
||||
connectionId: number,
|
||||
text: string,
|
||||
values: unknown[] | undefined,
|
||||
queryTimeout: PostgresQueryTimeout | undefined
|
||||
): Promise<pg.QueryResult> {
|
||||
const timeout = queryTimeout?.()
|
||||
if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values)
|
||||
// pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig.
|
||||
const config: pg.QueryConfig & { query_timeout: number } = {
|
||||
text,
|
||||
...(values === undefined ? {} : { values }),
|
||||
query_timeout: timeout
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
const timeout = queryTimeout?.()
|
||||
if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values)
|
||||
// pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig.
|
||||
const config: pg.QueryConfig & { query_timeout: number } = {
|
||||
text,
|
||||
...(values === undefined ? {} : { values }),
|
||||
query_timeout: timeout
|
||||
}
|
||||
return await client.query(config)
|
||||
} finally {
|
||||
noteStickySlotStatement(performance.now() - startedAt, connectionId)
|
||||
}
|
||||
return await client.query(config)
|
||||
}
|
||||
|
||||
class PostgresTransaction implements RelayDatabase {
|
||||
@@ -938,6 +945,7 @@ class PostgresTransaction implements RelayDatabase {
|
||||
|
||||
constructor(
|
||||
protected readonly client: pg.PoolClient,
|
||||
private readonly connectionId: number,
|
||||
private readonly queryTimeout?: PostgresQueryTimeout
|
||||
) {}
|
||||
|
||||
@@ -966,7 +974,7 @@ class PostgresTransaction implements RelayDatabase {
|
||||
`WITH final_write AS (${inlinePostgresParameters(sql, params, this.client)}) ` +
|
||||
'SELECT 1 / (SELECT count(*)::int FROM final_write); COMMIT'
|
||||
try {
|
||||
await timedQuery(this.client, message, undefined, this.queryTimeout)
|
||||
await timedQuery(this.client, this.connectionId, message, undefined, this.queryTimeout)
|
||||
} catch (error) {
|
||||
// Simple query stops at the first error, so any server error means COMMIT never ran:
|
||||
// retryable codes take the caller's normal rollback-and-retry path. A lost connection
|
||||
@@ -1017,7 +1025,13 @@ class PostgresTransaction implements RelayDatabase {
|
||||
this.assertNotCommitted()
|
||||
this.failIfPoisoned()
|
||||
try {
|
||||
const result = await timedQuery(this.client, postgresSql(sql), params, this.queryTimeout)
|
||||
const result = await timedQuery(
|
||||
this.client,
|
||||
this.connectionId,
|
||||
postgresSql(sql),
|
||||
params,
|
||||
this.queryTimeout
|
||||
)
|
||||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||||
} catch (error) {
|
||||
this.noteFailure(error)
|
||||
@@ -1177,6 +1191,9 @@ export class PostgresDatabase implements RelayDatabase {
|
||||
readonly dialect = 'postgres' as const
|
||||
private readonly pressure: PostgresPoolPressure
|
||||
private readonly holds = new CellInventoryHoldSamples()
|
||||
// pg-pool hands the same client object back for the same connection.
|
||||
private readonly connectionIds = new WeakMap<pg.PoolClient, number>()
|
||||
private nextConnectionId = 1
|
||||
|
||||
consumeHoldCounts(): CellInventoryHoldCounts {
|
||||
return this.holds.consumeCounts()
|
||||
@@ -1207,9 +1224,15 @@ export class PostgresDatabase implements RelayDatabase {
|
||||
let client: pg.PoolClient | undefined
|
||||
let lostReply: Error | undefined
|
||||
try {
|
||||
client = await this.pressure.connect(lane)
|
||||
client = await this.connect(lane)
|
||||
phase = 'execute'
|
||||
const result = await timedQuery(client, postgresSql(sql), params, this.queryTimeout)
|
||||
const result = await timedQuery(
|
||||
client,
|
||||
this.idOfConnection(client),
|
||||
postgresSql(sql),
|
||||
params,
|
||||
this.queryTimeout
|
||||
)
|
||||
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
|
||||
} catch (error) {
|
||||
markRelayDatabaseError(error)
|
||||
@@ -1257,15 +1280,18 @@ export class PostgresDatabase implements RelayDatabase {
|
||||
options: RelayTransactionOptions = {}
|
||||
): Promise<T> {
|
||||
for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
|
||||
const client = await this.pressure.connect()
|
||||
const transaction = new PostgresTransaction(client, this.queryTimeout)
|
||||
const client = await this.connect()
|
||||
const connectionId = this.idOfConnection(client)
|
||||
const transaction = new PostgresTransaction(client, connectionId, this.queryTimeout)
|
||||
let lostReply: Error | undefined
|
||||
try {
|
||||
await timedQuery(client, 'BEGIN', undefined, this.queryTimeout)
|
||||
await timedQuery(client, connectionId, 'BEGIN', undefined, this.queryTimeout)
|
||||
const result = await operation(transaction)
|
||||
// Even when the operation caught it: COMMIT must not run after a lost reply.
|
||||
if (transaction.poisonedBy) throw transaction.poisonedBy
|
||||
if (transaction.open) await timedQuery(client, 'COMMIT', undefined, this.queryTimeout)
|
||||
if (transaction.open) {
|
||||
await timedQuery(client, connectionId, 'COMMIT', undefined, this.queryTimeout)
|
||||
}
|
||||
recordMeasuredHold(this.holds, transaction)
|
||||
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
|
||||
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
|
||||
@@ -1322,6 +1348,24 @@ export class PostgresDatabase implements RelayDatabase {
|
||||
await this.pool.end()
|
||||
}
|
||||
|
||||
private async connect(lane?: PostgresPoolLane): Promise<pg.PoolClient> {
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
return await this.pressure.connect(lane)
|
||||
} finally {
|
||||
noteStickySlotPoolWait(performance.now() - startedAt)
|
||||
}
|
||||
}
|
||||
|
||||
private idOfConnection(client: pg.PoolClient): number {
|
||||
let id = this.connectionIds.get(client)
|
||||
if (id === undefined) {
|
||||
id = this.nextConnectionId++
|
||||
this.connectionIds.set(client, id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
consumePoolPressure(): PostgresPoolPressureCounts {
|
||||
return this.pressure.consumeCounts()
|
||||
}
|
||||
|
||||
@@ -32,7 +32,8 @@ const describePostgres = databaseUrl ? describe : describe.skip
|
||||
// asia-east2 cell to the us-central1 database, measured in production.
|
||||
const ASIA_ROUND_TRIP_MS = 171
|
||||
const LOCAL_ROUND_TRIP_MS = 1
|
||||
const WINDOW_MS = 10_000
|
||||
// Keep the measured contention rates; repeating them for longer only repeats the same locks.
|
||||
const WINDOW_MS = 5_000
|
||||
// Reconnecting hosts dialling the directors during the window. Above the
|
||||
// ~6-15/s the production tail sustained, so the director side is never idle.
|
||||
const DIAL_RATE_PER_SECOND = 20
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
import { createHash, createHmac } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import {
|
||||
buildHostProofMacInput,
|
||||
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
|
||||
RELAY_CLOSE_CODE
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { CellReserveBook } from './cell-reserve-book.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayCredentialStore } from './credential-store.js'
|
||||
import { HostSessionRegistry, type CellReserveAdmission } from './host-session-registry.js'
|
||||
import type { RelayRuntimeObserver } from './relay-observability.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
|
||||
|
||||
const CELL_ID = 'staging-gce-c1'
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly CLOSING = 2
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(''))
|
||||
})
|
||||
|
||||
asWebSocket(): WebSocket {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: FakeSocket implements the registry's WebSocket event and lifecycle surface.
|
||||
return this as unknown as WebSocket
|
||||
}
|
||||
|
||||
// The peer's close frame: ws leaves OPEN at once and emits 'close' later.
|
||||
hangUp(): void {
|
||||
this.readyState = this.CLOSING
|
||||
}
|
||||
}
|
||||
|
||||
function hostKey(): { keyPair: nacl.BoxKeyPair; identity: RelayTokenClaims } {
|
||||
const keyPair = nacl.box.keyPair()
|
||||
const relayHostId = createHash('sha256')
|
||||
.update(keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
return {
|
||||
keyPair,
|
||||
identity: {
|
||||
sub: 'user-1',
|
||||
prof: 'profile-1',
|
||||
relayHostId,
|
||||
purpose: 'host-control',
|
||||
exp: 4_102_444_800
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setup(input: {
|
||||
admitMode: 'db' | 'reserve'
|
||||
databaseSays: boolean
|
||||
databaseShedding?: boolean
|
||||
}) {
|
||||
let clock = 1_000_000
|
||||
const recordHostPreAuth = vi.fn<NonNullable<RelayRuntimeObserver['recordHostPreAuth']>>()
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn(),
|
||||
recordHostPreAuth
|
||||
} satisfies RelayRuntimeObserver
|
||||
const take = vi.fn<CellReserveBook['take']>((userId, relayHostId, epoch) => ({
|
||||
userId,
|
||||
relayHostId,
|
||||
epoch,
|
||||
directorId: 'director-1',
|
||||
expiresAt: clock + 60_000
|
||||
}))
|
||||
const reserveAdmission: CellReserveAdmission = {
|
||||
mode: () => input.admitMode,
|
||||
ticketEnforce: () => false,
|
||||
dryRunEnabled: () => false,
|
||||
databaseShedding: () => input.databaseShedding ?? false,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: beginProof calls only take() on the reserve book.
|
||||
book: { take } as unknown as CellReserveBook,
|
||||
verifyLease: vi.fn(async () => 'absent' as const)
|
||||
}
|
||||
let answerDatabase: (valid: boolean) => void = () => {}
|
||||
const databaseAnswered = new Promise<boolean>((resolve) => {
|
||||
answerDatabase = resolve
|
||||
})
|
||||
// Answers at once unless the test holds it with holdDatabase().
|
||||
let held = false
|
||||
const verifyCellAssignment = vi.fn(async () => (held ? databaseAnswered : input.databaseSays))
|
||||
const registry = new HostSessionRegistry(
|
||||
config(),
|
||||
vi.fn(),
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path never reads credentials.
|
||||
{} as RelayCredentialStore,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path calls only verifyCellAssignment().
|
||||
{ verifyCellAssignment } as unknown as RelayAssignmentStore,
|
||||
new ProcessQueuedByteBudget(),
|
||||
observer,
|
||||
() => clock,
|
||||
Math.random,
|
||||
'incarnation-1',
|
||||
undefined,
|
||||
reserveAdmission
|
||||
)
|
||||
return {
|
||||
registry,
|
||||
recordHostPreAuth,
|
||||
verifyCellAssignment,
|
||||
take,
|
||||
holdDatabase: () => {
|
||||
held = true
|
||||
},
|
||||
answerDatabase: (valid: boolean) => answerDatabase(valid),
|
||||
advance: (ms: number) => {
|
||||
clock += ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sendHello(socket: FakeSocket, keyPair: nacl.BoxKeyPair, relayHostId: string): void {
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-hello',
|
||||
v: 1,
|
||||
relayHostId,
|
||||
assignmentEpoch: 4,
|
||||
hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
|
||||
appVersion: '1.4.200'
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
function answerChallenge(socket: FakeSocket, keyPair: nacl.BoxKeyPair): void {
|
||||
const challenge: unknown = JSON.parse(String(socket.send.mock.calls[0]?.[0]))
|
||||
const field = (name: string): string => {
|
||||
const value: unknown =
|
||||
typeof challenge === 'object' && challenge !== null ? Reflect.get(challenge, name) : null
|
||||
if (typeof value !== 'string') throw new Error(`host-challenge has no ${name}`)
|
||||
return value
|
||||
}
|
||||
const plaintext = nacl.box.open(
|
||||
Buffer.from(field('ciphertextB64'), 'base64'),
|
||||
Buffer.from(field('nonceB64'), 'base64'),
|
||||
Buffer.from(field('relayEphemeralPublicKeyB64'), 'base64'),
|
||||
keyPair.secretKey
|
||||
)
|
||||
if (!plaintext) throw new Error('host-challenge did not open')
|
||||
const domainLength = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`).length
|
||||
const transcriptLength = new DataView(
|
||||
plaintext.buffer,
|
||||
plaintext.byteOffset + domainLength,
|
||||
4
|
||||
).getUint32(0, false)
|
||||
const transcriptStart = domainLength + 4
|
||||
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
|
||||
const secret = plaintext.slice(transcriptStart + transcriptLength)
|
||||
socket.emit(
|
||||
'message',
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
type: 'host-challenge-ack',
|
||||
challengeId: field('challengeId'),
|
||||
proofB64: createHmac('sha256', secret)
|
||||
.update(buildHostProofMacInput(transcript))
|
||||
.digest('base64')
|
||||
})
|
||||
),
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
describe('host pre-auth latency', () => {
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
it('records a booked hello on the reserve path, from accept to a verified proof', async () => {
|
||||
const h = setup({ admitMode: 'reserve', databaseSays: false })
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(30)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
|
||||
expect(h.recordHostPreAuth).not.toHaveBeenCalled()
|
||||
h.advance(45)
|
||||
answerChallenge(socket, keyPair)
|
||||
|
||||
expect(h.take).toHaveBeenCalledOnce()
|
||||
expect(h.verifyCellAssignment).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'admitted', 75)
|
||||
})
|
||||
|
||||
it('records a hello the database refuses on the db path', async () => {
|
||||
const h = setup({ admitMode: 'db', databaseSays: false })
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(120)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
|
||||
expect(h.take).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'refused', 120)
|
||||
})
|
||||
|
||||
it('records a hello timeout as a refusal before any check, once', () => {
|
||||
vi.useFakeTimers()
|
||||
const h = setup({ admitMode: 'db', databaseSays: true })
|
||||
const { identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(2_000)
|
||||
vi.advanceTimersByTime(2_000)
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
|
||||
'host hello timeout'
|
||||
)
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 2_000)
|
||||
})
|
||||
|
||||
it('records a host that hangs up before its hello as abandoned', () => {
|
||||
const h = setup({ admitMode: 'db', databaseSays: true })
|
||||
const { identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(300)
|
||||
socket.close()
|
||||
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'abandoned', 300)
|
||||
})
|
||||
|
||||
it.each([true, false])(
|
||||
'records a database answer (%s) that arrives after the host hung up as abandoned',
|
||||
async (valid) => {
|
||||
const h = setup({ admitMode: 'db', databaseSays: valid })
|
||||
h.holdDatabase()
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(h.verifyCellAssignment).toHaveBeenCalled())
|
||||
socket.hangUp()
|
||||
h.advance(900)
|
||||
h.answerDatabase(valid)
|
||||
await vi.waitFor(() => expect(h.recordHostPreAuth).toHaveBeenCalled())
|
||||
|
||||
expect(socket.send).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'abandoned', 900)
|
||||
}
|
||||
)
|
||||
|
||||
it('records a host that hangs up during its proof as abandoned on its path', async () => {
|
||||
const h = setup({ admitMode: 'reserve', databaseSays: false })
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
|
||||
h.advance(700)
|
||||
socket.close()
|
||||
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'abandoned', 700)
|
||||
})
|
||||
|
||||
it('records the pool-pressure shed as a refusal before any check', async () => {
|
||||
const h = setup({ admitMode: 'reserve', databaseSays: true, databaseShedding: true })
|
||||
h.take.mockReturnValue(null)
|
||||
const { keyPair, identity } = hostKey()
|
||||
const socket = new FakeSocket()
|
||||
h.registry.acceptControl(socket.asWebSocket(), identity)
|
||||
h.advance(40)
|
||||
sendHello(socket, keyPair, identity.relayHostId)
|
||||
await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
|
||||
|
||||
expect(h.verifyCellAssignment).not.toHaveBeenCalled()
|
||||
expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 40)
|
||||
})
|
||||
})
|
||||
|
||||
function config(): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://c1.relay.example.test',
|
||||
cellUrl: 'https://c1.relay.example.test',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new TextEncoder().encode('0123456789abcdef0123456789abcdef'),
|
||||
role: 'cell',
|
||||
cellId: CELL_ID,
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data'
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,8 @@ import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import {
|
||||
percentile,
|
||||
type HostPreAuthOutcome,
|
||||
type HostPreAuthPath,
|
||||
type RelayClientAcceptStage,
|
||||
type RelayClientAcceptTimedStage,
|
||||
type RelayRuntimeObserver
|
||||
@@ -186,26 +188,44 @@ function readControlFrame(
|
||||
socket: WebSocket,
|
||||
timeoutMs: number,
|
||||
timeoutReason: string,
|
||||
receive: (raw: RawData, isBinary: boolean) => void
|
||||
receive: (raw: RawData, isBinary: boolean) => void,
|
||||
ended?: (how: 'timeout' | 'closed') => void
|
||||
): void {
|
||||
if (socket.readyState !== socket.OPEN) return
|
||||
if (socket.readyState !== socket.OPEN) {
|
||||
ended?.('closed')
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
finish()
|
||||
ended?.('timeout')
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, timeoutReason)
|
||||
}, timeoutMs)
|
||||
function finish(): void {
|
||||
clearTimeout(timer)
|
||||
socket.off('message', onMessage)
|
||||
socket.off('close', finish)
|
||||
socket.off('close', onClose)
|
||||
}
|
||||
function onClose(): void {
|
||||
finish()
|
||||
ended?.('closed')
|
||||
}
|
||||
function onMessage(raw: RawData, isBinary: boolean): void {
|
||||
finish()
|
||||
receive(raw, isBinary)
|
||||
}
|
||||
socket.once('message', onMessage)
|
||||
socket.once('close', finish)
|
||||
socket.once('close', onClose)
|
||||
}
|
||||
|
||||
// Records one host control's pre-auth time, from the cell accepting the WebSocket to the
|
||||
// hello's admission and proof being decided, exactly once.
|
||||
type SettlePreAuth = (outcome: HostPreAuthOutcome, path: HostPreAuthPath) => void
|
||||
|
||||
const preAuthEnded =
|
||||
(settle: SettlePreAuth, path: HostPreAuthPath) =>
|
||||
(how: 'timeout' | 'closed'): void =>
|
||||
settle(how === 'timeout' ? 'refused' : 'abandoned', path)
|
||||
|
||||
// Hosts abandon connects after 15s; waiting much longer than that behind a
|
||||
// stalled predecessor only accumulates doomed sockets.
|
||||
const ACTIVATION_QUEUE_WAIT_MS = 30_000
|
||||
@@ -699,7 +719,7 @@ export class HostSessionRegistry {
|
||||
|
||||
// A director names this seat the loser of a duplicate: stop taking phones for it now, and
|
||||
// close it once its splices have had time to finish. Repeats are one demotion.
|
||||
demote(request: DemoteRequest): 'demoted' | 'already-demoted' | 'not-seated' | 'off' {
|
||||
demote(request: DemoteRequest): 'demoted' | 'already-demoted' | 'draining' | 'not-seated' | 'off' {
|
||||
// Only a reserve-mode cell takes demotions, so flipping admitMode back reverts the route.
|
||||
if (!this.reserveMode()) return 'off'
|
||||
const session = this.get(request)
|
||||
@@ -715,6 +735,8 @@ export class HostSessionRegistry {
|
||||
) {
|
||||
return 'not-seated'
|
||||
}
|
||||
// A drain already told this host to go: its own grace stands, never cut to the demotion's.
|
||||
if (this.drainSentHosts.has(session.relayHostId) || session.regionalDrainAttemptId) return 'draining'
|
||||
const socket = session.socket
|
||||
if (this.demotedSeats.get(key) === socket) return 'already-demoted'
|
||||
this.demotedSeats.set(key, socket)
|
||||
@@ -1212,35 +1234,64 @@ export class HostSessionRegistry {
|
||||
hostCapabilities?: ReadonlySet<string>,
|
||||
assignmentLease?: string
|
||||
): void {
|
||||
const settlePreAuth = this.startPreAuth(socket)
|
||||
const refuse = (code: number, reason: string): void => {
|
||||
settlePreAuth('refused', 'beforeCheck')
|
||||
socket.close(code, reason)
|
||||
}
|
||||
if (this.idleAttempts.has(identity.relayHostId)) {
|
||||
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
|
||||
refuse(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
|
||||
return
|
||||
}
|
||||
// Keyed by socket, not session: a rebind swaps the session's socket, and the
|
||||
// successor's own advertisement is the only one that describes its decoder.
|
||||
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
|
||||
if (this.draining) {
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
refuse(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
return
|
||||
}
|
||||
readControlFrame(socket, 2_000, 'host hello timeout', (raw, isBinary) => {
|
||||
if (isBinary) {
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello must be text')
|
||||
return
|
||||
}
|
||||
this.guardSessionTask(
|
||||
() =>
|
||||
this.beginProof(
|
||||
socket,
|
||||
identity,
|
||||
payload(raw, 'host-hello'),
|
||||
connectionInclusionWatermark,
|
||||
assignmentLease
|
||||
),
|
||||
socket,
|
||||
'host hello proof'
|
||||
)
|
||||
})
|
||||
readControlFrame(
|
||||
socket,
|
||||
2_000,
|
||||
'host hello timeout',
|
||||
(raw, isBinary) => {
|
||||
if (isBinary) {
|
||||
refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello must be text')
|
||||
return
|
||||
}
|
||||
this.guardSessionTask(
|
||||
() =>
|
||||
this.beginProof(
|
||||
socket,
|
||||
identity,
|
||||
payload(raw, 'host-hello'),
|
||||
settlePreAuth,
|
||||
connectionInclusionWatermark,
|
||||
assignmentLease
|
||||
).catch((error: unknown) => {
|
||||
// guardSessionTask closes the socket, so any throw refuses the hello as beforeCheck,
|
||||
// including one after the database answered (e.g. in send or the session lookup).
|
||||
settlePreAuth('refused', 'beforeCheck')
|
||||
throw error
|
||||
}),
|
||||
socket,
|
||||
'host hello proof'
|
||||
)
|
||||
},
|
||||
preAuthEnded(settlePreAuth, 'beforeCheck')
|
||||
)
|
||||
}
|
||||
|
||||
private startPreAuth(socket: WebSocket): SettlePreAuth {
|
||||
const startedAt = this.now()
|
||||
let settled = false
|
||||
return (outcome, path) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
// A decision made after the host hung up never reached it, so it counts as abandoned.
|
||||
const recorded = socket.readyState === socket.OPEN ? outcome : 'abandoned'
|
||||
this.observer.recordHostPreAuth?.(path, recorded, nonNegativeMs(this.now() - startedAt))
|
||||
}
|
||||
}
|
||||
|
||||
// A dependency failure (e.g. a database connect timeout) must cost one
|
||||
@@ -1379,29 +1430,34 @@ export class HostSessionRegistry {
|
||||
socket: WebSocket,
|
||||
identity: RelayTokenClaims,
|
||||
candidate: unknown,
|
||||
settlePreAuth: SettlePreAuth,
|
||||
connectionInclusionWatermark?: number,
|
||||
assignmentLease?: string
|
||||
): Promise<void> {
|
||||
// Which check decided the hello; refusals before one answers stay 'beforeCheck'.
|
||||
let path: HostPreAuthPath = 'beforeCheck'
|
||||
const refuse = (code: number, reason: string): void => {
|
||||
settlePreAuth('refused', path)
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(code, reason)
|
||||
}
|
||||
const hello = HostHelloSchema.safeParse(candidate)
|
||||
const hostPublicKey = hello.success
|
||||
? decodeCanonicalBase64(hello.data.hostPublicKeyB64, 32)
|
||||
: null
|
||||
if (!hello.success) {
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host hello')
|
||||
refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host hello')
|
||||
return
|
||||
}
|
||||
if (!hostPublicKey) {
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host public key')
|
||||
refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host public key')
|
||||
return
|
||||
}
|
||||
if (
|
||||
hello.data.relayHostId !== identity.relayHostId ||
|
||||
relayHostId(hostPublicKey) !== identity.relayHostId
|
||||
) {
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host key binding mismatch')
|
||||
refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host key binding mismatch')
|
||||
return
|
||||
}
|
||||
const leaseShadow = this.config.role === 'cell' ? this.assignmentLeaseShadow : undefined
|
||||
@@ -1434,12 +1490,12 @@ export class HostSessionRegistry {
|
||||
relayHostId: identity.relayHostId,
|
||||
helloEpoch: hello.data.assignmentEpoch
|
||||
})) === 'valid')
|
||||
if (admittedFromMemory) path = 'reserve'
|
||||
}
|
||||
if (reserveMode && !admittedFromMemory && this.reserveAdmission!.databaseShedding()) {
|
||||
// Memory cannot admit it (e.g. seated here, but at another epoch), and the wedged pool
|
||||
// would only time out its database check: shed it now, as the upgrade shed does.
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay temporarily unavailable')
|
||||
refuse(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay temporarily unavailable')
|
||||
return
|
||||
}
|
||||
const assignmentReadStartedAt = performance.now()
|
||||
@@ -1462,12 +1518,16 @@ export class HostSessionRegistry {
|
||||
performance.now() - assignmentReadStartedAt
|
||||
)
|
||||
}
|
||||
// The `combined` role compares the epoch without querying but is still labelled db.
|
||||
if (!admittedFromMemory) path = 'db'
|
||||
if (!assignmentValid) {
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
|
||||
refuse(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
|
||||
return
|
||||
}
|
||||
if (socket.readyState !== socket.OPEN) {
|
||||
settlePreAuth('abandoned', path)
|
||||
return
|
||||
}
|
||||
if (socket.readyState !== socket.OPEN) return
|
||||
|
||||
const key = this.key(identity.sub, identity.relayHostId)
|
||||
const existing = this.sessions.get(key)
|
||||
@@ -1513,40 +1573,46 @@ export class HostSessionRegistry {
|
||||
ciphertextB64: Buffer.from(ciphertext).toString('base64'),
|
||||
expiresAt
|
||||
})
|
||||
readControlFrame(socket, 10_000, 'host proof timeout', (raw, isBinary) => {
|
||||
const ack = isBinary
|
||||
? null
|
||||
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
|
||||
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
|
||||
if (
|
||||
!ack?.success ||
|
||||
ack.data.challengeId !== challengeId ||
|
||||
!proof ||
|
||||
this.now() > expiresAt ||
|
||||
!timingSafeEqual(proof, expectedProof)
|
||||
) {
|
||||
this.observer.recordAuth(false)
|
||||
socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host proof')
|
||||
return
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
this.guardSessionTask(
|
||||
() =>
|
||||
this.activate(
|
||||
socket,
|
||||
identity,
|
||||
existing ?? null,
|
||||
generation,
|
||||
rebind,
|
||||
hello.data.assignmentEpoch,
|
||||
hello.data.appVersion,
|
||||
connectionInclusionWatermark,
|
||||
admittedFromMemory ? { reservedBy } : undefined
|
||||
),
|
||||
socket,
|
||||
'host activation'
|
||||
)
|
||||
})
|
||||
readControlFrame(
|
||||
socket,
|
||||
10_000,
|
||||
'host proof timeout',
|
||||
(raw, isBinary) => {
|
||||
const ack = isBinary
|
||||
? null
|
||||
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
|
||||
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
|
||||
if (
|
||||
!ack?.success ||
|
||||
ack.data.challengeId !== challengeId ||
|
||||
!proof ||
|
||||
this.now() > expiresAt ||
|
||||
!timingSafeEqual(proof, expectedProof)
|
||||
) {
|
||||
refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host proof')
|
||||
return
|
||||
}
|
||||
settlePreAuth('admitted', path)
|
||||
this.observer.recordAuth(true)
|
||||
this.guardSessionTask(
|
||||
() =>
|
||||
this.activate(
|
||||
socket,
|
||||
identity,
|
||||
existing ?? null,
|
||||
generation,
|
||||
rebind,
|
||||
hello.data.assignmentEpoch,
|
||||
hello.data.appVersion,
|
||||
connectionInclusionWatermark,
|
||||
admittedFromMemory ? { reservedBy } : undefined
|
||||
),
|
||||
socket,
|
||||
'host activation'
|
||||
)
|
||||
},
|
||||
preAuthEnded(settlePreAuth, path)
|
||||
)
|
||||
}
|
||||
|
||||
private activate(
|
||||
|
||||
@@ -22,6 +22,7 @@ const PROBE_TABLE = 'relay_lost_reply_probe'
|
||||
async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
|
||||
const sockets = new Set<Socket>()
|
||||
const afterTrigger: Buffer[] = []
|
||||
let forwardingResumed = false
|
||||
const server: Server = createServer((client) => {
|
||||
const upstream = connect(Number(target.port || 5432), target.hostname)
|
||||
sockets.add(client)
|
||||
@@ -33,7 +34,7 @@ async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
|
||||
upstream.write(chunk)
|
||||
})
|
||||
upstream.on('data', (chunk: Buffer) => {
|
||||
if (!swallowing) client.write(chunk)
|
||||
if (!swallowing || forwardingResumed) client.write(chunk)
|
||||
else if (delayMs !== undefined) setTimeout(() => client.write(chunk), delayMs)
|
||||
})
|
||||
for (const socket of [client, upstream]) {
|
||||
@@ -52,6 +53,9 @@ async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
|
||||
url.port = String(address.port)
|
||||
return {
|
||||
url: url.toString(),
|
||||
resumeForwarding: () => {
|
||||
forwardingResumed = true
|
||||
},
|
||||
sentAfterTrigger: () => Buffer.concat(afterTrigger).toString('latin1'),
|
||||
close: async () => {
|
||||
for (const socket of sockets) socket.destroy()
|
||||
@@ -174,6 +178,7 @@ describePostgres('a lost PostgreSQL reply against a real server', () => {
|
||||
{ probe: 'stalled_reply_probe' }
|
||||
])
|
||||
expect(performance.now() - startedAt).toBeGreaterThan(6_000)
|
||||
proxy.resumeForwarding()
|
||||
// Not destroyed: the one pooled connection is still the same backend.
|
||||
expect(await pid()).toBe(before)
|
||||
}, 30_000)
|
||||
|
||||
@@ -233,17 +233,17 @@ describe('drain-return lane', () => {
|
||||
expect(redial.headers.get('retry-after')).toBe('2')
|
||||
})
|
||||
|
||||
it('admits a 2,500-host drain across five directors without a sticky or placement rejection', async () => {
|
||||
it('admits a paced drain across five directors without a sticky or placement rejection', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
// Five directors behind a load balancer that sends each request to any of
|
||||
// them. 2,500 drained hosts over the 300 s pace window, beside ordinary
|
||||
// reconnects (c29 baseline ~13/s) and a trickle of new placements.
|
||||
// them. A shorter cohort preserves the production arrival rates: ~8 drained
|
||||
// hosts/s, ~13 ordinary reconnects/s, and a trickle of new placements.
|
||||
const DIRECTORS = 5
|
||||
const DRAINED = 2_500
|
||||
const PACE_WINDOW_MS = 300_000
|
||||
const ORDINARY = 3_900
|
||||
const PLACEMENTS = 30
|
||||
const DRAINED = 250
|
||||
const PACE_WINDOW_MS = 30_000
|
||||
const ORDINARY = 390
|
||||
const PLACEMENTS = 3
|
||||
// Measured c29 2026-10-01: ~5.8 re-placements/s fleet-wide, bound by the
|
||||
// region's inventory lock, so ~172 ms of lock per re-placement.
|
||||
const INVENTORY_LOCK_MS = 172
|
||||
@@ -348,7 +348,7 @@ describe('drain-return lane', () => {
|
||||
}
|
||||
|
||||
let elapsedMs = 0
|
||||
while (inFlight > 0 && elapsedMs < 1_200_000) {
|
||||
while (inFlight > 0 && elapsedMs < 120_000) {
|
||||
await vi.advanceTimersByTimeAsync(20)
|
||||
elapsedMs += 20
|
||||
// Signing the lease completes off the timer queue.
|
||||
@@ -363,10 +363,11 @@ describe('drain-return lane', () => {
|
||||
expect(outcomes.get('sticky')).toBe(ORDINARY)
|
||||
expect(outcomes.get('placement')).toBe(PLACEMENTS)
|
||||
expect(outcomes.get('drain-return')).toBe(DRAINED)
|
||||
expect(outcomes.get('drain-return-deferred') ?? 0).toBeGreaterThan(0)
|
||||
// Paced returns: fewer than two deferrals per drained host, none past the cap.
|
||||
expect(outcomes.get('drain-return-deferred') ?? 0).toBeLessThan(2 * DRAINED)
|
||||
expect(Math.max(...retryAfters)).toBeLessThanOrEqual(300)
|
||||
// The cohort finishes at the inventory lock's rate, not minutes behind it.
|
||||
// Retry rounding and off-thread signing keep their original fixed allowance.
|
||||
expect(elapsedMs).toBeLessThan(DRAINED * INVENTORY_LOCK_MS + 60_000)
|
||||
}, 300_000)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentRowBusyError, type RelayAssignment } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { StickySlotSample } from './sticky-slot-statements.js'
|
||||
import {
|
||||
ScriptedPoolClient,
|
||||
scriptedPostgresDatabase
|
||||
} from './test-fixtures/scripted-postgres-pool.js'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
verifyRelayToken: vi.fn(async (token: string) => ({
|
||||
@@ -199,6 +204,64 @@ describe('public assignment reconnect lane', () => {
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('reports what each sticky slot spent its hold on', async () => {
|
||||
const first = 'ffffffffffffffff'
|
||||
const queued = 'qqqqqqqqqqqqqqqq'
|
||||
const newcomer = 'nnnnnnnnnnnnnnnn'
|
||||
const firstHost = new ScriptedPoolClient()
|
||||
const queuedHost = new ScriptedPoolClient()
|
||||
// first: verify, assign; newcomer: assign; queued: verify, assign.
|
||||
const database = scriptedPostgresDatabase([
|
||||
firstHost,
|
||||
firstHost,
|
||||
new ScriptedPoolClient(),
|
||||
queuedHost,
|
||||
queuedHost
|
||||
])
|
||||
const firstAssign = deferred<RelayAssignment>()
|
||||
const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
await database.query('SELECT verify')
|
||||
return assignment('cell-r', relayHostId)
|
||||
})
|
||||
const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
|
||||
await database.transaction(async (transaction) => {
|
||||
await transaction.query('SELECT pin')
|
||||
await transaction.query('UPDATE lease')
|
||||
})
|
||||
if (relayHostId === first) return await firstAssign.promise
|
||||
return assignment('cell-r', relayHostId)
|
||||
})
|
||||
const samples: StickySlotSample[] = []
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: { assign, resolve } as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true),
|
||||
recordStickySlot: (sample) => samples.push(sample)
|
||||
})
|
||||
|
||||
const holding = app.request('/v1/assign', assignmentRequest(first, { reconnect: true }))
|
||||
await vi.waitFor(() => expect(firstHost.statements).toHaveLength(5))
|
||||
const waiting = app.request('/v1/assign', assignmentRequest(queued, { reconnect: true }))
|
||||
// A placement request's statements are its own, even while a sticky slot is held.
|
||||
expect((await app.request('/v1/assign', assignmentRequest(newcomer))).status).toBe(200)
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 60))
|
||||
firstAssign.resolve(assignment('cell-r', first))
|
||||
expect((await holding).status).toBe(200)
|
||||
expect((await waiting).status).toBe(200)
|
||||
|
||||
// Verify, then BEGIN, two statements and COMMIT, each on that host's own connection.
|
||||
const roundTrips = (sample: StickySlotSample | undefined) =>
|
||||
[...(sample?.statementMsByConnection ?? [])].map(([id, ms]) => [id, ms.length])
|
||||
expect(samples).toHaveLength(2)
|
||||
expect(samples[0]).toMatchObject({ statements: 5 })
|
||||
expect(roundTrips(samples[0])).toEqual([[1, 5]])
|
||||
expect(samples[0]?.queueWaitMs).toBeLessThan(40)
|
||||
expect(samples[1]).toMatchObject({ statements: 5 })
|
||||
expect(roundTrips(samples[1])).toEqual([[3, 5]])
|
||||
expect(samples[1]?.queueWaitMs).toBeGreaterThanOrEqual(40)
|
||||
})
|
||||
|
||||
it('never probes for unhinted requests', async () => {
|
||||
const host = 'uuuuuuuuuuuuuuuu'
|
||||
const assign = vi.fn(async () => assignment('cell-u', host))
|
||||
|
||||
@@ -5,8 +5,10 @@ import { observeRelayDatabase } from './observed-relay-database.js'
|
||||
import {
|
||||
CONTROL_RTT_RESERVOIR_LIMIT,
|
||||
observedRelayRequests,
|
||||
PRE_AUTH_RESERVOIR_LIMIT,
|
||||
percentile,
|
||||
RelayObservability,
|
||||
STICKY_CONNECTION_LIMIT,
|
||||
type RelayProcessCounts
|
||||
} from './relay-observability.js'
|
||||
import { RELAY_FIX_LEVEL } from './relay-fix-level.js'
|
||||
@@ -283,6 +285,75 @@ describe('relay observability', () => {
|
||||
expect(entries[1]).not.toHaveProperty('drainReturnServiceMsP50')
|
||||
})
|
||||
|
||||
it('summarises sticky slots per window and per pooled connection', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'director', cellId: 'director', region: 'us-central1' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
for (let ms = 1; ms <= 100; ms++) {
|
||||
observability.recordStickySlot({
|
||||
queueWaitMs: ms * 20,
|
||||
poolWaitMs: ms,
|
||||
sqlMs: ms * 2,
|
||||
statements: 14,
|
||||
// The slow connection serves every other request.
|
||||
statementMsByConnection: new Map([ms % 2 === 0 ? [1, [300]] : [2, [5, 6]]])
|
||||
})
|
||||
}
|
||||
observability.flush(counts)
|
||||
observability.flush(counts)
|
||||
|
||||
expect(entries[0]).toMatchObject({
|
||||
stickySlotsDelta: 100,
|
||||
stickyQueueWaitMsP50: 1_000,
|
||||
stickyQueueWaitMsP99: 1_980,
|
||||
stickyQueueWaitMsMax: 2_000,
|
||||
stickyPoolWaitMsP50: 50,
|
||||
stickyPoolWaitMsP99: 99,
|
||||
stickySqlMsP50: 100,
|
||||
stickySqlMsP99: 198,
|
||||
stickyStatementsP50: 14,
|
||||
stickyStatementsMax: 14,
|
||||
stickySqlByConnection: {
|
||||
'1': {
|
||||
requests: 50,
|
||||
statements: 50,
|
||||
statementMsP50: 300,
|
||||
statementMsP99: 300,
|
||||
statementMsMax: 300
|
||||
},
|
||||
'2': { requests: 50, statements: 100, statementMsP50: 5, statementMsP99: 6 }
|
||||
},
|
||||
stickySqlConnectionSamplesDroppedDelta: 0
|
||||
})
|
||||
expect(entries[1]).not.toHaveProperty('stickySlotsDelta')
|
||||
expect(entries[1]).not.toHaveProperty('stickySqlByConnection')
|
||||
})
|
||||
|
||||
it('bounds the per-connection sticky summary', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'director', cellId: 'director', region: 'us-central1' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
for (let id = 1; id <= STICKY_CONNECTION_LIMIT + 3; id++) {
|
||||
observability.recordStickySlot({
|
||||
queueWaitMs: 0,
|
||||
poolWaitMs: 0,
|
||||
sqlMs: 1,
|
||||
statements: 1,
|
||||
statementMsByConnection: new Map([[id, [1]]])
|
||||
})
|
||||
}
|
||||
observability.flush(counts)
|
||||
|
||||
expect(Object.keys(entries[0]?.stickySqlByConnection as object)).toHaveLength(
|
||||
STICKY_CONNECTION_LIMIT
|
||||
)
|
||||
expect(entries[0]).toMatchObject({ stickySqlConnectionSamplesDroppedDelta: 3 })
|
||||
})
|
||||
|
||||
it('sums lock waiters per role and table over the samples taken', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
@@ -570,6 +641,109 @@ describe('relay observability', () => {
|
||||
expect(entries[0]!.controlRttMsMax).toBe(900)
|
||||
})
|
||||
|
||||
it('summarises host pre-auth latency per path and outcome', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
for (let ms = 1; ms <= 100; ms++) observability.recordHostPreAuth('db', 'admitted', ms)
|
||||
observability.recordHostPreAuth('db', 'abandoned', 6_000)
|
||||
observability.recordHostPreAuth('beforeCheck', 'refused', 2_000)
|
||||
observability.recordHostPreAuth('reserve', 'admitted', 12.3456)
|
||||
observability.flush(counts)
|
||||
observability.flush(counts)
|
||||
|
||||
const none = { admitted: { count: 0 }, refused: { count: 0 }, abandoned: { count: 0 } }
|
||||
const buckets = (filled: Partial<Record<string, number>>): Record<string, number> => ({
|
||||
'50': 0,
|
||||
'100': 0,
|
||||
'200': 0,
|
||||
'300': 0,
|
||||
'450': 0,
|
||||
'700': 0,
|
||||
'1000': 0,
|
||||
'2000': 0,
|
||||
'5000': 0,
|
||||
'+Inf': 0,
|
||||
...filled
|
||||
})
|
||||
expect(entries[0]!.preAuthLatencyMs).toEqual({
|
||||
beforeCheck: {
|
||||
...none,
|
||||
refused: {
|
||||
count: 1,
|
||||
p50: 2_000,
|
||||
p95: 2_000,
|
||||
p99: 2_000,
|
||||
max: 2_000,
|
||||
bucketsLeMs: buckets({ '2000': 1 })
|
||||
}
|
||||
},
|
||||
db: {
|
||||
...none,
|
||||
admitted: {
|
||||
count: 100,
|
||||
p50: 50,
|
||||
p95: 95,
|
||||
p99: 99,
|
||||
max: 100,
|
||||
bucketsLeMs: buckets({ '50': 50, '100': 50 })
|
||||
},
|
||||
abandoned: {
|
||||
count: 1,
|
||||
p50: 6_000,
|
||||
p95: 6_000,
|
||||
p99: 6_000,
|
||||
max: 6_000,
|
||||
bucketsLeMs: buckets({ '+Inf': 1 })
|
||||
}
|
||||
},
|
||||
reserve: {
|
||||
...none,
|
||||
admitted: {
|
||||
count: 1,
|
||||
p50: 12.346,
|
||||
p95: 12.346,
|
||||
p99: 12.346,
|
||||
max: 12.346,
|
||||
bucketsLeMs: buckets({ '50': 1 })
|
||||
}
|
||||
}
|
||||
})
|
||||
// The window resets, and an empty window publishes counts only.
|
||||
expect(entries[1]!.preAuthLatencyMs).toEqual({ beforeCheck: none, db: none, reserve: none })
|
||||
})
|
||||
|
||||
it('counts every hello in the pre-auth buckets, inclusive of each bound', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
const flooded = PRE_AUTH_RESERVOIR_LIMIT * 5
|
||||
for (let sample = 0; sample < flooded; sample++) {
|
||||
observability.recordHostPreAuth('reserve', 'admitted', 450)
|
||||
}
|
||||
observability.recordHostPreAuth('reserve', 'admitted', 450.001)
|
||||
observability.recordHostPreAuth('reserve', 'admitted', 9_000)
|
||||
observability.flush(counts)
|
||||
|
||||
expect(entries[0]).toMatchObject({
|
||||
preAuthLatencyMs: {
|
||||
reserve: {
|
||||
admitted: {
|
||||
// Exact past the reservoir cap, even when it declined the slowest hello.
|
||||
count: flooded + 2,
|
||||
max: 9_000,
|
||||
p50: 450,
|
||||
bucketsLeMs: { '300': 0, '450': flooded, '700': 1, '5000': 0, '+Inf': 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('observes successful and failed database calls including transactions', async () => {
|
||||
const recordSql = vi.fn()
|
||||
const underlying: RelayDatabase = {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
|
||||
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
|
||||
import type { RelayReadinessGraceEvent, RelayReadinessObservation } from './relay-readiness.js'
|
||||
import { RELAY_FIX_LEVEL } from './relay-fix-level.js'
|
||||
import type { StickySlotSample } from './sticky-slot-statements.js'
|
||||
|
||||
export type RelayRuntimeCounts = {
|
||||
totalConnections: number
|
||||
@@ -90,6 +91,7 @@ export interface RelayRuntimeObserver {
|
||||
recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void
|
||||
recordDrainReturnRetryAfter?(seconds: number): void
|
||||
recordAdmissionServiceMs?(lane: 'sticky' | 'drain-return', durationMs: number): void
|
||||
recordStickySlot?(sample: StickySlotSample): void
|
||||
recordAssignmentUnavailable?(cause: AssignmentUnavailableCause): void
|
||||
recordDatabaseLockWaitSample?(sample: DatabaseLockWaitSample): void
|
||||
recordRegionRequest?(region: RelayRegion | undefined): void
|
||||
@@ -103,6 +105,42 @@ export interface RelayRuntimeObserver {
|
||||
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
|
||||
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
|
||||
recordControlRtt?(rttMs: number): void
|
||||
recordHostPreAuth?(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void
|
||||
}
|
||||
|
||||
// Which check decided a host hello: the cell's memory in reserve mode (a booking, a seat it
|
||||
// remembers, or a lease), the database assignment check, or neither (beforeCheck: draining,
|
||||
// idle cutover, a bad or late hello, the pool shed, or a check that threw).
|
||||
export type HostPreAuthPath = 'beforeCheck' | 'db' | 'reserve'
|
||||
// Abandoned: the host hung up before a decision, or before the cell could act on one.
|
||||
export type HostPreAuthOutcome = 'admitted' | 'refused' | 'abandoned'
|
||||
|
||||
// Upper bounds in ms; the last bucket is unbounded. 450 is the step-5 t_preauth gate.
|
||||
const PRE_AUTH_BUCKET_BOUNDS_MS = [50, 100, 200, 300, 450, 700, 1_000, 2_000, 5_000] as const
|
||||
|
||||
type PreAuthLatencyGroup = {
|
||||
samplesMs: number[]
|
||||
observed: number
|
||||
maxMs: number
|
||||
// Non-cumulative; one more slot than there are bounds, for the unbounded bucket.
|
||||
bucketCounts: number[]
|
||||
}
|
||||
type PreAuthLatency = Record<HostPreAuthPath, Record<HostPreAuthOutcome, PreAuthLatencyGroup>>
|
||||
|
||||
const emptyPreAuthLatencyGroup = (): PreAuthLatencyGroup => ({
|
||||
samplesMs: [],
|
||||
observed: 0,
|
||||
maxMs: 0,
|
||||
bucketCounts: Array.from({ length: PRE_AUTH_BUCKET_BOUNDS_MS.length + 1 }, () => 0)
|
||||
})
|
||||
|
||||
const emptyPreAuthLatency = (): PreAuthLatency => {
|
||||
const byOutcome = (): Record<HostPreAuthOutcome, PreAuthLatencyGroup> => ({
|
||||
admitted: emptyPreAuthLatencyGroup(),
|
||||
refused: emptyPreAuthLatencyGroup(),
|
||||
abandoned: emptyPreAuthLatencyGroup()
|
||||
})
|
||||
return { beforeCheck: byOutcome(), db: byOutcome(), reserve: byOutcome() }
|
||||
}
|
||||
|
||||
// Which serialized accept step the phone had already hung up behind.
|
||||
@@ -148,6 +186,12 @@ type RelayMetricDeltas = {
|
||||
drainReturnRetryAfterSecondsSum: number
|
||||
drainReturnServiceMs: number[]
|
||||
stickyServiceMs: number[]
|
||||
stickySlotQueueWaitMs: number[]
|
||||
stickySlotPoolWaitMs: number[]
|
||||
stickySlotSqlMs: number[]
|
||||
stickySlotStatements: number[]
|
||||
stickySlotByConnection: Map<number, StickyConnectionStatements>
|
||||
stickySlotConnectionSamplesDropped: number
|
||||
assign503sByCause: Record<string, number>
|
||||
dbLockWaitSamples: number
|
||||
dbLockWaitersByKey: Record<string, number>
|
||||
@@ -170,11 +214,41 @@ type RelayMetricDeltas = {
|
||||
controlActivityRecoveries: number
|
||||
controlActivityRecoveryFailures: number
|
||||
hostHellosShed: number
|
||||
preAuthLatency: PreAuthLatency
|
||||
}
|
||||
|
||||
type StickyConnectionStatements = {
|
||||
requests: number
|
||||
statements: number
|
||||
maxMs: number
|
||||
samplesMs: number[]
|
||||
}
|
||||
|
||||
// A window sees each pooled connection the sticky lane used; a pool that keeps
|
||||
// replacing connections must not grow the line without bound.
|
||||
export const STICKY_CONNECTION_LIMIT = 64
|
||||
export const STICKY_STATEMENT_RESERVOIR_LIMIT = 1024
|
||||
|
||||
// A host chooses how often it answers a ping, so the process-wide window is a
|
||||
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
|
||||
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
|
||||
// Per path and outcome; a reconnect burst can put thousands of hellos in one window.
|
||||
export const PRE_AUTH_RESERVOIR_LIMIT = 1024
|
||||
|
||||
// Algorithm R: every sample in the window keeps an equal chance of being kept.
|
||||
function keepInReservoir(
|
||||
samples: number[],
|
||||
observedBefore: number,
|
||||
value: number,
|
||||
limit: number
|
||||
): void {
|
||||
if (samples.length < limit) {
|
||||
samples.push(value)
|
||||
return
|
||||
}
|
||||
const slot = Math.floor(Math.random() * (observedBefore + 1))
|
||||
if (slot < limit) samples[slot] = value
|
||||
}
|
||||
|
||||
type MetricWriter = (entry: Record<string, unknown>) => void
|
||||
|
||||
@@ -200,6 +274,12 @@ const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
drainReturnRetryAfterSecondsSum: 0,
|
||||
drainReturnServiceMs: [],
|
||||
stickyServiceMs: [],
|
||||
stickySlotQueueWaitMs: [],
|
||||
stickySlotPoolWaitMs: [],
|
||||
stickySlotSqlMs: [],
|
||||
stickySlotStatements: [],
|
||||
stickySlotByConnection: new Map(),
|
||||
stickySlotConnectionSamplesDropped: 0,
|
||||
assign503sByCause: {},
|
||||
dbLockWaitSamples: 0,
|
||||
dbLockWaitersByKey: {},
|
||||
@@ -227,7 +307,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
controlRenewalFlushRowsMax: 0,
|
||||
controlActivityRecoveries: 0,
|
||||
controlActivityRecoveryFailures: 0,
|
||||
hostHellosShed: 0
|
||||
hostHellosShed: 0,
|
||||
preAuthLatency: emptyPreAuthLatency()
|
||||
})
|
||||
|
||||
function ascending(values: number[]): number[] {
|
||||
@@ -252,12 +333,18 @@ function roundMs(value: number): number {
|
||||
// enough of it, so the maximum is folded instead. The fold is also not
|
||||
// interchangeable with the sorted last element: it is seeded with zero, so an
|
||||
// all-negative or NaN window reads differently.
|
||||
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
|
||||
// One sorted copy serves both ranks.
|
||||
function latencySummary(samples: number[]): {
|
||||
p50: number
|
||||
p95: number
|
||||
p99: number
|
||||
max: number
|
||||
} {
|
||||
// One sorted copy serves every rank.
|
||||
const sorted = samples.length === 0 ? samples : ascending(samples)
|
||||
return {
|
||||
p50: roundMs(nearestRank(sorted, 0.5)),
|
||||
p95: roundMs(nearestRank(sorted, 0.95)),
|
||||
p99: roundMs(nearestRank(sorted, 0.99)),
|
||||
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
|
||||
}
|
||||
}
|
||||
@@ -331,6 +418,36 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
else this.deltas.drainReturnServiceMs.push(durationMs)
|
||||
}
|
||||
|
||||
recordStickySlot(sample: StickySlotSample): void {
|
||||
const deltas = this.deltas
|
||||
deltas.stickySlotQueueWaitMs.push(sample.queueWaitMs)
|
||||
deltas.stickySlotPoolWaitMs.push(sample.poolWaitMs)
|
||||
deltas.stickySlotSqlMs.push(sample.sqlMs)
|
||||
deltas.stickySlotStatements.push(sample.statements)
|
||||
for (const [connectionId, samplesMs] of sample.statementMsByConnection) {
|
||||
let connection = deltas.stickySlotByConnection.get(connectionId)
|
||||
if (!connection) {
|
||||
if (deltas.stickySlotByConnection.size >= STICKY_CONNECTION_LIMIT) {
|
||||
deltas.stickySlotConnectionSamplesDropped++
|
||||
continue
|
||||
}
|
||||
connection = { requests: 0, statements: 0, maxMs: 0, samplesMs: [] }
|
||||
deltas.stickySlotByConnection.set(connectionId, connection)
|
||||
}
|
||||
connection.requests++
|
||||
for (const durationMs of samplesMs) {
|
||||
const observedBefore = connection.statements++
|
||||
connection.maxMs = Math.max(connection.maxMs, durationMs)
|
||||
keepInReservoir(
|
||||
connection.samplesMs,
|
||||
observedBefore,
|
||||
durationMs,
|
||||
STICKY_STATEMENT_RESERVOIR_LIMIT
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recordAssignmentUnavailable(cause: AssignmentUnavailableCause): void {
|
||||
increment(this.deltas.assign503sByCause, cause)
|
||||
}
|
||||
@@ -433,15 +550,23 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
}
|
||||
|
||||
recordControlRtt(rttMs: number): void {
|
||||
const samples = this.deltas.controlRttSamplesMs
|
||||
const observedBefore = this.deltas.controlRttObserved++
|
||||
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
|
||||
samples.push(rttMs)
|
||||
return
|
||||
}
|
||||
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
|
||||
const slot = Math.floor(Math.random() * (observedBefore + 1))
|
||||
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
|
||||
keepInReservoir(
|
||||
this.deltas.controlRttSamplesMs,
|
||||
observedBefore,
|
||||
rttMs,
|
||||
CONTROL_RTT_RESERVOIR_LIMIT
|
||||
)
|
||||
}
|
||||
|
||||
recordHostPreAuth(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void {
|
||||
const group = this.deltas.preAuthLatency[path][outcome]
|
||||
const observedBefore = group.observed++
|
||||
// Folded outside the reservoir so a dropped sample cannot hide the worst hello.
|
||||
group.maxMs = Math.max(group.maxMs, durationMs)
|
||||
const bucket = PRE_AUTH_BUCKET_BOUNDS_MS.findIndex((bound) => durationMs <= bound)
|
||||
group.bucketCounts[bucket === -1 ? PRE_AUTH_BUCKET_BOUNDS_MS.length : bucket]! += 1
|
||||
keepInReservoir(group.samplesMs, observedBefore, durationMs, PRE_AUTH_RESERVOIR_LIMIT)
|
||||
}
|
||||
|
||||
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
|
||||
@@ -527,6 +652,7 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
stickyServiceMsP50: roundMs(percentile(deltas.stickyServiceMs, 0.5)),
|
||||
stickyServiceMsP99: roundMs(percentile(deltas.stickyServiceMs, 0.99))
|
||||
}),
|
||||
...stickySlotFields(deltas),
|
||||
assign503sByCauseDelta: deltas.assign503sByCause,
|
||||
assignNonDrain503sDelta: Object.entries(deltas.assign503sByCause)
|
||||
.filter(([cause]) => cause !== 'drain-return-deferred')
|
||||
@@ -584,6 +710,7 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
|
||||
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
|
||||
hostHellosShedDelta: deltas.hostHellosShed,
|
||||
preAuthLatencyMs: preAuthLatencyFields(deltas.preAuthLatency),
|
||||
// Meaning changed when renewals began batching: for a batched row this is
|
||||
// the flush's duration, not that row's own statement latency. The
|
||||
// per-flush fields below are the ones to read for statement cost.
|
||||
@@ -619,6 +746,71 @@ function regionCounterFields(
|
||||
)
|
||||
}
|
||||
|
||||
// Count and buckets cover every hello in the window, so buckets sum exactly across windows and
|
||||
// cells; the percentiles summarise the reservoir and are omitted when empty.
|
||||
function preAuthLatencyFields(
|
||||
latency: PreAuthLatency
|
||||
): Record<HostPreAuthPath, Record<HostPreAuthOutcome, Record<string, unknown>>> {
|
||||
const summarize = (group: PreAuthLatencyGroup): Record<string, unknown> => {
|
||||
if (group.observed === 0) return { count: 0 }
|
||||
const { p50, p95, p99 } = latencySummary(group.samplesMs)
|
||||
const bucketsLeMs: Record<string, number> = {}
|
||||
PRE_AUTH_BUCKET_BOUNDS_MS.forEach((bound, index) => {
|
||||
bucketsLeMs[String(bound)] = group.bucketCounts[index]!
|
||||
})
|
||||
bucketsLeMs['+Inf'] = group.bucketCounts[PRE_AUTH_BUCKET_BOUNDS_MS.length]!
|
||||
return { count: group.observed, p50, p95, p99, max: roundMs(group.maxMs), bucketsLeMs }
|
||||
}
|
||||
const byOutcome = (
|
||||
groups: Record<HostPreAuthOutcome, PreAuthLatencyGroup>
|
||||
): Record<HostPreAuthOutcome, Record<string, unknown>> => ({
|
||||
admitted: summarize(groups.admitted),
|
||||
refused: summarize(groups.refused),
|
||||
abandoned: summarize(groups.abandoned)
|
||||
})
|
||||
return {
|
||||
beforeCheck: byOutcome(latency.beforeCheck),
|
||||
db: byOutcome(latency.db),
|
||||
reserve: byOutcome(latency.reserve)
|
||||
}
|
||||
}
|
||||
|
||||
// What the sticky slot spent its hold on: the queue before it, then pool waits and
|
||||
// round trips inside it; per connection, whether one slow connection keeps serving it.
|
||||
function stickySlotFields(deltas: RelayMetricDeltas): Record<string, unknown> {
|
||||
if (deltas.stickySlotQueueWaitMs.length === 0) return {}
|
||||
const queueWait = latencySummary(deltas.stickySlotQueueWaitMs)
|
||||
const poolWait = latencySummary(deltas.stickySlotPoolWaitMs)
|
||||
const sql = latencySummary(deltas.stickySlotSqlMs)
|
||||
const statements = latencySummary(deltas.stickySlotStatements)
|
||||
const byConnection: Record<string, Record<string, number>> = {}
|
||||
for (const [connectionId, connection] of deltas.stickySlotByConnection) {
|
||||
const { p50, p99 } = latencySummary(connection.samplesMs)
|
||||
byConnection[String(connectionId)] = {
|
||||
requests: connection.requests,
|
||||
statements: connection.statements,
|
||||
statementMsP50: p50,
|
||||
statementMsP99: p99,
|
||||
statementMsMax: roundMs(connection.maxMs)
|
||||
}
|
||||
}
|
||||
return {
|
||||
stickySlotsDelta: deltas.stickySlotQueueWaitMs.length,
|
||||
stickyQueueWaitMsP50: queueWait.p50,
|
||||
stickyQueueWaitMsP99: queueWait.p99,
|
||||
stickyQueueWaitMsMax: queueWait.max,
|
||||
stickyPoolWaitMsP50: poolWait.p50,
|
||||
stickyPoolWaitMsP99: poolWait.p99,
|
||||
stickySqlMsP50: sql.p50,
|
||||
stickySqlMsP99: sql.p99,
|
||||
stickyStatementsP50: statements.p50,
|
||||
stickyStatementsMax: statements.max,
|
||||
stickySqlByConnection: byConnection,
|
||||
// One per request-connection pair that found the window's table full.
|
||||
stickySqlConnectionSamplesDroppedDelta: deltas.stickySlotConnectionSamplesDropped
|
||||
}
|
||||
}
|
||||
|
||||
function lockWaiters(byKey: Record<string, number>, waiterRole: string): number {
|
||||
return Object.entries(byKey)
|
||||
.filter(([key]) => key.startsWith(`${waiterRole}:relay_cells:`))
|
||||
|
||||
@@ -255,6 +255,7 @@ export function createRelayServer(
|
||||
epoch: join.epoch
|
||||
})
|
||||
}
|
||||
directory.onOutranked = (seat) => reservePlacement?.supersedeOutranked(seat)
|
||||
reserveLedger.start()
|
||||
const reconcile = (): void => {
|
||||
const now = (options.now ?? Date.now)()
|
||||
@@ -469,6 +470,7 @@ export function createRelayServer(
|
||||
recordDrainReturnRetryAfter: (seconds) => observability.recordDrainReturnRetryAfter?.(seconds),
|
||||
recordAdmissionServiceMs: (lane, durationMs) =>
|
||||
observability.recordAdmissionServiceMs?.(lane, durationMs),
|
||||
recordStickySlot: (sample) => observability.recordStickySlot?.(sample),
|
||||
recordAssignmentUnavailable: (cause) => observability.recordAssignmentUnavailable?.(cause),
|
||||
recordRegionRequest: (region) => observability.recordRegionRequest?.(region),
|
||||
shadowSeats: shadowSeatPoller?.directory,
|
||||
|
||||
@@ -52,6 +52,7 @@ function setup(
|
||||
databaseBusy?: () => boolean
|
||||
mode?: 'off' | 'dry-run' | 'on'
|
||||
startedAt?: number
|
||||
log?: (line: string) => void
|
||||
} = {}
|
||||
) {
|
||||
const now = { value: 100_000 }
|
||||
@@ -88,7 +89,7 @@ function setup(
|
||||
...(options.databaseBusy ? { databaseBusy: options.databaseBusy } : {}),
|
||||
now: () => now.value,
|
||||
random: () => 0.25,
|
||||
log: () => undefined
|
||||
log: options.log ?? (() => undefined)
|
||||
})
|
||||
// Starts each cell's refill clock, as a director's first look does.
|
||||
for (const cell of assignment.placementCells()) placer.hasBudget(cell)
|
||||
@@ -350,6 +351,17 @@ describe('reserve assignment on a director', () => {
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
// The booking map had not seen the c1 seat (half-open, joined within a poll lag): the feed
|
||||
// demotes it once both joins are in the map, instead of the 75 s silence close.
|
||||
it('tells an old reserve seat the booking missed to go once a newer join shows in the feed', () => {
|
||||
const { assignment, directory, demoted, now } = setup()
|
||||
directory.onOutranked = (seat) => assignment.supersedeOutranked(seat)
|
||||
directory.apply('c2', feed('c2', { full: [seated(6, 3_000)] }), now.value, now.value)
|
||||
expect(demoted).toEqual([])
|
||||
directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
|
||||
expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
|
||||
})
|
||||
|
||||
it('answers a reconnect only at the newest epoch, newest join first', async () => {
|
||||
const { assignment, reserved } = setup({
|
||||
feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
|
||||
@@ -370,4 +382,38 @@ describe('reserve assignment on a director', () => {
|
||||
expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
|
||||
expect(reserved).toMatchObject([{ dryRun: true }])
|
||||
})
|
||||
|
||||
it('in dry run counts outcomes per picked cell, and a `full` refusal while the map showed room', async () => {
|
||||
const summaries: Array<{ dryRun: Record<string, unknown> }> = []
|
||||
const dryRunOn = async (units: number, outcome: string) => {
|
||||
const { assignment, now } = setup({
|
||||
mode: 'dry-run',
|
||||
cells: cellList(['c1']),
|
||||
feeds: [feed('c1', { counts: { seats: units, controls: 0, bookings: 0, units, ceiling: 500 } })],
|
||||
reserve: () => ({ outcome }),
|
||||
log: (line) => summaries.push(JSON.parse(line))
|
||||
})
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const plan = await assignment.plan(HOST, { reconnect: false, region: US })
|
||||
if (plan.kind === 'database') await plan.placeFresh?.()
|
||||
}
|
||||
// Lets the fire-and-forget dry-run reserves settle before the summary flushes.
|
||||
await Promise.resolve()
|
||||
now.value += 60_000
|
||||
await assignment.plan(HOST, { reconnect: false, region: US })
|
||||
}
|
||||
// 1 free seat of 500 is under the 1% bound; 250 free is far over it.
|
||||
await dryRunOn(499, 'full')
|
||||
await dryRunOn(250, 'full')
|
||||
await dryRunOn(250, 'ok')
|
||||
expect(summaries.map((summary) => summary.dryRun)).toMatchObject([
|
||||
{
|
||||
outcomes: { full: 2 },
|
||||
count: 2,
|
||||
cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 0, minFullFreeShare: 0.002 } }
|
||||
},
|
||||
{ outcomes: { full: 2 }, cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 2, minFullFreeShare: 0.5 } } },
|
||||
{ outcomes: { ok: 2 }, cells: { c1: { outcomes: { ok: 2 }, fullWithRoom: 0, minFullFreeShare: null } } }
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
type PlacementCell,
|
||||
type ReservePlacer
|
||||
} from './reserve-placement.js'
|
||||
import type { SeatFeedCell, ShadowSeat, ShadowSeatDirectory } from './shadow-seat-directory.js'
|
||||
import type { OutrankedSeat, SeatFeedCell, ShadowSeat, ShadowSeatDirectory } from './shadow-seat-directory.js'
|
||||
|
||||
// Step 5 on a director: answer a desktop from the in-memory map for cells that report
|
||||
// admitMode=reserve, and fall back to today's database path for everything else. The cell
|
||||
@@ -24,6 +24,8 @@ export const RESERVE_STICKY_RECENT_MS = 10 * 60_000
|
||||
// A starting director waits this long for every live cell before it answers.
|
||||
export const RESERVE_STARTUP_GATE_MS = 30_000
|
||||
const SUMMARY_INTERVAL_MS = 60_000
|
||||
// A dry-run `full` while the map showed more free room than this share of the ceiling is `fullWithRoom`.
|
||||
export const RESERVE_DRY_RUN_ROOM_BOUND = 0.01
|
||||
// A leaver for these reasons was moved off its cell; it is not sent back there.
|
||||
const NO_REJOIN_CLOSE_CODES: ReadonlySet<number> = new Set([
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
@@ -342,13 +344,23 @@ export class ReserveAssignment {
|
||||
const cell = this.input.placer.pick(placements, region, new Set())
|
||||
const target = cell && this.input.cells().find((entry) => entry.cellId === cell.cellId)
|
||||
if (!target) return null
|
||||
// The map's free room at pick time; pick only returns a cell with a ceiling.
|
||||
const freeShare = cell.ceiling ? this.input.placer.free(cell) / cell.ceiling : 0
|
||||
const startedAt = performance.now()
|
||||
void this.input.client
|
||||
.reserve(target, { ...identity, epoch: 1, ttlMs: CELL_RESERVE_TTL_MS }, true)
|
||||
.then((answer) => {
|
||||
const elapsedMs = performance.now() - startedAt
|
||||
this.summary.dryRun.outcomes[answer.outcome] = (this.summary.dryRun.outcomes[answer.outcome] ?? 0) + 1
|
||||
this.summary.dryRun.latenciesMs.push(elapsedMs)
|
||||
const { dryRun } = this.summary
|
||||
dryRun.outcomes[answer.outcome] = (dryRun.outcomes[answer.outcome] ?? 0) + 1
|
||||
dryRun.latenciesMs.push(elapsedMs)
|
||||
const perCell = dryRun.cells[cell.cellId] ?? { outcomes: {}, fullWithRoom: 0, minFullFreeShare: null }
|
||||
dryRun.cells[cell.cellId] = perCell
|
||||
perCell.outcomes[answer.outcome] = (perCell.outcomes[answer.outcome] ?? 0) + 1
|
||||
if (answer.outcome === 'full') {
|
||||
if (freeShare > RESERVE_DRY_RUN_ROOM_BOUND) perCell.fullWithRoom += 1
|
||||
perCell.minFullFreeShare = Math.min(perCell.minFullFreeShare ?? 1, freeShare)
|
||||
}
|
||||
})
|
||||
.catch(() => undefined)
|
||||
return null
|
||||
@@ -377,10 +389,10 @@ export class ReserveAssignment {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Fire-and-forget; the cell checks the seat's epoch and join itself.
|
||||
demoteSeat(seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number }): void {
|
||||
// Fire-and-forget; the cell checks the seat's epoch and join itself. False: no such cell listed.
|
||||
demoteSeat(seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number }): boolean {
|
||||
const cell = this.input.cells().find((entry) => entry.cellId === seat.cellId)
|
||||
if (!cell) return
|
||||
if (!cell) return false
|
||||
void this.input.client.demote(cell, {
|
||||
v: 1,
|
||||
userId: seat.userId,
|
||||
@@ -388,6 +400,7 @@ export class ReserveAssignment {
|
||||
epoch: seat.epoch,
|
||||
joinedAt: seat.joinedAt
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
// Supersede at placement: the old seats this director's map holds on reserve cells are told
|
||||
@@ -396,32 +409,39 @@ export class ReserveAssignment {
|
||||
for (const seat of seats) {
|
||||
if (seat.cellId === cellId || seat.epoch >= epoch) continue
|
||||
if (this.input.directory.admitModeOf(seat.cellId) !== 'reserve') continue
|
||||
const cell = this.input.cells().find((entry) => entry.cellId === seat.cellId)
|
||||
if (!cell) continue
|
||||
this.summary.supersedes += 1
|
||||
void this.input.client.demote(cell, {
|
||||
v: 1,
|
||||
userId: identity.userId,
|
||||
relayHostId: identity.relayHostId,
|
||||
epoch: seat.epoch,
|
||||
joinedAt: seat.joinedAt
|
||||
})
|
||||
this.input.log?.(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_reserve_supersede',
|
||||
relayHostIdDigest: relayHostLogDigest(identity.relayHostId),
|
||||
fromCellId: seat.cellId,
|
||||
toCellId: cellId
|
||||
})
|
||||
)
|
||||
this.demoteSuperseded({ ...identity, ...seat }, cellId, 'placement')
|
||||
}
|
||||
}
|
||||
|
||||
// The feed's half of supersede: a seat the booking map did not hold, found once both joins
|
||||
// are in this map. Without it that seat holds its cell until the 75 s silence close.
|
||||
supersedeOutranked(seat: OutrankedSeat): void {
|
||||
this.demoteSuperseded(seat, seat.byCellId, 'feed')
|
||||
}
|
||||
|
||||
private demoteSuperseded(
|
||||
seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number },
|
||||
toCellId: string,
|
||||
via: 'placement' | 'feed'
|
||||
): void {
|
||||
if (!this.demoteSeat(seat)) return
|
||||
this.summary.supersedes += 1
|
||||
this.input.log?.(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_reserve_supersede',
|
||||
relayHostIdDigest: relayHostLogDigest(seat.relayHostId),
|
||||
fromCellId: seat.cellId,
|
||||
toCellId,
|
||||
via
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
private flushSummary(): void {
|
||||
const now = this.now()
|
||||
if (now - this.summaryAt < SUMMARY_INTERVAL_MS) return
|
||||
this.summaryAt = now
|
||||
const { latenciesMs, outcomes } = this.summary.dryRun
|
||||
const { latenciesMs, outcomes, cells } = this.summary.dryRun
|
||||
latenciesMs.sort((left, right) => left - right)
|
||||
const percentile = (share: number) =>
|
||||
latenciesMs.length === 0
|
||||
@@ -432,7 +452,7 @@ export class ReserveAssignment {
|
||||
event: 'orca_relay_reserve_placement_summary',
|
||||
mode: this.input.mode,
|
||||
...this.summary,
|
||||
dryRun: { outcomes, count: latenciesMs.length, p50Ms: percentile(0.5), p99Ms: percentile(0.99) }
|
||||
dryRun: { outcomes, count: latenciesMs.length, p50Ms: percentile(0.5), p99Ms: percentile(0.99), cells }
|
||||
})
|
||||
)
|
||||
this.summary = emptySummary()
|
||||
@@ -448,7 +468,19 @@ type PlacementSummary = {
|
||||
supersedes: number
|
||||
outcomes: Record<string, number>
|
||||
retries: { mapIncomplete: number; database: number; paced: number }
|
||||
dryRun: { outcomes: Record<string, number>; latenciesMs: number[] }
|
||||
dryRun: {
|
||||
outcomes: Record<string, number>
|
||||
latenciesMs: number[]
|
||||
// Keyed by the picked cell, so bounded by the fleet.
|
||||
cells: Record<string, DryRunCellSummary>
|
||||
}
|
||||
}
|
||||
|
||||
type DryRunCellSummary = {
|
||||
outcomes: Record<string, number>
|
||||
fullWithRoom: number
|
||||
// Lowest map free share (of the ceiling) among this cell's `full` refusals.
|
||||
minFullFreeShare: number | null
|
||||
}
|
||||
|
||||
function emptySummary(): PlacementSummary {
|
||||
@@ -461,6 +493,6 @@ function emptySummary(): PlacementSummary {
|
||||
supersedes: 0,
|
||||
outcomes: {},
|
||||
retries: { mapIncomplete: 0, database: 0, paced: 0 },
|
||||
dryRun: { outcomes: {}, latenciesMs: [] }
|
||||
dryRun: { outcomes: {}, latenciesMs: [], cells: Object.create(null) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,8 +42,9 @@ function expectInvariants(report: SimulationReport): void {
|
||||
}
|
||||
|
||||
describe('step 5 deterministic simulation', () => {
|
||||
// Invariant 9 (dead-man) never trips here; its own tests below cover it.
|
||||
it.each(SEEDS)(
|
||||
'holds invariants 1-3 and 5-8 across mixed versions, lagging polls, restarts and database stalls (seed %i)',
|
||||
'holds invariants 1-8 across mixed versions, lagging polls, restarts and database stalls (seed %i)',
|
||||
async (seed) => {
|
||||
const report = await runReservePlacementSimulation({
|
||||
seed,
|
||||
@@ -174,6 +175,48 @@ describe('step 5 deterministic simulation', () => {
|
||||
120_000
|
||||
)
|
||||
|
||||
// A desktop seated at the same epoch on two cells (a seat half-open within a poll lag, then
|
||||
// placed again by a director that had not seen it): only the row breaks the tie. Short
|
||||
// sessions and all five directors restarting each minute put boot reconciles on those seats.
|
||||
const levelSeats = (seed: number, faults: SimulationConfig['faults']): SimulationConfig => ({
|
||||
seed,
|
||||
durationMs: 16 * 60_000,
|
||||
directors: 5,
|
||||
cells: Array.from({ length: 12 }, (_, index) => ({
|
||||
cellId: `us${String(index).padStart(2, '0')}`,
|
||||
region: 'us-central1' as const,
|
||||
hardCap: 600,
|
||||
ceiling: 480,
|
||||
intakePerSec: 20,
|
||||
mode: 'reserve' as const,
|
||||
preseated: 100
|
||||
})),
|
||||
hosts: 3_000,
|
||||
hostRegion: () => 'us-central1',
|
||||
arrivalPerSec: 20,
|
||||
meanSessionMs: 20_000,
|
||||
reconnectDelayMs: [500, 2_000],
|
||||
duplicateAssignShare: 0.05,
|
||||
halfOpenShare: 0.5,
|
||||
pollLagMs: (director) => [0, 200, 1_000, 2_000, 3_000][director]!,
|
||||
rttMs: rtt,
|
||||
directorRestarts: Array.from({ length: 75 }, (_, index) => ({
|
||||
at: 60_000 + Math.floor(index / 5) * 60_000,
|
||||
director: index % 5
|
||||
})),
|
||||
faults
|
||||
})
|
||||
|
||||
it.each(SEEDS)(
|
||||
'holds invariants 1-8 when directors contest level divergences (seed %i)',
|
||||
async (seed) => {
|
||||
const report = await runReservePlacementSimulation(levelSeats(seed, {}))
|
||||
expect(report.contestedDivergences).toBeGreaterThan(0)
|
||||
expectInvariants(report)
|
||||
},
|
||||
120_000
|
||||
)
|
||||
|
||||
// Each fault breaks one rule; the matching invariant must fire, or its check measures nothing.
|
||||
const mixed = (faults: SimulationConfig['faults']): SimulationConfig => ({
|
||||
seed: 11,
|
||||
@@ -251,6 +294,9 @@ describe('step 5 deterministic simulation', () => {
|
||||
[1, nearFull({ bookPastCap: true })],
|
||||
[2, drain({ intakeBurst: 200, directorShare: 0.05 })],
|
||||
[3, drainedWithoutSupersede()],
|
||||
// Half-open seats the booking map had not seen hold their cell until the silence close.
|
||||
[3, levelSeats(11, { skipFeedSupersede: true })],
|
||||
[4, levelSeats(11, { localTieBreak: true })],
|
||||
[5, mixed({ unguardedLedger: true })],
|
||||
// A director restart mid-queue loses its booked joins' rows until its boot reconcile.
|
||||
[5, restartedMidQueue()],
|
||||
@@ -271,13 +317,6 @@ describe('step 5 deterministic simulation', () => {
|
||||
// A director whose map has not seen the leave yet may re-place level with the row; the
|
||||
// WRONG_CELL row read keeps that a small share (the rest are repaired by one more reconcile).
|
||||
expect(report.badFirstReseats / report.rowAheadDemotions).toBeLessThan(0.1)
|
||||
const withoutRead = await runReservePlacementSimulation({ ...config, faults: { noWrongCellRowRead: true } })
|
||||
process.stderr.write(
|
||||
`row_ahead_repair ${JSON.stringify({
|
||||
withRead: { demotions: report.rowAheadDemotions, badFirstReseats: report.badFirstReseats, perHost: report.rowAheadDemotionsPerHost },
|
||||
withoutRead: { demotions: withoutRead.rowAheadDemotions, badFirstReseats: withoutRead.badFirstReseats, perHost: withoutRead.rowAheadDemotionsPerHost }
|
||||
})}\n`
|
||||
)
|
||||
}, 240_000)
|
||||
|
||||
// Every director misses its feed polls for 150 s: each reserve cell's dead-man trips (60-120 s)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import { RELAY_PROTOCOL_LIMITS, type RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import { CellReserveBook } from './cell-reserve-book.js'
|
||||
import { CELL_INTAKE_BURST, CELL_RESERVE_TTL_MS, type ReserveOutcome } from './cell-reserve-contract.js'
|
||||
import {
|
||||
@@ -9,10 +9,11 @@ import {
|
||||
ReservePlacer,
|
||||
type PlacementCell
|
||||
} from './reserve-placement.js'
|
||||
import { seatOutranks } from './shadow-seat-directory.js'
|
||||
|
||||
// A seeded discrete-event model of step 5 that drives the real CellReserveBook and
|
||||
// ReservePlacer. Everything else (sockets, the database, desktops) is modelled. The checks
|
||||
// are the design's invariants 1-8; a violation is recorded, never thrown, so one run reports
|
||||
// are the design's invariants 1-9; a violation is recorded, never thrown, so one run reports
|
||||
// every class it hit.
|
||||
|
||||
export function seededRandom(seed: number): () => number {
|
||||
@@ -98,8 +99,8 @@ const CONTROL_RENEWAL_MS = 25_000
|
||||
const RECONCILE_SETTLE_MS = 30_000
|
||||
// A drained seat lingers this long (its splices finishing) unless superseded first.
|
||||
const DRAIN_GRACE_MS = 30_000
|
||||
// A cell closes a control silent for 75 s, checked once per 25 s ping.
|
||||
const CONTROL_SILENCE_CLOSE_MS = 100_000
|
||||
// A cell closes a control silent for 75 s, checked once per ping (host-session-registry.ts heartbeat).
|
||||
const CONTROL_SILENCE_CLOSE_MS = 75_000 + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
// Longer than the slowest duplicate: a half-open seat's silence close.
|
||||
const QUIESCE_MS = 150_000
|
||||
|
||||
@@ -111,8 +112,10 @@ export type SimulationFaults = {
|
||||
intakeBurst?: number
|
||||
// Each director assumes this share of every cell's intake, instead of a fifth.
|
||||
directorShare?: number
|
||||
// Placement does not supersede the old seat: a duplicate then outlives its bound.
|
||||
// Nothing supersedes the old seat, at placement or from the feed: a duplicate outlives its bound.
|
||||
skipSupersede?: boolean
|
||||
// Only placement supersedes: a half-open seat the booking map had not seen lasts to its silence close.
|
||||
skipFeedSupersede?: boolean
|
||||
// A restarted director forgets its ledger queue and never reconciles.
|
||||
noBootReconcile?: boolean
|
||||
// The reconcile leaves a seat whose row is ahead of it alone.
|
||||
@@ -127,6 +130,8 @@ export type SimulationFaults = {
|
||||
noRawFeedMode?: boolean
|
||||
unguardedLedger?: boolean
|
||||
noEpochFloor?: boolean
|
||||
// On a level divergence each director keeps the seat its own cell order prefers, not the row's.
|
||||
localTieBreak?: boolean
|
||||
}
|
||||
|
||||
// relay_cell_admit_effective: each cell writes its raw mode every 15 s (cell-admit-effective-writer.ts),
|
||||
@@ -423,6 +428,11 @@ export type SimulationReport = {
|
||||
supersedes: number
|
||||
directorRestarts: number
|
||||
rowAheadDemotions: number
|
||||
// Demotions whose winner is level with the loser (same epoch, another cell): the tie-break's cases.
|
||||
levelDemotions: number
|
||||
// Level divergences (same epoch, two cells) two or more directors judged: the only cases where
|
||||
// invariant 4 can break, since a higher epoch wins for every director.
|
||||
contestedDivergences: number
|
||||
// Row-ahead demotions per demoted host: a repair that converges needs one.
|
||||
rowAheadDemotionsPerHost: { max: number; p99: number }
|
||||
deadManTrips: number
|
||||
@@ -503,8 +513,12 @@ export async function runReservePlacementSimulation(
|
||||
const random = seededRandom(config.seed)
|
||||
const clock = new Clock()
|
||||
const violations: SimulationViolation[] = []
|
||||
// Capped per invariant, so a flood of one class cannot hide another.
|
||||
const violationCounts = new Map<number, number>()
|
||||
const violate = (invariant: number, detail: string): void => {
|
||||
if (violations.length < 50) violations.push({ invariant, at: clock.now, detail })
|
||||
const count = violationCounts.get(invariant) ?? 0
|
||||
violationCounts.set(invariant, count + 1)
|
||||
if (count < 50) violations.push({ invariant, at: clock.now, detail })
|
||||
}
|
||||
const faults = config.faults ?? {}
|
||||
const database = new SimDatabase(violate, faults)
|
||||
@@ -550,6 +564,8 @@ export async function runReservePlacementSimulation(
|
||||
supersedes: 0,
|
||||
directorRestarts: 0,
|
||||
rowAheadDemotions: 0,
|
||||
levelDemotions: 0,
|
||||
contestedDivergences: 0,
|
||||
rowAheadDemotionsPerHost: { max: 0, p99: 0 },
|
||||
deadManTrips: 0,
|
||||
reserveCallsOnTrippedCells: 0,
|
||||
@@ -653,10 +669,15 @@ export async function runReservePlacementSimulation(
|
||||
placement: placementCell(cell, snapshot, requestedAt)
|
||||
}
|
||||
director.views.set(cell.cellId, current)
|
||||
supersedeOutranked(director, seats!.keys())
|
||||
} else if (current && current.incarnation === snapshot.incarnation) {
|
||||
for (const change of changes) applyChange(director, cell, current, change)
|
||||
current.cursor = snapshot.cursor
|
||||
current.placement = placementCell(cell, snapshot, requestedAt)
|
||||
supersedeOutranked(
|
||||
director,
|
||||
changes.filter((change) => change.kind === 'join').map((change) => change.host)
|
||||
)
|
||||
} else {
|
||||
// A restarted director's (or a stale) delta: the next poll asks for a full snapshot.
|
||||
return
|
||||
@@ -729,19 +750,95 @@ export async function runReservePlacementSimulation(
|
||||
// Supersede at placement: the director that books a host on `cellId` tells the old seats its
|
||||
// map holds on reserve cells to go. Each cell checks epoch and join itself.
|
||||
function supersede(director: SimDirector, host: string, cellId: string, epoch: number): void {
|
||||
const held = new Set<string>()
|
||||
for (const [seatCellId, view] of director.views) {
|
||||
const seat = view.seats.get(host)
|
||||
if (seat) held.add(`${seatCellId}\u0000${seat.joinedAt}`)
|
||||
}
|
||||
heldAtBooking.set(`${host}\u0000${cellId}\u0000${epoch}`, held)
|
||||
if (faults.skipSupersede) return
|
||||
for (const [seatCellId, view] of director.views) {
|
||||
const seat = view.seats.get(host)
|
||||
if (!seat || seat.demoted || seatCellId === cellId || seat.epoch >= epoch) continue
|
||||
if (!view.placement.reserve) continue
|
||||
const loser = cellById.get(seatCellId)!
|
||||
report.supersedes += 1
|
||||
clock.schedule(config.rttMs(loser.region), () =>
|
||||
loser.demote(host, seat.epoch, seat.joinedAt, () => onSeatClosed(host, seatCellId, seat.epoch))
|
||||
)
|
||||
demoteFor(director, host, seatCellId, seat, { cellId, epoch })
|
||||
}
|
||||
}
|
||||
|
||||
// As shadow-seat-directory.ts onOutranked: once both joins are in this map, a reserve seat behind
|
||||
// a later join at a newer epoch is told to go. A lower epoch that joined later is the row's.
|
||||
function supersedeOutranked(director: SimDirector, hosts: Iterable<string>): void {
|
||||
if (faults.skipSupersede || faults.skipFeedSupersede) return
|
||||
for (const host of new Set(hosts)) {
|
||||
const seats = [...director.views].flatMap(([cellId, view]) => {
|
||||
const seat = view.seats.get(host)
|
||||
return seat ? [{ cellId, view, seat }] : []
|
||||
})
|
||||
if (seats.length < 2) continue
|
||||
for (const { cellId, view, seat } of seats) {
|
||||
if (seat.demoted || !view.placement.reserve) continue
|
||||
const by = seats.find((other) => seatOutranks(other.seat, seat))
|
||||
if (!by) continue
|
||||
report.supersedes += 1
|
||||
demoteFor(director, host, cellId, seat, { cellId: by.cellId, epoch: by.seat.epoch })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The seats the booking director's map held when it placed each booking: only those it supersedes.
|
||||
const heldAtBooking = new Map<string, Set<string>>()
|
||||
|
||||
// Invariant 4: each demotion names its winner (the newer booking, or the row). Every director
|
||||
// names the same one per divergence, so no two seats of a host are each demoted for the other.
|
||||
const namedWinners = new Map<string, { won: string; by: string }>()
|
||||
const contested = new Set<string>()
|
||||
const demotedFor = new Map<string, string>()
|
||||
type JudgedSeat = { cellId: string; epoch: number }
|
||||
function judge(director: SimDirector, host: string, loserSeat: JudgedSeat, wonSeat: JudgedSeat): void {
|
||||
const loser = `${loserSeat.cellId}@${loserSeat.epoch}`
|
||||
const won = `${wonSeat.cellId}@${wonSeat.epoch}`
|
||||
const divergence = `${host}\u0000${[loser, won].sort().join('\u0000')}`
|
||||
const named = namedWinners.get(divergence)
|
||||
if (named === undefined) {
|
||||
namedWinners.set(divergence, { won, by: director.id })
|
||||
return
|
||||
}
|
||||
// A second director judged it, agreeing or not: the tie-break was exercised.
|
||||
if (named.by !== director.id && loserSeat.epoch === wonSeat.epoch && !contested.has(divergence)) {
|
||||
contested.add(divergence)
|
||||
report.contestedDivergences += 1
|
||||
}
|
||||
if (named.won !== won) {
|
||||
violate(4, `${director.id} names ${won} over ${loser} for ${host}; ${named.by} named ${named.won}`)
|
||||
}
|
||||
}
|
||||
function demoteFor(
|
||||
director: SimDirector,
|
||||
host: string,
|
||||
cellId: string,
|
||||
seat: { epoch: number; joinedAt: number },
|
||||
winner: { cellId: string; epoch: number }
|
||||
): void {
|
||||
const loser = `${cellId}@${seat.epoch}`
|
||||
const won = `${winner.cellId}@${winner.epoch}`
|
||||
judge(director, host, { cellId, epoch: seat.epoch }, winner)
|
||||
const cell = cellById.get(cellId)!
|
||||
clock.schedule(config.rttMs(cell.region), () => {
|
||||
const live = cell.seats.get(host)
|
||||
const wasLive = live !== undefined && !live.demoted
|
||||
cell.demote(host, seat.epoch, seat.joinedAt, () => onSeatClosed(host, cellId, seat.epoch))
|
||||
if (!wasLive || !live.demoted) return
|
||||
// Only a live seat demoted for a live level winner is a same-epoch duplicate resolved.
|
||||
const winnerSeat = cellById.get(winner.cellId)?.seats.get(host)
|
||||
if (winner.epoch === seat.epoch && winnerSeat?.epoch === winner.epoch && !winnerSeat.demoted) {
|
||||
report.levelDemotions += 1
|
||||
}
|
||||
demotedFor.set(`${host}\u0000${loser}`, won)
|
||||
if (demotedFor.get(`${host}\u0000${won}`) === loser) violate(4, `${host}: ${loser} and ${won} each demoted for the other`)
|
||||
})
|
||||
}
|
||||
|
||||
function onSeatClosed(hostId: string, cellId: string, epoch: number): void {
|
||||
const host = hostById.get(hostId)!
|
||||
if (host.current?.cellId === cellId && host.current.epoch === epoch) {
|
||||
@@ -766,6 +863,18 @@ export async function runReservePlacementSimulation(
|
||||
const demotedBehindRow = new Map<string, number>()
|
||||
// A host demoted behind its row: its next answer, from any director, is that row or above it.
|
||||
const reassignAbove = new Map<string, { cellId: string; epoch: number }>()
|
||||
// Only under localTieBreak: even directors keep the lower cell id of a level divergence, odd ones the higher.
|
||||
function faultKeepsThisSeat(
|
||||
director: SimDirector,
|
||||
cellId: string,
|
||||
epoch: number,
|
||||
row: { cellId: string; epoch: number }
|
||||
): boolean {
|
||||
return !!faults.localTieBreak && row.epoch === epoch && (cellId < row.cellId) === (director.index % 2 === 0)
|
||||
}
|
||||
function raiseRowFloor(director: SimDirector, host: string, epoch: number): void {
|
||||
director.rowFloors.set(host, Math.max(epoch, director.rowFloors.get(host) ?? 0))
|
||||
}
|
||||
function reconcileLedger(director: SimDirector): void {
|
||||
if (!database.up || director.old) return
|
||||
for (const [cellId, view] of director.views) {
|
||||
@@ -775,13 +884,32 @@ export async function runReservePlacementSimulation(
|
||||
const row = database.rows.get(host)
|
||||
const live = cellById.get(cellId)!.seats.get(host)
|
||||
const diverged = row && (row.epoch > seat.epoch || (row.epoch === seat.epoch && row.cellId !== cellId))
|
||||
// A map that has not yet seen another director's demotion still judges the seat; the
|
||||
// cell drops the demote, but the winner it names must be the same.
|
||||
// relay-server.ts raises the floor on every demote, so a dropped one raises it too.
|
||||
if (row && diverged && live && live.joinedAt === seat.joinedAt && live.demoted && !faults.noRowAheadDemote) {
|
||||
const thisSeat = { cellId, epoch: seat.epoch }
|
||||
const rowSeat = { cellId: row.cellId, epoch: row.epoch }
|
||||
if (faultKeepsThisSeat(director, cellId, seat.epoch, row)) judge(director, host, rowSeat, thisSeat)
|
||||
else {
|
||||
judge(director, host, thisSeat, rowSeat)
|
||||
raiseRowFloor(director, host, row.epoch)
|
||||
}
|
||||
}
|
||||
if (row && diverged && live && live.joinedAt === seat.joinedAt && !live.demoted) {
|
||||
const key = `${director.id}\u0000${host}\u0000${cellId}\u0000${seat.joinedAt}`
|
||||
if (behindRow.has(key)) violate(8, `${host}@${seat.epoch} on ${cellId} still behind row @${row.epoch}`)
|
||||
behindRow.set(key, clock.now)
|
||||
if (faults.noRowAheadDemote) continue
|
||||
if (faultKeepsThisSeat(director, cellId, seat.epoch, row)) {
|
||||
const rowSeat = director.views.get(row.cellId)?.seats.get(host)
|
||||
if (rowSeat && rowSeat.epoch === row.epoch && !rowSeat.demoted) {
|
||||
demoteFor(director, host, row.cellId, rowSeat, { cellId, epoch: seat.epoch })
|
||||
}
|
||||
continue
|
||||
}
|
||||
report.rowAheadDemotions += 1
|
||||
director.rowFloors.set(host, Math.max(row.epoch, director.rowFloors.get(host) ?? 0))
|
||||
raiseRowFloor(director, host, row.epoch)
|
||||
rowAheadDemotionsByHost.set(host, (rowAheadDemotionsByHost.get(host) ?? 0) + 1)
|
||||
// A director that had not seen the leave may re-place level with the row once; a later
|
||||
// divergence (a newer row) may demote the host again. One row never takes a third.
|
||||
@@ -790,10 +918,7 @@ export async function runReservePlacementSimulation(
|
||||
demotedBehindRow.set(sameRow, behindThisRow)
|
||||
if (behindThisRow > 2) violate(8, `${host} demoted ${behindThisRow} times behind its row ${row.cellId}@${row.epoch}`)
|
||||
reassignAbove.set(host, { ...row })
|
||||
const cell = cellById.get(cellId)!
|
||||
clock.schedule(config.rttMs(cell.region), () =>
|
||||
cell.demote(host, seat.epoch, seat.joinedAt, () => onSeatClosed(host, cellId, seat.epoch))
|
||||
)
|
||||
demoteFor(director, host, cellId, seat, row)
|
||||
continue
|
||||
}
|
||||
database.upsert(host, cellId, seat.epoch)
|
||||
@@ -1161,8 +1286,9 @@ export async function runReservePlacementSimulation(
|
||||
report.maxOccupancyShare[cell.cellId] = Math.max(report.maxOccupancyShare[cell.cellId] ?? 0, share)
|
||||
}
|
||||
}
|
||||
// A booking supersedes the old seat within a round trip; a duplicate nobody books past (a
|
||||
// half-open seat while today's path places the host) lasts until that seat's silence close.
|
||||
// A booking supersedes the old seats its director's map held within a round trip, and the feed
|
||||
// any seat a later join outranks. Any other duplicate (a half-open seat today's path placed past
|
||||
// at a lower epoch than it) has no lease to re-check and lasts until that seat's silence close.
|
||||
const supersedeBound =
|
||||
Math.max(...directors.map((director) => config.pollLagMs(director.index))) +
|
||||
2 * Math.max(...cells.map((cell) => config.rttMs(cell.region))) +
|
||||
@@ -1186,19 +1312,29 @@ export async function runReservePlacementSimulation(
|
||||
for (const [host, seats] of seatsByHost) {
|
||||
if (seats.length < 2) continue
|
||||
const newest = Math.max(...seats.map((entry) => entry.seat.epoch))
|
||||
// A seat outranked by a booking is superseded within the booking director's poll lag and
|
||||
// a round trip; anything else (today's path placed the host) waits for the silence close.
|
||||
const booked = seats.some((entry) => entry.seat.epoch === newest && entry.seat.reservedBy !== undefined)
|
||||
// By the newest seats' placement records, whatever their join says: no record, no supersede.
|
||||
const bookings = seats
|
||||
.filter((entry) => entry.seat.epoch === newest)
|
||||
.flatMap((entry) => heldAtBooking.get(`${host}\u0000${entry.cell.cellId}\u0000${newest}`) ?? [])
|
||||
const supersededBy = (cellId: string, joinedAt: number): boolean =>
|
||||
bookings.some((held) => held.has(`${cellId}\u0000${joinedAt}`))
|
||||
for (const { cell, seat } of seats) {
|
||||
if (cell.mode !== 'reserve' || seat.epoch === newest) continue
|
||||
// A leased seat goes by its renewal, as today.
|
||||
const bound = booked && !seat.leased ? supersedeBound : duplicateBound
|
||||
const tight =
|
||||
!seat.leased &&
|
||||
(supersededBy(cell.cellId, seat.joinedAt) || seats.some((other) => seatOutranks(other.seat, seat)))
|
||||
// The loose clock runs from the first outranking; the tight one from when supersede applies.
|
||||
const key = `${host}\u0000${cell.cellId}\u0000${seat.joinedAt}`
|
||||
live.add(key)
|
||||
const since = outrankedSince.get(key) ?? clock.now
|
||||
outrankedSince.set(key, since)
|
||||
if (clock.now - since > bound) {
|
||||
violate(3, `${host}@${seat.epoch} on ${cell.cellId} outranked for ${clock.now - since} ms`)
|
||||
const clocks: Array<[string, number]> = [[key, duplicateBound]]
|
||||
if (tight) clocks.push([`${key}\u0000tight`, supersedeBound])
|
||||
for (const [clockKey, bound] of clocks) {
|
||||
live.add(clockKey)
|
||||
const since = outrankedSince.get(clockKey) ?? clock.now
|
||||
outrankedSince.set(clockKey, since)
|
||||
if (clock.now - since > bound) {
|
||||
violate(3, `${host}@${seat.epoch} on ${cell.cellId} outranked for ${clock.now - since} ms`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,13 @@ function warmPlacer(now: { value: number }, random = () => 0.3) {
|
||||
}
|
||||
|
||||
describe('reserve placer', () => {
|
||||
it('reports free seats under the ceiling, never below zero or without a ceiling', () => {
|
||||
const placer = warmPlacer({ value: 0 })
|
||||
expect(placer.free(cell('c1', { seats: 60, bookings: 10 }))).toBe(30)
|
||||
expect(placer.free(cell('c1', { seats: 120 }))).toBe(0)
|
||||
expect(placer.free(cell('c1', { ceiling: undefined }))).toBe(0)
|
||||
})
|
||||
|
||||
it('places on the first cell that answers ok and counts one call', async () => {
|
||||
const now = { value: 0 }
|
||||
const placer = warmPlacer(now)
|
||||
|
||||
@@ -68,6 +68,11 @@ export class ReservePlacer {
|
||||
return cell.seats + cell.bookings + this.ownBookingsSince(cell)
|
||||
}
|
||||
|
||||
// Estimated free seats under the ceiling; 0 when the cell reports none.
|
||||
free(cell: PlacementCell): number {
|
||||
return cell.ceiling === undefined ? 0 : Math.max(0, cell.ceiling - this.load(cell))
|
||||
}
|
||||
|
||||
hasSeat(cell: PlacementCell): boolean {
|
||||
return cell.ceiling !== undefined && this.load(cell) < cell.ceiling
|
||||
}
|
||||
@@ -158,7 +163,7 @@ export class ReservePlacer {
|
||||
if (cell.region !== region || !cell.general || cell.draining) continue
|
||||
if (cell.reserve) {
|
||||
if (this.isCandidate(cell, region, now)) {
|
||||
reserveFree += Math.max(0, cell.ceiling! - this.load(cell))
|
||||
reserveFree += this.free(cell)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
classifyShadowSeat,
|
||||
SHADOW_COMPARE_FLUSH_MS,
|
||||
SHADOW_COMPARE_SAMPLES_PER_CLASS,
|
||||
SHADOW_COMPARE_SUPERSEDED_CLOSE_MS,
|
||||
ShadowDirectoryCompare
|
||||
} from './shadow-directory-compare.js'
|
||||
import { ShadowSeatDirectory, type SeatFeedResponse } from './shadow-seat-directory.js'
|
||||
@@ -27,8 +28,8 @@ function feed(cellId: string, overrides: Partial<SeatFeedResponse> = {}): SeatFe
|
||||
return { v: 1, cellId, incarnation: `inc-${cellId}`, seq: 1, at: 1, full: [], ...overrides }
|
||||
}
|
||||
|
||||
function seat(epoch: number, state = 'active', relayHostId = HOST) {
|
||||
return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt: 1 }
|
||||
function seat(epoch: number, state = 'active', relayHostId = HOST, joinedAt = 1) {
|
||||
return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt }
|
||||
}
|
||||
|
||||
function directoryWith(
|
||||
@@ -84,7 +85,7 @@ describe('classifyShadowSeat', () => {
|
||||
true
|
||||
],
|
||||
[
|
||||
directoryWith({ 'cell-a': [seat(3)], 'cell-b': [seat(2)] }),
|
||||
directoryWith({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] }),
|
||||
at('cell-a', 3),
|
||||
'duplicate-seat',
|
||||
false
|
||||
@@ -191,6 +192,178 @@ describe('classifyShadowSeat', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('a duplicate seat left by a superseded half-open control', () => {
|
||||
// Seated on cell-b at epoch 2 long ago; the director first saw epoch 3 on cell-a at firstSeen.
|
||||
const firstSeen = 1_000_000
|
||||
const directoryAt = (seats: Record<string, ReturnType<typeof seat>[]>) => {
|
||||
const directory = directoryWith(seats)
|
||||
directory.observeDatabaseEpoch('user-a', HOST, 3, firstSeen)
|
||||
return directory
|
||||
}
|
||||
const superseded = (joinedAt = firstSeen) =>
|
||||
directoryAt({
|
||||
'cell-a': [seat(3, 'active', HOST, joinedAt)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
|
||||
it('is explained while the old control may still await its silence close', () => {
|
||||
for (const now of [firstSeen + 1_700, firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS]) {
|
||||
expect(classifyShadowSeat(superseded(), IDENTITY, at('cell-a', 3), now)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: true
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
it('is unexplained just past the close bound', () => {
|
||||
const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
|
||||
expect(classifyShadowSeat(superseded(), IDENTITY, at('cell-a', 3), now)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('stays unexplained past the bound after a same-epoch rejoin resets joinedAt', () => {
|
||||
const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 30_000
|
||||
expect(classifyShadowSeat(superseded(now - 1_000), IDENTITY, at('cell-a', 3), now)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
const unseen = () =>
|
||||
directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
|
||||
it('is its own unexplained class with no sighting in the instance\'s first 100 s', () => {
|
||||
const now = firstSeen + 1_700
|
||||
const startedAt = now - SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
|
||||
expect(classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now, startedAt)).toEqual({
|
||||
class: 'superseded-unknown-age',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('is a plain unexplained duplicate with no sighting after the first 100 s', () => {
|
||||
const now = firstSeen + 1_700
|
||||
const startedAt = now - SHADOW_COMPARE_SUPERSEDED_CLOSE_MS - 1
|
||||
for (const classified of [
|
||||
classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now, startedAt),
|
||||
classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now)
|
||||
]) {
|
||||
expect(classified).toEqual({ class: 'duplicate-seat', explained: false })
|
||||
}
|
||||
})
|
||||
|
||||
it('stays a plain duplicate with no sighting when the current seat joined long ago', () => {
|
||||
const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), now)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('classifies before recording, so a later epoch bump is explained', () => {
|
||||
// Seated on cell-a at epoch 2 and on cell-b at epoch 1; the database names cell-a at 2.
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(2, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(1, 'active', HOST, 1)]
|
||||
})
|
||||
let now = firstSeen
|
||||
const compare = new ShadowDirectoryCompare(directory, () => now, () => {})
|
||||
expect(compare.compare('sticky-verify', IDENTITY, at('cell-a', 2))).toEqual({
|
||||
class: 'superseded-unknown-age',
|
||||
explained: false
|
||||
})
|
||||
// A re-assign to cell-b at epoch 3 supersedes cell-a's epoch-2 seat.
|
||||
now = firstSeen + 1_700
|
||||
directory.apply(
|
||||
'cell-b',
|
||||
feed('cell-b', { seq: 2, full: [seat(3, 'active', HOST, now)] }),
|
||||
now
|
||||
)
|
||||
expect(compare.compare('sticky-verify', IDENTITY, at('cell-b', 3))).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: true
|
||||
})
|
||||
})
|
||||
|
||||
it('is explained when the director last saw the host at a lower epoch', () => {
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
directory.observeDatabaseEpoch('user-a', HOST, 2, 1)
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: true
|
||||
})
|
||||
})
|
||||
|
||||
it('is unexplained after a lower-epoch sighting when the current seat joined long ago', () => {
|
||||
const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
directory.observeDatabaseEpoch('user-a', HOST, 2, 1)
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), now)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('is unexplained when the database answer regresses below the recorded epoch', () => {
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
directory.observeDatabaseEpoch('user-a', HOST, 4, firstSeen)
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('is unexplained when the database names a newer epoch than the seat on its cell', () => {
|
||||
const directory = directoryWith({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(2, 'active', HOST, 1)]
|
||||
})
|
||||
directory.observeDatabaseEpoch('user-a', HOST, 4, firstSeen)
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 4), firstSeen + 1_700)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
|
||||
it('is unexplained when the database names the lower epoch, or matches no seat', () => {
|
||||
for (const answer of [at('cell-b', 2), at('cell-a', 4), at('cell-b', 3)]) {
|
||||
expect(classifyShadowSeat(superseded(), IDENTITY, answer, firstSeen + 1_700)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
it('is unexplained on a level epoch, which only the row can break', () => {
|
||||
const directory = directoryAt({
|
||||
'cell-a': [seat(3, 'active', HOST, firstSeen)],
|
||||
'cell-b': [seat(3, 'active', HOST, 1)]
|
||||
})
|
||||
expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
|
||||
class: 'duplicate-seat',
|
||||
explained: false
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('answers map-incomplete until every listed cell has answered', () => {
|
||||
const directory = new ShadowSeatDirectory()
|
||||
directory.setCells(['cell-a', 'cell-b'])
|
||||
@@ -225,7 +398,8 @@ describe('ShadowDirectoryCompare', () => {
|
||||
class: 'map-only',
|
||||
explained: false,
|
||||
count: SHADOW_COMPARE_SAMPLES_PER_CLASS + 5,
|
||||
windowMs: SHADOW_COMPARE_FLUSH_MS
|
||||
windowMs: SHADOW_COMPARE_FLUSH_MS,
|
||||
countByCell: { 'cell-a': SHADOW_COMPARE_SAMPLES_PER_CLASS + 5 }
|
||||
},
|
||||
{
|
||||
event: 'orca_relay_shadow_compare',
|
||||
@@ -233,10 +407,55 @@ describe('ShadowDirectoryCompare', () => {
|
||||
class: 'agree',
|
||||
explained: true,
|
||||
count: 2,
|
||||
windowMs: SHADOW_COMPARE_FLUSH_MS
|
||||
windowMs: SHADOW_COMPARE_FLUSH_MS,
|
||||
countByCell: { 'cell-a': 2 }
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('counts a compare once on every cell it involves, and on none for a host neither side has', () => {
|
||||
const lines: Array<Record<string, unknown>> = []
|
||||
const compareOn = (seats: Record<string, ReturnType<typeof seat>[]>) =>
|
||||
new ShadowDirectoryCompare(directoryWith(seats), () => 0, (line) => lines.push(JSON.parse(line)))
|
||||
const unseen = compareOn({ 'cell-a': [], 'cell-b': [] })
|
||||
unseen.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
unseen.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
unseen.compare('sticky-verify', IDENTITY, assignment('cell-b'))
|
||||
unseen.compare('resolve', IDENTITY, null)
|
||||
unseen.flush(0)
|
||||
// The database says cell-a at epoch 3; the map seats the host on cell-b at a newer epoch.
|
||||
const mismatch = compareOn({ 'cell-a': [], 'cell-b': [seat(4)] })
|
||||
mismatch.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
mismatch.flush(0)
|
||||
// A level-epoch duplicate counts on the database's own cell once, and on the other seat cell.
|
||||
const duplicate = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] })
|
||||
duplicate.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
duplicate.flush(0)
|
||||
// A drain-only second seat is explained, so it counts only on the database's cell.
|
||||
const draining = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2, 'drain-only')] })
|
||||
draining.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
draining.flush(0)
|
||||
// A superseded older seat: the first-ever sighting has no age, so it is its own unexplained
|
||||
// class on both cells; seen before, it is explained and counts only on the database's cell.
|
||||
const superseded = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2)] })
|
||||
superseded.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
superseded.compare('sticky-verify', IDENTITY, assignment('cell-a'))
|
||||
superseded.flush(0)
|
||||
const mapOnly = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] })
|
||||
mapOnly.compare('resolve', IDENTITY, null)
|
||||
mapOnly.flush(0)
|
||||
const counts = lines.filter((line) => line.event === 'orca_relay_shadow_compare')
|
||||
expect(counts.map((line) => [line.class, line.explained, line.count, line.countByCell])).toEqual([
|
||||
['db-only-unseen', true, 3, { 'cell-a': 2, 'cell-b': 1 }],
|
||||
['agree-absent', true, 1, {}],
|
||||
['cell-mismatch', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
|
||||
['duplicate-seat', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
|
||||
['duplicate-seat', true, 1, { 'cell-a': 1 }],
|
||||
['superseded-unknown-age', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
|
||||
['duplicate-seat', true, 1, { 'cell-a': 1 }],
|
||||
['map-only', false, 1, { 'cell-a': 1, 'cell-b': 1 }]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('unexplained samples', () => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { ShadowSeatDirectory } from './shadow-seat-directory.js'
|
||||
|
||||
@@ -20,6 +21,8 @@ export type ShadowCompareClass =
|
||||
| 'cell-mismatch'
|
||||
| 'epoch-mismatch'
|
||||
| 'duplicate-seat'
|
||||
// A superseded-shaped duplicate with no prior sighting, in the instance's first 100 s.
|
||||
| 'superseded-unknown-age'
|
||||
| 'map-only'
|
||||
// The map seats the host on a cell the database does not call live, so resolve answered null.
|
||||
| 'map-only-cell-unlive'
|
||||
@@ -34,16 +37,28 @@ export const SHADOW_COMPARE_FLUSH_MS = 60_000
|
||||
// One poll, its timeout, and a reconnect (p95 ~6 s on 10-02). A move older than
|
||||
// this is not "the map catching up": a lost leave must not read as explained forever.
|
||||
export const SHADOW_COMPARE_LAG_BOUND_MS = 10_000
|
||||
// A superseded half-open control lingers until the cell's silence close (the timeout plus up
|
||||
// to one ping interval), then the map trails that close by a poll.
|
||||
export const SHADOW_COMPARE_SUPERSEDED_CLOSE_MS =
|
||||
RELAY_PROTOCOL_LIMITS.controlSilenceTimeoutMs +
|
||||
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs +
|
||||
SHADOW_COMPARE_LAG_BOUND_MS
|
||||
// Per class per flush window, so a systemic fault cannot flood the log.
|
||||
export const SHADOW_COMPARE_SAMPLES_PER_CLASS = 10
|
||||
|
||||
// `countByCell` is keyed by cell, so bounded by the fleet; prototype-free, so any cell id is a key.
|
||||
type ShadowCompareCount = { count: number; countByCell: Record<string, number> }
|
||||
|
||||
export function classifyShadowSeat(
|
||||
directory: ShadowSeatDirectory,
|
||||
identity: { userId: string; relayHostId: string },
|
||||
answer: ShadowCompareDatabaseAnswer,
|
||||
now: number
|
||||
now: number,
|
||||
// When this instance began comparing; omitted, a missing sighting is never excused.
|
||||
startedAt = -Infinity
|
||||
): ShadowCompareVerdict {
|
||||
// Pure: only ShadowDirectoryCompare records sightings. An epoch never seen is new.
|
||||
// Pure: only ShadowDirectoryCompare records sightings. An epoch never seen is new; unlike the
|
||||
// superseded rule, a wrong guess here only explains a mismatch for one lag bound.
|
||||
const movedAt = answer
|
||||
? (directory.databaseEpochFirstSeen(
|
||||
identity.userId,
|
||||
@@ -69,7 +84,31 @@ export function classifyShadowSeat(
|
||||
}
|
||||
if (seats.length > 1) {
|
||||
// Inside a drain or rehome grace the old seat is drain-only until it closes.
|
||||
return { class: 'duplicate-seat', explained: seats.some((seat) => seat.state === 'drain-only') }
|
||||
if (seats.some((seat) => seat.state === 'drain-only')) {
|
||||
return { class: 'duplicate-seat', explained: true }
|
||||
}
|
||||
// A newer DB seat supersedes older-epoch seats until their silence close;
|
||||
// a level epoch is a real tie.
|
||||
const current = seats.find(
|
||||
(seat) => seat.cellId === answer.cellId && seat.epoch === answer.assignmentEpoch
|
||||
)
|
||||
const supersededShape =
|
||||
current !== undefined &&
|
||||
seats.every((seat) => seat === current || seat.epoch < current.epoch) &&
|
||||
// A long-ago lower-epoch sighting dates the move to now; the join bounds the seat itself.
|
||||
now - current.joinedAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
|
||||
if (!supersededShape) return { class: 'duplicate-seat', explained: false }
|
||||
const known = directory.databaseEpochOf(identity.userId, identity.relayHostId)
|
||||
if (!known) {
|
||||
// Later, no sighting means another instance served the earlier compares, or the map evicted
|
||||
// the host, so a harmless supersede still reads as an unexplained duplicate-seat.
|
||||
const starting = now - startedAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
|
||||
return { class: starting ? 'superseded-unknown-age' : 'duplicate-seat', explained: false }
|
||||
}
|
||||
const supersededAt = movedSince(known, answer, now)
|
||||
const superseded =
|
||||
supersededAt !== undefined && now - supersededAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
|
||||
return { class: 'duplicate-seat', explained: superseded }
|
||||
}
|
||||
const seat = seats[0]
|
||||
if (!seat) {
|
||||
@@ -99,17 +138,54 @@ export function classifyShadowSeat(
|
||||
return { class: 'agree', explained: true }
|
||||
}
|
||||
|
||||
// When the move to the answer's epoch happened, from the sighting recorded before this compare.
|
||||
// A regressed epoch means unknown, not new.
|
||||
function movedSince(
|
||||
known: { epoch: number; firstSeenAt: number },
|
||||
answer: { assignmentEpoch: number },
|
||||
now: number
|
||||
): number | undefined {
|
||||
if (known.epoch > answer.assignmentEpoch) return undefined
|
||||
// A lower recorded epoch: the move is this sighting.
|
||||
return known.epoch < answer.assignmentEpoch ? now : known.firstSeenAt
|
||||
}
|
||||
|
||||
// The database's cell, or with no row every seat's cell; an unexplained cell-mismatch or
|
||||
// duplicate-seat also counts on every seat cell, so a stale seat feed shows on its own cell.
|
||||
// A compare with neither (agree-absent, or map-incomplete with no row and no seat) has none.
|
||||
function comparedCells(
|
||||
directory: ShadowSeatDirectory,
|
||||
identity: { userId: string; relayHostId: string },
|
||||
answer: ShadowCompareDatabaseAnswer,
|
||||
verdict: ShadowCompareVerdict
|
||||
): Set<string> {
|
||||
const seats = directory.seatsOf(identity.userId, identity.relayHostId)
|
||||
if (!answer) return new Set(seats.map((seat) => seat.cellId))
|
||||
const cells = new Set([answer.cellId])
|
||||
if (
|
||||
!verdict.explained &&
|
||||
(verdict.class === 'cell-mismatch' ||
|
||||
verdict.class === 'duplicate-seat' ||
|
||||
verdict.class === 'superseded-unknown-age')
|
||||
) {
|
||||
for (const seat of seats) cells.add(seat.cellId)
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
export class ShadowDirectoryCompare {
|
||||
private counts = new Map<string, number>()
|
||||
private counts = new Map<string, ShadowCompareCount>()
|
||||
private samples = new Map<ShadowCompareClass, number>()
|
||||
private windowStartedAt: number
|
||||
private readonly startedAt: number
|
||||
|
||||
constructor(
|
||||
private readonly directory: ShadowSeatDirectory,
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly log: (line: string) => void = (line) => console.warn(line)
|
||||
) {
|
||||
this.windowStartedAt = now()
|
||||
this.startedAt = now()
|
||||
this.windowStartedAt = this.startedAt
|
||||
}
|
||||
|
||||
compare(
|
||||
@@ -118,6 +194,8 @@ export class ShadowDirectoryCompare {
|
||||
answer: ShadowCompareDatabaseAnswer
|
||||
): ShadowCompareVerdict {
|
||||
const now = this.now()
|
||||
// Classify first, so it can tell a first-ever sighting from one seen before.
|
||||
const verdict = classifyShadowSeat(this.directory, identity, answer, now, this.startedAt)
|
||||
// Every answer, so a move's age counts from its first sighting on either route.
|
||||
if (answer) {
|
||||
this.directory.observeDatabaseEpoch(
|
||||
@@ -127,17 +205,22 @@ export class ShadowDirectoryCompare {
|
||||
now
|
||||
)
|
||||
}
|
||||
const verdict = classifyShadowSeat(this.directory, identity, answer, now)
|
||||
const key = `${route}\u0000${verdict.class}\u0000${verdict.explained}`
|
||||
this.counts.set(key, (this.counts.get(key) ?? 0) + 1)
|
||||
const entry = this.counts.get(key) ?? { count: 0, countByCell: Object.create(null) }
|
||||
this.counts.set(key, entry)
|
||||
entry.count += 1
|
||||
for (const cellId of comparedCells(this.directory, identity, answer, verdict)) {
|
||||
entry.countByCell[cellId] = (entry.countByCell[cellId] ?? 0) + 1
|
||||
}
|
||||
if (!verdict.explained) this.sample(route, identity, answer, verdict, now)
|
||||
if (now - this.windowStartedAt >= SHADOW_COMPARE_FLUSH_MS) this.flush(now)
|
||||
return verdict
|
||||
}
|
||||
|
||||
// One line per route and class per window; the log metric sums `count`.
|
||||
// One line per route and class per window; the log metric sums `count`. `countByCell`
|
||||
// counts each compare once on every cell it involves (see comparedCells).
|
||||
flush(now: number = this.now()): void {
|
||||
for (const [key, count] of this.counts) {
|
||||
for (const [key, { count, countByCell }] of this.counts) {
|
||||
const [route, verdictClass, explained] = key.split('\u0000')
|
||||
this.log(
|
||||
JSON.stringify({
|
||||
@@ -146,7 +229,8 @@ export class ShadowDirectoryCompare {
|
||||
class: verdictClass,
|
||||
explained: explained === 'true',
|
||||
count,
|
||||
windowMs: now - this.windowStartedAt
|
||||
windowMs: now - this.windowStartedAt,
|
||||
countByCell
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openInMemoryRelayDatabase } from './database.js'
|
||||
import {
|
||||
SHADOW_SEAT_OUTRANK_SKEW_MS,
|
||||
SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS,
|
||||
ShadowSeatDirectory,
|
||||
startShadowSeatPoller,
|
||||
@@ -577,3 +578,76 @@ describe('ShadowSeatDirectory.placementModeOf', () => {
|
||||
expect(directory.placementModeOf('cell-a')).toBe('reserve')
|
||||
})
|
||||
})
|
||||
|
||||
describe('ShadowSeatDirectory.onOutranked', () => {
|
||||
const reserve = { generation: 3, flags: { admitMode: 'reserve' } }
|
||||
const db = { generation: 3, flags: { admitMode: 'db' } }
|
||||
const full = (cellId: string, epoch: number, joinedAt: number, flags = reserve, state = 'active') =>
|
||||
feed({
|
||||
cellId,
|
||||
seq: 1,
|
||||
flagsApplied: flags,
|
||||
full: [{ userId: 'user-a', relayHostId: 'host-1', epoch, generation: 1, state, joinedAt }]
|
||||
})
|
||||
const directory = () => {
|
||||
const map = new ShadowSeatDirectory()
|
||||
map.setCells(['cell-a', 'cell-b'])
|
||||
const outranked: unknown[] = []
|
||||
map.onOutranked = (seat) => outranked.push(seat)
|
||||
return { map, outranked }
|
||||
}
|
||||
const halfOpen = {
|
||||
cellId: 'cell-a',
|
||||
userId: 'user-a',
|
||||
relayHostId: 'host-1',
|
||||
epoch: 3,
|
||||
joinedAt: 10,
|
||||
byCellId: 'cell-b'
|
||||
}
|
||||
|
||||
it('reports a reserve seat a later, newer join elsewhere outranks, in either feed order', () => {
|
||||
const first = directory()
|
||||
first.map.apply('cell-a', full('cell-a', 3, 10), 100)
|
||||
first.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
|
||||
expect(first.outranked).toEqual([halfOpen])
|
||||
const reversed = directory()
|
||||
reversed.map.apply('cell-b', full('cell-b', 4, 2_000), 100)
|
||||
reversed.map.apply('cell-a', full('cell-a', 3, 10), 101)
|
||||
expect(reversed.outranked).toEqual([halfOpen])
|
||||
const justPastSkew = directory()
|
||||
justPastSkew.map.apply('cell-a', full('cell-a', 3, 10), 100)
|
||||
justPastSkew.map.apply('cell-b', full('cell-b', 4, 11 + SHADOW_SEAT_OUTRANK_SKEW_MS), 101)
|
||||
expect(justPastSkew.outranked).toEqual([halfOpen])
|
||||
})
|
||||
|
||||
it('reports a seat once, even when a full resync re-judges its host', () => {
|
||||
const { map, outranked } = directory()
|
||||
map.apply('cell-a', full('cell-a', 3, 10), 100)
|
||||
map.apply('cell-b', full('cell-b', 4, 2_000), 101)
|
||||
map.apply('cell-b', { ...full('cell-b', 4, 2_000), incarnation: '22222222-2222-4222-8222-222222222222' }, 102)
|
||||
map.apply('cell-a', { ...full('cell-a', 3, 10), incarnation: '22222222-2222-4222-8222-222222222222' }, 103)
|
||||
expect(outranked).toEqual([halfOpen])
|
||||
})
|
||||
|
||||
it('leaves a database-cell seat, a drain-only seat, a later lower epoch, and a join within the skew', () => {
|
||||
const onDatabaseCell = directory()
|
||||
onDatabaseCell.map.apply('cell-a', full('cell-a', 3, 10, db), 100)
|
||||
onDatabaseCell.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
|
||||
const drainOnly = directory()
|
||||
drainOnly.map.apply('cell-a', full('cell-a', 3, 10, reserve, 'drain-only'), 100)
|
||||
drainOnly.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
|
||||
// Answered the old seat from a stale map after the newer join: the row decides that one.
|
||||
const staleSticky = directory()
|
||||
staleSticky.map.apply('cell-a', full('cell-a', 3, 3_000), 100)
|
||||
staleSticky.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
|
||||
const withinSkew = directory()
|
||||
withinSkew.map.apply('cell-a', full('cell-a', 3, 10), 100)
|
||||
withinSkew.map.apply('cell-b', full('cell-b', 4, 10 + SHADOW_SEAT_OUTRANK_SKEW_MS), 101)
|
||||
expect([onDatabaseCell, drainOnly, staleSticky, withinSkew].map(({ outranked }) => outranked)).toEqual([
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -16,6 +16,16 @@ export const SHADOW_SEAT_POLL_TIMEOUT_MS = 2_000
|
||||
export const SHADOW_SEAT_CELL_LIST_REFRESH_MS = 30_000
|
||||
export const SHADOW_SEAT_RECENTLY_LEFT_TTL_MS = 24 * 60 * 60 * 1_000
|
||||
export const SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS = 100_000
|
||||
// Cells stamp joinedAt by their own clocks: a later join must clear this to outrank across cells.
|
||||
export const SHADOW_SEAT_OUTRANK_SKEW_MS = 1_000
|
||||
const SHADOW_SEAT_OUTRANKED_REPORTS_MAX = 100_000
|
||||
|
||||
export function seatOutranks(
|
||||
newer: { epoch: number; joinedAt: number },
|
||||
seat: { epoch: number; joinedAt: number }
|
||||
): boolean {
|
||||
return newer.epoch > seat.epoch && newer.joinedAt > seat.joinedAt + SHADOW_SEAT_OUTRANK_SKEW_MS
|
||||
}
|
||||
const RECENTLY_LEFT_PER_HOST = 4
|
||||
// Google identity tokens live an hour; one per poll would be 26 metadata reads a second.
|
||||
const IDENTITY_TOKEN_REUSE_MS = 10 * 60_000
|
||||
@@ -172,6 +182,16 @@ export type BookedJoin = {
|
||||
reservedBy: string
|
||||
}
|
||||
|
||||
// A reserve seat behind a later, newer-epoch join of its host elsewhere that placement supersede missed.
|
||||
export type OutrankedSeat = {
|
||||
cellId: string
|
||||
userId: string
|
||||
relayHostId: string
|
||||
epoch: number
|
||||
joinedAt: number
|
||||
byCellId: string
|
||||
}
|
||||
|
||||
type CellCursor = SeatFeedCellState & { seats: Map<string, ShadowSeat> }
|
||||
|
||||
export type HeartbeatSnapshot = { readAt: number; expiresAt: ReadonlyMap<string, number> }
|
||||
@@ -180,6 +200,11 @@ function hostKey(userId: string, relayHostId: string): string {
|
||||
return `${userId}\u0000${relayHostId}`
|
||||
}
|
||||
|
||||
function parseHostKey(key: string): { userId: string; relayHostId: string } {
|
||||
const separator = key.indexOf('\u0000')
|
||||
return { userId: key.slice(0, separator), relayHostId: key.slice(separator + 1) }
|
||||
}
|
||||
|
||||
export class ShadowSeatDirectory {
|
||||
private readonly cells = new Map<string, CellCursor>()
|
||||
private readonly hosts = new Map<string, Map<string, ShadowSeat>>()
|
||||
@@ -193,6 +218,10 @@ export class ShadowSeatDirectory {
|
||||
// Per host, the newest booking any cell reported (step 5 epoch mint). Age-ordered.
|
||||
private readonly bookings = new Map<string, { cellId: string; epoch: number; at: number }>()
|
||||
onBookedJoin?: (join: BookedJoin) => void
|
||||
onOutranked?: (seat: OutrankedSeat) => void
|
||||
private readonly joinedSinceApply = new Set<string>()
|
||||
// Seats already reported, insertion-ordered: a full resync re-judges a host without a second demote.
|
||||
private readonly reportedOutranked = new Set<string>()
|
||||
|
||||
// Cells no longer listed are forgotten with their seats; new ones start pending.
|
||||
// `required` defaults to every listed cell.
|
||||
@@ -225,6 +254,11 @@ export class ShadowSeatDirectory {
|
||||
|
||||
// `polledAt` is when the request left; the reply holds every change before it.
|
||||
apply(cellId: string, response: SeatFeedResponse, now: number, polledAt = now): void {
|
||||
this.applyResponse(cellId, response, now, polledAt)
|
||||
this.reportOutranked()
|
||||
}
|
||||
|
||||
private applyResponse(cellId: string, response: SeatFeedResponse, now: number, polledAt: number): void {
|
||||
const cursor = this.cells.get(cellId)
|
||||
if (!cursor) return
|
||||
if (response.cellId !== cellId) {
|
||||
@@ -307,10 +341,7 @@ export class ShadowSeatDirectory {
|
||||
seatsOnCell(cellId: string): Array<{ userId: string; relayHostId: string; seat: ShadowSeat }> {
|
||||
const cursor = this.cells.get(cellId)
|
||||
if (!cursor) return []
|
||||
return [...cursor.seats].map(([key, seat]) => {
|
||||
const separator = key.indexOf('\u0000')
|
||||
return { userId: key.slice(0, separator), relayHostId: key.slice(separator + 1), seat }
|
||||
})
|
||||
return [...cursor.seats].map(([key, seat]) => ({ ...parseHostKey(key), seat }))
|
||||
}
|
||||
|
||||
// An old cell (404) has no feed; its last known seats stay, marked by status.
|
||||
@@ -373,6 +404,15 @@ export class ShadowSeatDirectory {
|
||||
return known && known.epoch >= epoch ? known.firstSeenAt : undefined
|
||||
}
|
||||
|
||||
// The newest epoch this director has seen the database name for the host, if still kept.
|
||||
databaseEpochOf(
|
||||
userId: string,
|
||||
relayHostId: string
|
||||
): { epoch: number; firstSeenAt: number } | undefined {
|
||||
const known = this.databaseEpochs.get(hostKey(userId, relayHostId))
|
||||
return known ? { ...known } : undefined
|
||||
}
|
||||
|
||||
// Records a sighting; returns when this director first saw the host at this epoch.
|
||||
observeDatabaseEpoch(userId: string, relayHostId: string, epoch: number, now: number): number {
|
||||
const key = hostKey(userId, relayHostId)
|
||||
@@ -558,6 +598,34 @@ export class ShadowSeatDirectory {
|
||||
this.hosts.set(key, seats)
|
||||
}
|
||||
seats.set(cursor.cellId, seat)
|
||||
if (this.onOutranked && seats.size > 1) this.joinedSinceApply.add(key)
|
||||
}
|
||||
|
||||
// After the whole response, so a cell's first snapshot is judged by the mode it carries. Only a
|
||||
// join triggers it, once per seat: a cell flipped to reserve later, or a lost demote, falls to
|
||||
// the silence close or the ledger reconcile, as does a lower epoch that joined later.
|
||||
private reportOutranked(): void {
|
||||
const report = this.onOutranked
|
||||
const keys = [...this.joinedSinceApply]
|
||||
this.joinedSinceApply.clear()
|
||||
if (!report) return
|
||||
for (const key of keys) {
|
||||
const seats = [...(this.hosts.get(key)?.values() ?? [])]
|
||||
for (const seat of seats) {
|
||||
if (seat.state !== 'active' || this.admitModeOf(seat.cellId) !== 'reserve') continue
|
||||
const by = seats.find((other) => seatOutranks(other, seat))
|
||||
if (!by) continue
|
||||
const { cellId, epoch, joinedAt } = seat
|
||||
const reported = `${key}\u0000${cellId}\u0000${epoch}\u0000${joinedAt}`
|
||||
if (this.reportedOutranked.has(reported)) continue
|
||||
this.reportedOutranked.add(reported)
|
||||
if (this.reportedOutranked.size > SHADOW_SEAT_OUTRANKED_REPORTS_MAX) {
|
||||
const oldest = this.reportedOutranked.values().next().value
|
||||
if (oldest !== undefined) this.reportedOutranked.delete(oldest)
|
||||
}
|
||||
report({ ...parseHostKey(key), cellId, epoch, joinedAt, byCellId: by.cellId })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private unseat(cursor: CellCursor, key: string): void {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { runInStickySlot, StickySlotStatements } from './sticky-slot-statements.js'
|
||||
import {
|
||||
ScriptedPoolClient,
|
||||
scriptedPostgresDatabase
|
||||
} from './test-fixtures/scripted-postgres-pool.js'
|
||||
|
||||
function gated() {
|
||||
let open!: () => void
|
||||
const opened = new Promise<void>((resolve) => (open = resolve))
|
||||
return { open, wait: async () => await opened }
|
||||
}
|
||||
|
||||
describe('sticky slot statements', () => {
|
||||
it('charges the slot every round trip it made, by pooled connection', async () => {
|
||||
const first = new ScriptedPoolClient()
|
||||
const second = new ScriptedPoolClient()
|
||||
const database = scriptedPostgresDatabase([first, second, first])
|
||||
const slot = new StickySlotStatements()
|
||||
await runInStickySlot(slot, async () => {
|
||||
await database.query('SELECT verify')
|
||||
await database.transaction(async (transaction) => {
|
||||
await transaction.query('SELECT pin')
|
||||
await transaction.queryLocked('SELECT row', [], { lockTimeoutMs: 200 })
|
||||
})
|
||||
})
|
||||
// Outside the slot: not this request's work.
|
||||
await database.query('SELECT other')
|
||||
|
||||
const sample = slot.close()
|
||||
// BEGIN, the statement, both lock_timeout SETs, the locked read and COMMIT.
|
||||
expect(second.statements).toHaveLength(6)
|
||||
expect(sample.statements).toBe(7)
|
||||
expect(sample.sqlMs).toBeGreaterThanOrEqual(0)
|
||||
expect(
|
||||
[...sample.statementMsByConnection].map(([id, samples]) => [id, samples.length])
|
||||
).toEqual([
|
||||
[1, 1],
|
||||
[2, 6]
|
||||
])
|
||||
})
|
||||
|
||||
it('leaves out statements another request runs while the slot is held', async () => {
|
||||
const otherGate = gated()
|
||||
const database = scriptedPostgresDatabase([
|
||||
new ScriptedPoolClient(otherGate.wait),
|
||||
new ScriptedPoolClient()
|
||||
])
|
||||
const slot = new StickySlotStatements()
|
||||
const other = database.query('SELECT other')
|
||||
await runInStickySlot(slot, async () => {
|
||||
await database.query('SELECT mine')
|
||||
otherGate.open()
|
||||
await other
|
||||
})
|
||||
|
||||
const sample = slot.close()
|
||||
expect(sample.statements).toBe(1)
|
||||
expect([...sample.statementMsByConnection.keys()]).toEqual([2])
|
||||
})
|
||||
|
||||
it('ignores work the request left running after it released the slot', async () => {
|
||||
const late = gated()
|
||||
const database = scriptedPostgresDatabase([new ScriptedPoolClient(), new ScriptedPoolClient()])
|
||||
const slot = new StickySlotStatements()
|
||||
let leftRunning: Promise<unknown> | undefined
|
||||
await runInStickySlot(slot, async () => {
|
||||
await database.query('SELECT 1')
|
||||
leftRunning = late.wait().then(async () => await database.query('SELECT late'))
|
||||
})
|
||||
const sample = slot.close()
|
||||
late.open()
|
||||
await leftRunning
|
||||
expect(sample.statements).toBe(1)
|
||||
})
|
||||
|
||||
it('reports the pool wait apart from the statements', async () => {
|
||||
const database = scriptedPostgresDatabase([new ScriptedPoolClient()], 30)
|
||||
const slot = new StickySlotStatements()
|
||||
await runInStickySlot(slot, async () => await database.query('SELECT 1'))
|
||||
|
||||
const sample = slot.close()
|
||||
expect(sample.poolWaitMs).toBeGreaterThanOrEqual(20)
|
||||
expect(sample.sqlMs).toBeLessThan(20)
|
||||
})
|
||||
|
||||
it('names each pooled connection once, however often it is checked out', async () => {
|
||||
const first = new ScriptedPoolClient()
|
||||
const second = new ScriptedPoolClient()
|
||||
const database = scriptedPostgresDatabase([first, second, first])
|
||||
const slot = new StickySlotStatements()
|
||||
await runInStickySlot(slot, async () => {
|
||||
for (let index = 0; index < 3; index++) await database.query('SELECT 1')
|
||||
})
|
||||
|
||||
expect(
|
||||
[...slot.close().statementMsByConnection].map(([id, samples]) => [id, samples.length])
|
||||
).toEqual([
|
||||
[1, 2],
|
||||
[2, 1]
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
|
||||
export type StickySlotSample = {
|
||||
queueWaitMs: number
|
||||
// Waiting for a pooled connection, apart from the statements run on one.
|
||||
poolWaitMs: number
|
||||
sqlMs: number
|
||||
statements: number
|
||||
statementMsByConnection: ReadonlyMap<number, readonly number[]>
|
||||
}
|
||||
|
||||
type StickySlotStatementTotals = Omit<StickySlotSample, 'queueWaitMs'>
|
||||
|
||||
// PostgreSQL round trips (all but ROLLBACK) made while one request holds the sticky
|
||||
// slot. Closed at release, so work the request left running is not charged to it.
|
||||
export class StickySlotStatements {
|
||||
private closed = false
|
||||
private poolWaitMs = 0
|
||||
private sqlMs = 0
|
||||
private statements = 0
|
||||
private readonly statementMsByConnection = new Map<number, number[]>()
|
||||
|
||||
notePoolWait(durationMs: number): void {
|
||||
if (!this.closed) this.poolWaitMs += durationMs
|
||||
}
|
||||
|
||||
noteStatement(durationMs: number, connectionId: number): void {
|
||||
if (this.closed) return
|
||||
this.statements++
|
||||
this.sqlMs += durationMs
|
||||
const samples = this.statementMsByConnection.get(connectionId)
|
||||
if (samples) samples.push(durationMs)
|
||||
else this.statementMsByConnection.set(connectionId, [durationMs])
|
||||
}
|
||||
|
||||
close(): StickySlotStatementTotals {
|
||||
this.closed = true
|
||||
return {
|
||||
poolWaitMs: this.poolWaitMs,
|
||||
sqlMs: this.sqlMs,
|
||||
statements: this.statements,
|
||||
statementMsByConnection: this.statementMsByConnection
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const activeStickySlot = new AsyncLocalStorage<StickySlotStatements>()
|
||||
|
||||
export function runInStickySlot<T>(
|
||||
statements: StickySlotStatements,
|
||||
work: () => Promise<T>
|
||||
): Promise<T> {
|
||||
return activeStickySlot.run(statements, work)
|
||||
}
|
||||
|
||||
export function noteStickySlotPoolWait(durationMs: number): void {
|
||||
activeStickySlot.getStore()?.notePoolWait(durationMs)
|
||||
}
|
||||
|
||||
export function noteStickySlotStatement(durationMs: number, connectionId: number): void {
|
||||
activeStickySlot.getStore()?.noteStatement(durationMs, connectionId)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PostgresDatabase } from '../database.js'
|
||||
|
||||
export class ScriptedPoolClient extends EventEmitter {
|
||||
readonly statements: string[] = []
|
||||
release: () => void = () => undefined
|
||||
|
||||
// Resolves each statement only once `gate` does, so a test can hold one mid-flight.
|
||||
constructor(private readonly gate: () => Promise<void> = async () => undefined) {
|
||||
super()
|
||||
}
|
||||
|
||||
query = async (sql: string | { text: string }) => {
|
||||
this.statements.push(typeof sql === 'string' ? sql : sql.text)
|
||||
await this.gate()
|
||||
return { rows: [], rowCount: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
// Hands out the listed clients in order, reusing an object the way pg-pool reuses a connection.
|
||||
export function scriptedPostgresDatabase(
|
||||
checkouts: ScriptedPoolClient[],
|
||||
connectDelayMs = 0
|
||||
): PostgresDatabase {
|
||||
const pool = {
|
||||
totalCount: checkouts.length,
|
||||
idleCount: 0,
|
||||
waitingCount: 0,
|
||||
connect: async () => {
|
||||
if (connectDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, connectDelayMs))
|
||||
const client = checkouts.shift()
|
||||
if (!client) throw new Error('scripted_pool_exhausted')
|
||||
// pg-pool assigns a fresh release on every checkout.
|
||||
client.release = () => undefined
|
||||
return client
|
||||
}
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: PostgresDatabase and its pool pressure read only connect, these counts and the optional options.
|
||||
return new PostgresDatabase(pool as never)
|
||||
}
|
||||
@@ -1,26 +1,22 @@
|
||||
import { readdirSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { defaultExclude, defineConfig } from 'vitest/config'
|
||||
|
||||
// Every test file that opens ORCA_RELAY_TEST_POSTGRES_URL shares one CI
|
||||
// database, and those tests take session-level locks with a 1s lock_timeout.
|
||||
// Running them alongside anything else collides into lock timeouts, capacity
|
||||
// exhaustion, and afterAll hangs. Keep them in their own serialized project so
|
||||
// only they give up file parallelism; the rest of the suite opens SQLite data
|
||||
// directories and stays fully parallel.
|
||||
// PostgreSQL files share session locks and a small connection budget.
|
||||
// The runner overlaps this serialized project with SQLite tests after other packages finish.
|
||||
const sourceDirectory = fileURLToPath(new URL('src', import.meta.url))
|
||||
const sharedPostgresTests = readdirSync(sourceDirectory)
|
||||
.filter((entry) => entry.endsWith('.test.ts'))
|
||||
.filter((entry) =>
|
||||
readFileSync(`${sourceDirectory}/${entry}`, 'utf8').includes(
|
||||
'ORCA_RELAY_TEST_POSTGRES_URL'
|
||||
)
|
||||
readFileSync(join(sourceDirectory, entry), 'utf8').includes('ORCA_RELAY_TEST_POSTGRES_URL')
|
||||
)
|
||||
.map((entry) => `src/${entry}`)
|
||||
|
||||
const timeouts = { testTimeout: 15_000, hookTimeout: 15_000 }
|
||||
|
||||
export default defineConfig({
|
||||
cacheDir: process.env.ORCA_RELAY_TEST_CACHE_DIR,
|
||||
test: {
|
||||
projects: [
|
||||
{
|
||||
|
||||
@@ -146,6 +146,7 @@
|
||||
"google_logging_metric.relay_event_loop_stall",
|
||||
"google_logging_metric.relay_long_cell_inventory_hold",
|
||||
"google_logging_metric.relay_reserve_placement_off_with_reserve_cells",
|
||||
"google_logging_metric.relay_reserve_watch_alarm",
|
||||
"google_logging_metric.relay_snapshot",
|
||||
"google_monitoring_alert_policy.relay_assignment_5xx",
|
||||
"google_monitoring_alert_policy.relay_assignment_edge_429",
|
||||
@@ -176,6 +177,7 @@
|
||||
"google_monitoring_alert_policy.relay_process_fatal",
|
||||
"google_monitoring_alert_policy.relay_region_hint_skew",
|
||||
"google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells",
|
||||
"google_monitoring_alert_policy.relay_reserve_watch_alarm",
|
||||
"google_monitoring_dashboard.relay_incident",
|
||||
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
|
||||
@@ -304,6 +306,10 @@
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_log_writer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
},
|
||||
"google_project_iam_member.github_logging_viewer": {
|
||||
"production": "relay",
|
||||
"staging": "apps"
|
||||
|
||||
@@ -244,7 +244,7 @@ function objectUrl(request, suffix = '') {
|
||||
}
|
||||
|
||||
// One transient 5xx or network failure is retried once, as every relay admin step does.
|
||||
async function readRuntime(fetchImpl, request, idToken, wait) {
|
||||
export async function readRuntime(fetchImpl, request, idToken, wait) {
|
||||
const response = await fetchAdminOnceMore(
|
||||
fetchImpl,
|
||||
`${request.cellOrigin}/v1/admin/runtime-status`,
|
||||
@@ -262,7 +262,7 @@ async function readRuntime(fetchImpl, request, idToken, wait) {
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
async function readCurrentObject(fetchImpl, request, accessToken) {
|
||||
export async function readCurrentObject(fetchImpl, request, accessToken) {
|
||||
const headers = { authorization: `Bearer ${accessToken}` }
|
||||
const metadata = await fetchImpl(objectUrl(request), { headers })
|
||||
if (metadata.status === 404) {
|
||||
@@ -278,6 +278,12 @@ async function readCurrentObject(fetchImpl, request, accessToken) {
|
||||
return { generation: String(generation), object: await media.json().catch(() => null) }
|
||||
}
|
||||
|
||||
function generationConflict(current, expected) {
|
||||
return Object.assign(new Error(`current generation is ${current}, not expected ${expected}`), {
|
||||
generationConflict: true
|
||||
})
|
||||
}
|
||||
|
||||
async function writeObject(fetchImpl, request, object, accessToken, audit) {
|
||||
const boundary = `relay-cell-flags-${Date.now()}`
|
||||
const metadata = {
|
||||
@@ -315,7 +321,8 @@ async function writeObject(fetchImpl, request, object, accessToken, audit) {
|
||||
if (response.status === 412) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), {
|
||||
notWritten: true
|
||||
notWritten: true,
|
||||
generationConflict: true
|
||||
})
|
||||
}
|
||||
if (!response.ok) {
|
||||
@@ -409,7 +416,7 @@ async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sle
|
||||
}
|
||||
const current = await readCurrentObject(fetchImpl, request, accessToken)
|
||||
if (current.generation !== request.expectedGeneration) {
|
||||
throw new Error(`current generation is ${current.generation}, not expected ${request.expectedGeneration}`)
|
||||
throw generationConflict(current.generation, request.expectedGeneration)
|
||||
}
|
||||
const object = desiredObject(request, current.object)
|
||||
const generation = await writeObject(fetchImpl, request, object, accessToken, audit)
|
||||
@@ -459,9 +466,7 @@ export async function operateCellFlags(request, dependencies) {
|
||||
// Before the generation check, so a dry run always shows the generation to pass next.
|
||||
log(JSON.stringify(plan))
|
||||
if (current.generation !== request.expectedGeneration) {
|
||||
throw new Error(
|
||||
`current generation is ${current.generation}, not expected ${request.expectedGeneration}`
|
||||
)
|
||||
throw generationConflict(current.generation, request.expectedGeneration)
|
||||
}
|
||||
// The cell applies every readable write; a generation it did not apply was voided there. A
|
||||
// write of an object this image accepts whole (checked above) repairs it.
|
||||
|
||||
@@ -33,6 +33,7 @@ test('no workflow names the retired generic production deploy identity', async (
|
||||
'operate-relay-asia-admission.yml',
|
||||
'operate-relay-production-cell-flags.yml',
|
||||
'operate-relay-production-rehome-job.yml',
|
||||
'operate-relay-production-reserve-watch.yml',
|
||||
'publish-relay-production.yml'
|
||||
].map((name) => relayWorkflowFile(name)).sort())
|
||||
})
|
||||
|
||||
@@ -922,18 +922,14 @@ test('the workflows offer exactly the closed set and scale the drain wait with i
|
||||
assert.match(job, /\n DRAIN_PACE_WINDOW_MS: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/)
|
||||
assert.doesNotMatch(job, /DRAIN_PACE_WINDOW_MS: '/)
|
||||
assert.match(job, /cell-class \\\n\s+--cell-id "\$\{TARGET_CELL_ID\}" --drain-pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
|
||||
// The 15-min migration lease plus the window, plus its excess over the default: restart-safe
|
||||
// waits a whole window of quiet after the last send, so a slow drain needs about twice it.
|
||||
// The 15-min migration lease plus the window: restart-safe passes once the window has ended.
|
||||
const timeout = /--timeout-ms "\$\(\((.+)\)\)"\n/.exec(job)[1]
|
||||
const timeoutMs = (pace) => Number(execFileSync(
|
||||
'bash', ['-c', `DRAIN_PACE_WINDOW_MS=${pace}; echo $((${timeout}))`], { encoding: 'utf8' }
|
||||
))
|
||||
// Exactly the timeouts every pace had before the slower ones existed.
|
||||
for (const pace of [300_000, 60_000, 30_000]) assert.equal(timeoutMs(pace), 900_000 + pace)
|
||||
for (const pace of SAME_CAP_DRAIN_PACE_WINDOWS_MS) {
|
||||
assert.ok(timeoutMs(pace) >= 2 * pace + 600_000, String(pace))
|
||||
assert.equal(timeoutMs(pace), 900_000 + pace, String(pace))
|
||||
}
|
||||
assert.equal(timeoutMs(1_200_000), 3_000_000)
|
||||
const slowest = timeoutMs(Math.max(...SAME_CAP_DRAIN_PACE_WINDOWS_MS))
|
||||
// The rest of the job keeps the 70 min it had beside the old 20-min drain step in 90.
|
||||
const jobMinutes = Number(/\n rollout:\n(?: [^\n]*\n)*? timeout-minutes: (\d+)\n/.exec(job)[1])
|
||||
|
||||
@@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml')
|
||||
const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
|
||||
|
||||
test('the copy carries every relay workflow', () => {
|
||||
assert.equal(relayWorkflows().length, 27)
|
||||
assert.equal(relayWorkflows().length, 28)
|
||||
})
|
||||
|
||||
// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming
|
||||
|
||||
@@ -12,6 +12,8 @@ const deadMan = read('../../apps/relay/src/cell-reserve-dead-man.ts')
|
||||
const registry = read('../../apps/relay/src/host-session-registry.ts')
|
||||
const server = read('../../apps/relay/src/relay-server.ts')
|
||||
const poller = read('../../apps/relay/src/shadow-seat-directory.ts')
|
||||
const watch = read('./watch-relay-reserve-cell.mjs')
|
||||
const githubActions = read('../../infra/terraform/relay-github-actions.tf')
|
||||
|
||||
const block = (kind, name) => {
|
||||
const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
|
||||
@@ -84,3 +86,24 @@ test('the withheld dead-cell verdict alert counts every director withheld line,
|
||||
assert.match(read('../../infra/terraform/variables.tf'), /variable "relay_dead_cell_verdict_alert" \{[\s\S]*?default = false/)
|
||||
assert.match(read('../../infra/terraform/environments/staging.tfvars'), /relay_dead_cell_verdict_alert\s+= true/)
|
||||
})
|
||||
|
||||
test('the reserve watch alarm pages on any line the watch runner writes, per cell', () => {
|
||||
// The runner writes a global-resource entry under its own log name.
|
||||
assert.match(watch, /export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'/)
|
||||
assert.match(watch, /export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'/)
|
||||
assert.match(watch, /JSON\.stringify\(\{ event: ALARM_EVENT, cellId, kind, reason \}\)/)
|
||||
const metric = block('google_logging_metric', 'relay_reserve_watch_alarm')
|
||||
assert.ok(
|
||||
metric.includes(
|
||||
'resource.type=\\"global\\" AND logName=\\"projects/${var.project_id}/logs/orca-relay-reserve-watch\\" AND jsonPayload.event=\\"orca_relay_reserve_watch_alarm\\"'
|
||||
)
|
||||
)
|
||||
assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"'))
|
||||
notifyingOnAnyLine('relay_reserve_watch_alarm', 'orca_relay_reserve_watch_alarm', 'global', 'cell_id')
|
||||
// The deploy identity the workflow runs as can write that line, and only in production.
|
||||
const grant = /resource "google_project_iam_member" "github_log_writer" \{([\s\S]*?)\n\}/.exec(githubActions)?.[1]
|
||||
assert.ok(grant, 'github_log_writer grant not found')
|
||||
assert.match(grant, /count = local\.relay_create_production_ops_identity \? 1 : 0/)
|
||||
assert.match(grant, /role = "roles\/logging\.logWriter"/)
|
||||
assert.match(grant, /member = google_service_account\.github_deploy\[0\]\.member/)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
// Stop rules for a cell watched in reserve mode (step 5 design §9, "Auto-abort per flip"). Pure:
|
||||
// it takes one already-read sample and the watch state, and returns the next state and whether
|
||||
// to flip the cell back. The reader lives in watch-relay-reserve-cell.mjs.
|
||||
import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-gate-verdict.mjs'
|
||||
|
||||
const MINUTE_MS = 60_000
|
||||
|
||||
export const RESERVE_WATCH_RULES = {
|
||||
// The roll ladder's warn line: max(1.5x, background + 20), so a zero background is not tripped
|
||||
// by one 503. Three complete minutes in a row, not two. The read spans more minutes than that so
|
||||
// a late publishing front still leaves three published minutes to judge.
|
||||
director503: {
|
||||
multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple,
|
||||
marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute,
|
||||
consecutiveMinutes: 3,
|
||||
readMinutes: 10,
|
||||
// Cloud Run publishes a minute's request count up to 3 minutes late.
|
||||
publishDelayMs: 3 * MINUTE_MS
|
||||
},
|
||||
// The cell closes a demoted seat after 60 s (DEMOTION_CLOSE_MS); 30 s more covers a director's
|
||||
// map still holding the closed seat.
|
||||
duplicateSeat: { graceMs: 60_000 + 30_000, lookbackMs: 15 * MINUTE_MS },
|
||||
// The floor keeps an almost empty cell's ordinary churn from reading as a 5% drop.
|
||||
hostDrop: { share: 0.05, windowMs: 5 * MINUTE_MS, minHosts: 10 },
|
||||
ledgerDrift: { share: 0.005, sustainMs: 2 * MINUTE_MS },
|
||||
// A placing director logs its ledger summary every 60 s; none in this long is no evidence.
|
||||
ledgerSummaryMaxAgeMs: 150_000,
|
||||
// A watch dispatched before the reserve write applies waits this long for the cell to report it.
|
||||
reserveApplyWaitMs: 10 * MINUTE_MS
|
||||
}
|
||||
|
||||
// Precedence: the first tripped rule names the verdict, so exact signals come before proxies. The
|
||||
// sample line's `rules` keys follow this order too.
|
||||
export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift']
|
||||
|
||||
export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) {
|
||||
return {
|
||||
v: 1,
|
||||
cellId,
|
||||
startedAt,
|
||||
deadlineAt: startedAt + durationMinutes * MINUTE_MS,
|
||||
state: 'watching',
|
||||
verdict: null,
|
||||
samples: 0,
|
||||
unverifiableStreak: 0,
|
||||
// The background is measured in the minutes before the flip, unless the operator supplied it.
|
||||
flippedAt: flippedAt ?? null,
|
||||
background:
|
||||
background503PerMinute === null ? null : { medianPerMinute: background503PerMinute, source: 'input' },
|
||||
hosts: [],
|
||||
reserveSeenAt: null,
|
||||
ledgerAboveSince: null,
|
||||
seatBehindRowFirstSeen: {}
|
||||
}
|
||||
}
|
||||
|
||||
// The background from the per-minute 503 counts before the flip. `publishedThrough` is the newest
|
||||
// minute any response code has a point for: a 503 minute at or before it with no point had none,
|
||||
// and the background waits (null) until it covers the last minute.
|
||||
export function measuredBackground(perMinute, minutes, publishedThrough) {
|
||||
if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null
|
||||
const background = backgroundOf({ minutes, series: minutes.map((minute) => perMinute[minute] ?? 0), unverified: false })
|
||||
return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' }
|
||||
}
|
||||
|
||||
const pass = (detail = {}) => ({ status: 'pass', ...detail })
|
||||
const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail })
|
||||
const unverifiable = (reason) => ({ status: 'unverifiable', reason })
|
||||
// Not readable yet by design, so it does not count toward the unverifiable streak.
|
||||
const pending = (reason) => ({ status: 'pending', reason })
|
||||
|
||||
function judgeDeadMan(cell) {
|
||||
if (!cell.ok) return unverifiable(cell.error)
|
||||
if (cell.admitModeEffective !== 'db' && cell.admitModeEffective !== 'reserve') {
|
||||
return unverifiable('runtime-status reports no admitModeEffective')
|
||||
}
|
||||
if (cell.admitModeEffective === 'db') {
|
||||
return trip('the cell tripped its dead-man (reserve configured, db effective); Postgres still records reserve')
|
||||
}
|
||||
return pass()
|
||||
}
|
||||
|
||||
// The cell ledger's total (seats, bookings, in-flight and phone units) against its hard cap. The
|
||||
// ledger refuses past the cap, so this is an invariant check: it trips only if that refusal broke.
|
||||
function judgeOverrun(cell) {
|
||||
if (!cell.ok) return unverifiable(cell.error)
|
||||
if (!Number.isInteger(cell.hardCap) || !Number.isInteger(cell.enforcedUnits)) {
|
||||
return unverifiable('runtime-status reports no hard cap or ledger units')
|
||||
}
|
||||
if (cell.enforcedUnits > cell.hardCap) {
|
||||
return trip(`ledger units ${cell.enforcedUnits} over hard cap ${cell.hardCap}`, {
|
||||
enforcedUnits: cell.enforcedUnits,
|
||||
hardCap: cell.hardCap
|
||||
})
|
||||
}
|
||||
return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap })
|
||||
}
|
||||
|
||||
// Fleet-wide director 503s: no existing signal attributes a 503 to the cell a host was bound for,
|
||||
// so a problem local to one cell can stay under the fleet's line. This proxy can miss.
|
||||
function judgeDirector503(read, state, at) {
|
||||
const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs } = RESERVE_WATCH_RULES.director503
|
||||
const { background } = state
|
||||
if (!background) {
|
||||
const publishesAt = state.flippedAt === null ? null : Math.floor(state.flippedAt / MINUTE_MS) * MINUTE_MS + publishDelayMs
|
||||
if (publishesAt !== null && at < publishesAt) return pending('the 503 background before the flip is not published yet')
|
||||
return unverifiable('no 503 background yet')
|
||||
}
|
||||
if (!read.ok) return unverifiable(read.error)
|
||||
const perMinute = background.medianPerMinute
|
||||
const above = Math.max(perMinute * multiple, perMinute + marginPerMinute)
|
||||
// Minute keys sort as time. Only minutes the publishing front has reached are judged: a 503
|
||||
// minute there with no point had none, one past it is not published yet and is left out.
|
||||
if (read.publishedThrough === null) return unverifiable('the director request series is empty')
|
||||
const recent = read.minutes.filter((minute) => minute <= read.publishedThrough).slice(-consecutiveMinutes)
|
||||
if (recent.length < consecutiveMinutes) {
|
||||
return unverifiable(`director requests are published only through ${read.publishedThrough}`)
|
||||
}
|
||||
const counts = recent.map((minute) => read.perMinute[minute] ?? 0)
|
||||
const detail = { backgroundPerMinute: perMinute, above, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) }
|
||||
if (counts.every((count) => count > above)) {
|
||||
return trip(`director 503s above ${above}/min for ${consecutiveMinutes} min (${counts.join(', ')})`, detail)
|
||||
}
|
||||
return pass(detail)
|
||||
}
|
||||
|
||||
// A seat the ledger reconcile found behind its row and found again after the demotion should
|
||||
// have closed it. The line names no host, so (director instance, cell, epoch, rowEpoch) stands in
|
||||
// for one: a proxy that can only over-count, when two hosts share that key.
|
||||
function judgeDuplicateSeat(read, firstSeen, at) {
|
||||
if (!read.ok) return { verdict: unverifiable(read.error), firstSeen }
|
||||
const { graceMs, lookbackMs } = RESERVE_WATCH_RULES.duplicateSeat
|
||||
const next = Object.fromEntries(
|
||||
Object.entries(firstSeen).filter(([, seenAt]) => seenAt >= at - 2 * lookbackMs)
|
||||
)
|
||||
for (const report of [...read.reports].sort((left, right) => left.at - right.at)) {
|
||||
next[report.key] = Math.min(next[report.key] ?? report.at, report.at)
|
||||
}
|
||||
for (const report of read.reports) {
|
||||
const ageMs = report.at - next[report.key]
|
||||
if (ageMs > graceMs) {
|
||||
return {
|
||||
verdict: trip(`a seat behind its ledger row (${report.key}) is still seated ${Math.round(ageMs / 1000)} s after first report`),
|
||||
firstSeen: next
|
||||
}
|
||||
}
|
||||
}
|
||||
return { verdict: pass({ reportsInLookback: read.reports.length }), firstSeen: next }
|
||||
}
|
||||
|
||||
function judgeHostDrop(hosts) {
|
||||
const { share, minHosts, windowMs } = RESERVE_WATCH_RULES.hostDrop
|
||||
const current = hosts.at(-1)
|
||||
if (!current) return unverifiable('no host count')
|
||||
if (hosts.some((entry) => entry.draining)) return pass({ controls: current.controls, draining: true })
|
||||
const peak = Math.max(...hosts.map((entry) => entry.controls))
|
||||
const drop = peak - current.controls
|
||||
const detail = { controls: current.controls, peakInWindow: peak }
|
||||
if (peak > 0 && drop >= minHosts && drop / peak > share) {
|
||||
return trip(`connected hosts fell ${peak} -> ${current.controls} within ${windowMs / MINUTE_MS} min outside a drain`, detail)
|
||||
}
|
||||
return pass(detail)
|
||||
}
|
||||
|
||||
// Rows the placing directors have not written (queued) or gave up on (dropped), per seat. The
|
||||
// summaries are fleet-wide: exact while this is the only reserve cell, an over-count otherwise.
|
||||
function judgeLedgerDrift(cell, ledger, aboveSince, at) {
|
||||
if (!cell.ok) return { verdict: unverifiable(cell.error), aboveSince }
|
||||
if (!ledger.ok) return { verdict: unverifiable(ledger.error), aboveSince }
|
||||
const { share, sustainMs } = RESERVE_WATCH_RULES.ledgerDrift
|
||||
const ratio = ledger.unwrittenRows / Math.max(cell.controls, 1)
|
||||
const detail = { unwrittenRows: ledger.unwrittenRows, directors: ledger.directors, ratio }
|
||||
if (ratio <= share) return { verdict: pass(detail), aboveSince: null }
|
||||
const since = aboveSince ?? at
|
||||
if (at - since >= sustainMs) {
|
||||
return {
|
||||
verdict: trip(`ledger drift ${(ratio * 100).toFixed(2)}% for ${Math.round((at - since) / 1000)} s`, detail),
|
||||
aboveSince: since
|
||||
}
|
||||
}
|
||||
return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since }
|
||||
}
|
||||
|
||||
/**
|
||||
* One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of
|
||||
* director 503s with the request series' publishing front (`publishedThrough`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this
|
||||
* cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read.
|
||||
*/
|
||||
export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
|
||||
const at = sample.at
|
||||
const next = { ...state, samples: state.samples + 1 }
|
||||
const { cell } = sample
|
||||
if (cell.ok && cell.admitModeConfigured !== 'reserve') {
|
||||
const seen = state.reserveSeenAt !== null
|
||||
if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) {
|
||||
// Not judged, and the streak holds: the reserve write may not have applied yet.
|
||||
const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) }
|
||||
return { state: next, rules, line: sampleLine(next, sample, rules) }
|
||||
}
|
||||
// Someone else flipped it back (or it never became reserve): nothing left to watch.
|
||||
next.state = seen ? 'stopped' : 'refused'
|
||||
next.verdict = seen
|
||||
? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})`
|
||||
: `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})`
|
||||
return { state: next, rules: {}, line: sampleLine(next, sample, {}) }
|
||||
}
|
||||
if (cell.ok) {
|
||||
next.reserveSeenAt = state.reserveSeenAt ?? at
|
||||
next.hosts = [...state.hosts, { at, controls: cell.controls, draining: cell.draining }].filter(
|
||||
(entry) => entry.at > at - RESERVE_WATCH_RULES.hostDrop.windowMs
|
||||
)
|
||||
}
|
||||
const duplicate = judgeDuplicateSeat(sample.seatBehindRow, state.seatBehindRowFirstSeen, at)
|
||||
next.seatBehindRowFirstSeen = duplicate.firstSeen
|
||||
const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at)
|
||||
next.ledgerAboveSince = ledger.aboveSince
|
||||
const rules = {
|
||||
deadMan: judgeDeadMan(cell),
|
||||
overrun: judgeOverrun(cell),
|
||||
director503: judgeDirector503(sample.director503, state, at),
|
||||
duplicateSeat: duplicate.verdict,
|
||||
hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error),
|
||||
ledgerDrift: ledger.verdict
|
||||
}
|
||||
const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip')
|
||||
if (tripped) {
|
||||
next.state = 'tripped'
|
||||
next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}`
|
||||
return { state: next, rules, line: sampleLine(next, sample, rules) }
|
||||
}
|
||||
const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable')
|
||||
const waiting = RULE_NAMES.some((name) => rules[name].status === 'pending')
|
||||
// A pending rule holds the streak where it was: neither evidence of health nor a gap.
|
||||
next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0
|
||||
if (next.unverifiableStreak >= unverifiableLimit) {
|
||||
next.state = 'tripped'
|
||||
next.verdict =
|
||||
`flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` +
|
||||
`read ${missing.join(', ')}`
|
||||
} else if (at >= state.deadlineAt) {
|
||||
next.state = 'completed'
|
||||
next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch`
|
||||
}
|
||||
return { state: next, rules, line: sampleLine(next, sample, rules) }
|
||||
}
|
||||
|
||||
function sampleLine(state, sample, rules) {
|
||||
return JSON.stringify({
|
||||
event: 'orca_relay_reserve_watch_sample',
|
||||
cellId: state.cellId,
|
||||
at: new Date(sample.at).toISOString(),
|
||||
sample: state.samples,
|
||||
state: state.state,
|
||||
unverifiableStreak: state.unverifiableStreak,
|
||||
rules,
|
||||
...(state.verdict ? { verdict: state.verdict } : {})
|
||||
})
|
||||
}
|
||||
@@ -122,7 +122,7 @@ async function retryingRead(reader, read) {
|
||||
return { failed: true, error: String(lastError?.message ?? lastError) }
|
||||
}
|
||||
|
||||
async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) {
|
||||
export async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) {
|
||||
const args = [
|
||||
'logging', 'read', filter,
|
||||
'--project', reader.projectId,
|
||||
@@ -154,21 +154,22 @@ async function readTimestampsOverWindow(reader, { filter, window }) {
|
||||
|
||||
// Cloud Run's own request counter, aligned per minute server-side: a brownout's tens of thousands
|
||||
// of 503s are counted, where a log read of them stops at its entry limit (10-01 c29).
|
||||
async function readDirector503PerMinute(reader, { config, span }) {
|
||||
async function readDirectorRequestCountSeries(reader, { config, span, filters = [], groupByFields }) {
|
||||
const url = new URL(`https://monitoring.googleapis.com/v3/projects/${config.projectId}/timeSeries`)
|
||||
url.searchParams.set('filter', [
|
||||
'metric.type="run.googleapis.com/request_count"',
|
||||
'resource.type="cloud_run_revision"',
|
||||
`resource.label."service_name"="${config.directorService}"`,
|
||||
'metric.label."response_code"="503"'
|
||||
...filters
|
||||
].join(' AND '))
|
||||
url.searchParams.set('interval.startTime', span.startedAt.toISOString())
|
||||
url.searchParams.set('interval.endTime', span.endedAt.toISOString())
|
||||
url.searchParams.set('aggregation.alignmentPeriod', '60s')
|
||||
url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_DELTA')
|
||||
url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_SUM')
|
||||
if (groupByFields) url.searchParams.set('aggregation.groupByFields', groupByFields)
|
||||
url.searchParams.set('pageSize', '1000')
|
||||
const read = await retryingRead(reader, async (timeoutMs) => {
|
||||
return await retryingRead(reader, async (timeoutMs) => {
|
||||
const { stdout } = await reader.runGcloud(['auth', 'print-access-token'], { timeoutMs })
|
||||
const response = await reader.fetch(url, {
|
||||
headers: { authorization: `Bearer ${stdout.trim()}` },
|
||||
@@ -179,17 +180,44 @@ async function readDirector503PerMinute(reader, { config, span }) {
|
||||
if (body.nextPageToken) throw new Error('Cloud Monitoring pagination is incomplete')
|
||||
return { timeSeries: body.timeSeries ?? [] }
|
||||
})
|
||||
const perMinute = {}
|
||||
for (const series of read.timeSeries ?? []) {
|
||||
for (const point of series.points ?? []) {
|
||||
// A delta point covers the minute that ends at its end time; a minute with no point had none.
|
||||
const minute = minuteKey(Date.parse(point.interval.endTime) - 60_000)
|
||||
perMinute[minute] = (perMinute[minute] ?? 0) + Number(point.value.int64Value ?? 0)
|
||||
}
|
||||
}
|
||||
|
||||
function addPointsPerMinute(perMinute, series) {
|
||||
for (const point of series.points ?? []) {
|
||||
// A delta point covers the minute that ends at its end time; a minute with no point had none.
|
||||
const minute = minuteKey(Date.parse(point.interval.endTime) - 60_000)
|
||||
perMinute[minute] = (perMinute[minute] ?? 0) + Number(point.value.int64Value ?? 0)
|
||||
}
|
||||
return perMinute
|
||||
}
|
||||
|
||||
export async function readDirector503PerMinute(reader, { config, span }) {
|
||||
const read = await readDirectorRequestCountSeries(reader, {
|
||||
config,
|
||||
span,
|
||||
filters: ['metric.label."response_code"="503"']
|
||||
})
|
||||
const perMinute = {}
|
||||
for (const series of read.timeSeries ?? []) addPointsPerMinute(perMinute, series)
|
||||
return { perMinute, failed: read.failed }
|
||||
}
|
||||
|
||||
// Every response code in one read, so a minute missing from the 503 series can be told apart from
|
||||
// one not published yet: the other codes' series carry the same publishing front.
|
||||
export async function readDirectorRequestsByCodePerMinute(reader, { config, span }) {
|
||||
const read = await readDirectorRequestCountSeries(reader, {
|
||||
config,
|
||||
span,
|
||||
groupByFields: 'metric.label."response_code"'
|
||||
})
|
||||
const byCode = {}
|
||||
for (const series of read.timeSeries ?? []) {
|
||||
const code = series.metric?.labels?.response_code ?? 'unknown'
|
||||
byCode[code] = addPointsPerMinute(byCode[code] ?? {}, series)
|
||||
}
|
||||
return { byCode, failed: read.failed }
|
||||
}
|
||||
|
||||
// Cells log through the COS container agent, so the text lives in jsonPayload.message; a
|
||||
// textPayload filter matches nothing here and returns zero without saying so.
|
||||
const CELL_LOG_SCOPE = 'resource.type="gce_instance" AND logName:"cos_containers"'
|
||||
@@ -419,8 +447,8 @@ export async function evaluateShadowGate(config, {
|
||||
appliedPaceWindowMs: config.drainAppliedPaceWindowMs,
|
||||
targetHosts: config.targetHosts,
|
||||
settledAt: config.drainSettledAt || null,
|
||||
// Isolate to restart-safe, which includes the quiet the restart wait holds after the cell
|
||||
// empties: (ceil(pace / 5 s) + 1) samples of 5 s.
|
||||
// Isolate to restart-safe: the later of the pace window's end and the cell emptying, plus
|
||||
// one 5 s sample.
|
||||
settledAfterSeconds: config.drainStartedAt && config.drainSettledAt
|
||||
? Math.round((Date.parse(config.drainSettledAt) - Date.parse(config.drainStartedAt)) / 1000)
|
||||
: null
|
||||
|
||||
@@ -42,9 +42,9 @@ const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c
|
||||
const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0'
|
||||
// C31 launches on the digest the director served when it was declared.
|
||||
const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269'
|
||||
// Staging C4 is declared on image F (main 3c1141f2, the six-cell dead-cell rerun); the staging
|
||||
// Staging C4 is declared on image G (main 2de73812, E-pre with the t_preauth metric); the staging
|
||||
// director deploy binds its digest to this line.
|
||||
const stagingC4Digest = '41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe'
|
||||
const stagingC4Digest = '07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d'
|
||||
|
||||
function cellBlock(tfvars, cellId) {
|
||||
const start = tfvars.indexOf(`"${cellId}"`)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import {
|
||||
forceTerminateProcessTree,
|
||||
signalProcessTree
|
||||
} from '@orca/process-host/process-tree-termination'
|
||||
import { mkdtempSync, mkdirSync, realpathSync, rmSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { availableParallelism, tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const vitest = join(dirname(require.resolve('vitest/package.json')), 'vitest.mjs')
|
||||
const relayDirectory = fileURLToPath(new URL('../../apps/relay', import.meta.url))
|
||||
|
||||
export async function runRelayTests({
|
||||
args = [],
|
||||
cwd = relayDirectory,
|
||||
env = process.env,
|
||||
parallelism = availableParallelism(),
|
||||
cli = vitest,
|
||||
run = runProcess,
|
||||
events = process,
|
||||
report = console.error,
|
||||
scratchParent = tmpdir()
|
||||
} = {}) {
|
||||
const workerBudget = Math.max(parallelism - 1, 1)
|
||||
// Explicit selectors and alternate database fixtures retain their existing execution path.
|
||||
const serial =
|
||||
args.length > 0 || workerBudget <= 1 || env.ORCA_IDLE_REHOME_POSTGRES_URL || env.DATABASE_URL
|
||||
const roles = serial
|
||||
? [{ name: 'all', args }]
|
||||
: [
|
||||
{ name: 'relay', args: ['--project=relay', `--maxWorkers=${workerBudget - 1}`] },
|
||||
{ name: 'relay-postgres', args: ['--project=relay-postgres', '--maxWorkers=1'] }
|
||||
]
|
||||
const scratch = realpathSync(mkdtempSync(join(scratchParent, 'orca-relay-tests-')))
|
||||
const abort = new AbortController()
|
||||
const closed = new Set()
|
||||
const terminatedTrees = new Set()
|
||||
let signalExitCode
|
||||
const onInterrupt = () => {
|
||||
signalExitCode ??= 130
|
||||
abort.abort()
|
||||
}
|
||||
const onTerminate = () => {
|
||||
signalExitCode ??= 143
|
||||
abort.abort()
|
||||
}
|
||||
events.on('SIGINT', onInterrupt)
|
||||
events.on('SIGTERM', onTerminate)
|
||||
|
||||
async function runRole(role) {
|
||||
const owned = join(scratch, role.name)
|
||||
try {
|
||||
mkdirSync(owned)
|
||||
return await run({
|
||||
program: process.execPath,
|
||||
args: [cli, 'run', ...role.args],
|
||||
cwd,
|
||||
env: {
|
||||
...env,
|
||||
TMPDIR: owned,
|
||||
TMP: owned,
|
||||
TEMP: owned,
|
||||
ORCA_BACKGROUND_LAUNCH: '1',
|
||||
ORCA_RELAY_TEST_CACHE_DIR: join(owned, 'cache')
|
||||
},
|
||||
stdio: 'inherit',
|
||||
timeoutMs: null,
|
||||
signal: abort.signal,
|
||||
terminationBarrier: {
|
||||
async signal(child, signal) {
|
||||
const terminated = await signalProcessTree(child, signal)
|
||||
if (process.platform === 'win32' && terminated) {
|
||||
terminatedTrees.add(role.name)
|
||||
}
|
||||
return terminated
|
||||
},
|
||||
async force(child) {
|
||||
const terminated = await forceTerminateProcessTree(child)
|
||||
if (terminated) {
|
||||
terminatedTrees.add(role.name)
|
||||
}
|
||||
return terminated
|
||||
}
|
||||
},
|
||||
onChildTerminated: () => closed.add(role.name)
|
||||
})
|
||||
} catch (error) {
|
||||
abort.abort()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
let exitCode = 0
|
||||
try {
|
||||
const results = await Promise.allSettled(roles.map(runRole))
|
||||
for (const result of results) {
|
||||
if (result.status === 'rejected') {
|
||||
report(result.reason)
|
||||
exitCode ||= 1
|
||||
} else if (result.value.code !== 0 || result.value.signal || result.value.timedOut) {
|
||||
exitCode ||= result.value.code || 1
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
events.removeListener('SIGINT', onInterrupt)
|
||||
events.removeListener('SIGTERM', onTerminate)
|
||||
// A cancelled root's close event alone does not prove its descendants stopped using scratch.
|
||||
if (
|
||||
closed.size === roles.length &&
|
||||
(!abort.signal.aborted || terminatedTrees.size === roles.length)
|
||||
) {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
} else {
|
||||
report(`Relay test process closure is unverifiable; retained ${scratch}`)
|
||||
exitCode ||= 1
|
||||
}
|
||||
}
|
||||
return signalExitCode ?? exitCode
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
process.exitCode = await runRelayTests({ args: process.argv.slice(2) })
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { setImmediate } from 'node:timers/promises'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import test from 'node:test'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { runRelayTests } from './run-relay-tests.mjs'
|
||||
|
||||
const success = { code: 0, signal: null, timedOut: false }
|
||||
const databaseUrl = 'postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test'
|
||||
const require = createRequire(import.meta.url)
|
||||
const cloud = resolve(import.meta.dirname, '../..')
|
||||
|
||||
function temporary(t) {
|
||||
const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca-relay-runner-contract-')))
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }))
|
||||
return directory
|
||||
}
|
||||
|
||||
function start(t, options = {}) {
|
||||
const scratchParent = temporary(t)
|
||||
const calls = []
|
||||
const reports = []
|
||||
const events = new EventEmitter()
|
||||
const env = { ORCA_RELAY_TEST_POSTGRES_URL: databaseUrl, PRESERVED: 'yes', ...options.env }
|
||||
const pending = runRelayTests({
|
||||
cwd: join(scratchParent, 'relay'),
|
||||
cli: join(scratchParent, 'vitest.mjs'),
|
||||
parallelism: 4,
|
||||
...options,
|
||||
env,
|
||||
events,
|
||||
scratchParent,
|
||||
report: (message) => reports.push(message),
|
||||
run: (spec) => {
|
||||
const deferred = Promise.withResolvers()
|
||||
calls.push({ spec, ...deferred })
|
||||
return deferred.promise
|
||||
}
|
||||
})
|
||||
return { pending, calls, reports, events, scratchParent, env }
|
||||
}
|
||||
|
||||
function finish(call, result = success, closed = true) {
|
||||
if (closed) {
|
||||
call.spec.onChildTerminated()
|
||||
}
|
||||
call.resolve(result)
|
||||
}
|
||||
|
||||
function assertClean(harness) {
|
||||
assert.deepEqual(readdirSync(harness.scratchParent), [])
|
||||
assert.equal(harness.events.listenerCount('SIGINT'), 0)
|
||||
assert.equal(harness.events.listenerCount('SIGTERM'), 0)
|
||||
}
|
||||
|
||||
test('overlaps both projects within the worker budget and isolates their scratch and caches', async (t) => {
|
||||
const h = start(t)
|
||||
assert.equal(h.calls.length, 2)
|
||||
const [regular, postgres] = h.calls.map((call) => call.spec)
|
||||
assert.deepEqual(regular.args.slice(1), ['run', '--project=relay', '--maxWorkers=2'])
|
||||
assert.deepEqual(postgres.args.slice(1), ['run', '--project=relay-postgres', '--maxWorkers=1'])
|
||||
assert.notEqual(regular.env.TMPDIR, postgres.env.TMPDIR)
|
||||
assert.notEqual(regular.env.ORCA_RELAY_TEST_CACHE_DIR, postgres.env.ORCA_RELAY_TEST_CACHE_DIR)
|
||||
for (const spec of [regular, postgres]) {
|
||||
assert.equal(spec.env.TMP, spec.env.TMPDIR)
|
||||
assert.equal(spec.env.TEMP, spec.env.TMPDIR)
|
||||
assert.equal(spec.env.ORCA_RELAY_TEST_CACHE_DIR, join(spec.env.TMPDIR, 'cache'))
|
||||
assert.equal(spec.env.ORCA_RELAY_TEST_POSTGRES_URL, databaseUrl)
|
||||
assert.equal(spec.env.PRESERVED, 'yes')
|
||||
assert.equal(spec.env.ORCA_BACKGROUND_LAUNCH, '1')
|
||||
assert.ok(existsSync(spec.env.TMPDIR))
|
||||
assert.equal(spec.timeoutMs, null)
|
||||
assert.equal(typeof spec.terminationBarrier.force, 'function')
|
||||
}
|
||||
assert.equal(regular.signal, postgres.signal)
|
||||
let settled = false
|
||||
h.pending.then(() => {
|
||||
settled = true
|
||||
})
|
||||
finish(h.calls[0], { ...success, code: 7 })
|
||||
await setImmediate()
|
||||
assert.equal(settled, false)
|
||||
assert.equal(postgres.signal.aborted, false)
|
||||
assert.ok(existsSync(postgres.env.TMPDIR))
|
||||
finish(h.calls[1])
|
||||
assert.equal(await h.pending, 7)
|
||||
assertClean(h)
|
||||
})
|
||||
|
||||
for (const [name, options] of [
|
||||
['selectors', { args: ['src/assignment.test.ts', '-t', 'requested behavior'] }],
|
||||
[
|
||||
'alternate PostgreSQL',
|
||||
{ env: { ORCA_IDLE_REHOME_POSTGRES_URL: 'postgres://localhost:55440/relay' } }
|
||||
],
|
||||
['application database', { env: { DATABASE_URL: databaseUrl } }],
|
||||
['one CPU', { parallelism: 1 }],
|
||||
['two CPUs', { parallelism: 2 }]
|
||||
]) {
|
||||
test(`retains the original single run for ${name}`, async (t) => {
|
||||
const h = start(t, options)
|
||||
assert.equal(h.calls.length, 1)
|
||||
const { spec } = h.calls[0]
|
||||
assert.deepEqual(spec.args.slice(1), ['run', ...(options.args ?? [])])
|
||||
for (const [key, value] of Object.entries(h.env)) {
|
||||
assert.equal(spec.env[key], value)
|
||||
}
|
||||
finish(h.calls[0])
|
||||
assert.equal(await h.pending, 0)
|
||||
assertClean(h)
|
||||
})
|
||||
}
|
||||
|
||||
for (const [name, result] of [
|
||||
['null exit code', { ...success, code: null }],
|
||||
['termination signal', { ...success, signal: 'SIGTERM' }],
|
||||
['timeout', { ...success, timedOut: true }]
|
||||
]) {
|
||||
test(`fails a ${name} even when other result fields indicate success`, async (t) => {
|
||||
const h = start(t)
|
||||
finish(h.calls[0], result)
|
||||
finish(h.calls[1])
|
||||
assert.equal(await h.pending, 1)
|
||||
assertClean(h)
|
||||
})
|
||||
}
|
||||
|
||||
test('fails and retains scratch when a child never reports closure', async (t) => {
|
||||
const h = start(t)
|
||||
finish(h.calls[0], success, false)
|
||||
finish(h.calls[1])
|
||||
assert.equal(await h.pending, 1)
|
||||
assert.ok(existsSync(h.calls[0].spec.env.TMPDIR))
|
||||
assert.match(String(h.reports[0]), /closure is unverifiable; retained/)
|
||||
assert.equal(h.events.listenerCount('SIGINT'), 0)
|
||||
assert.equal(h.events.listenerCount('SIGTERM'), 0)
|
||||
})
|
||||
|
||||
for (const [signal, exitCode, laterSignal] of [
|
||||
['SIGINT', 130, 'SIGTERM'],
|
||||
['SIGTERM', 143, 'SIGINT']
|
||||
]) {
|
||||
test(`${signal} cancels every child and retains scratch despite root close callbacks`, async (t) => {
|
||||
const h = start(t)
|
||||
h.events.emit(signal)
|
||||
h.events.emit(laterSignal)
|
||||
for (const call of h.calls) {
|
||||
assert.equal(call.spec.signal.aborted, true)
|
||||
finish(call)
|
||||
}
|
||||
assert.equal(await h.pending, exitCode)
|
||||
for (const call of h.calls) {
|
||||
assert.ok(existsSync(call.spec.env.TMPDIR))
|
||||
}
|
||||
assert.match(String(h.reports[0]), /closure is unverifiable; retained/)
|
||||
assert.equal(h.events.listenerCount('SIGINT'), 0)
|
||||
assert.equal(h.events.listenerCount('SIGTERM'), 0)
|
||||
})
|
||||
}
|
||||
|
||||
for (const verified of [1, 2]) {
|
||||
test(`requires verification of both cancelled trees (${verified} verified)`, async (t) => {
|
||||
const h = start(t)
|
||||
h.events.emit('SIGINT')
|
||||
for (const call of h.calls.slice(0, verified)) {
|
||||
// An unspawned handle proves absence without signalling a real process.
|
||||
assert.equal(await call.spec.terminationBarrier.force({ kill: () => true }), true)
|
||||
}
|
||||
for (const call of h.calls) {
|
||||
finish(call)
|
||||
}
|
||||
assert.equal(await h.pending, 130)
|
||||
if (verified === 2) {
|
||||
assertClean(h)
|
||||
} else {
|
||||
assert.ok(existsSync(h.calls[0].spec.env.TMPDIR))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('reports a spawn failure and waits for its cancelled peer before returning', async (t) => {
|
||||
const h = start(t)
|
||||
const failure = new Error('fixture spawn failed')
|
||||
h.calls[0].reject(failure)
|
||||
let settled = false
|
||||
h.pending.then(() => {
|
||||
settled = true
|
||||
})
|
||||
await setImmediate()
|
||||
assert.equal(h.calls[1].spec.signal.aborted, true)
|
||||
assert.equal(settled, false)
|
||||
finish(h.calls[1])
|
||||
assert.equal(await h.pending, 1)
|
||||
assert.equal(h.reports[0], failure)
|
||||
assert.ok(existsSync(h.calls[1].spec.env.TMPDIR))
|
||||
})
|
||||
|
||||
test('launches with only the Cloud dependency link and no root install or compiled host package', async (t) => {
|
||||
const root = temporary(t)
|
||||
const standalone = join(root, 'cloud')
|
||||
const manifest = JSON.parse(readFileSync(join(cloud, 'package.json'), 'utf8'))
|
||||
const dependency = manifest.devDependencies['@orca/process-host']
|
||||
assert.equal(dependency, 'link:../src/packages/process-host')
|
||||
const packageDir = resolve(standalone, dependency.slice('link:'.length))
|
||||
mkdirSync(packageDir, { recursive: true })
|
||||
for (const file of ['package.json', 'src']) {
|
||||
cpSync(join(cloud, '..', 'src', 'packages', 'process-host', file), join(packageDir, file), {
|
||||
recursive: true
|
||||
})
|
||||
}
|
||||
const scripts = join(standalone, 'dev', 'scripts')
|
||||
mkdirSync(scripts, { recursive: true })
|
||||
cpSync(join(cloud, 'dev', 'scripts', 'run-relay-tests.mjs'), join(scripts, 'run-relay-tests.mjs'))
|
||||
writeFileSync(join(standalone, 'package.json'), JSON.stringify(manifest))
|
||||
mkdirSync(join(standalone, 'apps', 'relay'), { recursive: true })
|
||||
mkdirSync(join(standalone, 'node_modules', '@orca'), { recursive: true })
|
||||
symlinkSync(
|
||||
packageDir,
|
||||
join(standalone, 'node_modules', '@orca', 'process-host'),
|
||||
process.platform === 'win32' ? 'junction' : 'dir'
|
||||
)
|
||||
const fakeVitest = join(standalone, 'node_modules', 'vitest')
|
||||
mkdirSync(fakeVitest)
|
||||
writeFileSync(join(fakeVitest, 'package.json'), '{"name":"vitest","version":"0.0.0"}')
|
||||
writeFileSync(
|
||||
join(fakeVitest, 'vitest.mjs'),
|
||||
'console.log(JSON.stringify({args:process.argv.slice(2), background:process.env.ORCA_BACKGROUND_LAUNCH, database:process.env.ORCA_RELAY_TEST_POSTGRES_URL}))'
|
||||
)
|
||||
assert.equal(existsSync(join(root, 'node_modules')), false)
|
||||
assert.equal(existsSync(join(packageDir, 'dist')), false)
|
||||
const result = await runProcess({
|
||||
program: process.execPath,
|
||||
args: [
|
||||
'--import',
|
||||
pathToFileURL(require.resolve('tsx')).href,
|
||||
'--conditions=orca-source',
|
||||
join(scripts, 'run-relay-tests.mjs'),
|
||||
'src/requested.test.ts'
|
||||
],
|
||||
cwd: standalone,
|
||||
timeoutMs: 15_000,
|
||||
terminationBarrier: true,
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_PATH: '',
|
||||
NODE_OPTIONS: '',
|
||||
ORCA_BACKGROUND_LAUNCH: '1',
|
||||
ORCA_RELAY_TEST_POSTGRES_URL: databaseUrl
|
||||
}
|
||||
})
|
||||
assert.equal(result.code, 0, result.stderr)
|
||||
assert.deepEqual(JSON.parse(result.stdout), {
|
||||
args: ['run', 'src/requested.test.ts'],
|
||||
background: '1',
|
||||
database: databaseUrl
|
||||
})
|
||||
assert.equal(existsSync(join(packageDir, 'dist')), false)
|
||||
})
|
||||
@@ -420,14 +420,17 @@ export async function verifyCapacityTransition(config, overrides = {}) {
|
||||
if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) {
|
||||
throw new Error('director is not capacity-protocol compatible')
|
||||
}
|
||||
const deadline = now() + config.timeoutMs
|
||||
// A paced drain can refill the runtime until its window ends, so the quiet must outlast it.
|
||||
// Each poll waits a full interval, so consecutive samples bound the quiet time from below.
|
||||
const requiredRestartSafeSamples =
|
||||
Math.max(Math.ceil((config.paceWindowMs ?? 0) / POLL_INTERVAL_MS) + 1, 2)
|
||||
const startedAt = now()
|
||||
const deadline = startedAt + config.timeoutMs
|
||||
// A paced drain can refill the runtime until its window ends, so the second of two consecutive
|
||||
// quiet samples must be taken at or after that end. This runs after the cell accepted the drain,
|
||||
// so the cell's window ends no later than this one.
|
||||
const paceWindowEndsAt = startedAt + (config.paceWindowMs ?? 0)
|
||||
let restartSafeSamples = 0
|
||||
let lastObservation = { runtimeAvailable: false }
|
||||
for (;;) {
|
||||
// Before the read, so a sample taken inside the window cannot pass.
|
||||
const sampledAt = now()
|
||||
const runtime = await cellRuntime(fetchImpl, config, token)
|
||||
lastObservation = { runtimeAvailable: runtime !== null }
|
||||
if ((runtime === null) === (config.runtime === 'unavailable')) {
|
||||
@@ -469,7 +472,7 @@ export async function verifyCapacityTransition(config, overrides = {}) {
|
||||
if (
|
||||
matches &&
|
||||
(config.activity !== 'restart-safe' ||
|
||||
restartSafeSamples + 1 >= requiredRestartSafeSamples)
|
||||
(restartSafeSamples >= 1 && sampledAt >= paceWindowEndsAt))
|
||||
) {
|
||||
return {
|
||||
cellId: status.cellId,
|
||||
@@ -491,15 +494,16 @@ export async function verifyCapacityTransition(config, overrides = {}) {
|
||||
restartSafeSamples = 0
|
||||
}
|
||||
if (config.activity === 'restart-safe') {
|
||||
const paceWindowRemainingMs = Math.max(paceWindowEndsAt - sampledAt, 0)
|
||||
lastObservation = {
|
||||
...lastObservation,
|
||||
restartSafeSamples,
|
||||
requiredRestartSafeSamples
|
||||
paceWindowRemainingMs
|
||||
}
|
||||
progress({
|
||||
event: 'relay_capacity_transition_restart_progress',
|
||||
restartSafeSamples,
|
||||
requiredRestartSafeSamples,
|
||||
paceWindowRemainingMs,
|
||||
totalConnections: lastObservation.runtime?.totalConnections ?? null,
|
||||
preAuthConnections: lastObservation.runtime?.preAuthConnections ?? null,
|
||||
runtimeAvailable: lastObservation.runtimeAvailable,
|
||||
|
||||
@@ -856,8 +856,7 @@ test('restart timeout reports one of two safe settling samples', async () => {
|
||||
{ fetch: base, token: 'masked-token' }
|
||||
),
|
||||
(error) => {
|
||||
assert.match(error.message, /"restartSafeSamples":1/)
|
||||
assert.match(error.message, /"requiredRestartSafeSamples":2/)
|
||||
assert.match(error.message, /"restartSafeSamples":1,"paceWindowRemainingMs":0/)
|
||||
return true
|
||||
}
|
||||
)
|
||||
@@ -1177,8 +1176,8 @@ const pacedRestartConfig = {
|
||||
timeoutMs: 1_200_000
|
||||
}
|
||||
|
||||
function fakeClock() {
|
||||
let now = 0
|
||||
function fakeClock(start = 0) {
|
||||
let now = start
|
||||
return {
|
||||
now: () => now,
|
||||
wait: async (milliseconds) => {
|
||||
@@ -1231,7 +1230,8 @@ test('passes after one pace window of an empty runtime and reports stranded leas
|
||||
},
|
||||
token: 'masked-token',
|
||||
progress: (line) => lines.push(line),
|
||||
...fakeClock()
|
||||
// A late start catches a window measured from zero instead of from the verifier's start.
|
||||
...fakeClock(1_000_000)
|
||||
})
|
||||
// One sample at each end of a full five-minute pace window.
|
||||
assert.equal(runtimeReads, 61)
|
||||
@@ -1246,12 +1246,15 @@ test('passes after one pace window of an empty runtime and reports stranded leas
|
||||
assert.equal(lines.length, 60)
|
||||
assert.ok(lines.every((line) =>
|
||||
line.event === 'relay_capacity_transition_restart_progress' &&
|
||||
line.requiredRestartSafeSamples === 61 &&
|
||||
line.stranded.restartBlockingActivityLeases === 4))
|
||||
assert.deepEqual(lines.slice(0, 2).map((line) => line.restartSafeSamples), [1, 2])
|
||||
assert.deepEqual(
|
||||
lines.slice(0, 2).map((line) => line.paceWindowRemainingMs),
|
||||
[300_000, 295_000]
|
||||
)
|
||||
})
|
||||
|
||||
test('the pace window resets on any non-zero runtime sample', async () => {
|
||||
test('a non-zero runtime sample restarts the quiet count', async () => {
|
||||
const quiet = harness(strandedCell)
|
||||
const busy = harness({ ...strandedCell, controls: 1 })
|
||||
let runtimeReads = 0
|
||||
@@ -1265,17 +1268,37 @@ test('the pace window resets on any non-zero runtime sample', async () => {
|
||||
progress: (line) => lines.push(line),
|
||||
...fakeClock()
|
||||
})
|
||||
assert.equal(runtimeReads, 91)
|
||||
// A mid-window reset adds no time; the quiet after it still reaches the window's end.
|
||||
assert.equal(runtimeReads, 61)
|
||||
assert.deepEqual(lines.slice(28, 31).map((line) => line.restartSafeSamples), [29, 0, 1])
|
||||
await assert.rejects(
|
||||
verifyCapacityTransition(
|
||||
{ ...pacedRestartConfig, timeoutMs: 295_000 },
|
||||
{ fetch: quiet, token: 'masked-token', progress: () => {}, ...fakeClock() }
|
||||
),
|
||||
/"restartSafeSamples":60,"requiredRestartSafeSamples":61/
|
||||
/"restartSafeSamples":60,"paceWindowRemainingMs":5000/
|
||||
)
|
||||
})
|
||||
|
||||
// c27 on 2026-10-13: the 20-minute paced drain emptied the runtime as its window closed, and the
|
||||
// old gate then waited a second full window of quiet samples before the restart.
|
||||
test('a runtime that empties as the pace window ends passes after two quiet samples', async () => {
|
||||
let runtimeReads = 0
|
||||
const lines = []
|
||||
await verifyCapacityTransition(pacedRestartConfig, {
|
||||
fetch: async (url, options) => {
|
||||
if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1
|
||||
return await harness({ ...strandedCell, controls: runtimeReads <= 60 ? 1 : 0 })(url, options)
|
||||
},
|
||||
token: 'masked-token',
|
||||
progress: (line) => lines.push(line),
|
||||
...fakeClock()
|
||||
})
|
||||
// Busy through 295 s; quiet at 300 s and 305 s, the first eligible pair.
|
||||
assert.equal(runtimeReads, 62)
|
||||
assert.deepEqual(lines.slice(59).map((line) => line.restartSafeSamples), [0, 1])
|
||||
})
|
||||
|
||||
test('restart-safe fails on live runtime work or an open migration regardless of leases', async (t) => {
|
||||
for (const blocker of [
|
||||
{ controls: 1 },
|
||||
@@ -1299,7 +1322,7 @@ test('restart-safe fails on live runtime work or an open migration regardless of
|
||||
}
|
||||
})
|
||||
|
||||
test('unauthenticated redials do not reset the pace window', async () => {
|
||||
test('unauthenticated redials do not restart the quiet count', async () => {
|
||||
let runtimeReads = 0
|
||||
const lines = []
|
||||
const result = await verifyCapacityTransition(pacedRestartConfig, {
|
||||
@@ -1439,7 +1462,7 @@ test('restart progress prints every counter the gate reads', async () => {
|
||||
assert.deepEqual(lines[0], {
|
||||
event: 'relay_capacity_transition_restart_progress',
|
||||
restartSafeSamples: 0,
|
||||
requiredRestartSafeSamples: 61,
|
||||
paceWindowRemainingMs: 300_000,
|
||||
totalConnections: 7,
|
||||
preAuthConnections: 1,
|
||||
runtimeAvailable: true,
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
#!/usr/bin/env node
|
||||
// Watches one cell already in reserve mode and, on the first stop rule it trips, flips it back to
|
||||
// db through the flag tool's own write (CAS, read-back, re-registration wait). Step 5 design §9.
|
||||
//
|
||||
// watch: one segment of the watch (the workflow re-authenticates between segments, since an
|
||||
// ID token lasts an hour). State carries over in --state-file. Writes nothing.
|
||||
// outcome: the job's state and verdict, from the state file.
|
||||
// flip-back: the flag tool's admitMode=db write (or its dry run), then exits non-zero with the
|
||||
// watch's verdict. Why it holds no flag-workflow group: docs/relay-workflows.md.
|
||||
// alarm: a refused watch or a flip back that did not land: an error annotation, and in write
|
||||
// mode a log line the reserve-watch alert pages on.
|
||||
import { execFile } from 'node:child_process'
|
||||
import { appendFile, readFile, writeFile } from 'node:fs/promises'
|
||||
import { parseArgs, promisify } from 'node:util'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
BREAK_GLASS_CONFIRMATION,
|
||||
operateCellFlags,
|
||||
parseCellFlagsRequest,
|
||||
readCurrentObject,
|
||||
readRuntime,
|
||||
WRITE_CONFIRMATION
|
||||
} from './operate-relay-cell-flags.mjs'
|
||||
import { readDirectorRequestsByCodePerMinute, readLogEntries } from './relay-same-cap-shadow-gate.mjs'
|
||||
import { minutesOf } from './relay-same-cap-shadow-gate-verdict.mjs'
|
||||
import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs'
|
||||
import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs'
|
||||
import {
|
||||
evaluateWatchSample,
|
||||
initialWatchState,
|
||||
measuredBackground,
|
||||
RESERVE_WATCH_RULES
|
||||
} from './relay-reserve-watch-rules.mjs'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const MINUTE_MS = 60_000
|
||||
export const SAMPLE_INTERVAL_MS = MINUTE_MS
|
||||
export const MAX_WATCH_MINUTES = 300
|
||||
// Below the hour an ID token lasts, with room for the segment's last sample.
|
||||
export const MAX_SEGMENT_MINUTES = 55
|
||||
const METRIC_DELAY_MS = RESERVE_WATCH_RULES.director503.publishDelayMs
|
||||
const BACKGROUND_MINUTES = 10
|
||||
// A flipped-at older than this is a typo or a stale dispatch, not this flip.
|
||||
const MAX_FLIPPED_AT_AGE_MS = 24 * 60 * MINUTE_MS
|
||||
export const FLIP_BACK_ATTEMPTS = 5
|
||||
const FLIP_BACK_BACKOFF_MS = 2_000
|
||||
// One slow runtime-status reply is not a dead cell: retried this often before break glass.
|
||||
export const FLIP_BACK_RUNTIME_ATTEMPTS = 4
|
||||
const SAMPLE_READ_BUDGET_MS = 40_000
|
||||
const LOG_ENTRY_LIMIT = 1_000
|
||||
|
||||
function integerIn(name, raw, min, max) {
|
||||
const value = Number(raw)
|
||||
if (!/^[0-9]+$/.test(raw ?? '') || value < min || value > max) {
|
||||
throw new Error(`--${name} must be an integer from ${min} to ${max}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// The flip back's request. Its generation is a placeholder: the flip back re-reads the current one.
|
||||
export function dbFlipRequest(values) {
|
||||
return parseCellFlagsRequest({ ...values, set: 'admitMode=db', 'expected-generation': '0' })
|
||||
}
|
||||
|
||||
// `now` is the watch's start: a later segment of a long watch must not age flipped-at out.
|
||||
function parseFlippedAt(raw, now) {
|
||||
if (raw === '') return null
|
||||
const at = Date.parse(raw)
|
||||
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,3})?)?(Z|[+-]\d{2}:\d{2})$/.test(raw) || !Number.isFinite(at)) {
|
||||
throw new Error('--flipped-at must be an ISO time with a zone, e.g. 2026-10-14T12:00:00Z')
|
||||
}
|
||||
if (at > now) throw new Error('--flipped-at is in the future')
|
||||
if (at < now - MAX_FLIPPED_AT_AGE_MS) throw new Error('--flipped-at is more than 24 hours ago')
|
||||
return at
|
||||
}
|
||||
|
||||
export function parseWatchArguments(values, now = Date.now()) {
|
||||
// The flag tool's own checks, so the watch is of exactly the cell the flip back would write;
|
||||
// a write-mode watch refuses a missing confirmation now, not hours later.
|
||||
const request = dbFlipRequest(values)
|
||||
if (!/^[a-z][a-z0-9-]{0,62}$/.test(values['director-service'] ?? '')) {
|
||||
throw new Error('--director-service is not a Cloud Run service name')
|
||||
}
|
||||
const background = values['background-503-per-minute'] ?? ''
|
||||
if (background !== '' && !/^(0|[1-9][0-9]{0,6})(\.[0-9]+)?$/.test(background)) {
|
||||
throw new Error('--background-503-per-minute must be empty (measure it) or a non-negative number')
|
||||
}
|
||||
// The background must predate the flip: a watch starts after it, so its own past is not one.
|
||||
const flippedAt = parseFlippedAt(values['flipped-at'] ?? '', now)
|
||||
if (background === '' && flippedAt === null) {
|
||||
throw new Error('--flipped-at (the background is the 10 minutes before it) or --background-503-per-minute is required')
|
||||
}
|
||||
if (!values['state-file']) throw new Error('--state-file is required')
|
||||
return {
|
||||
request,
|
||||
directorService: values['director-service'],
|
||||
durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES),
|
||||
segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES),
|
||||
unverifiableLimit: integerIn('unverifiable-limit', values['unverifiable-limit'], 1, 30),
|
||||
background503PerMinute: background === '' ? null : Number(background),
|
||||
flippedAt,
|
||||
stateFile: values['state-file'],
|
||||
githubOutput: values['github-output'] || null
|
||||
}
|
||||
}
|
||||
|
||||
const failedRead = (error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) })
|
||||
|
||||
export async function readCellSample(fetchImpl, request, idToken, wait) {
|
||||
try {
|
||||
const runtime = await readRuntime(fetchImpl, request, idToken, wait)
|
||||
if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) {
|
||||
return failedRead(`origin answers as ${runtime.role}/${runtime.cellId}`)
|
||||
}
|
||||
if (!runtime.flagsApplied) return failedRead('runtime-status reports no flagsApplied')
|
||||
const controls = runtime.runtime?.controls
|
||||
if (!Number.isInteger(controls)) return failedRead('runtime-status reports no control count')
|
||||
return {
|
||||
ok: true,
|
||||
admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db',
|
||||
admitModeEffective: runtime.admitModeEffective,
|
||||
draining: runtime.draining === true,
|
||||
controls,
|
||||
enforcedUnits: runtime.runtime?.enforcedConnectionUnits,
|
||||
hardCap: runtime.connectionCapacity?.hardCap
|
||||
}
|
||||
} catch (error) {
|
||||
return failedRead(error)
|
||||
}
|
||||
}
|
||||
|
||||
const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS
|
||||
const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null })
|
||||
|
||||
// The `minutes` whole minutes that end at the minute boundary `endedAt`: the 503 counts, and the
|
||||
// newest minute any response code has a point for. The director always serves something, so that
|
||||
// front is how far the 503 series is published; no front at all is no evidence. A code publishing
|
||||
// late only delays a trip: a later read sees its points.
|
||||
export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes }) {
|
||||
const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) }
|
||||
let read
|
||||
try {
|
||||
read = await readDirectorRequestsByCodePerMinute(reader, { config: { projectId, directorService }, span })
|
||||
} catch (error) {
|
||||
// A point with an unparseable end time has no minute.
|
||||
return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`)
|
||||
}
|
||||
if (read.failed) return failed503Read('director 503 metric read failed')
|
||||
const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort()
|
||||
return {
|
||||
ok: true,
|
||||
perMinute: read.byCode['503'] ?? {},
|
||||
minutes: minutesOf(span),
|
||||
publishedThrough: published.at(-1) ?? null
|
||||
}
|
||||
}
|
||||
|
||||
const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary'
|
||||
const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row'
|
||||
|
||||
function directorLogFilter(directorService, eventClause, sinceAt) {
|
||||
return [
|
||||
'resource.type="cloud_run_revision"',
|
||||
`resource.labels.service_name="${directorService}"`,
|
||||
eventClause,
|
||||
`timestamp>="${new Date(sinceAt).toISOString()}"`
|
||||
].join(' AND ')
|
||||
}
|
||||
|
||||
// Newest summary per director instance; queued plus dropped rows are the ones not in Postgres.
|
||||
export async function readLedgerSample(reader, { directorService, at }) {
|
||||
const read = await readLogEntries(reader, {
|
||||
filter: directorLogFilter(directorService, `jsonPayload.event="${LEDGER_SUMMARY_EVENT}"`, at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs),
|
||||
projection: 'json(timestamp,labels.instanceId,jsonPayload.pending,jsonPayload.dropped)',
|
||||
limit: LOG_ENTRY_LIMIT
|
||||
})
|
||||
if (read.failed) return failedRead(read.error)
|
||||
if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('ledger summary read truncated')
|
||||
if (read.entries.length === 0) return failedRead('no placing director logged a ledger summary')
|
||||
if (read.entries.some((entry) => !Number.isFinite(Date.parse(entry.timestamp)))) {
|
||||
return failedRead('ledger summary with an unparseable timestamp')
|
||||
}
|
||||
const newest = new Map()
|
||||
for (const entry of read.entries) {
|
||||
const instance = entry.labels?.instanceId ?? 'unknown'
|
||||
const previous = newest.get(instance)
|
||||
if (!previous || Date.parse(entry.timestamp) > Date.parse(previous.timestamp)) newest.set(instance, entry)
|
||||
}
|
||||
let unwrittenRows = 0
|
||||
for (const entry of newest.values()) {
|
||||
const pending = Number(entry.jsonPayload?.pending)
|
||||
const dropped = Number(entry.jsonPayload?.dropped)
|
||||
if (!Number.isFinite(pending) || !Number.isFinite(dropped)) return failedRead('ledger summary without counts')
|
||||
unwrittenRows += pending + dropped
|
||||
}
|
||||
return { ok: true, unwrittenRows, directors: newest.size }
|
||||
}
|
||||
|
||||
// The same read also takes the ledger summary every placing director logs each minute, as a
|
||||
// positive control: with no fresh summary in it, no seat-behind-row line is blindness, not zero.
|
||||
export async function readSeatBehindRowSample(reader, { directorService, cellId, at }) {
|
||||
const read = await readLogEntries(reader, {
|
||||
filter: directorLogFilter(
|
||||
directorService,
|
||||
`((jsonPayload.event="${SEAT_BEHIND_ROW_EVENT}" AND jsonPayload.cellId="${cellId}") OR jsonPayload.event="${LEDGER_SUMMARY_EVENT}")`,
|
||||
at - RESERVE_WATCH_RULES.duplicateSeat.lookbackMs
|
||||
),
|
||||
projection: 'json(timestamp,labels.instanceId,jsonPayload.event,jsonPayload.epoch,jsonPayload.rowEpoch)',
|
||||
limit: LOG_ENTRY_LIMIT
|
||||
})
|
||||
if (read.failed) return failedRead(read.error)
|
||||
if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('seat-behind-row read truncated')
|
||||
const reports = []
|
||||
let controlSeen = false
|
||||
for (const entry of read.entries) {
|
||||
const reportedAt = Date.parse(entry.timestamp)
|
||||
if (!Number.isFinite(reportedAt)) return failedRead('seat-behind-row line with an unparseable timestamp')
|
||||
if (entry.jsonPayload?.event === LEDGER_SUMMARY_EVENT) {
|
||||
controlSeen ||= reportedAt >= at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs
|
||||
continue
|
||||
}
|
||||
const { epoch, rowEpoch } = entry.jsonPayload ?? {}
|
||||
if (epoch === undefined || rowEpoch === undefined) return failedRead('seat-behind-row line without epochs')
|
||||
// Everything the line carries but its time, which differs on every report of one seat. A
|
||||
// proxy for the host: two hosts sharing it read as one seat, so it can only over-count.
|
||||
reports.push({ at: reportedAt, key: `${entry.labels?.instanceId ?? 'unknown'}/${cellId}/${epoch}:${rowEpoch}` })
|
||||
}
|
||||
if (!controlSeen) return failedRead('seat-behind-row read holds no fresh ledger summary (positive control)')
|
||||
return { ok: true, reports }
|
||||
}
|
||||
|
||||
function createReader(dependencies, projectId) {
|
||||
const now = dependencies.now
|
||||
return {
|
||||
runGcloud: dependencies.runGcloud,
|
||||
fetch: dependencies.fetchImpl,
|
||||
retryDelayMs: dependencies.retryDelayMs ?? 5_000,
|
||||
readTimeoutMs: 20_000,
|
||||
now,
|
||||
// Every sample gets its own budget, so a slow read costs one sample, not the watch.
|
||||
deadlineAt: now() + SAMPLE_READ_BUDGET_MS,
|
||||
projectId
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same-cap and capacity rolls hold the Cloud SQL rollout lease for the whole run; it names no
|
||||
* cell, so any live holder refuses the watch's start. Read only: holding it for hours would block
|
||||
* every rollout. An unreadable lease refuses too (fail closed, as the lease action does).
|
||||
*/
|
||||
export async function rolloutLeaseRefusal(readRolloutLease, now) {
|
||||
try {
|
||||
const existing = await readRolloutLease()
|
||||
if (!existing || existing.record.expires_at <= now) return null
|
||||
return (
|
||||
`a rollout holds the Cloud SQL rollout lease until ${new Date(existing.record.expires_at).toISOString()} ` +
|
||||
`(${existing.holder}); start the watch after it ends`
|
||||
)
|
||||
} catch (error) {
|
||||
return `the Cloud SQL rollout lease is unreadable (${error instanceof Error ? error.message : error}); refusing to start`
|
||||
}
|
||||
}
|
||||
|
||||
// The lease action's own reader, resolved from the repository root because this tree's depth
|
||||
// differs between the two repositories' copies.
|
||||
async function readProductionRolloutLease() {
|
||||
const action = new URL('.github/actions/cloud-sql-rollout-lease/', RELAY_REPOSITORY_ROOT)
|
||||
const [{ CloudSqlRolloutLease, describeHolder }, { createAccessTokenSource }] = await Promise.all([
|
||||
import(new URL('storage-lease.mjs', action).href),
|
||||
import(new URL('gcloud-access-token.mjs', action).href)
|
||||
])
|
||||
const lease = new CloudSqlRolloutLease({
|
||||
bucket: PRODUCTION_LEASE.bucket,
|
||||
objectName: PRODUCTION_LEASE.object,
|
||||
accessToken: createAccessTokenSource()
|
||||
})
|
||||
const existing = await lease.read()
|
||||
return existing && { record: existing.record, holder: describeHolder(existing.record) }
|
||||
}
|
||||
|
||||
// Only a missing file is a fresh watch. Anything else that is not a state fails the segment, which
|
||||
// the outcome step reads as failed and flips back: a corrupt file must not restart or refuse the watch.
|
||||
export async function loadState(path) {
|
||||
const text = await readFile(path, 'utf8').catch((error) => (error?.code === 'ENOENT' ? null : Promise.reject(error)))
|
||||
if (text === null) return null
|
||||
const state = JSON.parse(text)
|
||||
const isObject = state !== null && typeof state === 'object' && !Array.isArray(state)
|
||||
if (!isObject || typeof state.state !== 'string') throw new Error(`state file ${path} holds no watch state`)
|
||||
if (state.state === 'watching' && (typeof state.cellId !== 'string' || !Number.isFinite(state.startedAt))) {
|
||||
throw new Error(`state file ${path} has no cell or start`)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// A segment's config. Later segments check flipped-at against the watch's start; bad inputs before
|
||||
// the first sample are refused, so no flip back follows a typo.
|
||||
export async function parseWatchSegment(values, now = Date.now()) {
|
||||
const existing = values['state-file'] ? await loadState(values['state-file']) : null
|
||||
try {
|
||||
return parseWatchArguments(values, Number.isFinite(existing?.startedAt) ? existing.startedAt : now)
|
||||
} catch (error) {
|
||||
if (values['state-file'] && existing === null) {
|
||||
await writeFile(values['state-file'], JSON.stringify({ state: 'refused', verdict: error.message }))
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function watchSegment(config, dependencies) {
|
||||
const now = dependencies.now ?? Date.now
|
||||
const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
const log = dependencies.log ?? console.log
|
||||
const deps = { ...dependencies, now, sleep, fetchImpl: dependencies.fetchImpl ?? fetch }
|
||||
const { request, directorService } = config
|
||||
let state = await loadState(config.stateFile)
|
||||
if (state === null) {
|
||||
const refusal = await rolloutLeaseRefusal(dependencies.readRolloutLease, now())
|
||||
if (refusal) {
|
||||
state = { state: 'refused', verdict: refusal }
|
||||
await writeFile(config.stateFile, JSON.stringify(state))
|
||||
if (config.githubOutput) await writeOutcome(config.githubOutput, state)
|
||||
return state
|
||||
}
|
||||
state = initialWatchState({
|
||||
cellId: request.cellId,
|
||||
startedAt: now(),
|
||||
durationMinutes: config.durationMinutes,
|
||||
background503PerMinute: config.background503PerMinute,
|
||||
flippedAt: config.flippedAt
|
||||
})
|
||||
}
|
||||
if (state.cellId !== request.cellId) throw new Error(`state file is for ${state.cellId}, not ${request.cellId}`)
|
||||
const segmentEndsAt = now() + config.segmentMinutes * MINUTE_MS
|
||||
while (state.state === 'watching' && now() < segmentEndsAt) {
|
||||
const at = now()
|
||||
const baselineEndsAt = state.flippedAt === null ? null : minuteFloor(state.flippedAt)
|
||||
if (!state.background && baselineEndsAt !== null && at >= baselineEndsAt + METRIC_DELAY_MS) {
|
||||
// The minutes just before the flip, once published; retried every sample until one reads.
|
||||
const read = await readDirector503Minutes(createReader(deps, request.projectId), {
|
||||
projectId: request.projectId,
|
||||
directorService,
|
||||
endedAt: baselineEndsAt,
|
||||
minutes: BACKGROUND_MINUTES
|
||||
})
|
||||
state = { ...state, background: read.ok ? measuredBackground(read.perMinute, read.minutes, read.publishedThrough) : null }
|
||||
}
|
||||
const reader = createReader(deps, request.projectId)
|
||||
// Serialised: concurrent Logging reads are what earn a 429.
|
||||
const sample = {
|
||||
at,
|
||||
cell: await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep),
|
||||
director503: await readDirector503Minutes(reader, {
|
||||
projectId: request.projectId,
|
||||
directorService,
|
||||
// Complete minutes only, held back past the publishing delay.
|
||||
endedAt: minuteFloor(at - METRIC_DELAY_MS),
|
||||
minutes: RESERVE_WATCH_RULES.director503.readMinutes
|
||||
}),
|
||||
ledger: await readLedgerSample(reader, { directorService, at }),
|
||||
seatBehindRow: await readSeatBehindRowSample(reader, { directorService, cellId: request.cellId, at })
|
||||
}
|
||||
const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit })
|
||||
state = result.state
|
||||
log(result.line)
|
||||
await writeFile(config.stateFile, JSON.stringify(state))
|
||||
if (state.state !== 'watching') break
|
||||
await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now()))
|
||||
}
|
||||
if (config.githubOutput) await writeOutcome(config.githubOutput, state)
|
||||
return state
|
||||
}
|
||||
|
||||
function writeOutcome(githubOutput, { state, verdict }) {
|
||||
return appendFile(githubOutput, `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\n`)
|
||||
}
|
||||
|
||||
// The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the
|
||||
// workflow flips back in write mode: missing data is never healthy.
|
||||
export async function recordOutcome(stateFile, githubOutput) {
|
||||
const state = await loadState(stateFile).catch(() => null)
|
||||
const outcome =
|
||||
state && state.state !== 'watching' ? state : { state: 'failed', verdict: 'the watch stopped before a verdict' }
|
||||
await writeOutcome(githubOutput, outcome)
|
||||
return outcome
|
||||
}
|
||||
|
||||
function breakGlassInputs(request, generation) {
|
||||
return {
|
||||
mode: 'write',
|
||||
'cell-id': request.cellId,
|
||||
set: 'admitMode=default',
|
||||
'break-glass': `${BREAK_GLASS_CONFIRMATION} ${request.cellId}`,
|
||||
'expected-generation': generation,
|
||||
confirmation: `${WRITE_CONFIRMATION} ${request.cellId}`
|
||||
}
|
||||
}
|
||||
|
||||
async function readRuntimeRetrying(fetchImpl, request, idToken, sleep, log) {
|
||||
for (let attempt = 1; ; attempt += 1) {
|
||||
try {
|
||||
return await readRuntime(fetchImpl, request, idToken, sleep)
|
||||
} catch (error) {
|
||||
if (attempt >= FLIP_BACK_RUNTIME_ATTEMPTS) throw error
|
||||
log(JSON.stringify({ event: 'orca_relay_reserve_watch_runtime_retry', cellId: request.cellId, attempt, message: error instanceof Error ? error.message : String(error) }))
|
||||
await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The flag tool's admitMode=db write at the generation read just now. A generation conflict (some
|
||||
* other write landed first) is re-read and retried while the switch file still says reserve; a
|
||||
* file that already says db is done. A cell that still does not answer after the runtime retries
|
||||
* needs break glass, which only an operator may type.
|
||||
*/
|
||||
export async function flipBack(values, dependencies) {
|
||||
const log = dependencies.log ?? console.log
|
||||
const fetchImpl = dependencies.fetchImpl ?? fetch
|
||||
const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
||||
const base = dbFlipRequest(values)
|
||||
const record = (result) =>
|
||||
log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_reserve_watch_flip_back',
|
||||
mode: base.mode,
|
||||
cellId: base.cellId,
|
||||
written: result.written,
|
||||
generation: result.generation ?? null,
|
||||
...(result.alreadyDb ? { alreadyDb: true } : {})
|
||||
})
|
||||
)
|
||||
for (let attempt = 1; ; attempt += 1) {
|
||||
const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken)
|
||||
if ((current.object?.flags?.admitMode ?? 'db') !== 'reserve') {
|
||||
const result = { written: false, alreadyDb: true, generation: current.generation }
|
||||
record(result)
|
||||
return result
|
||||
}
|
||||
try {
|
||||
await readRuntimeRetrying(fetchImpl, base, dependencies.idToken, sleep, log)
|
||||
} catch (error) {
|
||||
const inputs = breakGlassInputs(base, current.generation)
|
||||
log(JSON.stringify({ event: 'orca_relay_reserve_watch_break_glass_required', cellId: base.cellId, inputs }))
|
||||
throw new Error(
|
||||
`${base.cellId} does not answer runtime-status (${error instanceof Error ? error.message : error}), so ` +
|
||||
'the flag tool cannot flip it back. Run Operate Relay Production Cell Flags with break glass: ' +
|
||||
Object.entries(inputs).map(([key, value]) => `${key}=${value}`).join(', ')
|
||||
)
|
||||
}
|
||||
try {
|
||||
const result = await operateCellFlags(
|
||||
{ ...base, expectedGeneration: current.generation },
|
||||
{ ...dependencies, fetchImpl, log, sleep }
|
||||
)
|
||||
record(result)
|
||||
return result
|
||||
} catch (error) {
|
||||
if (!error?.generationConflict || attempt >= FLIP_BACK_ATTEMPTS) throw error
|
||||
log(JSON.stringify({ event: 'orca_relay_reserve_watch_flip_back_conflict', cellId: base.cellId, attempt, message: error.message }))
|
||||
await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'
|
||||
export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'
|
||||
export const ALARM_KINDS = ['refused', 'flip-back-failed']
|
||||
|
||||
// GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another.
|
||||
export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
|
||||
const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/g, '%3A').replace(/,/g, '%2C')
|
||||
|
||||
/**
|
||||
* Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a
|
||||
* dry-run watch never flips back, so its refusal or failure changes nothing for the cell.
|
||||
*/
|
||||
export async function raiseAlarm({ mode, cellId, projectId, kind, reason }, { runGcloud, log = console.log }) {
|
||||
if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`)
|
||||
const message =
|
||||
kind === 'refused'
|
||||
? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.`
|
||||
: `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.`
|
||||
log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`)
|
||||
if (mode !== 'write') return { paged: false }
|
||||
const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason })
|
||||
try {
|
||||
// The flag tool's project check: a missing --project-id must not reach gcloud as "undefined".
|
||||
if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId ?? '')) throw new Error('--project-id is not a GCP project id')
|
||||
await runGcloud(
|
||||
['logging', 'write', ALARM_LOG_NAME, entry, '--payload-type=json', '--severity=ERROR', '--project', projectId],
|
||||
{ timeoutMs: 30_000 }
|
||||
)
|
||||
} catch (error) {
|
||||
log(`::error title=Reserve watch page failed::${escapeWorkflowCommand(`the alarm log line was not written, so nothing pages: ${error instanceof Error ? error.message : error}`)}`)
|
||||
return { paged: false }
|
||||
}
|
||||
return { paged: true }
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
||||
const [command, ...rest] = process.argv.slice(2)
|
||||
const { values } = parseArgs({
|
||||
args: rest,
|
||||
options: Object.fromEntries(
|
||||
[
|
||||
'mode',
|
||||
'cell-id',
|
||||
'cell-origin',
|
||||
'project-id',
|
||||
'director-origin',
|
||||
'director-service',
|
||||
'confirmation',
|
||||
'duration-minutes',
|
||||
'segment-minutes',
|
||||
'unverifiable-limit',
|
||||
'background-503-per-minute',
|
||||
'flipped-at',
|
||||
'state-file',
|
||||
'github-output',
|
||||
'verdict',
|
||||
'kind'
|
||||
].map((name) => [name, { type: 'string' }])
|
||||
)
|
||||
})
|
||||
const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
|
||||
const runGcloud = (args, { timeoutMs }) =>
|
||||
execFileAsync('gcloud', args, { maxBuffer: 64 * 1024 * 1024, timeout: timeoutMs, killSignal: 'SIGKILL' })
|
||||
try {
|
||||
if (command === 'alarm') {
|
||||
await raiseAlarm(
|
||||
{ mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '' },
|
||||
{ runGcloud }
|
||||
)
|
||||
// The alarm step is red, whatever the page did.
|
||||
process.exitCode = 1
|
||||
} else if (command === 'outcome') {
|
||||
if (!values['state-file'] || !values['github-output']) throw new Error('--state-file and --github-output are required')
|
||||
console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_outcome', ...(await recordOutcome(values['state-file'], values['github-output'])) }))
|
||||
} else if (!idToken) {
|
||||
throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required')
|
||||
} else if (command === 'watch') {
|
||||
const config = await parseWatchSegment(values)
|
||||
const state = await watchSegment(config, {
|
||||
idToken,
|
||||
readRolloutLease: readProductionRolloutLease,
|
||||
runGcloud
|
||||
})
|
||||
// A cell that was never reserve is the operator's mistake; one flipped back by hand is not.
|
||||
if (state.state === 'refused') {
|
||||
console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_refused', verdict: state.verdict }))
|
||||
process.exitCode = 1
|
||||
}
|
||||
} else if (command === 'flip-back') {
|
||||
const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN
|
||||
if (!accessToken) throw new Error('ORCA_RELAY_CONTROL_ACCESS_TOKEN is required')
|
||||
if (!values.verdict) throw new Error('--verdict is required')
|
||||
await flipBack(values, {
|
||||
accessToken,
|
||||
idToken,
|
||||
audit: {
|
||||
writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`,
|
||||
runId: process.env.GITHUB_RUN_ID ?? 'local',
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0'
|
||||
}
|
||||
})
|
||||
// A trip is a failed watch even once the cell is safely back on db.
|
||||
console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_verdict', mode: values.mode, verdict: values.verdict }))
|
||||
process.exitCode = 1
|
||||
} else {
|
||||
throw new Error('command must be watch, outcome, flip-back or alarm')
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_reserve_watch_failed',
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,958 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
evaluateWatchSample,
|
||||
initialWatchState,
|
||||
measuredBackground
|
||||
} from './relay-reserve-watch-rules.mjs'
|
||||
import {
|
||||
ALARM_EVENT,
|
||||
ALARM_LOG_NAME,
|
||||
FLIP_BACK_ATTEMPTS,
|
||||
FLIP_BACK_RUNTIME_ATTEMPTS,
|
||||
flipBack,
|
||||
parseWatchArguments,
|
||||
parseWatchSegment,
|
||||
raiseAlarm,
|
||||
readDirector503Minutes,
|
||||
readSeatBehindRowSample,
|
||||
recordOutcome,
|
||||
rolloutLeaseRefusal,
|
||||
watchSegment
|
||||
} from './watch-relay-reserve-cell.mjs'
|
||||
import { readRelayWorkflow } from './relay-repository.mjs'
|
||||
|
||||
const CELL = 'production-gce-c26'
|
||||
const START = Date.parse('2026-10-14T12:00:00Z')
|
||||
const MINUTE = 60_000
|
||||
const LIMIT = { unverifiableLimit: 3 }
|
||||
|
||||
const healthyCell = (overrides = {}) => ({
|
||||
ok: true,
|
||||
admitModeConfigured: 'reserve',
|
||||
admitModeEffective: 'reserve',
|
||||
draining: false,
|
||||
controls: 1_000,
|
||||
enforcedUnits: 1_100,
|
||||
hardCap: 2_000,
|
||||
...overrides
|
||||
})
|
||||
const minutesBefore = (at, count) =>
|
||||
Array.from({ length: count }, (_, index) =>
|
||||
new Date(Math.floor(at / MINUTE) * MINUTE - (count + 3 - index) * MINUTE).toISOString().slice(0, 16)
|
||||
)
|
||||
const quiet503 = (at, perMinute = 10) => {
|
||||
const minutes = minutesBefore(at, 3)
|
||||
return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1) }
|
||||
}
|
||||
const sampleAt = (at, overrides = {}) => ({
|
||||
at,
|
||||
cell: healthyCell(),
|
||||
director503: quiet503(at),
|
||||
ledger: { ok: true, unwrittenRows: 0, directors: 5 },
|
||||
seatBehindRow: { ok: true, reports: [] },
|
||||
...overrides
|
||||
})
|
||||
const freshState = (overrides = {}) => ({
|
||||
...initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 10 }),
|
||||
...overrides
|
||||
})
|
||||
|
||||
// Runs samples one minute apart; returns every result.
|
||||
function run(samples, state = freshState()) {
|
||||
const results = []
|
||||
for (const sample of samples) {
|
||||
const result = evaluateWatchSample(state, sample, LIMIT)
|
||||
results.push(result)
|
||||
state = result.state
|
||||
if (state.state !== 'watching') break
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
test('a healthy cell keeps watching and completes at the deadline', () => {
|
||||
const results = run(Array.from({ length: 61 }, (_, minute) => sampleAt(START + minute * MINUTE)))
|
||||
assert.equal(results.length, 61)
|
||||
assert.ok(results.slice(0, 60).every((result) => result.state.state === 'watching'))
|
||||
assert.equal(results.at(-1).state.state, 'completed')
|
||||
const line = JSON.parse(results[0].line)
|
||||
assert.equal(line.event, 'orca_relay_reserve_watch_sample')
|
||||
assert.deepEqual(Object.keys(line.rules), ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift'])
|
||||
})
|
||||
|
||||
test('503s trip only after three complete minutes over max(1.5x background, background + 20)', () => {
|
||||
// Background 100: the line is 150. Two minutes over and one under is not a trip.
|
||||
const state = freshState({ background: { medianPerMinute: 100, source: 'input' } })
|
||||
const series = (counts) => {
|
||||
const minutes = minutesBefore(START, 3)
|
||||
return {
|
||||
ok: true,
|
||||
minutes,
|
||||
perMinute: Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])),
|
||||
publishedThrough: minutes.at(-1)
|
||||
}
|
||||
}
|
||||
const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT)
|
||||
assert.equal(twoOver.state.state, 'watching')
|
||||
const atLine = evaluateWatchSample(state, sampleAt(START, { director503: series([150, 150, 150]) }), LIMIT)
|
||||
assert.equal(atLine.state.state, 'watching')
|
||||
const threeOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 160, 170]) }), LIMIT)
|
||||
assert.equal(threeOver.state.state, 'tripped')
|
||||
assert.match(threeOver.state.verdict, /^flip back production-gce-c26: director503: /)
|
||||
// A zero background still needs 20 a minute, not one 503.
|
||||
const zero = freshState({ background: { medianPerMinute: 0, source: 'input' } })
|
||||
assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([5, 5, 5]) }), LIMIT).state.state, 'watching')
|
||||
assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([21, 21, 21]) }), LIMIT).state.state, 'tripped')
|
||||
// A published minute with no 503 point had none.
|
||||
const gap = series([151, 160, 170])
|
||||
delete gap.perMinute[gap.minutes[1]]
|
||||
assert.equal(evaluateWatchSample(state, sampleAt(START, { director503: gap }), LIMIT).state.state, 'watching')
|
||||
})
|
||||
|
||||
test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => {
|
||||
const minutes = minutesBefore(START, 10)
|
||||
const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1) }
|
||||
const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } }))
|
||||
assert.equal(result.rules.director503.status, 'pass')
|
||||
assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0])
|
||||
assert.equal(result.state.unverifiableStreak, 0)
|
||||
assert.deepEqual(measuredBackground({}, minutes, minutes.at(-1)), { medianPerMinute: 0, minutes: 10, source: 'measured' })
|
||||
})
|
||||
|
||||
test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => {
|
||||
const state = freshState({ background: { medianPerMinute: 100, source: 'input' } })
|
||||
const minutes = minutesBefore(START, 10)
|
||||
const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT)
|
||||
// The front is two minutes behind: the three minutes before it are judged, the two after are not.
|
||||
const over = Object.fromEntries(minutes.map((minute) => [minute, 500]))
|
||||
const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7] })
|
||||
assert.equal(behind.rules.director503.status, 'trip')
|
||||
assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8))
|
||||
// A sparse 503 series: minutes after its last point but within the front are zero, not late.
|
||||
const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1) })
|
||||
assert.equal(sparse.rules.director503.status, 'pass')
|
||||
assert.equal(sparse.state.unverifiableStreak, 0)
|
||||
// No request series at all.
|
||||
const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null })
|
||||
assert.equal(empty.rules.director503.status, 'unverifiable')
|
||||
assert.match(empty.rules.director503.reason, /request series is empty/)
|
||||
assert.equal(empty.state.unverifiableStreak, 1)
|
||||
// A front too far behind to leave three published minutes in the read.
|
||||
const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1] })
|
||||
assert.equal(stale.rules.director503.status, 'unverifiable')
|
||||
assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`))
|
||||
})
|
||||
|
||||
const SEAT_EVENT = 'orca_relay_reserve_ledger_seat_behind_row'
|
||||
const SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary'
|
||||
const summaryLine = (at, instanceId = 'a') => ({
|
||||
timestamp: new Date(at).toISOString(),
|
||||
labels: { instanceId },
|
||||
jsonPayload: { event: SUMMARY_EVENT, pending: 0, dropped: 0 }
|
||||
})
|
||||
|
||||
// A Cloud Monitoring or Cloud Logging reader over fakes, as the watch builds one.
|
||||
const fakeReader = ({ fetch: fetchImpl = async () => new Response('{}'), runGcloud = async () => ({ stdout: 'token' }) } = {}) => ({
|
||||
fetch: fetchImpl,
|
||||
runGcloud,
|
||||
retryDelayMs: 0,
|
||||
readTimeoutMs: 1_000,
|
||||
now: () => START,
|
||||
deadlineAt: START + 10_000,
|
||||
projectId: 'onorca-cloud'
|
||||
})
|
||||
|
||||
test('an unparseable timestamp makes the read unverifiable', async () => {
|
||||
const badPoint = {
|
||||
timeSeries: [{ points: [{ interval: { endTime: 'not a time' }, value: { int64Value: '3' } }] }]
|
||||
}
|
||||
const read503 = await readDirector503Minutes(
|
||||
fakeReader({ fetch: async () => new Response(JSON.stringify(badPoint)) }),
|
||||
{ projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt: START, minutes: 3 }
|
||||
)
|
||||
assert.equal(read503.ok, false)
|
||||
assert.match(read503.error, /unreadable/)
|
||||
const seats = await readSeatBehindRowSample(
|
||||
fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([summaryLine(START), { timestamp: 'garbage', jsonPayload: { epoch: 1, rowEpoch: 2 } }]) }) }),
|
||||
{ directorService: 'orca-cloud-relay', cellId: CELL, at: START }
|
||||
)
|
||||
assert.equal(seats.ok, false)
|
||||
assert.match(seats.error, /unparseable timestamp/)
|
||||
})
|
||||
|
||||
test('a seat-behind-row key carries the director instance and cell beside the epochs', async () => {
|
||||
const entry = (instanceId) => ({ timestamp: new Date(START).toISOString(), labels: { instanceId }, jsonPayload: { event: SEAT_EVENT, epoch: 7, rowEpoch: 8 } })
|
||||
const read = await readSeatBehindRowSample(
|
||||
fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([entry('a'), summaryLine(START), entry('b')]) }) }),
|
||||
{ directorService: 'orca-cloud-relay', cellId: CELL, at: START }
|
||||
)
|
||||
assert.deepEqual(read.reports.map((report) => report.key), [`a/${CELL}/7:8`, `b/${CELL}/7:8`])
|
||||
})
|
||||
|
||||
test('the duplicate read carries a positive control: no fresh ledger summary in it is unverifiable, not zero', async () => {
|
||||
const filters = []
|
||||
const readWith = (entries) =>
|
||||
readSeatBehindRowSample(
|
||||
fakeReader({ runGcloud: async (args) => (filters.push(args[2]), { stdout: JSON.stringify(entries) }) }),
|
||||
{ directorService: 'orca-cloud-relay', cellId: CELL, at: START }
|
||||
)
|
||||
// One read, one window: the control rides the same filter as the seat lines.
|
||||
assert.deepEqual(await readWith([summaryLine(START - 30_000)]), { ok: true, reports: [] })
|
||||
assert.match(filters[0], new RegExp(`\\(\\(jsonPayload\\.event="${SEAT_EVENT}" AND jsonPayload\\.cellId="${CELL}"\\) OR jsonPayload\\.event="${SUMMARY_EVENT}"\\)`))
|
||||
assert.match(filters[0], /resource\.labels\.service_name="orca-cloud-relay"/)
|
||||
for (const entries of [[], [summaryLine(START - 10 * MINUTE)]]) {
|
||||
const blind = await readWith(entries)
|
||||
assert.equal(blind.ok, false)
|
||||
assert.match(blind.error, /no fresh ledger summary \(positive control\)/)
|
||||
}
|
||||
// Blind is a gap in the sample, which counts toward the unverifiable streak.
|
||||
const [gap] = run([sampleAt(START, { seatBehindRow: await readWith([]) })])
|
||||
assert.equal(gap.rules.duplicateSeat.status, 'unverifiable')
|
||||
assert.equal(gap.state.unverifiableStreak, 1)
|
||||
})
|
||||
|
||||
test('the measured background is the median of the minutes before the flip, published gaps as zero', () => {
|
||||
const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49']
|
||||
const front = minutes[2]
|
||||
assert.equal(measuredBackground({ '2026-10-14T11:48': 30 }, minutes, front).medianPerMinute, 0)
|
||||
assert.equal(measuredBackground({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, minutes, front).medianPerMinute, 20)
|
||||
assert.equal(measuredBackground({}, minutes, '2026-10-14T11:52').medianPerMinute, 0)
|
||||
// Not published through the last minute yet, or no request series: no background yet.
|
||||
assert.equal(measuredBackground({ [minutes[0]]: 10 }, minutes, minutes[1]), null)
|
||||
assert.equal(measuredBackground({}, minutes, null), null)
|
||||
assert.equal(measuredBackground({}, [], front), null)
|
||||
})
|
||||
|
||||
test('any overrun trips at once', () => {
|
||||
const [result] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_001 }) })])
|
||||
assert.equal(result.state.state, 'tripped')
|
||||
assert.match(result.state.verdict, /overrun: ledger units 2001 over hard cap 2000/)
|
||||
const [atCap] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_000 }) })])
|
||||
assert.equal(atCap.state.state, 'watching')
|
||||
})
|
||||
|
||||
test('a seat reported behind its row again past the grace trips; inside the grace it does not', () => {
|
||||
const report = (at) => ({ at, key: '7:8' })
|
||||
const within = run([
|
||||
sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
|
||||
sampleAt(START + MINUTE, { seatBehindRow: { ok: true, reports: [report(START - 30_000), report(START + 50_000)] } })
|
||||
])
|
||||
assert.equal(within.at(-1).state.state, 'watching')
|
||||
// Carried across samples: the second read no longer holds the first report.
|
||||
const past = run([
|
||||
sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
|
||||
sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [report(START + 70_000)] } })
|
||||
])
|
||||
assert.equal(past.at(-1).state.state, 'tripped')
|
||||
assert.match(past.at(-1).state.verdict, /duplicateSeat: a seat behind its ledger row \(7:8\)/)
|
||||
// Another seat's report is not the same seat.
|
||||
const other = run([
|
||||
sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
|
||||
sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [{ at: START + 70_000, key: '3:4' }] } })
|
||||
])
|
||||
assert.equal(other.at(-1).state.state, 'watching')
|
||||
})
|
||||
|
||||
test('a host drop over 5% in 5 minutes trips outside a drain only', () => {
|
||||
const controls = [1_000, 990, 980, 960, 949]
|
||||
const samples = (draining) =>
|
||||
controls.map((count, minute) =>
|
||||
sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count, draining: draining && minute === 2 }) })
|
||||
)
|
||||
assert.equal(run(samples(false)).at(-1).state.state, 'tripped')
|
||||
assert.match(run(samples(false)).at(-1).state.verdict, /hostDrop: connected hosts fell 1000 -> 949/)
|
||||
assert.equal(run(samples(true)).at(-1).state.state, 'watching')
|
||||
// A drop spread over more than 5 minutes is not one.
|
||||
const slow = [1_000, 990, 980, 970, 960, 955, 950].map((count, minute) =>
|
||||
sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) })
|
||||
)
|
||||
assert.equal(run(slow).at(-1).state.state, 'watching')
|
||||
// Under the floor on a nearly empty cell: 100 -> 92 is 8%, but only 8 hosts.
|
||||
const small = [100, 92].map((count, minute) => sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) }))
|
||||
assert.equal(run(small).at(-1).state.state, 'watching')
|
||||
})
|
||||
|
||||
test('ledger drift over 0.5% trips once it has lasted 2 minutes', () => {
|
||||
const drift = (rows) => ({ ok: true, unwrittenRows: rows, directors: 5 })
|
||||
const brief = run([6, 6, 0, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
|
||||
assert.equal(brief.at(-1).state.state, 'watching')
|
||||
const sustained = run([6, 6, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
|
||||
assert.equal(sustained.length, 3)
|
||||
assert.equal(sustained.at(-1).state.state, 'tripped')
|
||||
assert.match(sustained.at(-1).state.verdict, /ledgerDrift: ledger drift 0\.60% for 120 s/)
|
||||
const atLine = run([5, 5, 5, 5].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
|
||||
assert.equal(atLine.at(-1).state.state, 'watching')
|
||||
})
|
||||
|
||||
test('a tripped dead-man is a trip: Postgres still records reserve until a db write', () => {
|
||||
const [result] = run([sampleAt(START, { cell: healthyCell({ admitModeEffective: 'db' }) })])
|
||||
assert.equal(result.state.state, 'tripped')
|
||||
assert.match(result.state.verdict, /deadMan: the cell tripped its dead-man/)
|
||||
})
|
||||
|
||||
test('unreadable data is never healthy: N samples in a row with any gap trip', () => {
|
||||
const unreadable = { ok: false, error: 'cell runtime status returned 502' }
|
||||
const gaps = [
|
||||
{ cell: unreadable },
|
||||
{ ledger: { ok: false, error: 'no placing director logged a ledger summary' } },
|
||||
{ director503: { ok: false, error: 'director 503 metric read failed', perMinute: {}, minutes: [], publishedThrough: null } }
|
||||
]
|
||||
const results = run(gaps.map((gap, minute) => sampleAt(START + minute * MINUTE, gap)))
|
||||
assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [1, 2, 3])
|
||||
assert.equal(results.at(-1).state.state, 'tripped')
|
||||
assert.match(results.at(-1).state.verdict, /unverifiable: 3 samples in a row could not read director503/)
|
||||
// One whole sample resets the streak.
|
||||
const reset = run(
|
||||
[gaps[0], {}, gaps[0], gaps[0]].map((gap, minute) => sampleAt(START + minute * MINUTE, gap))
|
||||
)
|
||||
assert.deepEqual(reset.map((result) => result.state.unverifiableStreak), [1, 0, 1, 2])
|
||||
assert.equal(reset.at(-1).state.state, 'watching')
|
||||
// No background yet is a gap too.
|
||||
const noBackground = freshState({ background: null })
|
||||
assert.equal(evaluateWatchSample(noBackground, sampleAt(START), LIMIT).state.unverifiableStreak, 1)
|
||||
// A measured trip still wins over a gap elsewhere.
|
||||
const [both] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_500 }), ledger: gaps[1].ledger })])
|
||||
assert.equal(both.state.state, 'tripped')
|
||||
assert.match(both.state.verdict, /overrun/)
|
||||
})
|
||||
|
||||
test('a cell no longer in reserve stops the watch without a write; one never in reserve is refused', () => {
|
||||
const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'reserve' })
|
||||
const later = run([sampleAt(START), sampleAt(START + MINUTE, { cell: dbCell })])
|
||||
assert.equal(later.at(-1).state.state, 'stopped')
|
||||
assert.match(later.at(-1).state.verdict, /is not in reserve mode \(switch file says db\)/)
|
||||
// Never reserve within the wait: refused, which the workflow does not flip back.
|
||||
const never = run(Array.from({ length: 12 }, (_, minute) => sampleAt(START + minute * MINUTE, { cell: dbCell })))
|
||||
assert.equal(never.at(-1).state.state, 'refused')
|
||||
assert.equal(never.length, 11)
|
||||
assert.match(never.at(-1).state.verdict, /did not report reserve mode within 10 min \(switch file says db\)/)
|
||||
})
|
||||
|
||||
test('a watch dispatched before the reserve write applied waits for it, then judges as usual', () => {
|
||||
const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'db' })
|
||||
const results = run([
|
||||
sampleAt(START, { cell: dbCell, ledger: { ok: false, error: 'not yet' } }),
|
||||
sampleAt(START + MINUTE, { cell: dbCell }),
|
||||
sampleAt(START + 2 * MINUTE),
|
||||
sampleAt(START + 3 * MINUTE, { cell: healthyCell({ enforcedUnits: 2_500 }) })
|
||||
])
|
||||
// Waiting samples judge nothing: not a dead-man trip on a db cell, not a gap.
|
||||
assert.deepEqual(results.slice(0, 2).map((result) => result.state.state), ['watching', 'watching'])
|
||||
assert.equal(results[0].rules.reserve.status, 'pending')
|
||||
assert.deepEqual(Object.keys(results[0].rules), ['reserve'])
|
||||
assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [0, 0, 0, 0])
|
||||
assert.equal(results[2].state.reserveSeenAt, START + 2 * MINUTE)
|
||||
assert.equal(results[2].rules.overrun.status, 'pass')
|
||||
assert.equal(results[3].state.state, 'tripped')
|
||||
// Once seen in reserve, a later db reads as a hand flip back, not a refusal.
|
||||
const after = run([sampleAt(START + 2 * MINUTE), sampleAt(START + 3 * MINUTE, { cell: dbCell })], results[0].state)
|
||||
assert.equal(after.at(-1).state.state, 'stopped')
|
||||
})
|
||||
|
||||
const watchValues = (overrides = {}) => ({
|
||||
mode: 'dry-run',
|
||||
'cell-id': CELL,
|
||||
'cell-origin': 'https://c26.relay.onorca.dev',
|
||||
'project-id': 'onorca-cloud',
|
||||
'director-origin': 'https://relay.onorca.dev',
|
||||
'director-service': 'orca-cloud-relay',
|
||||
'duration-minutes': '120',
|
||||
'segment-minutes': '55',
|
||||
'unverifiable-limit': '3',
|
||||
'background-503-per-minute': '',
|
||||
'flipped-at': new Date(START - 5 * MINUTE).toISOString(),
|
||||
'state-file': '/unused',
|
||||
...overrides
|
||||
})
|
||||
const parseAt = (values) => parseWatchArguments(values, START)
|
||||
|
||||
test('watch arguments reuse the flag tool checks, so a write watch needs its confirmation up front', () => {
|
||||
assert.equal(parseAt(watchValues()).background503PerMinute, null)
|
||||
assert.equal(parseAt(watchValues()).flippedAt, START - 5 * MINUTE)
|
||||
assert.equal(parseAt(watchValues({ 'background-503-per-minute': '12.5', 'flipped-at': '' })).background503PerMinute, 12.5)
|
||||
assert.throws(() => parseAt(watchValues({ mode: 'write' })), /WRITE_RELAY_CELL_FLAGS production-gce-c26/)
|
||||
assert.doesNotThrow(() => parseAt(watchValues({ mode: 'write', confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}` })))
|
||||
assert.throws(() => parseAt(watchValues({ 'cell-origin': 'https://c27.relay.onorca.dev' })), /names another cell/)
|
||||
assert.throws(() => parseAt(watchValues({ 'duration-minutes': '301' })), /duration-minutes/)
|
||||
assert.throws(() => parseAt(watchValues({ 'segment-minutes': '56' })), /segment-minutes/)
|
||||
assert.throws(() => parseAt(watchValues({ 'background-503-per-minute': '-1' })), /background/)
|
||||
})
|
||||
|
||||
test('the 503 background needs flipped-at or an explicit value; flipped-at must be a recent zoned time', () => {
|
||||
assert.throws(() => parseAt(watchValues({ 'flipped-at': '' })), /--flipped-at .* or --background-503-per-minute is required/)
|
||||
assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T11:55:00' })), /ISO time with a zone/)
|
||||
assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T12:01:00Z' })), /in the future/)
|
||||
assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-13T11:00:00Z' })), /more than 24 hours/)
|
||||
assert.equal(parseAt(watchValues({ 'flipped-at': '2026-10-14T07:55:00-04:00' })).flippedAt, START - 5 * MINUTE)
|
||||
})
|
||||
|
||||
// A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample.
|
||||
function fakeSources({
|
||||
cell = () => ({}),
|
||||
// Logged at each sample's minute, so it stays fresh for the positive control.
|
||||
ledger = (sample) => [{ timestamp: new Date(START + (sample - 1) * MINUTE).toISOString(), labels: { instanceId: 'a' }, jsonPayload: { pending: 0, dropped: 0 } }],
|
||||
per503 = () => 10,
|
||||
// Requests of every other code per minute; zero is a minute with no traffic at all.
|
||||
perOther = () => 1_000,
|
||||
seatControl = true,
|
||||
rolloutLease = null
|
||||
} = {}) {
|
||||
let samples = 0
|
||||
const monitoringSpans = []
|
||||
const fetchImpl = async (url) => {
|
||||
const text = String(url)
|
||||
if (text.endsWith('/v1/admin/runtime-status')) {
|
||||
samples += 1
|
||||
const overrides = cell(samples)
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
role: 'cell',
|
||||
cellId: CELL,
|
||||
draining: false,
|
||||
flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true } },
|
||||
admitModeEffective: 'reserve',
|
||||
connectionCapacity: { hardCap: 2_000 },
|
||||
runtime: { controls: 1_000, enforcedConnectionUnits: 1_100 },
|
||||
...overrides
|
||||
})
|
||||
)
|
||||
}
|
||||
if (text.startsWith('https://monitoring.googleapis.com/')) {
|
||||
const params = new URL(text).searchParams
|
||||
const span = { startedAt: Date.parse(params.get('interval.startTime')), endedAt: Date.parse(params.get('interval.endTime')) }
|
||||
monitoringSpans.push(span)
|
||||
assert.equal(params.get('aggregation.groupByFields'), 'metric.label."response_code"')
|
||||
assert.doesNotMatch(params.get('filter'), /response_code/)
|
||||
// Cloud Monitoring returns no point for a minute with no requests of that code.
|
||||
const seriesOf = (code, count) => {
|
||||
const points = []
|
||||
for (let minute = span.startedAt; minute < span.endedAt; minute += MINUTE) {
|
||||
if (count(minute) > 0) {
|
||||
points.push({ interval: { endTime: new Date(minute + MINUTE).toISOString() }, value: { int64Value: String(count(minute)) } })
|
||||
}
|
||||
}
|
||||
return points.length === 0 ? [] : [{ metric: { labels: { response_code: code } }, points }]
|
||||
}
|
||||
return new Response(JSON.stringify({ timeSeries: [...seriesOf('200', perOther), ...seriesOf('503', per503)] }))
|
||||
}
|
||||
throw new Error(`unexpected fetch ${text}`)
|
||||
}
|
||||
const runGcloud = async (args) => {
|
||||
if (args[0] === 'auth') return { stdout: 'token' }
|
||||
const filter = args[2]
|
||||
const summaries = ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } }))
|
||||
if (filter.includes(SEAT_EVENT)) return { stdout: JSON.stringify(seatControl ? summaries : []) }
|
||||
if (filter.includes(SUMMARY_EVENT)) return { stdout: JSON.stringify(ledger(samples)) }
|
||||
return { stdout: '[]' }
|
||||
}
|
||||
return { fetchImpl, runGcloud, monitoringSpans, readRolloutLease: async () => rolloutLease }
|
||||
}
|
||||
|
||||
test('a segment samples once a minute, carries state to the next segment, and stops on a trip', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
const lines = []
|
||||
const config = parseAt(
|
||||
watchValues({ 'state-file': join(directory, 'state.json'), 'github-output': join(directory, 'out'), 'segment-minutes': '3' })
|
||||
)
|
||||
const sources = fakeSources({ cell: (sample) => (sample === 5 ? { runtime: { controls: 1_000, enforcedConnectionUnits: 2_001 } } : {}) })
|
||||
const dependencies = {
|
||||
...sources,
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
}
|
||||
const first = await watchSegment(config, dependencies)
|
||||
assert.equal(first.state, 'watching')
|
||||
assert.equal(lines.length, 3)
|
||||
assert.equal(first.background.source, 'measured')
|
||||
assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\n')
|
||||
const second = await watchSegment(config, dependencies)
|
||||
assert.equal(second.state, 'tripped')
|
||||
assert.equal(second.samples, 5)
|
||||
assert.deepEqual(lines.map((line) => line.sample), [1, 2, 3, 4, 5])
|
||||
assert.equal(lines.at(-1).rules.overrun.status, 'trip')
|
||||
assert.match(await readFile(join(directory, 'out'), 'utf8'), /state=tripped\nverdict=flip back production-gce-c26: overrun: /)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a sample that cannot read its sources counts as unverifiable rather than crashing the watch', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10' }))
|
||||
const sources = fakeSources({ ledger: () => [] })
|
||||
const state = await watchSegment(config, {
|
||||
...sources,
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: () => {}
|
||||
})
|
||||
assert.equal(state.state, 'tripped')
|
||||
assert.equal(state.samples, 3)
|
||||
// No summary also blinds the duplicate rule, whose positive control it is.
|
||||
assert.match(state.verdict, /unverifiable: 3 samples in a row could not read duplicateSeat, ledgerDrift/)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a watch that ended without a verdict records failed, which the workflow flips back', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
const output = join(directory, 'out')
|
||||
assert.equal((await recordOutcome(join(directory, 'missing.json'), output)).state, 'failed')
|
||||
await writeFile(join(directory, 'watching.json'), JSON.stringify(freshState()))
|
||||
assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed')
|
||||
await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' })))
|
||||
assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped')
|
||||
assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\n$/)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
// Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload`
|
||||
// returns 'conflict' to play another writer landing first, optionally with its own admitMode.
|
||||
function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = true } = {}) {
|
||||
const calls = []
|
||||
let uploads = 0
|
||||
let object = { generation: Number(generation), flags: { admitMode: 'reserve', readinessLocal: true } }
|
||||
let applied = object
|
||||
const fetchImpl = async (url, init = {}) => {
|
||||
const text = String(url)
|
||||
calls.push({ url: text, method: init.method ?? 'GET', body: init.body })
|
||||
if (text.includes('/upload/storage/v1/')) {
|
||||
uploads += 1
|
||||
assert.match(text, new RegExp(`ifGenerationMatch=${object.generation}$`))
|
||||
const outcome = onUpload(uploads)
|
||||
if (outcome !== 'ok') {
|
||||
object = { generation: object.generation + 1, flags: { ...object.flags, admitMode: outcome.admitMode ?? 'reserve' } }
|
||||
applied = object
|
||||
return new Response('', { status: 412 })
|
||||
}
|
||||
object = { generation: object.generation + 1, flags: JSON.parse(init.body.split('\r\n')[7]).flags }
|
||||
applied = object
|
||||
return new Response(JSON.stringify({ generation: String(object.generation) }))
|
||||
}
|
||||
if (text.includes('?alt=media')) {
|
||||
return new Response(JSON.stringify({ v: 1, cellId: CELL, flags: object.flags }))
|
||||
}
|
||||
if (text.includes('/storage/v1/b/')) return new Response(JSON.stringify({ generation: String(object.generation) }))
|
||||
if (text.endsWith('/v1/admin/runtime-status')) {
|
||||
if (!cellAnswers) return new Response('', { status: 502 })
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
role: 'cell',
|
||||
cellId: CELL,
|
||||
flagsApplied: applied,
|
||||
supportedFlags: {
|
||||
admitMode: { type: 'enum', values: ['db', 'reserve'] },
|
||||
readinessLocal: { type: 'boolean' }
|
||||
},
|
||||
admitModeEffective: applied.flags.admitMode,
|
||||
runtime: { controls: 10 }
|
||||
})
|
||||
)
|
||||
}
|
||||
if (text.endsWith('/v1/admin/cell-admit-mode')) {
|
||||
const admitMode = JSON.parse(init.body).admitMode
|
||||
return new Response(JSON.stringify({ v: 1, cellId: CELL, admitMode: admitMode ?? 'reserve', heartbeatFresh: true }))
|
||||
}
|
||||
throw new Error(`unexpected fetch ${text}`)
|
||||
}
|
||||
return { fetchImpl, calls, uploads: () => uploads }
|
||||
}
|
||||
|
||||
const flipDependencies = (fetchImpl, lines, sleeps = []) => {
|
||||
let clock = START
|
||||
return {
|
||||
fetchImpl,
|
||||
accessToken: 'access',
|
||||
idToken: 'id',
|
||||
audit: { writtenBy: 'reserve-watch/test', runId: '1', runAttempt: '1' },
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { sleeps.push(ms); clock += ms },
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
}
|
||||
}
|
||||
|
||||
const flipValues = (overrides = {}) => ({
|
||||
mode: 'write',
|
||||
'cell-id': CELL,
|
||||
'cell-origin': 'https://c26.relay.onorca.dev',
|
||||
'project-id': 'onorca-cloud',
|
||||
'director-origin': 'https://relay.onorca.dev',
|
||||
confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}`,
|
||||
...overrides
|
||||
})
|
||||
|
||||
test('the flip back is the flag tool write: admitMode=db at the generation just read, then Postgres db', async () => {
|
||||
const { fetchImpl, calls } = fakeFlipBack()
|
||||
const lines = []
|
||||
const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines))
|
||||
assert.equal(result.written, true)
|
||||
const upload = calls.find((call) => call.url.includes('/upload/storage/v1/'))
|
||||
assert.match(upload.body, /"flags":\{"admitMode":"db","readinessLocal":true\}/)
|
||||
assert.ok(calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode === 'db'))
|
||||
assert.deepEqual(lines.at(-1), {
|
||||
event: 'orca_relay_reserve_watch_flip_back',
|
||||
mode: 'write',
|
||||
cellId: CELL,
|
||||
written: true,
|
||||
generation: '8'
|
||||
})
|
||||
})
|
||||
|
||||
test('a dry-run flip back plans the write and writes nothing', async () => {
|
||||
const { fetchImpl, calls } = fakeFlipBack()
|
||||
const lines = []
|
||||
const result = await flipBack(flipValues({ mode: 'dry-run', confirmation: '' }), {
|
||||
fetchImpl,
|
||||
accessToken: 'access',
|
||||
idToken: 'id',
|
||||
audit: {},
|
||||
sleep: async () => {},
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
assert.equal(result.written, false)
|
||||
assert.ok(!calls.some((call) => call.url.includes('/upload/')))
|
||||
assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode))
|
||||
assert.equal(lines.find((line) => line.event === 'orca_relay_cell_flags_plan').desired.flags.admitMode, 'db')
|
||||
})
|
||||
|
||||
test('a write flip back without the typed confirmation is refused before any read', async () => {
|
||||
const { fetchImpl, calls } = fakeFlipBack()
|
||||
await assert.rejects(
|
||||
flipBack(flipValues({ confirmation: '' }), { fetchImpl, accessToken: 'a', idToken: 'i', audit: {}, log: () => {} }),
|
||||
/WRITE_RELAY_CELL_FLAGS production-gce-c26/
|
||||
)
|
||||
assert.equal(calls.length, 0)
|
||||
})
|
||||
|
||||
test('a generation conflict is re-read and retried while the switch file still says reserve', async () => {
|
||||
const { fetchImpl, uploads } = fakeFlipBack({ onUpload: (n) => (n < 3 ? { admitMode: 'reserve' } : 'ok') })
|
||||
const lines = []
|
||||
const sleeps = []
|
||||
const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps))
|
||||
assert.equal(result.written, true)
|
||||
assert.equal(uploads(), 3)
|
||||
assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_flip_back_conflict').map((line) => line.attempt), [1, 2])
|
||||
assert.deepEqual(sleeps.slice(0, 1), [2_000])
|
||||
assert.equal(lines.at(-1).generation, '10')
|
||||
})
|
||||
|
||||
test('a conflict whose writer already flipped the cell to db ends the flip back without a write', async () => {
|
||||
const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'db' }) })
|
||||
const lines = []
|
||||
const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines))
|
||||
assert.equal(uploads(), 1)
|
||||
assert.equal(result.written, false)
|
||||
assert.equal(result.alreadyDb, true)
|
||||
assert.deepEqual(lines.at(-1), { event: 'orca_relay_reserve_watch_flip_back', mode: 'write', cellId: CELL, written: false, generation: '8', alreadyDb: true })
|
||||
})
|
||||
|
||||
test('conflicts past the retry bound fail the flip back', async () => {
|
||||
const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'reserve' }) })
|
||||
await assert.rejects(flipBack(flipValues(), flipDependencies(fetchImpl, [])), /object changed since expected-generation/)
|
||||
assert.equal(uploads(), FLIP_BACK_ATTEMPTS)
|
||||
})
|
||||
|
||||
test('an unreachable cell fails with the exact break-glass inputs and writes nothing', async () => {
|
||||
const { fetchImpl, uploads } = fakeFlipBack({ cellAnswers: false })
|
||||
const lines = []
|
||||
await assert.rejects(
|
||||
flipBack(flipValues(), flipDependencies(fetchImpl, lines)),
|
||||
(error) => {
|
||||
assert.match(error.message, /production-gce-c26 does not answer runtime-status \(cell runtime status returned 502\)/)
|
||||
assert.match(error.message, /set=admitMode=default, break-glass=BREAK GLASS production-gce-c26, expected-generation=7/)
|
||||
return true
|
||||
}
|
||||
)
|
||||
assert.equal(uploads(), 0)
|
||||
assert.deepEqual(lines.at(-1).inputs, {
|
||||
mode: 'write',
|
||||
'cell-id': CELL,
|
||||
set: 'admitMode=default',
|
||||
'break-glass': `BREAK GLASS ${CELL}`,
|
||||
'expected-generation': '7',
|
||||
confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}`
|
||||
})
|
||||
})
|
||||
|
||||
test('the 503 background is the 10 minutes before flipped-at, read once they are published', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
// Flipped one minute ago: the minutes before it are not published until 3 minutes after it.
|
||||
const flippedAt = START - MINUTE + 20_000
|
||||
const config = parseAt(
|
||||
watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '3', 'flipped-at': new Date(flippedAt).toISOString() })
|
||||
)
|
||||
const sources = fakeSources({ per503: (minute) => (minute < START - MINUTE ? 4 : 40) })
|
||||
const lines = []
|
||||
const state = await watchSegment(config, {
|
||||
...sources,
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
// Pending until then, and pending is not a gap: the streak stays at zero.
|
||||
assert.deepEqual(lines.map((line) => line.rules.director503.status), ['pending', 'pending', 'pass'])
|
||||
assert.match(lines[0].rules.director503.reason, /background before the flip is not published yet/)
|
||||
assert.deepEqual(lines.map((line) => line.unverifiableStreak), [0, 0, 0])
|
||||
const baseline = sources.monitoringSpans.find((span) => span.endedAt === START - MINUTE)
|
||||
assert.equal(baseline.startedAt, START - 11 * MINUTE)
|
||||
assert.deepEqual(state.background, { medianPerMinute: 4, minutes: 10, source: 'measured' })
|
||||
// An explicit value overrides the measurement.
|
||||
const explicit = initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 9, flippedAt })
|
||||
assert.deepEqual(explicit.background, { medianPerMinute: 9, source: 'input' })
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a quiet fleet with zero director 503s watches on and never trips', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
const config = parseAt(
|
||||
watchValues({
|
||||
'state-file': join(directory, 'state.json'),
|
||||
'segment-minutes': '20',
|
||||
'flipped-at': new Date(START - 2 * MINUTE).toISOString()
|
||||
})
|
||||
)
|
||||
const lines = []
|
||||
const state = await watchSegment(config, {
|
||||
...fakeSources({ per503: () => 0 }),
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: (line) => lines.push(JSON.parse(line))
|
||||
})
|
||||
assert.equal(state.state, 'watching')
|
||||
assert.equal(lines.length, 20)
|
||||
assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' })
|
||||
assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass'))
|
||||
assert.ok(lines.every((line) => line.unverifiableStreak === 0))
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('no request series at all is unverifiable and trips at the limit', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
let clock = START
|
||||
const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'background-503-per-minute': '0', 'flipped-at': '' }))
|
||||
const state = await watchSegment(config, {
|
||||
...fakeSources({ per503: () => 0, perOther: () => 0 }),
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: () => {}
|
||||
})
|
||||
assert.equal(state.state, 'tripped')
|
||||
assert.match(state.verdict, /unverifiable: 3 samples in a row could not read director503/)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a later segment judges flipped-at against the watch start, so a long watch does not age it out', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
const stateFile = join(directory, 'state.json')
|
||||
const values = watchValues({ 'state-file': stateFile, 'flipped-at': new Date(START - 23 * 60 * MINUTE).toISOString() })
|
||||
const later = START + 4 * 60 * MINUTE
|
||||
await writeFile(stateFile, JSON.stringify(freshState()))
|
||||
assert.equal((await parseWatchSegment(values, later)).flippedAt, START - 23 * 60 * MINUTE)
|
||||
// The first segment (no state file yet) still refuses it, and records refused.
|
||||
await rm(stateFile)
|
||||
await assert.rejects(parseWatchSegment(values, later), /more than 24 hours/)
|
||||
assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused')
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a corrupt state file fails the segment, never refuses or restarts it, and the outcome reads failed', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
const stateFile = join(directory, 'state.json')
|
||||
const output = join(directory, 'out')
|
||||
const values = watchValues({ 'state-file': stateFile })
|
||||
for (const corrupt of ['null', '[]', '7', JSON.stringify({ state: 'watching', cellId: 'c7' })]) {
|
||||
await writeFile(stateFile, corrupt)
|
||||
await assert.rejects(parseWatchSegment(values, START), /state file/)
|
||||
assert.equal(await readFile(stateFile, 'utf8'), corrupt)
|
||||
assert.equal((await recordOutcome(stateFile, output)).state, 'failed')
|
||||
}
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('a slow runtime-status reply is retried before the flip back calls for break glass', async () => {
|
||||
const { fetchImpl: answering, uploads } = fakeFlipBack()
|
||||
let runtimeCalls = 0
|
||||
// fetchAdminOnceMore makes two tries per read; the first read's two both fail.
|
||||
const fetchImpl = async (url, init) => {
|
||||
if (String(url).endsWith('/v1/admin/runtime-status') && (runtimeCalls += 1) <= 2) return new Response('', { status: 503 })
|
||||
return answering(url, init)
|
||||
}
|
||||
const lines = []
|
||||
const sleeps = []
|
||||
const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps))
|
||||
assert.equal(result.written, true)
|
||||
assert.equal(uploads(), 1)
|
||||
assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_runtime_retry').map((line) => line.attempt), [1])
|
||||
assert.ok(!lines.some((line) => line.event === 'orca_relay_reserve_watch_break_glass_required'))
|
||||
assert.ok(sleeps.includes(2_000))
|
||||
assert.ok(FLIP_BACK_RUNTIME_ATTEMPTS > 1)
|
||||
})
|
||||
|
||||
const watchWorkflow = readRelayWorkflow('operate-relay-production-reserve-watch.yml')
|
||||
const jobOf = (name) => {
|
||||
const start = watchWorkflow.indexOf(`\n ${name}:\n`)
|
||||
assert.notEqual(start, -1, `job ${name} is missing`)
|
||||
const next = watchWorkflow.slice(start + 1).search(/\n [a-z-]+:\n/)
|
||||
return watchWorkflow.slice(start, next === -1 ? undefined : start + 1 + next)
|
||||
}
|
||||
|
||||
test('workflow: neither job takes the flag workflow group, which would let a newer run cancel a pending flip back', () => {
|
||||
assert.doesNotMatch(watchWorkflow, /group: relay-production-cell-flags/)
|
||||
assert.doesNotMatch(jobOf('flip-back'), /^\s+concurrency:/m)
|
||||
assert.doesNotMatch(jobOf('watch'), /^\s+concurrency:/m)
|
||||
})
|
||||
|
||||
test('workflow: a watch that failed or was cancelled without a verdict flips back too', () => {
|
||||
const flip = jobOf('flip-back')
|
||||
// A cancel re-evaluates each step: every flip-back step must run on always() too.
|
||||
const steps = flip.slice(flip.indexOf('\n steps:\n')).split(/\n - /).slice(1)
|
||||
assert.equal(steps.length, 7)
|
||||
for (const step of steps) assert.match(step, /\n\s+if: \$\{\{ always\(\) \}\}/, step.split('\n')[0])
|
||||
assert.match(flip, /if: \$\{\{ always\(\) && vars\.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs\.watch\.result != 'skipped'/)
|
||||
assert.match(flip, /!contains\(fromJSON\('\["completed","stopped","refused"\]'\), needs\.watch\.outputs\.state\)/)
|
||||
// Every segment checks its ID token's shape, and runs the one shared command.
|
||||
const segments = watchWorkflow.match(/run: bash -c "\$\{WATCH_SEGMENT\}"/g) ?? []
|
||||
assert.equal(segments.length, 6)
|
||||
assert.match(jobOf('watch'), /process\.env\.ORCA_RELAY_ADMIN_ID_TOKEN/)
|
||||
})
|
||||
|
||||
test('a live rollout lease refuses the watch before its first sample; an expired one does not', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
|
||||
try {
|
||||
const stateFile = join(directory, 'state.json')
|
||||
const output = join(directory, 'out')
|
||||
const config = parseAt(watchValues({ 'state-file': stateFile, 'github-output': output, 'segment-minutes': '1' }))
|
||||
const lease = (expiresAt) => ({ record: { expires_at: expiresAt }, holder: 'Deploy Relay Production Same-Cap (run 9)' })
|
||||
let clock = START
|
||||
const dependencies = (rolloutLease) => ({
|
||||
...fakeSources({ rolloutLease }),
|
||||
idToken: 'id',
|
||||
retryDelayMs: 0,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => { clock += ms },
|
||||
log: () => {}
|
||||
})
|
||||
const held = await watchSegment(config, dependencies(lease(START + MINUTE)))
|
||||
assert.equal(held.state, 'refused')
|
||||
assert.match(held.verdict, /a rollout holds the Cloud SQL rollout lease until .*Same-Cap \(run 9\)/)
|
||||
assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused')
|
||||
assert.match(await readFile(output, 'utf8'), /^state=refused\n/)
|
||||
await rm(stateFile)
|
||||
assert.equal((await watchSegment(config, dependencies(lease(START - 1)))).state, 'watching')
|
||||
// Only the first segment checks: a later one carries on from its state file.
|
||||
assert.equal((await watchSegment(config, dependencies(lease(clock + MINUTE)))).state, 'watching')
|
||||
// A lease it cannot read refuses too, as the lease action fails closed.
|
||||
assert.match(await rolloutLeaseRefusal(async () => { throw new Error('403') }, START), /lease is unreadable \(403\)/)
|
||||
assert.equal(await rolloutLeaseRefusal(async () => null, START), null)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
const stepOf = (job, name) => {
|
||||
const text = jobOf(job)
|
||||
const start = text.indexOf(`- name: ${name}\n`)
|
||||
assert.notEqual(start, -1, `step ${name} is missing`)
|
||||
const end = text.indexOf('\n - ', start + 1)
|
||||
return text.slice(start, end === -1 ? undefined : end)
|
||||
}
|
||||
const ALARM_COMMAND = /node dev\/scripts\/watch-relay-reserve-cell\.mjs alarm --kind KIND \\\n\s+--mode "\$\{MODE\}" --cell-id "\$\{CELL_ID\}" --project-id "\$\{GCP_PROJECT_ID\}" --verdict "\$\{VERDICT_VAR\}"/
|
||||
const alarmCommand = (kind, verdict) => new RegExp(ALARM_COMMAND.source.replace('KIND', kind).replace('VERDICT_VAR', verdict))
|
||||
|
||||
test('workflow: a flip back that did not land and a refused watch both raise the alarm', () => {
|
||||
const alarm = stepOf('flip-back', 'Alarm when the flip back did not land')
|
||||
assert.match(alarm, /if: \$\{\{ always\(\) \}\}/)
|
||||
assert.match(alarm, /grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "\$\{RUNNER_TEMP\}\/relay-reserve-flip-back\.jsonl"/)
|
||||
assert.match(alarm, alarmCommand('flip-back-failed', 'VERDICT'))
|
||||
const refused = stepOf('watch', 'Alarm on a refused watch')
|
||||
assert.match(refused, /if: \$\{\{ always\(\) && steps\.outcome\.outputs\.state == 'refused' \}\}/)
|
||||
assert.match(refused, /WATCH_VERDICT: \$\{\{ steps\.outcome\.outputs\.verdict \}\}/)
|
||||
assert.match(refused, alarmCommand('refused', 'WATCH_VERDICT'))
|
||||
// After the outcome step, so it reads the recorded state.
|
||||
assert.ok(jobOf('watch').indexOf('- name: Alarm on a refused watch') > jobOf('watch').indexOf('- id: outcome'))
|
||||
// The flip-back line names only the write's own event, never its conflict retries.
|
||||
assert.match(readFileSync(new URL('./watch-relay-reserve-cell.mjs', import.meta.url), 'utf8'), /event: 'orca_relay_reserve_watch_flip_back',\n/)
|
||||
})
|
||||
|
||||
test('an alarm is an escaped error annotation everywhere, and in write mode a paging log line', async () => {
|
||||
const calls = []
|
||||
const lines = []
|
||||
const runGcloud = async (args, options) => (calls.push({ args, options }), { stdout: '' })
|
||||
const alarm = (overrides) =>
|
||||
raiseAlarm(
|
||||
{ mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'refused', reason: '50%\r\nlease held', ...overrides },
|
||||
{ runGcloud, log: (line) => lines.push(line) }
|
||||
)
|
||||
assert.deepEqual(await alarm(), { paged: true })
|
||||
assert.equal(
|
||||
lines[0],
|
||||
`::error title=Reserve watch refused for ${CELL}::${CELL} may be in reserve mode and unwatched: the watch was refused (50%25%0D%0Alease held). ` +
|
||||
'Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.'
|
||||
)
|
||||
assert.deepEqual(calls[0].args, [
|
||||
'logging', 'write', ALARM_LOG_NAME,
|
||||
JSON.stringify({ event: ALARM_EVENT, cellId: CELL, kind: 'refused', reason: '50%\r\nlease held' }),
|
||||
'--payload-type=json', '--severity=ERROR', '--project', 'onorca-cloud'
|
||||
])
|
||||
// A dry run never flips back, so its alarm annotates but does not page.
|
||||
assert.deepEqual(await alarm({ mode: 'dry-run', kind: 'flip-back-failed' }), { paged: false })
|
||||
assert.equal(calls.length, 1)
|
||||
assert.match(lines[1], /^::error title=Reserve watch flip-back-failed for production-gce-c26::.*did not land/)
|
||||
// A title property escapes its own separators too.
|
||||
await alarm({ mode: 'dry-run', cellId: 'a:b,c' })
|
||||
assert.match(lines[2], /^::error title=Reserve watch refused for a%3Ab%2Cc::/)
|
||||
// A failed log write is named, never swallowed silently.
|
||||
const failed = await raiseAlarm(
|
||||
{ mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x' },
|
||||
{ runGcloud: async () => { throw new Error('PERMISSION_DENIED') }, log: (line) => lines.push(line) }
|
||||
)
|
||||
assert.deepEqual(failed, { paged: false })
|
||||
assert.match(lines.at(-1), /^::error title=Reserve watch page failed::.*nothing pages: PERMISSION_DENIED/)
|
||||
await assert.rejects(alarm({ kind: 'other' }), /--kind must be one of/)
|
||||
// A missing project still annotates, then names the page it could not send.
|
||||
const before = calls.length
|
||||
assert.deepEqual(await alarm({ projectId: undefined }), { paged: false })
|
||||
assert.equal(calls.length, before)
|
||||
assert.match(lines.at(-2), /^::error title=Reserve watch refused for /)
|
||||
assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/)
|
||||
})
|
||||
@@ -34,7 +34,7 @@ const EXPECTED_CONDITIONS = {
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'",
|
||||
|
||||
github:
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-reserve-watch.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
|
||||
github_monitor:
|
||||
"assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'",
|
||||
github_fence:
|
||||
|
||||
@@ -109,7 +109,7 @@ identity bootstrap apply. The providers require their exact workflows on
|
||||
`refs/heads/main` with the `production` environment. Missing values fail
|
||||
closed; no dedicated operations identity falls back to the shared deploy identity.
|
||||
|
||||
The shared production identity is restricted to seven named direct Relay callers plus the exact
|
||||
The shared production identity is restricted to nine named direct Relay callers plus the exact
|
||||
regional-rehome and same-cap reusable wrapper/job pairs on `main` in the `production` environment.
|
||||
Its Artifact Registry and Cloud Run mutation permissions are scoped to the Orca repository, Relay
|
||||
director, and Relay fence broker; it cannot mutate the API or auth services.
|
||||
@@ -461,8 +461,9 @@ margin covers. Each cell is isolated, drained until restart-safe, replaced
|
||||
from a targeted saved plan, and restored only after a new incarnation reports the exact digest, cap,
|
||||
heartbeat, and rehome protocol. Restart-safe means the cell runtime itself carries nothing live (no
|
||||
controls, in-flight connections, reserved connection units, splices, or queued bytes) and no
|
||||
migration is open, for a whole drain pace window, which every live restart-safe call must pass.
|
||||
Pre-auth and total connections are printed but do not reset the window: they include
|
||||
migration is open, on two consecutive samples the second of which is taken at or after the end
|
||||
of the drain pace window. Every live restart-safe call supplies that window (`--pace-window-ms`).
|
||||
Pre-auth and total connections are printed but do not restart the count: they include
|
||||
unauthenticated redials that lose nothing on a restart. Director activity leases left by hosts that already went or cannot
|
||||
be placed do not hold the restart; every `relay_capacity_transition_restart_progress` sample and
|
||||
the final verified line report them under `stranded`. The durable
|
||||
@@ -539,11 +540,10 @@ drained. Then read the cell's live runtime image from
|
||||
The `drain-pace-window-ms` input sets the window the cell spreads its drain sends over. Each
|
||||
drained host re-dials the director as soon as it reads `drain`, so the window sets the re-placement
|
||||
arrival rate: hosts / window, about 1.8-2.7 hosts/s for a US cell at the default. The window also
|
||||
sets two waits: restart-safe needs an empty runtime for (ceil(window / 5 s) + 1) consecutive
|
||||
5-second samples, and the drain step's overall timeout is the 15-minute migration lease plus the
|
||||
window, plus the window's excess over `300000` (20 minutes at the default, 50 at `1200000`).
|
||||
Because the quiet starts only once the last host has left, a drain step takes about twice the
|
||||
window: about 30 minutes at `900000` and 40 at `1200000`.
|
||||
sets two waits: restart-safe needs an empty runtime on two consecutive 5-second samples, the
|
||||
second at or after the window's end, and the drain step's overall timeout is the 15-minute
|
||||
migration lease plus the window (20 minutes at the default, 35 at `1200000`). The runtime empties
|
||||
as the window closes, so a drain step takes about the window plus a few seconds to half a minute.
|
||||
|
||||
- Allowed values are `300000` (the default, and every wave before this input), `60000`,
|
||||
`30000`, `900000`, and `1200000`. The wave validator refuses anything else, and each cell job
|
||||
@@ -634,7 +634,7 @@ monitor already excludes 503s from its director 5xx rule.
|
||||
- its `paceVerdict` is PASS, sealed only from a report on that cell that drained at least 400
|
||||
hosts at that pace;
|
||||
- time to empty is within window + hosts / 50 s + 10 s. Take `settledAfterSeconds` minus the
|
||||
restart-safe quiet; this is an upper bound, since it includes the isolate.
|
||||
one 5-second confirming sample; this is an upper bound, since it includes the isolate.
|
||||
|
||||
The batch check enforces the PASS. A canary that sealed anything else authorizes only
|
||||
`300000` batches. The 400-host floor exists because a pace is an arrival rate (hosts /
|
||||
@@ -801,6 +801,76 @@ It ends with the single command that finishes from where it stopped. That comman
|
||||
state. Each run writes a timestamped log under `~/.orca/relay-director-deploy/`
|
||||
(`--log-directory` overrides it).
|
||||
|
||||
## Reserve watch
|
||||
|
||||
`Operate Relay Production Reserve Watch` watches one cell already switched to reserve mode and, on
|
||||
the first step 5 stop rule it trips, flips it back to db through the flag tool's own write
|
||||
(`operate-relay-cell-flags.mjs`: CAS, read-back, re-registration wait, Postgres record). It is
|
||||
manual only and inert until dispatched; `dry-run` logs the flag tool's plan instead of writing.
|
||||
`write` takes the flag workflow's confirmation, `WRITE_RELAY_CELL_FLAGS <cell-id>`.
|
||||
|
||||
It samples once a minute for up to 300 minutes, re-authenticating every 55-minute segment because
|
||||
an ID token lasts an hour, and logs one JSON line per sample. The 503 background must predate the
|
||||
flip: give `flipped-at` (when the reserve write applied; the background is the median of the 10
|
||||
minutes before it) or `background-503-per-minute`, which overrides it. With neither, the watch is
|
||||
refused. A watch dispatched before the reserve write applied waits up to 10 minutes for the cell to
|
||||
report reserve, judging nothing meanwhile; a cell that never does ends the watch refused, without a
|
||||
write. It also refuses to start while any run holds the Cloud SQL rollout lease (same-cap and
|
||||
capacity rolls hold it for their whole run, and it names no cell), or when that lease cannot be
|
||||
read. It only reads the lease: holding it for hours would block every rollout. Pausing on a roll
|
||||
that starts mid-watch was not built, since a pause blinds the stop rules for the roll's length and
|
||||
a flip back mid-roll races its drain, so do not start a roll on a watched cell. It trips on, in
|
||||
this order:
|
||||
|
||||
| Rule | Signal | Exact or proxy |
|
||||
|---|---|---|
|
||||
| Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact |
|
||||
| Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units |
|
||||
| 503s | Director `request_count` 503s above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line |
|
||||
| Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat |
|
||||
| Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact |
|
||||
| Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell |
|
||||
|
||||
Any rule it cannot read in a sample makes that sample unverifiable; `unverifiable-limit` such
|
||||
samples in a row is a trip of its own. The 503 read takes every response code in one query, and the
|
||||
newest minute any code has a point for is how far the series is published: a 503 minute at or
|
||||
before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not
|
||||
counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable
|
||||
timestamp is unverifiable. This assumes the codes publish together; one code lagging the others
|
||||
delays a trip by its lag but cannot hide one, because a later read sees its points. Until the
|
||||
minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does
|
||||
not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which
|
||||
every placing director logs each minute, in the same query and window: with no summary from the
|
||||
last 150 s in it, the read is blind and the sample unverifiable, not zero duplicates. The duplicate
|
||||
rule keys a seat by director instance, so a director restart (a new instance id) starts its
|
||||
first-seen clock over. A cell whose switch file no longer says reserve ends the watch without a
|
||||
write.
|
||||
|
||||
In `write` mode a watch that fails, times out or is cancelled before a verdict flips the cell back
|
||||
as a trip would; the flip-back job and each of its steps run on `always()`, so a cancel during the
|
||||
lease wait does not skip the switch-file write. That cancel can still kill the process before
|
||||
Postgres records db, leaving the file at db and Postgres at reserve; a force-cancel can stop it
|
||||
outright. The workflow's only group is
|
||||
its own per-cell one, which a running run never yields. It stays out of the flag workflow's group
|
||||
because GitHub keeps one pending run per group and a newer one cancels it, so a flip back queued
|
||||
there could be dropped by any later flag dispatch, and a watch held there would block manual writes
|
||||
for hours. The flag tool's generation check (CAS) orders it against other writes instead; on a
|
||||
conflict it re-reads and retries, up to 5 times, while the switch file still says reserve, and stops
|
||||
if it already says db. A cell that still does not answer runtime-status after 4 tries with backoff
|
||||
cannot take the ordinary flip back: the run fails and prints the exact break-glass inputs for
|
||||
`Operate Relay Production Cell Flags` (`set=admitMode=default`, `break-glass=BREAK GLASS <cell-id>`).
|
||||
|
||||
An operator watches the run. A refused watch and a flip back that did not land (no ID token, a
|
||||
failed login, a cell that does not answer) can each leave the cell in reserve with nothing watching
|
||||
it, so both raise an `::error::` annotation naming the cell, as the clock-skew monitor does. In
|
||||
`write` mode both also log `orca_relay_reserve_watch_alarm` (log `orca-relay-reserve-watch`,
|
||||
resource `global`) with the deploy identity, and the `Orca Relay: reserve cell may be unwatched`
|
||||
alert pages on any such line. A failed login cannot page: the runner then has no credentials for
|
||||
`gcloud logging write`, so only the `::error::` and the operator watching the run report it. Apply
|
||||
the root Terraform that adds the workflow to the shared deploy identity's provider condition and
|
||||
grants that identity `roles/logging.logWriter` before the first `write` run; until then the
|
||||
workflow cannot log in and no alarm line can be written.
|
||||
|
||||
## Mobile push gateway
|
||||
|
||||
`Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path
|
||||
|
||||
@@ -48,8 +48,8 @@ relay_gce_cells = {
|
||||
boot_disk_gb = 30
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
|
||||
capacity_requests = 4000
|
||||
# Step-5 image F (main 3c1141f2). Never 2d0f6e6: it cannot report capped telemetry.
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
|
||||
# Step-5 image G (main 2de73812, reports preAuthLatencyMs). Never 2d0f6e6: it cannot report capped telemetry.
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
|
||||
# Step-5 staging e2e: c1 is the second US reserve cell, so it needs a cap and rehome trust.
|
||||
connection_hard_cap = 600
|
||||
connection_unobserved_bound = 60
|
||||
@@ -86,7 +86,7 @@ relay_gce_cells = {
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 10
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
|
||||
initially_enabled = false
|
||||
connection_hard_cap = 3000
|
||||
connection_unobserved_bound = 60
|
||||
@@ -100,7 +100,7 @@ relay_gce_cells = {
|
||||
boot_disk_gb = 30
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
|
||||
capacity_requests = 4000
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
|
||||
connection_hard_cap = 600
|
||||
connection_unobserved_bound = 60
|
||||
}
|
||||
@@ -111,7 +111,7 @@ relay_gce_cells = {
|
||||
boot_disk_gb = 30
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
|
||||
capacity_requests = 4000
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
|
||||
connection_hard_cap = 600
|
||||
connection_unobserved_bound = 60
|
||||
}
|
||||
@@ -122,7 +122,7 @@ relay_gce_cells = {
|
||||
boot_disk_gb = 30
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
|
||||
capacity_requests = 4000
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
|
||||
connection_hard_cap = 600
|
||||
connection_unobserved_bound = 60
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ locals {
|
||||
"deploy-relay-production.yml",
|
||||
"operate-relay-asia-admission.yml",
|
||||
"operate-relay-production-cell-flags.yml",
|
||||
"operate-relay-production-reserve-watch.yml",
|
||||
"publish-relay-production.yml"
|
||||
|
||||
]
|
||||
@@ -380,6 +381,16 @@ resource "google_project_iam_member" "github_logging_viewer" {
|
||||
member = google_service_account.github_deploy[0].member
|
||||
}
|
||||
|
||||
# The reserve watch logs its paging alarm line (orca_relay_reserve_watch_alarm) from the runner.
|
||||
# Accepted risk: Cloud Logging cannot narrow writes to one log, so this identity can write any line.
|
||||
resource "google_project_iam_member" "github_log_writer" {
|
||||
count = local.relay_create_production_ops_identity ? 1 : 0
|
||||
|
||||
project = var.project_id
|
||||
role = "roles/logging.logWriter"
|
||||
member = google_service_account.github_deploy[0].member
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "github_cloudsql_viewer" {
|
||||
count = local.relay_create_production_ops_identity ? 1 : 0
|
||||
|
||||
|
||||
@@ -1748,6 +1748,74 @@ resource "google_monitoring_alert_policy" "relay_cell_reserve_dead_man_tripped"
|
||||
depends_on = [google_logging_metric.relay_cell_reserve_dead_man_tripped]
|
||||
}
|
||||
|
||||
# The reserve watch workflow logs this from its runner (resource global) when a watch is refused or
|
||||
# a flip back did not land in write mode: either can leave a cell in reserve unwatched.
|
||||
resource "google_logging_metric" "relay_reserve_watch_alarm" {
|
||||
project = var.project_id
|
||||
name = "orca_relay_reserve_watch_alarm"
|
||||
description = "Reserve watch runs that were refused or whose flip back to db did not land, so the cell may be in reserve mode with nothing watching it."
|
||||
filter = "resource.type=\"global\" AND logName=\"projects/${var.project_id}/logs/orca-relay-reserve-watch\" AND jsonPayload.event=\"orca_relay_reserve_watch_alarm\""
|
||||
label_extractors = {
|
||||
cell_id = "EXTRACT(jsonPayload.cellId)"
|
||||
kind = "EXTRACT(jsonPayload.kind)"
|
||||
}
|
||||
|
||||
metric_descriptor {
|
||||
metric_kind = "DELTA"
|
||||
value_type = "INT64"
|
||||
unit = "1"
|
||||
|
||||
labels {
|
||||
key = "cell_id"
|
||||
value_type = "STRING"
|
||||
description = "Durable relay cell identifier."
|
||||
}
|
||||
|
||||
labels {
|
||||
key = "kind"
|
||||
value_type = "STRING"
|
||||
description = "refused or flip-back-failed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_reserve_watch_alarm" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: reserve cell may be unwatched"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "A reserve watch was refused or its flip back did not land"
|
||||
|
||||
condition_threshold {
|
||||
filter = "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_watch_alarm\""
|
||||
comparison = "COMPARISON_GT"
|
||||
threshold_value = 0
|
||||
duration = "0s"
|
||||
|
||||
aggregations {
|
||||
alignment_period = "300s"
|
||||
per_series_aligner = "ALIGN_SUM"
|
||||
cross_series_reducer = "REDUCE_SUM"
|
||||
group_by_fields = ["metric.label.\"cell_id\""]
|
||||
}
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "A write-mode `Operate Relay Production Reserve Watch` run was refused (a rollout held the Cloud SQL rollout lease, the lease was unreadable, bad inputs, or the cell never reported reserve within 10 minutes) or its flip back to db did not land (no ID token, a failed login, or a cell that does not answer). The cell may be in reserve mode with nothing watching it. Read the cell's `runtime-status`; if its switch file says reserve, either rerun the watch or run `cloud-operate-relay-production-cell-flags.yml` with `admitMode=db` for that cell, using the break-glass inputs the failed run printed if the cell does not answer."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_reserve_watch_alarm]
|
||||
}
|
||||
|
||||
# host-session-registry.ts: a flip back closed a control it could not re-register (4409), after 20 refusals.
|
||||
resource "google_logging_metric" "relay_cell_reregistration_gave_up" {
|
||||
project = var.project_id
|
||||
|
||||
+12
-10
@@ -1,12 +1,7 @@
|
||||
{
|
||||
"name": "orca-cloud",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"packageManager": "pnpm@10.34.6",
|
||||
"engines": {
|
||||
"node": ">=24 <27",
|
||||
"pnpm": ">=10"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "pnpm -r build",
|
||||
"dev": "pnpm --filter @orca-cloud/relay dev",
|
||||
@@ -21,14 +16,21 @@
|
||||
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
|
||||
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
|
||||
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck"
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner",
|
||||
"test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck",
|
||||
"test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@orca/process-host": "link:../src/packages/process-host",
|
||||
"@types/node": "^24.19.0",
|
||||
"tsx": "^4.23.15",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.1.11"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=24 <27",
|
||||
"pnpm": ">=10"
|
||||
},
|
||||
"packageManager": "pnpm@10.34.6"
|
||||
}
|
||||
|
||||
Generated
+3
@@ -8,6 +8,9 @@ importers:
|
||||
|
||||
.:
|
||||
devDependencies:
|
||||
'@orca/process-host':
|
||||
specifier: link:../src/packages/process-host
|
||||
version: link:../src/packages/process-host
|
||||
'@types/node':
|
||||
specifier: ^24.19.0
|
||||
version: 24.19.0
|
||||
|
||||
@@ -168,7 +168,7 @@ try {
|
||||
$report.nativeInputs=$verified
|
||||
Write-Stage 'preview-native-input-verification-complete'
|
||||
} else {
|
||||
Install-WindowsInboxSshCapability $Arch $report {param($stage) Write-Stage $stage}
|
||||
Install-WindowsInboxSshCapability $Arch $report {param($stage) Write-Stage $stage} ([IO.Path]::GetDirectoryName([IO.Path]::GetFullPath($Receipt)))
|
||||
$verified=@()
|
||||
foreach($binary in @('sshd.exe','ssh.exe','ssh-keygen.exe','sftp.exe','sftp-server.exe')){
|
||||
$path=Join-Path $sshDir $binary
|
||||
|
||||
@@ -38,22 +38,52 @@ try {
|
||||
function Assert-IsolatedWindowsSshCi([string]$Arch){if($script:refuseCapabilityHost){throw 'Injected host refusal'}}
|
||||
function Assert-WindowsSshGlobalServerDormant([string]$Server){$script:globalChecks++;if($script:globalChecks -eq $script:refuseGlobalCheck){throw 'Injected global server refusal'}}
|
||||
function Assert-WindowsSshStockShell {$script:shellChecks++;if($script:shellChecks -eq $script:refuseShellCheck){throw 'Injected shell refusal'}}
|
||||
function Get-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityQueries++;return @{State=$script:capabilityState}}
|
||||
function Add-WindowsCapability([switch]$Online,[string]$Name){$script:capabilityAdds++;if($script:failCapabilityInstall){throw 'Injected capability install failure'}}
|
||||
function Assert-CapabilityCommand([switch]$Online,[string]$Name,[string]$LogPath,[object]$ErrorAction,[string]$Operation){
|
||||
if(-not $Online -or $Name -ne 'OpenSSH.Server~~~~0.0.1.0' -or $ErrorAction -ne 'Stop' -or [IO.Path]::GetDirectoryName($LogPath) -ne $script:capabilityLogs -or [IO.Path]::GetFileName($LogPath) -notlike "inbox-capability-$Operation-*.log"){throw 'Capability command changed its online identity, error handling or receipt log boundary'}
|
||||
$script:capabilityLogPaths.Add($LogPath)
|
||||
[IO.File]::WriteAllText($LogPath,"$Operation fixture")
|
||||
}
|
||||
function Get-WindowsCapability([switch]$Online,[string]$Name,[string]$LogPath,[object]$ErrorAction){
|
||||
$script:capabilityQueries++;Assert-CapabilityCommand -Online:$Online -Name $Name -LogPath $LogPath -ErrorAction $ErrorAction -Operation 'query'
|
||||
if($script:failCapabilityQuery){Write-Error 'Injected capability query failure' -ErrorAction $ErrorAction}
|
||||
return @{State=$script:capabilityState}
|
||||
}
|
||||
function Add-WindowsCapability([switch]$Online,[string]$Name,[string]$LogPath,[object]$ErrorAction){
|
||||
$script:capabilityAdds++;Assert-CapabilityCommand -Online:$Online -Name $Name -LogPath $LogPath -ErrorAction $ErrorAction -Operation 'install'
|
||||
if($script:activeCapabilityReceipt){
|
||||
$script:installObservedReceipt=Get-Content -LiteralPath $script:activeCapabilityReceipt -Raw | ConvertFrom-Json
|
||||
if($script:installObservedReceipt.inboxCapabilityInitialState -ne 'NotPresent' -or $script:installObservedReceipt.inboxCapabilityOperations[0].status -ne 'passed' -or $script:installObservedReceipt.inboxCapabilityOperations[1].status -ne 'running'){throw 'Initial capability state was not durably recorded before installation'}
|
||||
}
|
||||
if($script:failCapabilityInstall){Write-Error 'Injected capability install failure' -ErrorAction $ErrorAction}
|
||||
return @{Path='C:\';Online=$true;RestartNeeded=$script:capabilityRestartNeeded}
|
||||
}
|
||||
function Reset-CapabilityControl([string]$State){
|
||||
$script:capabilityState=$State;$script:capabilityQueries=0;$script:capabilityAdds=0
|
||||
$script:globalChecks=0;$script:shellChecks=0;$script:refuseGlobalCheck=0;$script:refuseShellCheck=0
|
||||
$script:refuseCapabilityHost=$false;$script:failCapabilityInstall=$false
|
||||
$script:refuseCapabilityHost=$false;$script:failCapabilityInstall=$false;$script:failCapabilityQuery=$false
|
||||
$script:activeCapabilityReceipt=$null;$script:installObservedReceipt=$null;$script:capabilityRestartNeeded=$false
|
||||
$script:capabilityStages=[Collections.Generic.List[string]]::new()
|
||||
$script:capabilityLogPaths=[Collections.Generic.List[string]]::new()
|
||||
}
|
||||
$script:capabilityLogs=Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString('N'))
|
||||
try {
|
||||
foreach($state in @('Installed','NotPresent')){
|
||||
Reset-CapabilityControl $state
|
||||
$capabilityReport=@{}
|
||||
Install-WindowsInboxSshCapability 'x64' $capabilityReport {param($name) $script:capabilityStages.Add($name)}
|
||||
Install-WindowsInboxSshCapability 'x64' $capabilityReport {param($name) $script:capabilityStages.Add($name)} $script:capabilityLogs
|
||||
$expectedAdds=if($state -eq 'Installed'){0}else{1}
|
||||
if($capabilityReport.inboxCapabilityInitialState -ne $state -or $script:capabilityAdds -ne $expectedAdds -or $script:globalChecks -ne 2 -or $script:shellChecks -ne 2 -or ($script:capabilityStages -join ',') -ne 'inbox-capability-start,inbox-capability-complete'){throw 'Shared capability preparation did not preserve the install and guard boundaries'}
|
||||
$expectedStages=@('inbox-capability-start','inbox-capability-query-start','inbox-capability-query-finished')
|
||||
if($expectedAdds){$expectedStages+=@('inbox-capability-install-start','inbox-capability-install-finished')}
|
||||
$expectedStages+=@('inbox-capability-guards-start','inbox-capability-complete')
|
||||
if($capabilityReport.inboxCapabilityInitialState -ne $state -or $script:capabilityQueries -ne 1 -or $script:capabilityAdds -ne $expectedAdds -or $script:globalChecks -ne 2 -or $script:shellChecks -ne 2 -or ($script:capabilityStages -join ',') -ne ($expectedStages -join ',')){throw 'Shared capability preparation did not preserve the install and guard boundaries'}
|
||||
if($capabilityReport.inboxCapabilityOperations.Count -ne 1+$expectedAdds){throw 'Capability operations missing'}
|
||||
foreach($operation in $capabilityReport.inboxCapabilityOperations){
|
||||
if($operation.status -ne 'passed' -or $operation.elapsedMs -isnot [long] -or $operation.elapsedMs -lt 0 -or -not [IO.File]::Exists((Join-Path $script:capabilityLogs $operation.log))){throw 'Capability operation did not retain its timing, verdict or diagnostic log'}
|
||||
}
|
||||
if($expectedAdds -and ($capabilityReport.inboxCapabilityInstallResult.online -ne $true -or $capabilityReport.inboxCapabilityInstallResult.path -ne 'C:\' -or $capabilityReport.inboxCapabilityInstallResult.restartNeeded -ne $false)){throw 'Capability install result omitted'}
|
||||
if(-not $expectedAdds -and $capabilityReport.ContainsKey('inboxCapabilityInstallResult')){throw 'Installed capability invented an install result'}
|
||||
}
|
||||
foreach($fault in @('host','global-before','shell-before','global-after','shell-after','install')){
|
||||
foreach($fault in @('host','global-before','shell-before','global-after','shell-after','query','install')){
|
||||
Reset-CapabilityControl 'NotPresent'
|
||||
switch($fault){
|
||||
'host' {$script:refuseCapabilityHost=$true}
|
||||
@@ -61,26 +91,45 @@ foreach($fault in @('host','global-before','shell-before','global-after','shell-
|
||||
'shell-before' {$script:refuseShellCheck=1}
|
||||
'global-after' {$script:refuseGlobalCheck=2}
|
||||
'shell-after' {$script:refuseShellCheck=2}
|
||||
'query' {$script:failCapabilityQuery=$true}
|
||||
'install' {$script:failCapabilityInstall=$true}
|
||||
}
|
||||
$rejected=$false
|
||||
try {Install-WindowsInboxSshCapability 'x64' @{} {param($name) $script:capabilityStages.Add($name)}} catch {$rejected=$true}
|
||||
$faultReport=@{}
|
||||
try {Install-WindowsInboxSshCapability 'x64' $faultReport {param($name) $script:capabilityStages.Add($name)} $script:capabilityLogs} catch {$rejected=$true}
|
||||
if(-not $rejected -or $script:capabilityStages.Contains('inbox-capability-complete')){throw "Capability fault did not fail closed: $fault"}
|
||||
if($fault -in @('host','global-before','shell-before') -and $script:capabilityAdds){throw 'Capability mutation preceded its host guards'}
|
||||
if($fault -eq 'query' -and $script:capabilityAdds){throw 'Failed query proceeded to installation'}
|
||||
if($fault -in @('query','install') -and ($faultReport.inboxCapabilityOperations[-1].status -ne 'failed' -or $faultReport.inboxCapabilityOperations[-1].elapsedMs -isnot [long])){throw 'Failed servicing operation lost its verdict or duration'}
|
||||
}
|
||||
Reset-CapabilityControl 'NotPresent'
|
||||
foreach($invocation in 1..2){Install-WindowsInboxSshCapability 'x64' @{} {param($name) $script:capabilityStages.Add($name)} $script:capabilityLogs}
|
||||
if($script:capabilityLogPaths.Count -ne 4 -or ($script:capabilityLogPaths | Select-Object -Unique).Count -ne 4){throw 'Separate servicing operations or invocations reused a diagnostic log'}
|
||||
} finally {Remove-Item -LiteralPath $script:capabilityLogs -Recurse -Force -ErrorAction SilentlyContinue}
|
||||
$capabilityRoot=Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString('N'))
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $capabilityRoot | Out-Null
|
||||
$capabilityReceipt=Join-Path $capabilityRoot 'capability.json'
|
||||
$script:capabilityLogs=$capabilityRoot
|
||||
Reset-CapabilityControl 'Installed'
|
||||
Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt
|
||||
$completed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if($completed.status -ne 'passed' -or $completed.inboxCapabilityInitialState -ne 'Installed'){throw 'Capability success receipt missing its observed initial state'}
|
||||
Reset-CapabilityControl 'NotPresent';$script:failCapabilityInstall=$true
|
||||
if($completed.status -ne 'passed' -or $completed.inboxCapabilityInitialState -ne 'Installed' -or $completed.preparationMode -ne 'overlap'){throw 'Capability success receipt missing its observed initial state or default mode'}
|
||||
Reset-CapabilityControl 'NotPresent';$script:capabilityLogs=$capabilityRoot;$script:activeCapabilityReceipt=$capabilityReceipt;$script:capabilityRestartNeeded=$true
|
||||
Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt 'serial'
|
||||
$installed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if($installed.status -ne 'passed' -or $installed.preparationMode -ne 'serial' -or $installed.inboxCapabilityInstallResult.restartNeeded -ne $true -or -not $script:installObservedReceipt){throw 'Serial preparation lost its mode, restart result or pre-install receipt'}
|
||||
Reset-CapabilityControl 'NotPresent';$script:activeCapabilityReceipt=$capabilityReceipt;$script:failCapabilityInstall=$true
|
||||
$rejected=$false
|
||||
try {Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt} catch {$rejected=$true}
|
||||
$failed=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if(-not $rejected -or $failed.status -ne 'failed' -or $failed.inboxCapabilityInitialState -ne 'NotPresent' -or $failed.error -ne 'Injected capability install failure'){throw 'Failed capability preparation masqueraded as a passing receipt'}
|
||||
if($failed.inboxCapabilityOperations[-1].status -ne 'failed' -or -not $failed.stages.stage.Contains('inbox-capability-install-finished')){throw 'Failed preparation omitted its servicing result'}
|
||||
Reset-CapabilityControl 'NotPresent';$script:failCapabilityQuery=$true
|
||||
$rejected=$false
|
||||
try {Initialize-WindowsInboxSshCapability 'x64' $capabilityReceipt} catch {$rejected=$true}
|
||||
$failedQuery=Get-Content -LiteralPath $capabilityReceipt -Raw | ConvertFrom-Json
|
||||
if(-not $rejected -or $failedQuery.status -ne 'failed' -or $failedQuery.inboxCapabilityOperations[0].status -ne 'failed' -or $failedQuery.inboxCapabilityOperations.Count -ne 1 -or $script:capabilityAdds -or $failedQuery.inboxCapabilityInitialState){throw 'Failed query receipt invented a capability state or proceeded to installation'}
|
||||
} finally {Remove-Item -LiteralPath $capabilityRoot -Recurse -Force -ErrorAction SilentlyContinue}
|
||||
& {
|
||||
param($ProofAst)
|
||||
|
||||
@@ -33,26 +33,48 @@ function Write-WindowsSshReceiptStage([hashtable]$Report,[string]$Receipt,[strin
|
||||
return $written
|
||||
}
|
||||
|
||||
function Install-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[hashtable]$Report,[scriptblock]$Stage) {
|
||||
function Install-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[hashtable]$Report,[scriptblock]$Stage,[Parameter(Mandatory=$true)][string]$LogDirectory) {
|
||||
Assert-IsolatedWindowsSshCi $Arch
|
||||
Assert-WindowsSshGlobalServerDormant 'inbox'
|
||||
Assert-WindowsSshStockShell
|
||||
& $Stage 'inbox-capability-start'
|
||||
$capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0'
|
||||
$Report.inboxCapabilityInitialState=[string]$capability.State
|
||||
if($capability.State -ne 'Installed'){Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' | Out-Null}
|
||||
New-Item -ItemType Directory -Force -Path $LogDirectory | Out-Null
|
||||
$invocation=[Guid]::NewGuid().ToString('N')
|
||||
$query=@{operation='query';status='running';log="inbox-capability-query-$invocation.log"}
|
||||
$Report.inboxCapabilityOperations=@($query)
|
||||
& $Stage 'inbox-capability-query-start'
|
||||
$watch=[Diagnostics.Stopwatch]::StartNew()
|
||||
try {
|
||||
$capability=Get-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' -LogPath (Join-Path $LogDirectory $query.log) -ErrorAction Stop
|
||||
$Report.inboxCapabilityInitialState=[string]$capability.State
|
||||
$query.status='passed'
|
||||
} catch {$query.status='failed';$query.error=$_.Exception.Message;throw}
|
||||
finally {$watch.Stop();$query.elapsedMs=$watch.ElapsedMilliseconds;& $Stage 'inbox-capability-query-finished'}
|
||||
if($capability.State -ne 'Installed'){
|
||||
$install=@{operation='install';status='running';log="inbox-capability-install-$invocation.log"}
|
||||
$Report.inboxCapabilityOperations+=@($install)
|
||||
& $Stage 'inbox-capability-install-start'
|
||||
$watch=[Diagnostics.Stopwatch]::StartNew()
|
||||
try {
|
||||
$result=Add-WindowsCapability -Online -Name 'OpenSSH.Server~~~~0.0.1.0' -LogPath (Join-Path $LogDirectory $install.log) -ErrorAction Stop
|
||||
$Report.inboxCapabilityInstallResult=@{path=$result.Path;online=$result.Online;restartNeeded=$result.RestartNeeded}
|
||||
$install.status='passed'
|
||||
} catch {$install.status='failed';$install.error=$_.Exception.Message;throw}
|
||||
finally {$watch.Stop();$install.elapsedMs=$watch.ElapsedMilliseconds;& $Stage 'inbox-capability-install-finished'}
|
||||
}
|
||||
& $Stage 'inbox-capability-guards-start'
|
||||
Assert-WindowsSshGlobalServerDormant 'inbox'
|
||||
Assert-WindowsSshStockShell
|
||||
& $Stage 'inbox-capability-complete'
|
||||
}
|
||||
|
||||
function Initialize-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[string]$Receipt) {
|
||||
$report=@{scope='Windows inbox OpenSSH.Server capability preparation only';arch=$Arch;sourceSha=$env:GITHUB_SHA;runId=$env:GITHUB_RUN_ID;runAttempt=$env:GITHUB_RUN_ATTEMPT;runnerName=$env:RUNNER_NAME;imageVersion=$env:ImageVersion;status='running';stages=@();globalBootstrapCleanup='Disposable CI VM destruction is the boundary; no global sshd is started'}
|
||||
function Initialize-WindowsInboxSshCapability([ValidateSet('arm64','x64')][string]$Arch,[string]$Receipt,[ValidateSet('overlap','serial')][string]$PreparationMode='overlap') {
|
||||
$report=@{scope='Windows inbox OpenSSH.Server capability preparation only';arch=$Arch;sourceSha=$env:GITHUB_SHA;runId=$env:GITHUB_RUN_ID;runAttempt=$env:GITHUB_RUN_ATTEMPT;runnerName=$env:RUNNER_NAME;imageVersion=$env:ImageVersion;preparationMode=$PreparationMode;status='running';stages=@();globalBootstrapCleanup='Disposable CI VM destruction is the boundary; no global sshd is started'}
|
||||
try {
|
||||
if(-not (Write-WindowsSshReceiptStage $report $Receipt 'preparation-start')){throw 'Initial progress receipt unavailable; refuse provisioning'}
|
||||
Install-WindowsInboxSshCapability $Arch $report {param($name)
|
||||
if(-not (Write-WindowsSshReceiptStage $report $Receipt $name)){throw 'Capability preparation progress receipt unavailable'}
|
||||
}
|
||||
} ([IO.Path]::GetDirectoryName([IO.Path]::GetFullPath($Receipt)))
|
||||
$report.status='passed'
|
||||
} catch {$report.status='failed';$report.error=$_.Exception.Message;throw}
|
||||
finally {
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# Prune after removing reads: pnpm check:focus-read-ratchet --prune
|
||||
2 src/renderer/src/App.tsx
|
||||
5 src/renderer/src/app-shell/use-app-shell-services.ts
|
||||
9 src/renderer/src/components/JiraIssueWorkspace.tsx
|
||||
1 src/renderer/src/components/JiraIssueWorkspace.tsx
|
||||
1 src/renderer/src/components/Landing.tsx
|
||||
2 src/renderer/src/components/LinearIssueTextEditor.tsx
|
||||
3 src/renderer/src/components/LinearIssueWorkspace.tsx
|
||||
@@ -219,7 +219,7 @@
|
||||
1 src/renderer/src/components/tab-group/workspace-tab-close-commands.ts
|
||||
1 src/renderer/src/components/task-page-jira-status-order.ts
|
||||
2 src/renderer/src/components/task-page-linear-issue-model.tsx
|
||||
11 src/renderer/src/components/task-page/TaskPage.tsx
|
||||
9 src/renderer/src/components/task-page/TaskPage.tsx
|
||||
2 src/renderer/src/components/task-page/jira/IssueAssigneeField.tsx
|
||||
1 src/renderer/src/components/task-page/linear/Filters.tsx
|
||||
1 src/renderer/src/components/terminal-pane/TerminalPaneSurface.tsx
|
||||
@@ -254,8 +254,7 @@
|
||||
2 src/renderer/src/components/use-new-linear-issue-projects.ts
|
||||
4 src/renderer/src/components/use-task-page-jira-creation-metadata.ts
|
||||
4 src/renderer/src/components/use-task-page-jira-issue-creation.ts
|
||||
2 src/renderer/src/components/use-task-page-jira-list-effects.ts
|
||||
2 src/renderer/src/components/use-task-page-jira-list-state.ts
|
||||
1 src/renderer/src/components/use-task-page-jira-list-effects.ts
|
||||
4 src/renderer/src/components/use-task-page-linear-board.ts
|
||||
6 src/renderer/src/components/use-task-page-linear-creation-state.ts
|
||||
4 src/renderer/src/components/use-task-page-linear-issue-creation.ts
|
||||
@@ -464,6 +463,7 @@
|
||||
1 src/renderer/src/store/slices/tabs/tabs-close-actions.ts
|
||||
2 src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts
|
||||
2 src/renderer/src/store/slices/terminal-tab-retirement.ts
|
||||
3 src/renderer/src/store/slices/workspace-attention/workspace-unread-characterization-fixture.ts
|
||||
1 src/renderer/src/store/slices/worktree-visibility-owner-settings.ts
|
||||
3 src/renderer/src/store/slices/worktree-visibility-settings-write.ts
|
||||
3 src/renderer/src/store/slices/worktrees-slice-test-harness.ts
|
||||
@@ -485,7 +485,7 @@
|
||||
2 src/renderer/src/store/terminals/terminal-active-workspace-creation.ts
|
||||
1 src/renderer/src/store/terminals/terminal-pane-hibernation.ts
|
||||
1 src/renderer/src/store/terminals/terminal-shutdown.ts
|
||||
1 src/renderer/src/store/terminals/terminal-tab-attention.ts
|
||||
1 src/renderer/src/store/terminals/terminal-tab-appearance.ts
|
||||
1 src/renderer/src/store/terminals/terminal-tab-close-providers.ts
|
||||
1 src/renderer/src/store/terminals/terminal-workspace-routing.ts
|
||||
1 src/renderer/src/web/preload-api/web-app-api.ts
|
||||
|
||||
@@ -54,19 +54,21 @@
|
||||
3 src/main/runtime/mobile-session-layout-projection.ts
|
||||
2 src/main/runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts
|
||||
2 src/main/runtime/orca-runtime-apply-mobile-session-tab-navigation.ts
|
||||
2 src/main/runtime/orca-runtime-attach-window.ts
|
||||
2 src/main/runtime/session-graph/session-graph-promotion.ts
|
||||
2 src/main/runtime/orca-runtime-build-headless-mobile-session-browser-tabs.ts
|
||||
1 src/main/runtime/session-tabs/structured-session-tab-close.ts
|
||||
1 src/main/runtime/orca-runtime-get-runtime-id.ts
|
||||
2 src/main/runtime/orca-runtime-move-headless-mobile-session-tab.ts
|
||||
6 src/main/runtime/orca-runtime-persist-headless-session-tab-props.ts
|
||||
2 src/main/runtime/orca-runtime-persist-headless-terminal-title.ts
|
||||
1 src/main/runtime/orca-runtime-persist-terminal-surface-retirements.ts
|
||||
1 src/main/runtime/session-tabs/terminal-surface-retirements.ts
|
||||
1 src/main/runtime/orca-runtime-pty-foreground-process-reads.ts
|
||||
3 src/main/runtime/orca-runtime-stop-terminals-for-worktree.ts
|
||||
1 src/main/runtime/orca-runtime-terminal-drivers.ts
|
||||
4 src/main/runtime/runtime-legacy-worker-terminal-recovery-persistence.ts
|
||||
2 src/main/runtime/runtime-pane-admission.ts
|
||||
7 src/main/runtime/runtime-terminal-orphan-session-adoption.ts
|
||||
1 src/main/runtime/runtime-workspace-session-controller.ts
|
||||
2 src/main/runtime/session-tabs/headless-tab-group-moves.ts
|
||||
6 src/main/runtime/session-tabs/headless-tab-properties.ts
|
||||
4 src/main/runtime/workspace-session-worktree-id.ts
|
||||
1 src/main/ssh/ssh-target-id-migration.ts
|
||||
2 src/main/terminal-scrollback-snapshots.ts
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// Flags Enter handling on raw text-entry surfaces that skips the shared IME guard, so the Enter
|
||||
// that confirms a CJK candidate cannot submit half-composed text. <Input>/<Textarea>/ImeInput/
|
||||
// ImeTextarea are capitalized components and never match; only intrinsic elements are checked.
|
||||
|
||||
const KEY_HANDLER_PROPS = new Set([
|
||||
'onKeyDown',
|
||||
'onKeyDownCapture',
|
||||
'onKeyPress',
|
||||
'onKeyPressCapture'
|
||||
])
|
||||
const COMPOSITION_PROPS = new Set([
|
||||
'onCompositionStart',
|
||||
'onCompositionStartCapture',
|
||||
'onCompositionEnd',
|
||||
'onCompositionEndCapture'
|
||||
])
|
||||
const NON_TEXT_INPUT_TYPES = new Set([
|
||||
'button',
|
||||
'checkbox',
|
||||
'color',
|
||||
'date',
|
||||
'datetime-local',
|
||||
'file',
|
||||
'hidden',
|
||||
'image',
|
||||
'month',
|
||||
'radio',
|
||||
'range',
|
||||
'reset',
|
||||
'submit',
|
||||
'time',
|
||||
'week'
|
||||
])
|
||||
const KEY_EVENT_NAMES = new Set(['keydown', 'keypress'])
|
||||
|
||||
const ENTER_TEST_PATTERNS = [
|
||||
/(['"`])(?:Numpad)?Enter\1/,
|
||||
/\b(?:keyCode|which|charCode)\s*[!=]==?\s*13\b/,
|
||||
/\b13\s*[!=]==?\s*[\w$.?]*\b(?:keyCode|which|charCode)\b/
|
||||
]
|
||||
const IME_GUARD_PATTERN =
|
||||
/\b(?:isImeOwnedKeyboardEvent|isImeCompositionKeyDown|isComposing|ownsKeyDown|useImeTextFieldProps)\b|\b229\b/
|
||||
const TEXT_ELEMENT_RECEIVER_PATTERN =
|
||||
/(?:^|[^A-Za-z])(?:input|textarea|editable)|Input|Textarea|TextArea|Editable/
|
||||
const CREATE_TEXT_ELEMENT_PATTERN = /createElement\(\s*(['"`])(?:input|textarea)\1/
|
||||
|
||||
function jsxAttributeName(attribute) {
|
||||
if (attribute.type !== 'JSXAttribute') {
|
||||
return null
|
||||
}
|
||||
return attribute.name.type === 'JSXIdentifier' ? attribute.name.name : null
|
||||
}
|
||||
|
||||
function jsxAttribute(node, name) {
|
||||
return node.attributes.find((attribute) => jsxAttributeName(attribute) === name) ?? null
|
||||
}
|
||||
|
||||
function attributeLiteral(attribute) {
|
||||
const value = attribute?.value
|
||||
if (value?.type === 'Literal') {
|
||||
return value.value
|
||||
}
|
||||
if (value?.type === 'JSXExpressionContainer' && value.expression.type === 'Literal') {
|
||||
return value.expression.value
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function isTextEntryElement(node) {
|
||||
if (node.name.type !== 'JSXIdentifier') {
|
||||
return false
|
||||
}
|
||||
const tag = node.name.name
|
||||
if (tag === 'textarea') {
|
||||
return true
|
||||
}
|
||||
if (tag === 'input') {
|
||||
const type = attributeLiteral(jsxAttribute(node, 'type'))
|
||||
return typeof type !== 'string' || !NON_TEXT_INPUT_TYPES.has(type.toLowerCase())
|
||||
}
|
||||
const editable = jsxAttribute(node, 'contentEditable') ?? jsxAttribute(node, 'contenteditable')
|
||||
if (!editable) {
|
||||
return false
|
||||
}
|
||||
const value = editable.value === null ? true : attributeLiteral(editable)
|
||||
return value !== false && value !== 'false'
|
||||
}
|
||||
|
||||
function testsEnter(text) {
|
||||
return ENTER_TEST_PATTERNS.some((pattern) => pattern.test(text))
|
||||
}
|
||||
|
||||
function createImeEnterGuardRule(context) {
|
||||
const localFunctionTexts = new Map()
|
||||
const textElementNames = new Set()
|
||||
const candidates = []
|
||||
|
||||
const text = (node) => context.sourceCode.getText(node)
|
||||
|
||||
// Unions the handler with every same-file function it names, so a guard in a helper counts.
|
||||
const reachableText = (rootText) => {
|
||||
const seen = new Set()
|
||||
const pending = [rootText]
|
||||
const parts = []
|
||||
while (pending.length > 0) {
|
||||
const current = pending.pop()
|
||||
parts.push(current)
|
||||
for (const word of current.match(/[A-Za-z_$][\w$]*/g) ?? []) {
|
||||
if (!seen.has(word) && localFunctionTexts.has(word)) {
|
||||
seen.add(word)
|
||||
pending.push(...localFunctionTexts.get(word))
|
||||
}
|
||||
}
|
||||
}
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
const rememberFunction = (name, node) => {
|
||||
const texts = localFunctionTexts.get(name) ?? []
|
||||
texts.push(text(node))
|
||||
localFunctionTexts.set(name, texts)
|
||||
}
|
||||
|
||||
return {
|
||||
FunctionDeclaration(node) {
|
||||
if (node.id) {
|
||||
rememberFunction(node.id.name, node)
|
||||
}
|
||||
},
|
||||
VariableDeclarator(node) {
|
||||
if (node.id.type !== 'Identifier' || !node.init) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
['ArrowFunctionExpression', 'FunctionExpression', 'CallExpression'].includes(node.init.type)
|
||||
) {
|
||||
rememberFunction(node.id.name, node.init)
|
||||
}
|
||||
if (CREATE_TEXT_ELEMENT_PATTERN.test(text(node.init))) {
|
||||
textElementNames.add(node.id.name)
|
||||
}
|
||||
},
|
||||
JSXOpeningElement(node) {
|
||||
if (!isTextEntryElement(node)) {
|
||||
return
|
||||
}
|
||||
if (node.attributes.some((attribute) => COMPOSITION_PROPS.has(jsxAttributeName(attribute)))) {
|
||||
return
|
||||
}
|
||||
for (const attribute of node.attributes) {
|
||||
if (
|
||||
KEY_HANDLER_PROPS.has(jsxAttributeName(attribute)) &&
|
||||
attribute.value?.type === 'JSXExpressionContainer'
|
||||
) {
|
||||
candidates.push({ node: attribute, handler: attribute.value.expression })
|
||||
}
|
||||
}
|
||||
},
|
||||
CallExpression(node) {
|
||||
const callee = node.callee
|
||||
if (
|
||||
callee.type !== 'MemberExpression' ||
|
||||
callee.property.type !== 'Identifier' ||
|
||||
callee.property.name !== 'addEventListener'
|
||||
) {
|
||||
return
|
||||
}
|
||||
const [eventName, listener] = node.arguments
|
||||
if (eventName?.type !== 'Literal' || !KEY_EVENT_NAMES.has(eventName.value) || !listener) {
|
||||
return
|
||||
}
|
||||
const receiver = text(callee.object)
|
||||
const rootName = receiver.match(/^[A-Za-z_$][\w$]*/)?.[0]
|
||||
if (TEXT_ELEMENT_RECEIVER_PATTERN.test(receiver) || textElementNames.has(rootName)) {
|
||||
candidates.push({ node, handler: listener })
|
||||
}
|
||||
},
|
||||
'Program:exit'() {
|
||||
for (const candidate of candidates) {
|
||||
const handler = candidate.handler
|
||||
let handlerText
|
||||
if (['ArrowFunctionExpression', 'FunctionExpression'].includes(handler.type)) {
|
||||
handlerText = text(handler)
|
||||
} else if (handler.type === 'Identifier' && localFunctionTexts.has(handler.name)) {
|
||||
handlerText = handler.name
|
||||
} else {
|
||||
// Props, hook results, and imports are judged where they are defined.
|
||||
continue
|
||||
}
|
||||
const reachable = reachableText(handlerText)
|
||||
if (testsEnter(reachable) && !IME_GUARD_PATTERN.test(reachable)) {
|
||||
context.report({
|
||||
node: candidate.node,
|
||||
message:
|
||||
'Enter handling on a raw text field must ignore IME composition: use ImeInput/ImeTextarea (or <Input>/<Textarea>), or guard with isImeOwnedKeyboardEvent from @/lib/ime-composition-keyboard-event.'
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
meta: { name: 'ime-enter-guard' },
|
||||
rules: {
|
||||
'require-ime-guard': { create: createImeEnterGuardRule }
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
diff --git a/dist/cjs/extractor/core/extractor.js b/dist/cjs/extractor/core/extractor.js
|
||||
index 6d61ade2471c20bf1a253e3784bfadcb01440545..c1b001e78a215f89a8d4da1756a87562a5a6e723 100644
|
||||
index 6d61ade2471..23ad9f5ba49 100644
|
||||
--- a/dist/cjs/extractor/core/extractor.js
|
||||
+++ b/dist/cjs/extractor/core/extractor.js
|
||||
@@ -143,6 +143,8 @@ const extractionSiteRegexes = new WeakMap();
|
||||
@@ -20,8 +20,185 @@ index 6d61ade2471c20bf1a253e3784bfadcb01440545..c1b001e78a215f89a8d4da1756a87562
|
||||
extractionSiteRegexes.set(config, re);
|
||||
}
|
||||
return re.test(code);
|
||||
@@ -185,13 +187,15 @@ function mayContainExtractionSite(code, config) {
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
-async function processFile(file, plugins, astVisitors, pluginContext, config, logger$1 = new logger.ConsoleLogger(), fileErrors) {
|
||||
+async function processFile(file, plugins, astVisitors, pluginContext, config, logger$1 = new logger.ConsoleLogger(), fileErrors, parsedFiles) {
|
||||
try {
|
||||
// Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
|
||||
const fileStat = await promises.stat(file);
|
||||
if (fileStat.isDirectory())
|
||||
return;
|
||||
- let code = await promises.readFile(file, 'utf-8');
|
||||
+ const cached = parsedFiles?.get(file);
|
||||
+ parsedFiles?.delete(file);
|
||||
+ let code = cached?.code ?? await promises.readFile(file, 'utf-8');
|
||||
// Run onLoad hooks from plugins with error handling.
|
||||
// Track whether any plugin actually transformed the code so we can make
|
||||
// the right parse decision for non-native file types (e.g. .svelte, .vue).
|
||||
@@ -241,59 +245,61 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
|
||||
const isTypeScriptFile = effectiveExt === '.ts' || effectiveExt === '.tsx' || effectiveExt === '.mts' || effectiveExt === '.cts';
|
||||
const isTSX = effectiveExt === '.tsx';
|
||||
const isJSX = effectiveExt === '.jsx';
|
||||
- let ast;
|
||||
- try {
|
||||
- ast = await core.parse(code, {
|
||||
- syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
|
||||
- tsx: isTSX,
|
||||
- jsx: isJSX,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- }
|
||||
- catch (err) {
|
||||
- // Fallback for .ts files with JSX (already present)
|
||||
- if (effectiveExt === '.ts' && !isTSX) {
|
||||
- try {
|
||||
- ast = await core.parse(code, {
|
||||
- syntax: 'typescript',
|
||||
- tsx: true,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- logger$1.info?.(`Parsed ${file} using TSX fallback`);
|
||||
- }
|
||||
- catch (err2) {
|
||||
- throw new validation.ExtractorError('Failed to process file', file, err2);
|
||||
- }
|
||||
- // Fallback for .js files with JSX
|
||||
+ let ast = cached?.ast;
|
||||
+ if (!cached) {
|
||||
+ try {
|
||||
+ ast = await core.parse(code, {
|
||||
+ syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
|
||||
+ tsx: isTSX,
|
||||
+ jsx: isJSX,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
}
|
||||
- else if (effectiveExt === '.js' && !isJSX) {
|
||||
- try {
|
||||
- ast = await core.parse(code, {
|
||||
- syntax: 'ecmascript',
|
||||
- jsx: true,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- logger$1.info?.(`Parsed ${file} using JSX fallback`);
|
||||
+ catch (err) {
|
||||
+ // Fallback for .ts files with JSX (already present)
|
||||
+ if (effectiveExt === '.ts' && !isTSX) {
|
||||
+ try {
|
||||
+ ast = await core.parse(code, {
|
||||
+ syntax: 'typescript',
|
||||
+ tsx: true,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
+ logger$1.info?.(`Parsed ${file} using TSX fallback`);
|
||||
+ }
|
||||
+ catch (err2) {
|
||||
+ throw new validation.ExtractorError('Failed to process file', file, err2);
|
||||
+ }
|
||||
+ // Fallback for .js files with JSX
|
||||
}
|
||||
- catch (err2) {
|
||||
- throw new validation.ExtractorError('Failed to process file', file, err2);
|
||||
+ else if (effectiveExt === '.js' && !isJSX) {
|
||||
+ try {
|
||||
+ ast = await core.parse(code, {
|
||||
+ syntax: 'ecmascript',
|
||||
+ jsx: true,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
+ logger$1.info?.(`Parsed ${file} using JSX fallback`);
|
||||
+ }
|
||||
+ catch (err2) {
|
||||
+ throw new validation.ExtractorError('Failed to process file', file, err2);
|
||||
+ }
|
||||
+ }
|
||||
+ else {
|
||||
+ throw new validation.ExtractorError('Failed to process file', file, err);
|
||||
}
|
||||
}
|
||||
- else {
|
||||
- throw new validation.ExtractorError('Failed to process file', file, err);
|
||||
- }
|
||||
+ // Normalize SWC span offsets so every span is a file-relative (0-based)
|
||||
+ // character index. SWC accumulates byte offsets across successive parse()
|
||||
+ // calls and Module.span.start points to the first token, NOT to byte 0 of
|
||||
+ // the source.
|
||||
+ astUtils.normalizeSpansToCharIndices(ast, code);
|
||||
}
|
||||
- // Normalize SWC span offsets so every span is a file-relative (0-based)
|
||||
- // character index. SWC accumulates byte offsets across successive parse()
|
||||
- // calls and Module.span.start points to the first token, NOT to byte 0 of
|
||||
- // the source.
|
||||
- astUtils.normalizeSpansToCharIndices(ast, code);
|
||||
// "Wire up" the visitor's scope method to the context.
|
||||
// This avoids a circular dependency while giving plugins access to the scope.
|
||||
pluginContext.getVarFromScope = astVisitors.getVarFromScope.bind(astVisitors);
|
||||
@@ -344,7 +350,7 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
|
||||
* @param logger - Logger for warnings/errors
|
||||
* @param fileErrors - Optional array to collect per-file error messages
|
||||
*/
|
||||
-async function preScanFile(file, astVisitors, config, logger$1 = new logger.ConsoleLogger(), fileErrors) {
|
||||
+async function preScanFile(file, astVisitors, config, logger$1 = new logger.ConsoleLogger(), fileErrors, parsedFiles) {
|
||||
try {
|
||||
// Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
|
||||
const fileStat = await promises.stat(file);
|
||||
@@ -396,6 +402,8 @@ async function preScanFile(file, astVisitors, config, logger$1 = new logger.Cons
|
||||
astUtils.normalizeSpansToCharIndices(ast, code);
|
||||
astVisitors.setCurrentFile(file, code);
|
||||
astVisitors.preScanForConstants(ast);
|
||||
+ if (parsedFiles && mayContainExtractionSite(code, config))
|
||||
+ parsedFiles.set(file, { code, ast });
|
||||
}
|
||||
catch (error) {
|
||||
if (error instanceof pluginManager.ConflictError)
|
||||
diff --git a/dist/cjs/extractor/core/key-finder.js b/dist/cjs/extractor/core/key-finder.js
|
||||
index 1411f287157..200b6b974b6 100644
|
||||
--- a/dist/cjs/extractor/core/key-finder.js
|
||||
+++ b/dist/cjs/extractor/core/key-finder.js
|
||||
@@ -40,6 +40,8 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
|
||||
const plugins = pluginsOrUndefined || [];
|
||||
const sourceFiles = await processSourceFiles(config);
|
||||
const allKeys = new Map();
|
||||
+ // Retain only translation-site trees; plugins can transform code between passes.
|
||||
+ const parsedFiles = plugins.length === 0 ? new Map() : undefined;
|
||||
// 1. Create the base context with config and logger.
|
||||
const pluginContext = pluginManager.createPluginContext(allKeys, plugins, otherConfig, logger$1);
|
||||
// 2. Create hooks for plugins to hook into AST
|
||||
@@ -92,7 +94,7 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
|
||||
// correctly even when the defining constants file is processed after the file that
|
||||
// uses the identifier.
|
||||
for (const file of sourceFiles) {
|
||||
- await extractor.preScanFile(file, astVisitors$1, otherConfig, logger$1, fileErrors);
|
||||
+ await extractor.preScanFile(file, astVisitors$1, otherConfig, logger$1, fileErrors, parsedFiles);
|
||||
}
|
||||
// 6b. Mirror declarations onto the local names they were imported as, so that
|
||||
// `import { ResourceStatus as Status }` resolves like the unaliased import.
|
||||
@@ -100,7 +102,7 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
|
||||
// 7. Extraction pass: all shared tables are now fully populated, so every
|
||||
// identifier reference can be resolved regardless of file order.
|
||||
for (const file of sourceFiles) {
|
||||
- await extractor.processFile(file, plugins, astVisitors$1, pluginContext, otherConfig, logger$1, fileErrors);
|
||||
+ await extractor.processFile(file, plugins, astVisitors$1, pluginContext, otherConfig, logger$1, fileErrors, parsedFiles);
|
||||
}
|
||||
// 7. Run onEnd hooks
|
||||
for (const plugin of plugins) {
|
||||
diff --git a/dist/esm/extractor/core/extractor.js b/dist/esm/extractor/core/extractor.js
|
||||
index 910bdb6e25e26c01ea6127191f1a00c7089693c1..00da4a6a7acb6c3da8d4b6fe7be106c409542f8d 100644
|
||||
index 910bdb6e25e..a3868d29950 100644
|
||||
--- a/dist/esm/extractor/core/extractor.js
|
||||
+++ b/dist/esm/extractor/core/extractor.js
|
||||
@@ -141,6 +141,8 @@ const extractionSiteRegexes = new WeakMap();
|
||||
@@ -42,3 +219,180 @@ index 910bdb6e25e26c01ea6127191f1a00c7089693c1..00da4a6a7acb6c3da8d4b6fe7be106c4
|
||||
extractionSiteRegexes.set(config, re);
|
||||
}
|
||||
return re.test(code);
|
||||
@@ -183,13 +185,15 @@ function mayContainExtractionSite(code, config) {
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
-async function processFile(file, plugins, astVisitors, pluginContext, config, logger = new ConsoleLogger(), fileErrors) {
|
||||
+async function processFile(file, plugins, astVisitors, pluginContext, config, logger = new ConsoleLogger(), fileErrors, parsedFiles) {
|
||||
try {
|
||||
// Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
|
||||
const fileStat = await stat(file);
|
||||
if (fileStat.isDirectory())
|
||||
return;
|
||||
- let code = await readFile(file, 'utf-8');
|
||||
+ const cached = parsedFiles?.get(file);
|
||||
+ parsedFiles?.delete(file);
|
||||
+ let code = cached?.code ?? await readFile(file, 'utf-8');
|
||||
// Run onLoad hooks from plugins with error handling.
|
||||
// Track whether any plugin actually transformed the code so we can make
|
||||
// the right parse decision for non-native file types (e.g. .svelte, .vue).
|
||||
@@ -239,59 +243,61 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
|
||||
const isTypeScriptFile = effectiveExt === '.ts' || effectiveExt === '.tsx' || effectiveExt === '.mts' || effectiveExt === '.cts';
|
||||
const isTSX = effectiveExt === '.tsx';
|
||||
const isJSX = effectiveExt === '.jsx';
|
||||
- let ast;
|
||||
- try {
|
||||
- ast = await parse(code, {
|
||||
- syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
|
||||
- tsx: isTSX,
|
||||
- jsx: isJSX,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- }
|
||||
- catch (err) {
|
||||
- // Fallback for .ts files with JSX (already present)
|
||||
- if (effectiveExt === '.ts' && !isTSX) {
|
||||
- try {
|
||||
- ast = await parse(code, {
|
||||
- syntax: 'typescript',
|
||||
- tsx: true,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- logger.info?.(`Parsed ${file} using TSX fallback`);
|
||||
- }
|
||||
- catch (err2) {
|
||||
- throw new ExtractorError('Failed to process file', file, err2);
|
||||
- }
|
||||
- // Fallback for .js files with JSX
|
||||
+ let ast = cached?.ast;
|
||||
+ if (!cached) {
|
||||
+ try {
|
||||
+ ast = await parse(code, {
|
||||
+ syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
|
||||
+ tsx: isTSX,
|
||||
+ jsx: isJSX,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
}
|
||||
- else if (effectiveExt === '.js' && !isJSX) {
|
||||
- try {
|
||||
- ast = await parse(code, {
|
||||
- syntax: 'ecmascript',
|
||||
- jsx: true,
|
||||
- decorators: true,
|
||||
- dynamicImport: true,
|
||||
- comments: true,
|
||||
- });
|
||||
- logger.info?.(`Parsed ${file} using JSX fallback`);
|
||||
+ catch (err) {
|
||||
+ // Fallback for .ts files with JSX (already present)
|
||||
+ if (effectiveExt === '.ts' && !isTSX) {
|
||||
+ try {
|
||||
+ ast = await parse(code, {
|
||||
+ syntax: 'typescript',
|
||||
+ tsx: true,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
+ logger.info?.(`Parsed ${file} using TSX fallback`);
|
||||
+ }
|
||||
+ catch (err2) {
|
||||
+ throw new ExtractorError('Failed to process file', file, err2);
|
||||
+ }
|
||||
+ // Fallback for .js files with JSX
|
||||
}
|
||||
- catch (err2) {
|
||||
- throw new ExtractorError('Failed to process file', file, err2);
|
||||
+ else if (effectiveExt === '.js' && !isJSX) {
|
||||
+ try {
|
||||
+ ast = await parse(code, {
|
||||
+ syntax: 'ecmascript',
|
||||
+ jsx: true,
|
||||
+ decorators: true,
|
||||
+ dynamicImport: true,
|
||||
+ comments: true,
|
||||
+ });
|
||||
+ logger.info?.(`Parsed ${file} using JSX fallback`);
|
||||
+ }
|
||||
+ catch (err2) {
|
||||
+ throw new ExtractorError('Failed to process file', file, err2);
|
||||
+ }
|
||||
+ }
|
||||
+ else {
|
||||
+ throw new ExtractorError('Failed to process file', file, err);
|
||||
}
|
||||
}
|
||||
- else {
|
||||
- throw new ExtractorError('Failed to process file', file, err);
|
||||
- }
|
||||
+ // Normalize SWC span offsets so every span is a file-relative (0-based)
|
||||
+ // character index. SWC accumulates byte offsets across successive parse()
|
||||
+ // calls and Module.span.start points to the first token, NOT to byte 0 of
|
||||
+ // the source.
|
||||
+ normalizeSpansToCharIndices(ast, code);
|
||||
}
|
||||
- // Normalize SWC span offsets so every span is a file-relative (0-based)
|
||||
- // character index. SWC accumulates byte offsets across successive parse()
|
||||
- // calls and Module.span.start points to the first token, NOT to byte 0 of
|
||||
- // the source.
|
||||
- normalizeSpansToCharIndices(ast, code);
|
||||
// "Wire up" the visitor's scope method to the context.
|
||||
// This avoids a circular dependency while giving plugins access to the scope.
|
||||
pluginContext.getVarFromScope = astVisitors.getVarFromScope.bind(astVisitors);
|
||||
@@ -342,7 +348,7 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
|
||||
* @param logger - Logger for warnings/errors
|
||||
* @param fileErrors - Optional array to collect per-file error messages
|
||||
*/
|
||||
-async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger(), fileErrors) {
|
||||
+async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger(), fileErrors, parsedFiles) {
|
||||
try {
|
||||
// Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
|
||||
const fileStat = await stat(file);
|
||||
@@ -394,6 +400,8 @@ async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger
|
||||
normalizeSpansToCharIndices(ast, code);
|
||||
astVisitors.setCurrentFile(file, code);
|
||||
astVisitors.preScanForConstants(ast);
|
||||
+ if (parsedFiles && mayContainExtractionSite(code, config))
|
||||
+ parsedFiles.set(file, { code, ast });
|
||||
}
|
||||
catch (error) {
|
||||
if (error instanceof ConflictError)
|
||||
diff --git a/dist/esm/extractor/core/key-finder.js b/dist/esm/extractor/core/key-finder.js
|
||||
index 86612731b2d..4e724e84538 100644
|
||||
--- a/dist/esm/extractor/core/key-finder.js
|
||||
+++ b/dist/esm/extractor/core/key-finder.js
|
||||
@@ -38,6 +38,8 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
|
||||
const plugins = pluginsOrUndefined || [];
|
||||
const sourceFiles = await processSourceFiles(config);
|
||||
const allKeys = new Map();
|
||||
+ // Retain only translation-site trees; plugins can transform code between passes.
|
||||
+ const parsedFiles = plugins.length === 0 ? new Map() : undefined;
|
||||
// 1. Create the base context with config and logger.
|
||||
const pluginContext = createPluginContext(allKeys, plugins, otherConfig, logger);
|
||||
// 2. Create hooks for plugins to hook into AST
|
||||
@@ -90,7 +92,7 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
|
||||
// correctly even when the defining constants file is processed after the file that
|
||||
// uses the identifier.
|
||||
for (const file of sourceFiles) {
|
||||
- await preScanFile(file, astVisitors, otherConfig, logger, fileErrors);
|
||||
+ await preScanFile(file, astVisitors, otherConfig, logger, fileErrors, parsedFiles);
|
||||
}
|
||||
// 6b. Mirror declarations onto the local names they were imported as, so that
|
||||
// `import { ResourceStatus as Status }` resolves like the unaliased import.
|
||||
@@ -98,7 +100,7 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
|
||||
// 7. Extraction pass: all shared tables are now fully populated, so every
|
||||
// identifier reference can be resolved regardless of file order.
|
||||
for (const file of sourceFiles) {
|
||||
- await processFile(file, plugins, astVisitors, pluginContext, otherConfig, logger, fileErrors);
|
||||
+ await processFile(file, plugins, astVisitors, pluginContext, otherConfig, logger, fileErrors, parsedFiles);
|
||||
}
|
||||
// 7. Run onEnd hooks
|
||||
for (const plugin of plugins) {
|
||||
|
||||
@@ -1401,14 +1401,14 @@
|
||||
"invariant": "Headless startup installs and disposes push delivery; desktop notification categories remain authoritative, the three-minute away policy is preserved, and host activity cannot extend the seven-day mobile lease.",
|
||||
"oracle": "Require registration after RPC identity initialization and shutdown cleanup; idle 179/180/0 yields false/true/false in retained event metadata and exactly one gateway push. Legacy socket subscriptions preserve notification filtering, while opted-in push clients can request dismissal reconciliation. Desktop categories remain authoritative. Persisted lease expires exactly at seven days and only explicit registration renews it.",
|
||||
"commands": [
|
||||
"ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/mobile-notification-dismissal-store.test.ts src/renderer/src/hooks/useAutoAckViewedAgent.away.test.ts",
|
||||
"ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/mobile-notification-dismissal-store.test.ts src/renderer/src/hooks/useReadOnScreenAttention.away.test.ts",
|
||||
"ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/orcad/orcad-push-startup.test.ts src/main/runtime/push/push-policy-pipeline.integration.test.ts"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/main/orcad/orcad-push-startup.test.ts",
|
||||
"src/main/runtime/push/push-policy-pipeline.integration.test.ts",
|
||||
"src/main/runtime/mobile-notification-dismissal-store.test.ts",
|
||||
"src/renderer/src/hooks/useAutoAckViewedAgent.away.test.ts"
|
||||
"src/renderer/src/hooks/useReadOnScreenAttention.away.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
1 src/renderer/src/components/status-bar/workspace-space-decision-inputs.ts
|
||||
2 src/renderer/src/components/tab-bar/terminal-tab-activity-status.ts
|
||||
3 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model-worktree-write-probe.ts
|
||||
23 src/renderer/src/components/tab-group/tab-drag-preview-activation.ts
|
||||
1 src/renderer/src/components/tab-group/useTabGroupWorkspaceModel.ts
|
||||
1 src/renderer/src/components/terminal-pane/pty-connection-test-deps.ts
|
||||
4 src/renderer/src/components/terminal-pane/terminal-recovery-ledger-test-store.ts
|
||||
@@ -51,7 +50,6 @@
|
||||
1 src/renderer/src/runtime/web-session-terminal-orphan-recovery-surface.ts
|
||||
2 src/renderer/src/runtime/web-session-terminal-orphan-topology.ts
|
||||
3 src/renderer/src/runtime/web-session-terminal-recovery-snapshot-validation.ts
|
||||
9 src/renderer/src/store/repos/repo-removal.ts
|
||||
10 src/renderer/src/store/slices/browser/browser-close-actions.ts
|
||||
6 src/renderer/src/store/slices/browser/browser-hydration-actions.ts
|
||||
5 src/renderer/src/store/slices/browser/browser-page-focus-actions.ts
|
||||
@@ -60,7 +58,6 @@
|
||||
4 src/renderer/src/store/slices/browser/browser-tab-focus-actions.ts
|
||||
2 src/renderer/src/store/slices/browser/create-browser-slice.ts
|
||||
1 src/renderer/src/store/slices/direct-ssh-terminal-recovery.ts
|
||||
1 src/renderer/src/store/slices/editor/actions/chat-visual-tab-actions.ts
|
||||
9 src/renderer/src/store/slices/editor/actions/check-run-details-actions.ts
|
||||
7 src/renderer/src/store/slices/editor/actions/close-file-action.ts
|
||||
1 src/renderer/src/store/slices/editor/actions/git-status-actions.ts
|
||||
@@ -82,13 +79,13 @@
|
||||
2 src/renderer/src/store/slices/editor/file-ids/hydrated-editor-file-selection.ts
|
||||
1 src/renderer/src/store/slices/editor/file-ids/open-file-path-rekey.ts
|
||||
4 src/renderer/src/store/slices/editor/tabs/editor-open-target-group.ts
|
||||
2 src/renderer/src/store/slices/ssh-target-cleanup.ts
|
||||
2 src/renderer/src/store/slices/tab-group-reference-repair.ts
|
||||
4 src/renderer/src/store/slices/tab-group-state.ts
|
||||
2 src/renderer/src/store/slices/tabs-hydration.ts
|
||||
4 src/renderer/src/store/slices/tabs/create-tabs-slice.ts
|
||||
17 src/renderer/src/store/slices/tabs/tabs-close-actions.ts
|
||||
14 src/renderer/src/store/slices/tabs/tabs-create-actions.ts
|
||||
23 src/renderer/src/store/slices/tabs/tabs-drag-preview-actions.ts
|
||||
11 src/renderer/src/store/slices/tabs/tabs-drop-actions.ts
|
||||
4 src/renderer/src/store/slices/tabs/tabs-focus-actions.ts
|
||||
13 src/renderer/src/store/slices/tabs/tabs-group-actions.ts
|
||||
@@ -96,17 +93,15 @@
|
||||
8 src/renderer/src/store/slices/tabs/tabs-label-actions.ts
|
||||
10 src/renderer/src/store/slices/tabs/tabs-move-actions.ts
|
||||
9 src/renderer/src/store/slices/tabs/tabs-reconciliation.ts
|
||||
1 src/renderer/src/store/slices/tabs/tabs-secondary-actions.ts
|
||||
4 src/renderer/src/store/slices/tabs/tabs-session-actions.ts
|
||||
1 src/renderer/src/store/slices/tabs/tabs-split-ratio-actions.ts
|
||||
5 src/renderer/src/store/slices/tabs/tabs-surface.ts
|
||||
33 src/renderer/src/store/slices/tabs/worktree-layout-lifecycle.ts
|
||||
6 src/renderer/src/store/slices/terminal-orphan-helpers.ts
|
||||
4 src/renderer/src/store/slices/terminal-tab-title-batch.ts
|
||||
6 src/renderer/src/store/slices/terminals.ts
|
||||
7 src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts
|
||||
15 src/renderer/src/store/slices/worktrees/session/worktree-identity-rename-state.ts
|
||||
5 src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts
|
||||
2 src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts
|
||||
18 src/renderer/src/store/slices/worktrees/teardown/remove-worktree-store-cleanup.ts
|
||||
18 src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts
|
||||
5 src/renderer/src/store/terminals/terminal-layout-state.ts
|
||||
2 src/renderer/src/store/terminals/terminal-pane-hibernation.ts
|
||||
3 src/renderer/src/store/terminals/terminal-pty-bindings.ts
|
||||
@@ -114,7 +109,7 @@
|
||||
1 src/renderer/src/store/terminals/terminal-shutdown-guards.ts
|
||||
3 src/renderer/src/store/terminals/terminal-shutdown-state.ts
|
||||
1 src/renderer/src/store/terminals/terminal-tab-agent-launch-pane.ts
|
||||
2 src/renderer/src/store/terminals/terminal-tab-attention.ts
|
||||
2 src/renderer/src/store/terminals/terminal-tab-appearance.ts
|
||||
6 src/renderer/src/store/terminals/terminal-tab-close.ts
|
||||
10 src/renderer/src/store/terminals/terminal-tab-creation.ts
|
||||
7 src/renderer/src/store/terminals/terminal-tab-navigation.ts
|
||||
|
||||
@@ -0,0 +1,518 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, relative, resolve, sep } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { buildCounterbalancedSchedule } from './counterbalanced-benchmark-schedule.mjs'
|
||||
import { summarizeBenchmarkSamples } from './benchmark-sample-summary.mjs'
|
||||
import { resolvePnpmCliInvocation } from './pnpm-cli-invocation.mjs'
|
||||
import { hasWindowsCompilerSelection } from './initialize-windows-native-toolchain.mjs'
|
||||
import {
|
||||
compilerEnvironmentKey,
|
||||
environmentReceipt,
|
||||
hash
|
||||
} from './windows-native-benchmark-observer.mjs'
|
||||
|
||||
const addonNames = [
|
||||
'node-pty',
|
||||
'@orca/windows-registry',
|
||||
'@orca/windows-security',
|
||||
'@vscode/windows-process-tree'
|
||||
]
|
||||
const candidate = 'initialize-windows-native-toolchain.mjs'
|
||||
const control = 'ensure-native-runtime.mjs'
|
||||
const unchangedFiles = [
|
||||
`.github/actions/prepare-native-runtime/action.yml`,
|
||||
`config/scripts/${control}`,
|
||||
`config/scripts/${candidate}`,
|
||||
'config/scripts/capture-windows-native-environment.cmd',
|
||||
'config/scripts/windows-native-benchmark-observer.mjs',
|
||||
'config/scripts/benchmark-windows-native-preparation.mjs'
|
||||
]
|
||||
|
||||
export function validateObservedPreparation(receipt, variant, warm = false) {
|
||||
assert.equal(
|
||||
receipt.parentBefore.hash,
|
||||
receipt.parentAfter.hash,
|
||||
'Outer compiler environment changed'
|
||||
)
|
||||
const calls = (phase) => receipt.calls.filter((call) => call.phase === phase)
|
||||
const captured = variant !== 'control' && !warm
|
||||
assert.equal(calls('discovery').length, captured ? 1 : 0, 'Unexpected compiler discovery count')
|
||||
assert.equal(calls('capture').length, captured ? 1 : 0, 'Unexpected developer command count')
|
||||
assert.equal(
|
||||
calls('probe').length,
|
||||
variant === 'control' ? 0 : 1,
|
||||
'Missing helper preflight observation'
|
||||
)
|
||||
for (const call of receipt.calls) {
|
||||
assert(
|
||||
!call.error && !call.timedOut && !call.outputTruncated,
|
||||
'Incomplete compiler observation'
|
||||
)
|
||||
}
|
||||
if (variant === 'control') {
|
||||
assert.equal(calls('addon-build').length, warm ? 0 : 4, 'Expected all four cold addon builds')
|
||||
assert(
|
||||
receipt.calls.every((call) => call.code === 0),
|
||||
'Control preparation failed'
|
||||
)
|
||||
return
|
||||
}
|
||||
const [prepare] = calls('preparation')
|
||||
assert.equal(calls('preparation').length, 1, 'Missing unchanged preparation child')
|
||||
assert.equal(prepare.code, 0, 'Native preparation failed')
|
||||
assert.equal(calls('probe')[0].code, warm ? 0 : 1, 'Unexpected helper cold/warm probe')
|
||||
if (warm || variant === 'fallback') {
|
||||
assert.equal(
|
||||
prepare.environment.hash,
|
||||
receipt.parentBefore.hash,
|
||||
'Fallback/warm changed compiler selection'
|
||||
)
|
||||
assert.deepEqual(prepare.changedNames, [], 'Fallback/warm altered child environment')
|
||||
} else {
|
||||
assert.equal(calls('discovery')[0].code, 0)
|
||||
assert.equal(calls('capture')[0].code, 0)
|
||||
assert(prepare.changedNames.length > 0, 'Captured environment was not applied')
|
||||
assert(
|
||||
prepare.changedNames.every((name) => compilerEnvironmentKey.test(name)),
|
||||
'Unrelated environment changed'
|
||||
)
|
||||
assert(prepare.environment.compiler.vcinstalldir, 'Prepared child lacks compiler selection')
|
||||
}
|
||||
if (variant === 'fallback' && !warm) {
|
||||
assert.equal(calls('discovery')[0].code, 0)
|
||||
assert.equal(calls('capture')[0].code, 7, 'Fixed failure fixture did not run')
|
||||
}
|
||||
}
|
||||
|
||||
export function assertNativeColdProbe(result) {
|
||||
assert.equal(result.code, 1, 'Cold fixture unexpectedly passed')
|
||||
for (const name of addonNames) {
|
||||
assert(result.stderr.includes(`${name}:`), `Cold fixture did not fail ${name}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function assertNoHeaderDownloads(result) {
|
||||
assert(
|
||||
!/gyp http (GET|200|download)/i.test(`${result.stdout}\n${result.stderr}`),
|
||||
'Timed preparation downloaded headers/libraries'
|
||||
)
|
||||
}
|
||||
|
||||
function digestFiles(root, paths) {
|
||||
return Object.fromEntries(
|
||||
paths
|
||||
.filter((path) => existsSync(join(root, path)))
|
||||
.map((path) => [path, hash(readFileSync(join(root, path)))])
|
||||
)
|
||||
}
|
||||
|
||||
function treeDigest(root) {
|
||||
const digest = createHash('sha256')
|
||||
const files = []
|
||||
const visit = (path) => {
|
||||
for (const name of readdirSync(path).sort()) {
|
||||
const child = join(path, name)
|
||||
if (statSync(child).isDirectory()) {
|
||||
visit(child)
|
||||
} else {
|
||||
files.push(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
visit(root)
|
||||
for (const file of files) {
|
||||
digest.update(relative(root, file)).update('\0').update(readFileSync(file))
|
||||
}
|
||||
return { sha256: digest.digest('hex'), files: files.length }
|
||||
}
|
||||
|
||||
function nativeDirectories(snapshot) {
|
||||
const require = createRequire(join(snapshot, 'package.json'))
|
||||
const root = realpathSync(snapshot)
|
||||
return addonNames.map((name) => {
|
||||
const directory = realpathSync(dirname(require.resolve(`${name}/package.json`)))
|
||||
assert(directory.startsWith(`${root}${sep}`), `Addon escapes its isolated copy: ${name}`)
|
||||
return { name, directory }
|
||||
})
|
||||
}
|
||||
|
||||
function binaryReceipt(snapshot) {
|
||||
const require = createRequire(join(snapshot, 'package.json'))
|
||||
const paths = [
|
||||
join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty.node'),
|
||||
join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty/conpty.dll'),
|
||||
join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty/OpenConsole.exe'),
|
||||
join(snapshot, 'native/windows-registry/build/Release/orca_windows_registry.node'),
|
||||
join(snapshot, 'native/windows-security/build/Release/orca_windows_security.node'),
|
||||
join(
|
||||
dirname(require.resolve('@vscode/windows-process-tree/package.json')),
|
||||
'build/Release/windows_process_tree.node'
|
||||
)
|
||||
]
|
||||
return paths.map((path) => {
|
||||
assert(existsSync(path), `Missing prepared artifact: ${path}`)
|
||||
return {
|
||||
path: relative(snapshot, path),
|
||||
bytes: statSync(path).size,
|
||||
sha256: hash(readFileSync(path))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export async function benchmarkWindowsNativePreparation(
|
||||
root = resolve(import.meta.dirname, '../..')
|
||||
) {
|
||||
assert.equal(process.platform, 'win32', 'This hosted comparison requires Windows')
|
||||
assert(['x64', 'arm64'].includes(process.arch), 'Unsupported Windows runner')
|
||||
assert.equal(process.versions.node, '24.21.0', 'Comparison requires pinned Node')
|
||||
assert(
|
||||
!hasWindowsCompilerSelection(process.env),
|
||||
'Explicit compiler selection prevents a qualified automatic-capture comparison'
|
||||
)
|
||||
assert(/^[0-9a-f]{40}$/.test(process.env.SOURCE_SHA ?? ''), 'SOURCE_SHA must be an exact commit')
|
||||
const output = join(root, '.build/windows-native-benchmark')
|
||||
mkdirSync(output, { recursive: true })
|
||||
const temporary = mkdtempSync(
|
||||
join(process.env.RUNNER_TEMP || tmpdir(), 'orca-native-comparison-')
|
||||
)
|
||||
const archive = join(temporary, 'source.tar')
|
||||
const headers = join(temporary, 'headers')
|
||||
const pnpm = resolvePnpmCliInvocation()
|
||||
const environment = {
|
||||
...process.env,
|
||||
ORCA_BACKGROUND_LAUNCH: '1',
|
||||
npm_config_devdir: headers,
|
||||
npm_package_config_node_gyp_devdir: headers
|
||||
}
|
||||
const parentBefore = environmentReceipt(process.env)
|
||||
const githubEnvironmentBefore = existsSync(process.env.GITHUB_ENV ?? '')
|
||||
? hash(readFileSync(process.env.GITHUB_ENV))
|
||||
: null
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
startedAt: new Date().toISOString(),
|
||||
qualification: 'incomplete',
|
||||
firstRecordedJobStep: JSON.parse(readFileSync(join(output, 'job-start.json'), 'utf8')),
|
||||
phases: [],
|
||||
samples: [],
|
||||
scope: {
|
||||
warm: 'Same-build warm preparation; no Actions native-cache restore/save is performed.',
|
||||
metric:
|
||||
'Cold addon preparation with identical diagnostic preload; installs, header downloads, standalone qualification probes and artifact upload are recorded separately; the helper preflight remains included.',
|
||||
occupancy:
|
||||
'Use complete hosted job started_at/completed_at, including setup and upload/post steps, for actual runner occupancy. This artifact only measures its explicit window.',
|
||||
excluded:
|
||||
'No Node-to-Electron, cross-target template, exact-key restore, or CI workload gain is claimed.'
|
||||
}
|
||||
}
|
||||
const writeReport = () =>
|
||||
writeFileSync(join(output, 'receipt.json'), `${JSON.stringify(report, null, 2)}\n`)
|
||||
const command = async (label, program, args, cwd = root, env = environment, accepted = [0]) => {
|
||||
const startedAt = new Date().toISOString()
|
||||
const start = performance.now()
|
||||
const result = await runProcess({
|
||||
program,
|
||||
args,
|
||||
cwd,
|
||||
env,
|
||||
timeoutMs: 1_200_000,
|
||||
maxOutputBytes: 64 * 1024 * 1024,
|
||||
terminationBarrier: true
|
||||
})
|
||||
const elapsedMs = performance.now() - start
|
||||
const stdoutFile = `${label}.stdout.log`
|
||||
const stderrFile = `${label}.stderr.log`
|
||||
writeFileSync(join(output, stdoutFile), result.stdout)
|
||||
writeFileSync(join(output, stderrFile), result.stderr)
|
||||
report.phases.push({
|
||||
label,
|
||||
program,
|
||||
args,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
durationMs: elapsedMs,
|
||||
code: result.code,
|
||||
signal: result.signal,
|
||||
timedOut: result.timedOut,
|
||||
outputTruncated: result.outputTruncated,
|
||||
stdoutFile,
|
||||
stderrFile,
|
||||
stdoutSha256: hash(result.stdout),
|
||||
stderrSha256: hash(result.stderr)
|
||||
})
|
||||
writeReport()
|
||||
assert(
|
||||
!result.timedOut &&
|
||||
!result.signal &&
|
||||
!result.outputTruncated &&
|
||||
accepted.includes(result.code),
|
||||
`${label} failed: ${result.stderr.slice(-4000)}`
|
||||
)
|
||||
return { ...result, durationMs: elapsedMs }
|
||||
}
|
||||
const pnpmCommand = (label, args, cwd) =>
|
||||
command(label, pnpm.command, [...pnpm.prefixArgs, ...args], cwd)
|
||||
try {
|
||||
const head = (await command('source-head', 'git', ['rev-parse', 'HEAD'])).stdout.trim()
|
||||
assert.equal(head, process.env.SOURCE_SHA, 'Checkout differs from frozen source')
|
||||
await command('source-clean', 'git', ['diff', '--exit-code', 'HEAD', '--'])
|
||||
const tree = (await command('source-tree', 'git', ['rev-parse', 'HEAD^{tree}'])).stdout.trim()
|
||||
const pnpmVersion = (await pnpmCommand('pnpm-version', ['--version'], root)).stdout.trim()
|
||||
assert.equal(pnpmVersion, '12.8.1', 'Comparison requires pinned pnpm')
|
||||
const action = readFileSync(
|
||||
join(root, '.github/actions/prepare-native-runtime/action.yml'),
|
||||
'utf8'
|
||||
)
|
||||
const hashExpression = action.match(/NATIVE_SOURCE_HASH: (.+)/)?.[1]
|
||||
assert(hashExpression, 'Missing production source identity')
|
||||
const inputs = [...hashExpression.matchAll(/'([^']+)'/g)].map((match) => match[1])
|
||||
assert(
|
||||
inputs.includes(`config/scripts/${candidate}`),
|
||||
'Frozen source lacks the candidate native-cache fingerprint'
|
||||
)
|
||||
const requiredScriptHashes = digestFiles(root, unchangedFiles)
|
||||
assert.equal(
|
||||
Object.keys(requiredScriptHashes).length,
|
||||
unchangedFiles.length,
|
||||
'Frozen source lacks required scripts'
|
||||
)
|
||||
const sourceInputs = digestFiles(root, inputs)
|
||||
report.identity = {
|
||||
sourceSha: head,
|
||||
tree,
|
||||
runtime: 'node',
|
||||
node: process.versions.node,
|
||||
nodeAbi: process.versions.modules,
|
||||
arch: process.arch,
|
||||
pnpm: pnpmVersion,
|
||||
runner: process.env.BENCHMARK_RUNNER,
|
||||
imageOS: process.env.ImageOS,
|
||||
imageVersion: process.env.ImageVersion,
|
||||
workflowRef: process.env.GITHUB_WORKFLOW_REF,
|
||||
workflowSha: process.env.GITHUB_WORKFLOW_SHA,
|
||||
runId: process.env.GITHUB_RUN_ID,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT,
|
||||
sourceInputs,
|
||||
requiredScriptHashes,
|
||||
sourceInputDigest: hash(JSON.stringify(sourceInputs)),
|
||||
productionHashExpression: hashExpression,
|
||||
parentCompilerEnvironment: parentBefore.compiler
|
||||
}
|
||||
await command('archive-source', 'git', ['archive', '--format=tar', '--output', archive, 'HEAD'])
|
||||
report.identity.archiveSha256 = hash(readFileSync(archive))
|
||||
const require = createRequire(join(root, 'package.json'))
|
||||
const nodeGyp = process.env.npm_config_node_gyp || require.resolve('node-gyp/bin/node-gyp.js')
|
||||
report.identity.nodeGyp = JSON.parse(
|
||||
readFileSync(resolve(dirname(nodeGyp), '..', 'package.json'), 'utf8')
|
||||
).version
|
||||
report.identity.nodeGypEntrySha256 = hash(readFileSync(nodeGyp))
|
||||
await command('download-headers-before-measurement', process.execPath, [
|
||||
nodeGyp,
|
||||
'install',
|
||||
`--target=${process.versions.node}`,
|
||||
`--arch=${process.arch}`,
|
||||
`--devdir=${headers}`
|
||||
])
|
||||
const headersBefore = treeDigest(headers)
|
||||
report.identity.headers = headersBefore
|
||||
const physicalDirectories = new Set()
|
||||
const schedule = buildCounterbalancedSchedule(2, 'control', 'candidate').flat()
|
||||
report.schedule = schedule
|
||||
for (const [index, variant] of [...schedule, 'fallback'].entries()) {
|
||||
const label = `sample-${String(index + 1).padStart(2, '0')}`
|
||||
const snapshot = join(temporary, label)
|
||||
mkdirSync(snapshot)
|
||||
await command(`${label}-extract`, 'tar', ['-xf', archive, '-C', snapshot])
|
||||
await pnpmCommand(
|
||||
`${label}-install`,
|
||||
[
|
||||
'install',
|
||||
'--offline',
|
||||
'--frozen-lockfile',
|
||||
'--ignore-scripts',
|
||||
'--package-import-method=copy',
|
||||
'--side-effects-cache=false'
|
||||
],
|
||||
snapshot
|
||||
)
|
||||
await pnpmCommand(`${label}-internal-packages`, ['run', 'build:packages'], snapshot)
|
||||
const directories = nativeDirectories(snapshot)
|
||||
for (const { directory } of directories) {
|
||||
assert(
|
||||
!physicalDirectories.has(directory.toLowerCase()),
|
||||
'Native copies share a build directory'
|
||||
)
|
||||
physicalDirectories.add(directory.toLowerCase())
|
||||
rmSync(join(directory, 'build'), { recursive: true, force: true })
|
||||
}
|
||||
const initial = await command(
|
||||
`${label}-cold-probe`,
|
||||
process.execPath,
|
||||
[join(snapshot, 'config/scripts', control), '--check-only'],
|
||||
snapshot,
|
||||
environment,
|
||||
[1]
|
||||
)
|
||||
assertNativeColdProbe(initial)
|
||||
const snapshotScriptHashes = digestFiles(snapshot, unchangedFiles)
|
||||
assert.deepEqual(
|
||||
snapshotScriptHashes,
|
||||
requiredScriptHashes,
|
||||
'Ordinary snapshot source/config drift'
|
||||
)
|
||||
const originalCapture = readFileSync(
|
||||
join(snapshot, 'config/scripts/capture-windows-native-environment.cmd')
|
||||
)
|
||||
if (variant === 'fallback') {
|
||||
writeFileSync(
|
||||
join(snapshot, 'config/scripts/capture-windows-native-environment.cmd'),
|
||||
'@echo off\r\nexit /b 7\r\n'
|
||||
)
|
||||
}
|
||||
const sample = {
|
||||
label,
|
||||
variant,
|
||||
sourceHashesBeforeInjection: snapshotScriptHashes,
|
||||
originalCaptureSha256: hash(originalCapture),
|
||||
addonDirectories: directories,
|
||||
measurements: []
|
||||
}
|
||||
report.samples.push(sample)
|
||||
for (const warm of [false, true]) {
|
||||
const stage = `${label}-${warm ? 'warm' : 'cold'}`
|
||||
const traceFile = join(output, `${stage}.trace.json`)
|
||||
const script = join(snapshot, 'config/scripts', variant === 'control' ? control : candidate)
|
||||
const args = [
|
||||
'--import',
|
||||
pathToFileURL(join(snapshot, 'config/scripts/windows-native-benchmark-observer.mjs'))
|
||||
.href,
|
||||
script,
|
||||
...(variant === 'control' ? ['--runtime=node'] : [])
|
||||
]
|
||||
const result = await command(stage, process.execPath, args, snapshot, {
|
||||
...environment,
|
||||
ORCA_NATIVE_BENCHMARK_TRACE: traceFile
|
||||
})
|
||||
assertNoHeaderDownloads(result)
|
||||
const receipt = JSON.parse(readFileSync(traceFile, 'utf8'))
|
||||
validateObservedPreparation(receipt, variant, warm)
|
||||
if (variant === 'fallback' && !warm) {
|
||||
assert(result.stderr.includes('retaining node-gyp discovery'), 'Fallback warning missing')
|
||||
}
|
||||
const final = await command(
|
||||
`${stage}-final-probe`,
|
||||
process.execPath,
|
||||
[join(snapshot, 'config/scripts', control), '--check-only'],
|
||||
snapshot
|
||||
)
|
||||
assert.equal(final.code, 0)
|
||||
const calls = receipt.calls
|
||||
const milliseconds = (phase) =>
|
||||
calls
|
||||
.filter((call) => call.phase === phase)
|
||||
.reduce((total, call) => total + call.durationMs, 0)
|
||||
sample.measurements.push({
|
||||
warm,
|
||||
totalPreparationMs: result.durationMs,
|
||||
helperDiscoveryMs: milliseconds('discovery'),
|
||||
helperCaptureMs: milliseconds('capture'),
|
||||
helperPreprobeMs: milliseconds('probe'),
|
||||
unchangedPreparationChildMs: variant === 'control' ? null : milliseconds('preparation'),
|
||||
helperOutsidePreparationChildMs:
|
||||
variant === 'control' ? null : result.durationMs - milliseconds('preparation'),
|
||||
traceFile: relative(output, traceFile),
|
||||
traceSha256: hash(readFileSync(traceFile)),
|
||||
binaries: binaryReceipt(snapshot)
|
||||
})
|
||||
assert.deepEqual(
|
||||
treeDigest(headers),
|
||||
headersBefore,
|
||||
'Timed preparation changed prefilled headers/libraries'
|
||||
)
|
||||
assert.equal(
|
||||
environmentReceipt(process.env).hash,
|
||||
parentBefore.hash,
|
||||
'Parent environment changed'
|
||||
)
|
||||
assert.equal(
|
||||
existsSync(process.env.GITHUB_ENV ?? '')
|
||||
? hash(readFileSync(process.env.GITHUB_ENV))
|
||||
: null,
|
||||
githubEnvironmentBefore,
|
||||
'Compiler setup wrote GITHUB_ENV'
|
||||
)
|
||||
writeReport()
|
||||
}
|
||||
sample.isolatedCopyWindowMs = report.phases
|
||||
.filter((phase) => phase.label.startsWith(`${label}-`))
|
||||
.reduce((total, phase) => total + phase.durationMs, 0)
|
||||
sample.injectedCaptureSha256 =
|
||||
variant === 'fallback'
|
||||
? hash(
|
||||
readFileSync(join(snapshot, 'config/scripts/capture-windows-native-environment.cmd'))
|
||||
)
|
||||
: null
|
||||
}
|
||||
report.comparison = Object.fromEntries(
|
||||
[false, true].map((warm) => {
|
||||
const timings = (variant) =>
|
||||
report.samples
|
||||
.filter((sample) => sample.variant === variant)
|
||||
.map(
|
||||
(sample) =>
|
||||
sample.measurements.find((measurement) => measurement.warm === warm)
|
||||
.totalPreparationMs
|
||||
)
|
||||
const baseline = timings('control')
|
||||
const proposed = timings('candidate')
|
||||
return [
|
||||
warm ? 'sameBuildWarm' : 'cold',
|
||||
{
|
||||
control: summarizeBenchmarkSamples(baseline),
|
||||
candidate: summarizeBenchmarkSamples(proposed),
|
||||
pairDeltaMs: baseline.map((value, index) => proposed[index] - value),
|
||||
role: 'Two counterbalanced pairs are descriptive; no significance or ordinary CI hit-rate estimate.'
|
||||
}
|
||||
]
|
||||
})
|
||||
)
|
||||
report.parentEnvironment = {
|
||||
before: parentBefore,
|
||||
after: environmentReceipt(process.env),
|
||||
githubEnvironmentBefore,
|
||||
githubEnvironmentAfter: githubEnvironmentBefore
|
||||
}
|
||||
report.qualification = 'passed'
|
||||
} catch (error) {
|
||||
report.failure = error.message
|
||||
report.qualification = 'failed'
|
||||
throw error
|
||||
} finally {
|
||||
rmSync(temporary, { recursive: true, force: true })
|
||||
report.finishedAt = new Date().toISOString()
|
||||
report.explicitWindowMs = Date.parse(report.finishedAt) - Date.parse(report.startedAt)
|
||||
report.explicitJobWindowMs =
|
||||
Date.parse(report.finishedAt) - Date.parse(report.firstRecordedJobStep.startedAt)
|
||||
writeReport()
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
await benchmarkWindowsNativePreparation()
|
||||
}
|
||||
@@ -56,6 +56,7 @@ import { findSlotProblems, readManifest } from './orcad-prebuild-slot-contents.m
|
||||
import { orcadAgentBrowserNativeName } from '../../src/shared/orcad-agent-browser-name.ts'
|
||||
import { runProcessSync } from '@orca/process-host'
|
||||
import { describeProcessFailure } from './process-failure-message.mjs'
|
||||
import { resolveBuildSourceCommit } from './build-source-commit.mjs'
|
||||
|
||||
const ROOT = join(import.meta.dirname, '..', '..')
|
||||
const OUT_DIR = process.env.ORCAD_OUT_DIR
|
||||
@@ -232,10 +233,14 @@ const childResults = await Promise.all(
|
||||
const result = await buildOrcadEntry(SERVER_OUT_FILE)
|
||||
const cliResult = await buildOrcadCli(join(OUT_DIR, ORCAD_CLI_ENTRY_FILENAME))
|
||||
const { version: cliVersion } = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8'))
|
||||
writeFileSync(
|
||||
join(OUT_DIR, ORCAD_CLI_PACKAGE_FILENAME),
|
||||
`${JSON.stringify({ type: 'commonjs', private: true, version: cliVersion })}\n`
|
||||
)
|
||||
// Why commit: `orca skills install|update` pins official skills to it, as the desktop CLI does.
|
||||
const cliPackage = {
|
||||
type: 'commonjs',
|
||||
private: true,
|
||||
version: cliVersion,
|
||||
commit: resolveBuildSourceCommit()
|
||||
}
|
||||
writeFileSync(join(OUT_DIR, ORCAD_CLI_PACKAGE_FILENAME), `${JSON.stringify(cliPackage)}\n`)
|
||||
const launcherResult = await buildOrcadLauncher(OUT_FILE)
|
||||
|
||||
const output = Object.values(result.metafile.outputs).find(
|
||||
|
||||
@@ -12,6 +12,8 @@ import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { parse } from 'yaml'
|
||||
import { runProcessSync } from '@orca/process-host'
|
||||
import { buildWaylandClipboard } from './build-wayland-clipboard.mjs'
|
||||
import glibcVerification from './verify-linux-glibc-floor.cjs'
|
||||
|
||||
@@ -148,6 +150,50 @@ describe('Wayland clipboard build', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('Wayland clipboard CI cache recovery', () => {
|
||||
it.skipIf(process.platform === 'win32').each([
|
||||
['healthy cache', true, 0, 0, 1],
|
||||
['invalid cache', true, 1, 0, 2],
|
||||
['failed replacement build', true, 1, 1, 2],
|
||||
['failed uncached build', false, 1, 1, 1]
|
||||
])('%s preserves build validation and failures', (_label, hit, initial, rebuild, calls) => {
|
||||
const action = parse(
|
||||
readFileSync('.github/actions/prepare-wayland-clipboard/action.yml', 'utf8')
|
||||
)
|
||||
const build = action.runs.steps.find(
|
||||
(step) => step.name === 'Build or validate Wayland clipboard helper'
|
||||
)
|
||||
mkdirSync(join(root, 'native', 'wayland-clipboard', '.build'), { recursive: true })
|
||||
writeFileSync(join(root, 'native', 'wayland-clipboard', '.build', 'restored-cache'), '')
|
||||
const script = `
|
||||
node() {
|
||||
echo attempt >> calls
|
||||
if [ -e native/wayland-clipboard/.build/restored-cache ]; then
|
||||
return "$INITIAL_STATUS"
|
||||
fi
|
||||
return "$REBUILD_STATUS"
|
||||
}
|
||||
${build.run}
|
||||
`
|
||||
const result = runProcessSync({
|
||||
program: 'bash',
|
||||
args: ['-e', '-c', script],
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
CACHE_HIT: String(hit),
|
||||
INITIAL_STATUS: String(initial),
|
||||
REBUILD_STATUS: String(rebuild)
|
||||
}
|
||||
})
|
||||
expect(result.code).toBe(initial === 0 ? 0 : hit ? rebuild : initial)
|
||||
expect(readFileSync(join(root, 'calls'), 'utf8').trim().split('\n')).toHaveLength(calls)
|
||||
expect(existsSync(join(root, 'native', 'wayland-clipboard', '.build', 'restored-cache'))).toBe(
|
||||
calls === 1
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Wayland clipboard packaging', () => {
|
||||
it.each([
|
||||
[1, 'x64'],
|
||||
|
||||
@@ -11,7 +11,8 @@ import {
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { runProcessSync } from '@orca/process-host'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
shouldReuseCompiledWindowsRustExecutable,
|
||||
windowsRustExecutableFileVersion,
|
||||
@@ -45,6 +46,43 @@ function itWindows(name, test) {
|
||||
runner(name, { timeout: 300_000 }, test)
|
||||
}
|
||||
|
||||
let sharedLauncherRoot = null
|
||||
let sharedLauncherPath = null
|
||||
let sharedLauncherBuild = null
|
||||
|
||||
// Compile once; each launch case gets a private copy so it can own its fixture tree.
|
||||
beforeAll(() => {
|
||||
if (process.platform !== 'win32') {
|
||||
return
|
||||
}
|
||||
sharedLauncherRoot = mkdtempSync(join(tmpdir(), 'orca cli launcher build '))
|
||||
sharedLauncherPath = join(sharedLauncherRoot, 'orca.exe')
|
||||
const result = runProcessSync({
|
||||
program: process.execPath,
|
||||
args: ['config/scripts/build-windows-cli-launcher.mjs', '--output', sharedLauncherPath],
|
||||
cwd: projectRoot,
|
||||
timeoutMs: 300_000
|
||||
})
|
||||
sharedLauncherBuild = { status: result.code, stdout: result.stdout, stderr: result.stderr }
|
||||
if (result.code !== 0) {
|
||||
throw new Error(`${result.stdout}\n${result.stderr}`)
|
||||
}
|
||||
}, 300_000)
|
||||
|
||||
afterAll(() => {
|
||||
if (sharedLauncherRoot) {
|
||||
removeFixtureTree(sharedLauncherRoot)
|
||||
}
|
||||
})
|
||||
|
||||
function copySharedLauncher(target) {
|
||||
if (!sharedLauncherPath) {
|
||||
throw new Error('shared Windows CLI launcher was not built')
|
||||
}
|
||||
mkdirSync(dirname(target), { recursive: true })
|
||||
copyFileSync(sharedLauncherPath, target)
|
||||
}
|
||||
|
||||
describe('Windows CLI launcher', () => {
|
||||
it('reuses restored builds only while all embedded inputs and the release version match', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-cli-launcher-reuse-'))
|
||||
@@ -107,12 +145,12 @@ describe('Windows CLI launcher', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca launcher metadata '))
|
||||
try {
|
||||
const launcherPath = join(root, 'orca.exe')
|
||||
const build = spawnSync(
|
||||
process.execPath,
|
||||
['config/scripts/build-windows-cli-launcher.mjs', '--output', launcherPath],
|
||||
{ cwd: projectRoot, encoding: 'utf8' }
|
||||
)
|
||||
expect(build.status, `${build.stdout}\n${build.stderr}`).toBe(0)
|
||||
expect(sharedLauncherBuild).not.toBeNull()
|
||||
expect(
|
||||
sharedLauncherBuild.status,
|
||||
`${sharedLauncherBuild.stdout}\n${sharedLauncherBuild.stderr}`
|
||||
).toBe(0)
|
||||
copySharedLauncher(launcherPath)
|
||||
const inspect = spawnSync(
|
||||
'powershell.exe',
|
||||
[
|
||||
@@ -183,12 +221,7 @@ describe('Windows CLI launcher', () => {
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const build = spawnSync(
|
||||
process.execPath,
|
||||
['config/scripts/build-windows-cli-launcher.mjs', '--output', launcherPath],
|
||||
{ cwd: projectRoot, encoding: 'utf8' }
|
||||
)
|
||||
expect(build.status, `${build.stdout}\n${build.stderr}`).toBe(0)
|
||||
copySharedLauncher(launcherPath)
|
||||
|
||||
const body = 'paragraph one line one\nparagraph one line two\n\nparagraph two'
|
||||
const powershell = spawnSync(
|
||||
@@ -247,12 +280,7 @@ describe('Windows CLI launcher', () => {
|
||||
}))\n`,
|
||||
'utf8'
|
||||
)
|
||||
const build = spawnSync(
|
||||
process.execPath,
|
||||
['config/scripts/build-windows-cli-launcher.mjs', '--output', launcherPath],
|
||||
{ cwd: projectRoot, encoding: 'utf8' }
|
||||
)
|
||||
expect(build.status, `${build.stdout}\n${build.stderr}`).toBe(0)
|
||||
copySharedLauncher(launcherPath)
|
||||
writeFileSync(
|
||||
join(dirname(launcherPath), 'orca-launcher.cfg'),
|
||||
`node=${process.execPath}\ncli=${cliPath}\nuser-data=${profile}\n`
|
||||
@@ -326,12 +354,7 @@ describe('Windows CLI launcher', () => {
|
||||
}))\n`,
|
||||
'utf8'
|
||||
)
|
||||
const build = spawnSync(
|
||||
process.execPath,
|
||||
['config/scripts/build-windows-cli-launcher.mjs', '--output', launcherPath],
|
||||
{ cwd: projectRoot, encoding: 'utf8' }
|
||||
)
|
||||
expect(build.status, `${build.stdout}\n${build.stderr}`).toBe(0)
|
||||
copySharedLauncher(launcherPath)
|
||||
|
||||
const compiler = findFrameworkCompiler()
|
||||
expect(compiler).not.toBeNull()
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
call "%ORCA_WINDOWS_VSDEVCMD%" -no_logo -arch=%ORCA_WINDOWS_NATIVE_ARCH% -host_arch=%ORCA_WINDOWS_NATIVE_ARCH%
|
||||
if errorlevel 1 exit /b 1
|
||||
echo ORCA_WINDOWS_NATIVE_ENVIRONMENT
|
||||
set
|
||||
@@ -1,162 +1,579 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { build } from 'esbuild'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { createCheckLogFixture } from './check-job-log-test-fixture.mjs'
|
||||
import {
|
||||
assertWatchdogExit,
|
||||
assertMeasurementRuntime,
|
||||
validateChildProtocol,
|
||||
adverseCaseKey,
|
||||
auditBaselineCensoring
|
||||
} from './check-job-log-watchdog-protocol.mjs'
|
||||
import {
|
||||
BASELINE_REF,
|
||||
BASELINE_SOURCE_HASHES,
|
||||
createManifest,
|
||||
selectRequiredDescriptors
|
||||
} from './check-job-log-performance-cases.mjs'
|
||||
|
||||
// Pipe the baseline check-job-log-tail-slice.ts on stdin; both arms use the actual UTF-8 implementation.
|
||||
const entry = path.resolve('src/shared/check-job-log-tail-slice.ts')
|
||||
const sources = [readFileSync(0, 'utf8'), readFileSync(entry, 'utf8')]
|
||||
assert(sources.every((source) => source.includes('export function sliceCheckLogTail')))
|
||||
const shared = Object.keys(BASELINE_SOURCE_HASHES)
|
||||
export const hash = (value) => createHash('sha256').update(value).digest('hex')
|
||||
export const nodeExecutable = () => {
|
||||
const executable = process.versions.bun
|
||||
? (process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.env.npm_node_execpath)
|
||||
: process.execPath
|
||||
assert(
|
||||
executable && path.isAbsolute(executable),
|
||||
'A verified absolute Node executable is required'
|
||||
)
|
||||
return executable
|
||||
}
|
||||
const observationSource = `
|
||||
const counters = {};
|
||||
export function record(kind, value) {
|
||||
const bytes = Buffer.byteLength(value); const item = counters[kind] ??= { calls: 0, maxBytes: 0, totalBytes: 0 };
|
||||
item.calls++; item.maxBytes = Math.max(item.maxBytes, bytes); item.totalBytes += bytes;
|
||||
}
|
||||
export function observations() { return structuredClone(counters); }
|
||||
let phase = null;
|
||||
export function setPhase(value) { phase = value; }
|
||||
export function enterStrip() { if (phase) { process.stdout.write(JSON.stringify({ type: 'entered-strip', phase }) + '\\n'); phase = null; } }
|
||||
export function resetObservations() { for (const key of Object.keys(counters)) delete counters[key]; }
|
||||
`
|
||||
|
||||
async function load(source) {
|
||||
const result = await build({
|
||||
entryPoints: [entry],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
write: false,
|
||||
plugins: [
|
||||
{
|
||||
name: 'log-excerpt-source',
|
||||
setup(builder) {
|
||||
builder.onLoad({ filter: /check-job-log-tail-slice\.ts$/ }, () => ({
|
||||
contents: source,
|
||||
loader: 'ts',
|
||||
resolveDir: path.dirname(entry)
|
||||
}))
|
||||
export async function prepareBenchmark(
|
||||
root = process.cwd(),
|
||||
{ profile = false, compareBaseline = true } = {}
|
||||
) {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'orca-check-log-benchmark-'))
|
||||
try {
|
||||
const baseline = new Map()
|
||||
const current = new Map()
|
||||
for (const name of [...shared, 'check-job-log-excerpt.ts']) {
|
||||
const filename = path.join(root, 'src', 'shared', name)
|
||||
current.set(name, await readFile(filename, 'utf8'))
|
||||
if (!compareBaseline || !shared.includes(name)) {
|
||||
continue
|
||||
}
|
||||
const result = await runProcess({
|
||||
program: 'git',
|
||||
args: ['show', `${BASELINE_REF}:src/shared/${name}`],
|
||||
cwd: root,
|
||||
timeoutMs: 10_000,
|
||||
maxOutputBytes: 64 * 1024,
|
||||
env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
|
||||
})
|
||||
assert(
|
||||
result.code === 0 && !result.signal && !result.timedOut && !result.outputTruncated,
|
||||
'Baseline source snapshot failed'
|
||||
)
|
||||
baseline.set(name, result.stdout)
|
||||
assert.equal(
|
||||
hash(result.stdout),
|
||||
BASELINE_SOURCE_HASHES[name],
|
||||
`Baseline source changed: ${name}`
|
||||
)
|
||||
if (!shared.slice(0, 2).includes(name)) {
|
||||
assert.equal(hash(result.stdout), hash(current.get(name)), `Dependency drift: ${name}`)
|
||||
}
|
||||
}
|
||||
const identity = {
|
||||
baselineRef: BASELINE_REF,
|
||||
measurementScriptHashes: Object.fromEntries(
|
||||
await Promise.all(
|
||||
[
|
||||
'check-job-log-byte-cap-benchmark.mjs',
|
||||
'check-job-log-performance-cases.mjs',
|
||||
'check-job-log-test-fixture.mjs',
|
||||
'check-job-log-performance-child.mjs',
|
||||
'check-job-log-differential-cases.mjs',
|
||||
'check-job-log-watchdog-protocol.mjs'
|
||||
].map(async (name) => [
|
||||
name,
|
||||
hash(await readFile(path.join(root, 'config', 'scripts', name)))
|
||||
])
|
||||
)
|
||||
),
|
||||
baselineSourceHashes: Object.fromEntries(
|
||||
[...baseline].map(([name, value]) => [name, hash(value)])
|
||||
),
|
||||
proposedSourceHashes: Object.fromEntries(
|
||||
[...current].map(([name, value]) => [name, hash(value)])
|
||||
)
|
||||
}
|
||||
const bundles = {}
|
||||
for (const arm of compareBaseline ? ['baseline', 'proposed'] : ['proposed']) {
|
||||
for (const instrumented of arm === 'baseline' ? [false, true, 'negative'] : [false, true]) {
|
||||
const sources = arm === 'baseline' ? baseline : current
|
||||
const result = await build({
|
||||
stdin: {
|
||||
contents: `export { sliceCheckLogTail } from './check-job-log-tail-slice'; export { stripAnsiEscapeSequences, TERMINAL_CONTROL_CHARACTER_PATTERN } from './ansi-escape-sequences'; export { ownRetainedString, resetOwnRetainedStringCopier } from './own-retained-string'; ${arm === 'proposed' ? "export { sanitizeCheckJobLogExcerpt } from './check-job-log-excerpt';" : ''} ${profile && arm === 'proposed' ? "export { hasErrorMarker } from './check-job-log-tail-slice'; export { stripControlStrings } from './ansi-escape-sequences'; export { isUtf8ByteLengthWithinLimit } from './utf8-byte-limits';" : ''} export { observations, resetObservations, setPhase } from 'check-log-observations';`,
|
||||
resolveDir: path.join(root, 'src', 'shared'),
|
||||
loader: 'ts'
|
||||
},
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
write: false,
|
||||
plugins: [
|
||||
{
|
||||
name: 'source-pinned-log-text',
|
||||
setup(builder) {
|
||||
builder.onResolve({ filter: /^check-log-observations$/ }, () => ({
|
||||
path: 'check-log-observations',
|
||||
namespace: 'observations'
|
||||
}))
|
||||
builder.onLoad({ filter: /.*/, namespace: 'observations' }, () => ({
|
||||
contents: observationSource,
|
||||
loader: 'js'
|
||||
}))
|
||||
builder.onLoad(
|
||||
{
|
||||
filter:
|
||||
/[\\/](?:ansi-escape-sequences|check-job-log-tail-slice|check-job-log-excerpt|utf8-byte-limits|own-retained-string|owned-utf16-suffix)\.ts$/
|
||||
},
|
||||
({ path: filename }) => {
|
||||
const name = path.basename(filename)
|
||||
let contents = sources.get(name)
|
||||
assert(contents, `Unexpected module in ${arm} graph: ${name}`)
|
||||
if (profile && arm === 'proposed') {
|
||||
contents = contents
|
||||
.replace('function hasErrorMarker(', 'export function hasErrorMarker(')
|
||||
.replace(
|
||||
'function stripControlStrings(',
|
||||
'export function stripControlStrings('
|
||||
)
|
||||
}
|
||||
if (instrumented === 'negative') {
|
||||
if (name === 'ansi-escape-sequences.ts') {
|
||||
contents =
|
||||
`import { enterStrip } from 'check-log-observations';\n${contents}`.replace(
|
||||
'export function stripAnsiEscapeSequences(value: string): string {',
|
||||
'export function stripAnsiEscapeSequences(value: string): string {\nenterStrip();'
|
||||
)
|
||||
}
|
||||
} else if (instrumented) {
|
||||
contents = `import { record as __record } from 'check-log-observations';\n${contents}`
|
||||
if (name === 'ansi-escape-sequences.ts') {
|
||||
contents = contents.replace(
|
||||
'export function stripAnsiEscapeSequences(value: string): string {',
|
||||
'export function stripAnsiEscapeSequences(value: string): string {\n__record("strip", value);'
|
||||
)
|
||||
}
|
||||
if (name === 'utf8-byte-limits.ts') {
|
||||
contents = contents.replace(
|
||||
'export function getUtf8ByteLength(text: string): number {',
|
||||
'export function getUtf8ByteLength(text: string): number {\n__record("exact", text);'
|
||||
)
|
||||
}
|
||||
if (name === 'check-job-log-tail-slice.ts') {
|
||||
assert.equal(
|
||||
contents.split('ERROR_LINE_PATTERN.test(').length,
|
||||
2,
|
||||
'Actual matcher injection changed'
|
||||
)
|
||||
contents = `${contents
|
||||
.replace(
|
||||
'stripAnsiEscapeSequences(candidate)',
|
||||
'(__record("candidate", candidate), stripAnsiEscapeSequences(candidate))'
|
||||
)
|
||||
.replace(
|
||||
'ERROR_LINE_PATTERN.test(',
|
||||
'__match('
|
||||
)}\nfunction __match(value: string): boolean { __record("matcher", value); return ERROR_LINE_PATTERN.test(value); }`
|
||||
}
|
||||
}
|
||||
return { contents, loader: 'ts', resolveDir: path.dirname(filename) }
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
const text = result.outputFiles[0].text
|
||||
const key = `${arm}${instrumented === 'negative' ? '-negative' : instrumented ? '-observed' : ''}`
|
||||
const filename = path.join(directory, `${key}.mjs`)
|
||||
await writeFile(filename, text)
|
||||
bundles[key] = { filename, hash: hash(text) }
|
||||
}
|
||||
}
|
||||
return {
|
||||
root,
|
||||
directory,
|
||||
identity,
|
||||
bundles,
|
||||
cleanup: () => rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
} catch (error) {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function runDescriptor(context, descriptor) {
|
||||
const baseline = ['baseline', 'baseline-bounded', 'full-log'].includes(descriptor.arm)
|
||||
const key = `${baseline ? 'baseline' : 'proposed'}${descriptor.baselineTimeoutAllowed ? '-negative' : descriptor.mode === 'coverage' ? '-observed' : ''}`
|
||||
const bundle = context.bundles[key]
|
||||
const identity = { ...context.identity, bundleHash: bundle.hash, id: descriptor.id }
|
||||
const censored = (context.censoredBaselines ??= new Map())
|
||||
const initialTimeout = descriptor.baselineTimeoutAllowed
|
||||
? censored.get(adverseCaseKey(descriptor))
|
||||
: undefined
|
||||
if (initialTimeout && descriptor.sample > 0) {
|
||||
assertWatchdogExit(initialTimeout.watchdogTermination)
|
||||
assertMeasurementRuntime(initialTimeout.runtime)
|
||||
return {
|
||||
...descriptor,
|
||||
capturePath: undefined,
|
||||
status: 'baseline_censored',
|
||||
attempted: false,
|
||||
initialTimeoutId: initialTimeout.id,
|
||||
identity,
|
||||
runtime: initialTimeout.runtime,
|
||||
inputBytes: initialTimeout.inputBytes,
|
||||
codeUnits: initialTimeout.codeUnits,
|
||||
inputHash: initialTimeout.inputHash,
|
||||
unavailableSamples: descriptor.samples,
|
||||
samples: []
|
||||
}
|
||||
}
|
||||
const started = performance.now()
|
||||
const controller = new AbortController()
|
||||
let outerTimer
|
||||
const outer = new Promise((_resolve, reject) => {
|
||||
outerTimer = setTimeout(() => {
|
||||
controller.abort()
|
||||
reject(new Error('Child exceeded its outer settlement budget'))
|
||||
}, 15_000)
|
||||
})
|
||||
let result
|
||||
try {
|
||||
result = await Promise.race([
|
||||
runProcess({
|
||||
signal: controller.signal,
|
||||
program: nodeExecutable(),
|
||||
args: [
|
||||
'--expose-gc',
|
||||
path.join(context.root, 'config', 'scripts', 'check-job-log-performance-child.mjs'),
|
||||
bundle.filename,
|
||||
JSON.stringify(descriptor),
|
||||
JSON.stringify(identity)
|
||||
],
|
||||
cwd: context.root,
|
||||
env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' },
|
||||
timeoutMs: 10_000,
|
||||
maxOutputBytes: 64 * 1024
|
||||
}),
|
||||
outer
|
||||
])
|
||||
} finally {
|
||||
clearTimeout(outerTimer)
|
||||
}
|
||||
const settlementMs = performance.now() - started
|
||||
assert(settlementMs < 15_000, 'Child exceeded its outer settlement budget')
|
||||
assert(!result.outputTruncated, 'Child protocol truncated')
|
||||
assert(result.stderr.length === 0, 'Child emitted unexpected stderr')
|
||||
const { ready, entered, completed, warmups } = validateChildProtocol(result.stdout, descriptor)
|
||||
assert(!result.timedOut || completed.length === 0, 'Completed work cannot be censored')
|
||||
assert(
|
||||
ready && ready.identity.id === descriptor.id && ready.identity.bundleHash === bundle.hash,
|
||||
'Missing or mismatched pre-work identity'
|
||||
)
|
||||
assert.deepEqual(ready.identity, identity)
|
||||
assert.equal(ready.identity.baselineRef, BASELINE_REF)
|
||||
assert(Number.isSafeInteger(ready.inputBytes) && ready.inputBytes >= 0)
|
||||
assert(Number.isSafeInteger(ready.codeUnits) && ready.codeUnits >= 0)
|
||||
assert(ready.inputBytes >= ready.codeUnits && ready.inputBytes <= ready.codeUnits * 3)
|
||||
assert(typeof ready.inputHash === 'string' && /^[a-f0-9]{64}$/.test(ready.inputHash))
|
||||
if (
|
||||
descriptor.mode === 'direct' ||
|
||||
descriptor.mode === 'counts' ||
|
||||
(descriptor.mode === 'cpu' && descriptor.family === 'short-lines')
|
||||
) {
|
||||
assert.equal(ready.inputBytes, descriptor.bytes, 'Generator byte size changed')
|
||||
}
|
||||
if (descriptor.mode === 'cpu' && ['actual', 'fixture'].includes(descriptor.family)) {
|
||||
const input =
|
||||
descriptor.family === 'actual'
|
||||
? await readFile(descriptor.capturePath, 'utf8')
|
||||
: createCheckLogFixture().rawLog
|
||||
assert.equal(ready.inputBytes, Buffer.byteLength(input), 'Captured input bytes changed')
|
||||
assert.equal(ready.codeUnits, input.length, 'Captured input code units changed')
|
||||
assert.equal(ready.inputHash, hash(input), 'Captured input hash changed')
|
||||
}
|
||||
if (descriptor.mode === 'watchdog') {
|
||||
assert.equal(entered?.phase, 'watchdog', 'Watchdog did not enter the synchronous child')
|
||||
assertWatchdogExit(result)
|
||||
return {
|
||||
...descriptor,
|
||||
status: 'watchdog_passed',
|
||||
attempted: true,
|
||||
settlementMs,
|
||||
signal: result.signal,
|
||||
code: result.code,
|
||||
watchdogTermination: { timedOut: true, signal: result.signal, code: result.code },
|
||||
identity,
|
||||
inputBytes: ready.inputBytes,
|
||||
codeUnits: ready.codeUnits,
|
||||
inputHash: ready.inputHash,
|
||||
runtime: ready.runtime
|
||||
}
|
||||
}
|
||||
if (result.timedOut && descriptor.baselineTimeoutAllowed) {
|
||||
assertWatchdogExit(result)
|
||||
assert(
|
||||
descriptor.mode === 'direct' &&
|
||||
descriptor.arm === 'baseline' &&
|
||||
entered &&
|
||||
['warmup', 'sample'].includes(entered.phase),
|
||||
'Invalid adverse baseline censoring'
|
||||
)
|
||||
const record = {
|
||||
...descriptor,
|
||||
capturePath: undefined,
|
||||
status: 'baseline_timeout',
|
||||
identity,
|
||||
runtime: ready.runtime,
|
||||
inputBytes: ready.inputBytes,
|
||||
codeUnits: ready.codeUnits,
|
||||
inputHash: ready.inputHash,
|
||||
phase: entered.phase,
|
||||
deadlineMs: 10_000,
|
||||
settlementMs,
|
||||
watchdogTermination: { timedOut: true, signal: result.signal, code: result.code },
|
||||
unavailableSamples: descriptor.samples,
|
||||
samples: []
|
||||
}
|
||||
record.attempted = true
|
||||
if (descriptor.sample === 0) {
|
||||
censored.set(adverseCaseKey(descriptor), record)
|
||||
}
|
||||
return record
|
||||
}
|
||||
assert(
|
||||
!result.timedOut && result.code === 0 && !result.signal,
|
||||
'Workload child failed or timed out'
|
||||
)
|
||||
assert.equal(completed.length, 1, 'Missing result protocol')
|
||||
if (descriptor.mode === 'direct' || descriptor.mode === 'cpu') {
|
||||
assert.equal(completed[0].samples.length, descriptor.samples, 'Incomplete timed samples')
|
||||
assert.equal(warmups, descriptor.warmups ?? 3)
|
||||
for (const sample of completed[0].samples) {
|
||||
assert(Number.isSafeInteger(sample.calls) && sample.calls > 0)
|
||||
for (const value of [
|
||||
sample.wallMs,
|
||||
sample.cpu.user,
|
||||
sample.cpu.system,
|
||||
sample.timerLatenessMs,
|
||||
sample.histogramMaxMs,
|
||||
sample.utilization.active,
|
||||
sample.utilization.idle,
|
||||
sample.utilization.utilization
|
||||
]) {
|
||||
assert(Number.isFinite(value) && value >= 0)
|
||||
}
|
||||
assert(sample.utilization.active > 0 && sample.utilization.utilization <= 1)
|
||||
assert(sample.samplesBeforeWork > 0 && sample.finalCount > sample.postWorkCount)
|
||||
if (descriptor.mode === 'direct' && descriptor.bytes <= 32768) {
|
||||
assert(sample.wallMs >= 30, 'Tiny input batch did not reach its timing floor')
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
...descriptor,
|
||||
capturePath: undefined,
|
||||
...completed[0],
|
||||
...completed[0].inputIdentity,
|
||||
identity,
|
||||
runtime: ready.runtime,
|
||||
settlementMs,
|
||||
status: 'completed',
|
||||
attempted: true
|
||||
}
|
||||
}
|
||||
|
||||
export function median(values) {
|
||||
assert(values.length > 0 && values.every(Number.isFinite), 'Missing finite timing samples')
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
return sorted[Math.floor(sorted.length / 2)]
|
||||
}
|
||||
|
||||
export function auditResults(manifest, records, { compareBaseline = true } = {}) {
|
||||
assert.equal(
|
||||
new Set(manifest.map((item) => item.id)).size,
|
||||
manifest.length,
|
||||
'Duplicate manifest IDs'
|
||||
)
|
||||
assert.equal(records.length, manifest.length, 'Incomplete or duplicate descriptor coverage')
|
||||
const byId = new Map(records.map((record) => [record.id, record]))
|
||||
assert.equal(byId.size, manifest.length, 'Incomplete or duplicate descriptor coverage')
|
||||
const expectedIds = new Set(manifest.map((item) => item.id))
|
||||
assert(
|
||||
records.every((record) => expectedIds.has(record.id)),
|
||||
'Unexpected descriptor record'
|
||||
)
|
||||
for (const descriptor of manifest) {
|
||||
const record = byId.get(descriptor.id)
|
||||
assert(
|
||||
record &&
|
||||
['completed', 'baseline_timeout', 'baseline_censored', 'watchdog_passed'].includes(
|
||||
record.status
|
||||
),
|
||||
`Failed descriptor: ${descriptor.id}`
|
||||
)
|
||||
for (const field of ['mode', 'arm', 'family', 'width', 'bytes', 'sample']) {
|
||||
assert.equal(record[field], descriptor[field], `Descriptor identity changed: ${field}`)
|
||||
}
|
||||
if (descriptor.arm === 'proposed') {
|
||||
assert(!['baseline_timeout', 'baseline_censored'].includes(record.status))
|
||||
}
|
||||
}
|
||||
auditBaselineCensoring(manifest, byId)
|
||||
const differentials = records.filter((record) => record.mode === 'differential')
|
||||
assert.equal(differentials.length, compareBaseline ? 2 : 1)
|
||||
assert.equal(differentials[0].cases, 5126)
|
||||
if (compareBaseline) {
|
||||
assert.equal(
|
||||
differentials[0].outputHash,
|
||||
differentials[1].outputHash,
|
||||
'Plain/LF differential changed'
|
||||
)
|
||||
}
|
||||
const candidateGroups = new Map()
|
||||
for (const descriptor of manifest.filter(
|
||||
(item) => item.mode === 'coverage' && item.arm === 'proposed' && item.bytes >= 32 * 1024
|
||||
)) {
|
||||
const key = `${descriptor.family}:${descriptor.width}:${descriptor.position}:${descriptor.variant}:${descriptor.terminator}`
|
||||
const values = candidateGroups.get(key) ?? []
|
||||
values.push(byId.get(descriptor.id).observations.candidate)
|
||||
candidateGroups.set(key, values)
|
||||
}
|
||||
for (const values of candidateGroups.values()) {
|
||||
assert(values.every((value) => value && value.maxBytes <= 8192))
|
||||
assert(
|
||||
Math.max(...values.map((value) => value.totalBytes)) <=
|
||||
Math.min(...values.map((value) => value.totalBytes)) + 64,
|
||||
'Candidate bytes grew with a raw line'
|
||||
)
|
||||
assert(
|
||||
Math.max(...values.map((value) => value.calls)) ===
|
||||
Math.min(...values.map((value) => value.calls)),
|
||||
'Candidate calls grew with a raw line'
|
||||
)
|
||||
}
|
||||
const scaling = []
|
||||
const groups = new Map()
|
||||
for (const descriptor of manifest.filter(
|
||||
(item) => item.mode === 'direct' && item.arm === 'proposed'
|
||||
)) {
|
||||
const key = `${descriptor.family}:${descriptor.width}`
|
||||
const sizes = groups.get(key) ?? new Map()
|
||||
const values = sizes.get(descriptor.bytes) ?? []
|
||||
values.push(...byId.get(descriptor.id).samples.map((sample) => sample.wallMs / sample.calls))
|
||||
sizes.set(descriptor.bytes, values)
|
||||
groups.set(key, sizes)
|
||||
}
|
||||
for (const [family, sizes] of groups) {
|
||||
const keys = [...sizes.keys()].sort((a, b) => a - b)
|
||||
for (let index = 0; index < keys.length; index += 2) {
|
||||
const small = sizes.get(keys[index])
|
||||
const large = sizes.get(keys[index + 1])
|
||||
assert.equal(small.length, 7)
|
||||
assert.equal(large.length, 7)
|
||||
const first = median(small)
|
||||
const second = median(large)
|
||||
assert(second <= 3 * first + 1, `Linear doubling gate failed: ${family} ${keys[index]}`)
|
||||
scaling.push({ family, bytes: keys.slice(index, index + 2), medianMs: [first, second] })
|
||||
}
|
||||
}
|
||||
const retained = records.filter((record) => record.mode === 'retention')
|
||||
const pinned = retained.find((record) => record.arm === 'pinned')
|
||||
for (const record of retained.filter(
|
||||
(item) => item.family === 'eight-parents' && item.arm !== 'pinned'
|
||||
)) {
|
||||
assert(record.growth.heapUsed < 4 * 1024 * 1024)
|
||||
assert(record.growth.heapUsed < pinned.growth.heapUsed / 8, 'Owned strings pin raw parents')
|
||||
}
|
||||
const ordinary = []
|
||||
for (const family of compareBaseline ? ['actual', 'fixture', 'short-lines'] : []) {
|
||||
for (const bytes of new Set(
|
||||
manifest
|
||||
.filter((item) => item.mode === 'cpu' && item.family === family)
|
||||
.map((item) => item.bytes)
|
||||
)) {
|
||||
const comparisons = {}
|
||||
for (const arm of ['baseline', 'proposed', 'baseline-bounded', 'full-log']) {
|
||||
const measured = records
|
||||
.filter(
|
||||
(record) =>
|
||||
record.mode === 'cpu' &&
|
||||
record.family === family &&
|
||||
record.bytes === bytes &&
|
||||
record.arm === arm
|
||||
)
|
||||
.flatMap((record) => record.samples)
|
||||
comparisons[arm] = {
|
||||
wallMs: median(measured.map((sample) => sample.wallMs)),
|
||||
cpuMs: median(measured.map((sample) => (sample.cpu.user + sample.cpu.system) / 1000)),
|
||||
timerLatenessMs: median(measured.map((sample) => sample.timerLatenessMs))
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
const bundled = `${result.outputFiles[0].text}\n//# sourceURL=check-log-byte-cap-benchmark-bundle.js`
|
||||
return import(`data:text/javascript;base64,${Buffer.from(bundled).toString('base64')}`)
|
||||
}
|
||||
|
||||
const modules = await Promise.all(sources.map(load))
|
||||
const arms = modules.map((module) => module.sliceCheckLogTail)
|
||||
const limit = modules[0].PR_CHECK_LOG_TAIL_BYTES
|
||||
assert.equal(modules[1].PR_CHECK_LOG_TAIL_BYTES, limit)
|
||||
let seed = 0xc0ffee16
|
||||
function random(max) {
|
||||
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0
|
||||
return (seed >>> 8) % max
|
||||
}
|
||||
|
||||
let comparisons = 0
|
||||
function compare(text) {
|
||||
const expected = arms[0](text)
|
||||
assert.equal(arms[1](text), expected)
|
||||
assert(Buffer.byteLength(expected, 'utf8') <= limit)
|
||||
comparisons++
|
||||
return expected
|
||||
}
|
||||
|
||||
const units = ['x', 'é', '界', '😀', '\ud83d', '\udc00', 'x\ud83d界\udc00']
|
||||
for (const unit of units) {
|
||||
for (let delta = -4; delta <= 4; delta++) {
|
||||
const text = unit.repeat(Math.floor(limit / Buffer.byteLength(unit)) + delta)
|
||||
compare(text)
|
||||
compare(`error: ${text}\n${'recent\n'.repeat(103)}`)
|
||||
}
|
||||
}
|
||||
const endings = ['\n', '\r\n', '\r', '']
|
||||
const tokens = [
|
||||
'plain text',
|
||||
'##[error]',
|
||||
'::error::',
|
||||
'error:',
|
||||
'FAILED',
|
||||
'exit code',
|
||||
'ENOENT',
|
||||
'EACCES',
|
||||
'panic:',
|
||||
'AssertionError',
|
||||
'emoji 😀',
|
||||
'\ud83d',
|
||||
'\udc00',
|
||||
'\0',
|
||||
'界',
|
||||
'é',
|
||||
'\r'
|
||||
]
|
||||
for (let iteration = 0; iteration < 5000; iteration++) {
|
||||
const rows = Array.from({ length: random(250) }, (_, index) => {
|
||||
const token = tokens[random(tokens.length)]
|
||||
if (index === 0 && iteration % 20 === 0) {
|
||||
return `${token}${units[random(units.length)].repeat(limit + random(4))}`
|
||||
}
|
||||
return `${token} ${index} ${units[random(units.length)].repeat(random(30))}`
|
||||
})
|
||||
compare(rows.join(endings[random(endings.length)]) + endings[random(endings.length)])
|
||||
}
|
||||
console.log(`${comparisons} full-output differential cases passed`)
|
||||
|
||||
const workloads = [
|
||||
['short ASCII', 'log '.repeat(16)],
|
||||
['short Unicode', '🦀界'.repeat(20)],
|
||||
['8KiB ASCII', 'x'.repeat(8192)],
|
||||
['16KiB ASCII exact cap', 'x'.repeat(limit)],
|
||||
['8Ki code units / 24KiB Unicode', '界'.repeat(8192)],
|
||||
['2MiB ASCII line', 'x'.repeat(2 * 1024 * 1024)],
|
||||
['8MiB ASCII line', 'x'.repeat(8 * 1024 * 1024)],
|
||||
['2MiB Unicode line', '界'.repeat(Math.floor((2 * 1024 * 1024) / 3))],
|
||||
[
|
||||
'2MiB earlier error context',
|
||||
`error: ${'x'.repeat(2 * 1024 * 1024)}\n${'recent\n'.repeat(103)}`
|
||||
],
|
||||
[
|
||||
'220 ordinary lines',
|
||||
Array.from({ length: 220 }, (_, i) => `line ${i} ${'text'.repeat(8)}`).join('\n')
|
||||
],
|
||||
[
|
||||
'220 lines / small earlier error',
|
||||
Array.from(
|
||||
{ length: 220 },
|
||||
(_, i) => `${i === 30 ? 'error:' : 'line'} ${i} ${'text'.repeat(8)}`
|
||||
).join('\n')
|
||||
]
|
||||
]
|
||||
|
||||
function sample(arm, input, expected, repeats) {
|
||||
const started = performance.now()
|
||||
let output
|
||||
for (let i = 0; i < repeats; i++) {
|
||||
output = arm(input)
|
||||
}
|
||||
const elapsed = (performance.now() - started) / repeats
|
||||
assert.equal(output, expected)
|
||||
return elapsed
|
||||
}
|
||||
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
pairs: 8,
|
||||
unit: 'ms'
|
||||
})
|
||||
)
|
||||
for (const [name, input] of workloads) {
|
||||
const expected = compare(input)
|
||||
for (const arm of arms) {
|
||||
const until = performance.now() + 80
|
||||
while (performance.now() < until) {
|
||||
sample(arm, input, expected, 1)
|
||||
const before = comparisons.baseline
|
||||
const after = comparisons.proposed
|
||||
const threshold = bytes === 63 * 1024 * 1024 ? 15 : 5
|
||||
const review = ['wallMs', 'cpuMs', 'timerLatenessMs'].some(
|
||||
(key) => after[key] > before[key] * 1.25 && after[key] - before[key] > threshold
|
||||
)
|
||||
ordinary.push({ family, bytes, comparisons, reviewRequired: review })
|
||||
}
|
||||
}
|
||||
const repeats = Math.max(1, Math.min(100000, Math.ceil(40 / sample(arms[0], input, expected, 1))))
|
||||
/** @type {number[][]} */
|
||||
const samples = [[], []]
|
||||
for (let pair = 0; pair < 8; pair++) {
|
||||
for (const index of pair % 2 ? [1, 0] : [0, 1]) {
|
||||
samples[index].push(sample(arms[index], input, expected, repeats))
|
||||
return {
|
||||
descriptors: manifest.length,
|
||||
proposed: manifest.filter((item) => item.arm === 'proposed').length,
|
||||
censoredBaselines: records.filter((record) => record.status === 'baseline_timeout').length,
|
||||
censoredSampleSlots: records.filter((record) => record.status === 'baseline_censored').length,
|
||||
attemptedDescriptors: records.filter((record) => record.attempted !== false).length,
|
||||
scaling,
|
||||
ordinary
|
||||
}
|
||||
}
|
||||
|
||||
// Manual before/after comparison requires the pinned baseline in local Git history.
|
||||
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
|
||||
const context = await prepareBenchmark()
|
||||
const manifest = selectRequiredDescriptors(
|
||||
createManifest({ capturePath: process.env.ORCA_CHECK_LOG_CAPTURE })
|
||||
)
|
||||
const records = []
|
||||
try {
|
||||
for (const descriptor of manifest) {
|
||||
try {
|
||||
records.push(await runDescriptor(context, descriptor))
|
||||
} catch (error) {
|
||||
records.push({ id: descriptor.id, status: 'failed', reason: error.message.split('\n')[0] })
|
||||
}
|
||||
if (process.env.ORCA_CHECK_LOG_REPORT) {
|
||||
await writeFile(
|
||||
process.env.ORCA_CHECK_LOG_REPORT,
|
||||
JSON.stringify({ identity: context.identity, records })
|
||||
)
|
||||
}
|
||||
}
|
||||
const summary = auditResults(manifest, records)
|
||||
if (process.env.ORCA_CHECK_LOG_REPORT) {
|
||||
await writeFile(
|
||||
process.env.ORCA_CHECK_LOG_REPORT,
|
||||
JSON.stringify({ identity: context.identity, summary, records })
|
||||
)
|
||||
}
|
||||
console.log(JSON.stringify(summary))
|
||||
} finally {
|
||||
await context.cleanup()
|
||||
}
|
||||
const median = samples.map((values) => {
|
||||
values.sort((a, b) => a - b)
|
||||
return (values[3] + values[4]) / 2
|
||||
})
|
||||
console.log(JSON.stringify({ name, repeats, median, samples }))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { Session } from 'node:inspector/promises'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { readFile, writeFile } from 'node:fs/promises'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { prepareBenchmark, hash } from './check-job-log-byte-cap-benchmark.mjs'
|
||||
import { workload } from './check-job-log-performance-cases.mjs'
|
||||
|
||||
assert(!process.versions.bun && typeof globalThis.gc === 'function')
|
||||
const context = await prepareBenchmark(process.cwd(), { profile: true })
|
||||
const records = []
|
||||
const profileNames = [
|
||||
'isUtf8ByteLengthWithinLimit',
|
||||
'hasErrorMarker',
|
||||
'matches',
|
||||
'matchesErrorMarker',
|
||||
'stripAnsiEscapeSequences',
|
||||
'stripControlStrings',
|
||||
'appendSpan',
|
||||
'ownRetainedString',
|
||||
'copyUtf16SuffixToOwnedString'
|
||||
]
|
||||
async function measure(name, operation, calls, input) {
|
||||
for (let warmup = 0; warmup < 3; warmup++) {
|
||||
operation()
|
||||
}
|
||||
globalThis.gc()
|
||||
const session = new Session()
|
||||
session.connect()
|
||||
try {
|
||||
await session.post('Profiler.enable')
|
||||
await session.post('Profiler.setSamplingInterval', { interval: 100 })
|
||||
await session.post('Profiler.start')
|
||||
const cpuBefore = process.cpuUsage()
|
||||
const started = performance.now()
|
||||
for (let call = 0; call < calls; call++) {
|
||||
operation()
|
||||
}
|
||||
const wallMs = performance.now() - started
|
||||
const cpu = process.cpuUsage(cpuBefore)
|
||||
const { profile } = await session.post('Profiler.stop')
|
||||
const nodes = new Map(profile.nodes.map((node) => [node.id, node]))
|
||||
const parents = new Map(
|
||||
profile.nodes.flatMap((node) => (node.children ?? []).map((id) => [id, node.id]))
|
||||
)
|
||||
const timings = Object.fromEntries(
|
||||
profileNames.map((fn) => [fn, { selfMs: 0, inclusiveMs: 0 }])
|
||||
)
|
||||
for (let sample = 0; sample < (profile.samples?.length ?? 0); sample++) {
|
||||
let id = profile.samples[sample]
|
||||
const duration = profile.timeDeltas[sample] / 1000
|
||||
const fn = nodes.get(id)?.callFrame.functionName
|
||||
if (timings[fn]) {
|
||||
timings[fn].selfMs += duration
|
||||
}
|
||||
const names = new Set()
|
||||
while (id !== undefined) {
|
||||
names.add(nodes.get(id)?.callFrame.functionName)
|
||||
id = parents.get(id)
|
||||
}
|
||||
for (const fn of names) {
|
||||
if (timings[fn]) {
|
||||
timings[fn].inclusiveMs += duration
|
||||
}
|
||||
}
|
||||
}
|
||||
records.push({
|
||||
name,
|
||||
calls,
|
||||
wallMs,
|
||||
cpuMs: (cpu.user + cpu.system) / 1000,
|
||||
inputBytes: Buffer.byteLength(input),
|
||||
codeUnits: input.length,
|
||||
inputHash: hash(input),
|
||||
timings
|
||||
})
|
||||
} finally {
|
||||
session.disconnect()
|
||||
}
|
||||
}
|
||||
try {
|
||||
const module = await import(pathToFileURL(context.bundles.proposed.filename).href)
|
||||
const line = workload({ mode: 'cpu', family: 'short-lines', bytes: 8 * 1024 * 1024 }).text.split(
|
||||
'\n'
|
||||
)[0]
|
||||
const count = 100000
|
||||
await measure(
|
||||
'bounded-byte-fit',
|
||||
() => module.isUtf8ByteLengthWithinLimit(line, 16384),
|
||||
count,
|
||||
line
|
||||
)
|
||||
await measure('candidate-stripper', () => module.stripAnsiEscapeSequences(line), count, line)
|
||||
const plain = module.stripAnsiEscapeSequences(line)
|
||||
await measure(
|
||||
'candidate-control-cleanup',
|
||||
() => plain.replace(module.TERMINAL_CONTROL_CHARACTER_PATTERN, ''),
|
||||
count,
|
||||
plain
|
||||
)
|
||||
await measure('candidate-total', () => module.hasErrorMarker(line), count, line)
|
||||
for (const bytes of [8 * 1024 * 1024, 63 * 1024 * 1024]) {
|
||||
let input = workload({ mode: 'cpu', family: 'short-lines', bytes }).text
|
||||
await measure(
|
||||
`extraction-short-lines-${bytes}`,
|
||||
() => module.sliceCheckLogTail(input),
|
||||
3,
|
||||
input
|
||||
)
|
||||
input = ''
|
||||
}
|
||||
for (const family of ['dense-osc', 'dense-string', 'sparse', 'mixed']) {
|
||||
let input = workload({
|
||||
mode: 'direct',
|
||||
family,
|
||||
width: family === 'mixed' ? 0 : 7,
|
||||
bytes: 8 * 1024 * 1024,
|
||||
variant: 'unfinished'
|
||||
}).text
|
||||
await measure(`collector-${family}`, () => module.stripAnsiEscapeSequences(input), 3, input)
|
||||
input = ''
|
||||
}
|
||||
if (process.env.ORCA_CHECK_LOG_CAPTURE) {
|
||||
let input = await readFile(process.env.ORCA_CHECK_LOG_CAPTURE, 'utf8')
|
||||
await measure('extraction-actual', () => module.sliceCheckLogTail(input), 15, input)
|
||||
input = ''
|
||||
}
|
||||
const report = {
|
||||
identity: context.identity,
|
||||
profilerScriptHash: hash(await readFile(new URL(import.meta.url))),
|
||||
runtime: { node: process.version, platform: process.platform, arch: process.arch },
|
||||
purpose: 'Targeted sampled diagnosis; instrumentation timings are not acceptance medians',
|
||||
records
|
||||
}
|
||||
assert(process.env.ORCA_CHECK_LOG_PROFILE_REPORT)
|
||||
await writeFile(process.env.ORCA_CHECK_LOG_PROFILE_REPORT, JSON.stringify(report, null, 2))
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
records.map(({ name, wallMs, cpuMs, timings }) => ({ name, wallMs, cpuMs, timings }))
|
||||
)
|
||||
)
|
||||
} finally {
|
||||
await context.cleanup()
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { build } from 'esbuild'
|
||||
|
||||
// Runtime bundling keeps independent consumer assertions outside shared project type graphs.
|
||||
export async function loadAnsiConsumers(root = process.cwd()) {
|
||||
const entries = [
|
||||
['parseAnsiSegments', 'renderer/src/components/editor/ipynb-ansi'],
|
||||
[
|
||||
'restoredSnapshotPaintsPrintableContent',
|
||||
'renderer/src/components/terminal-pane/restored-snapshot-coverage'
|
||||
],
|
||||
['parseCodexLoginAuthUrl', 'main/codex-accounts/codex-login-auth-url'],
|
||||
['TerminalCommandPaint', 'main/runtime/terminal-command-paint'],
|
||||
['claudePermissionSubject', 'main/claude/claude-permission-presentation'],
|
||||
['terminalSafe', 'cli/agent-session-search-format'],
|
||||
[
|
||||
'createAutomationRunOutputSnapshotBuffer',
|
||||
'renderer/src/components/automations/automation-run-output-snapshot'
|
||||
]
|
||||
]
|
||||
const bundle = await build({
|
||||
stdin: {
|
||||
contents: entries
|
||||
.map(([name, filename]) => `export { ${name} } from './${filename}';`)
|
||||
.join('\n'),
|
||||
resolveDir: path.join(root, 'src'),
|
||||
loader: 'ts'
|
||||
},
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
write: false
|
||||
})
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'orca-check-log-consumers-'))
|
||||
try {
|
||||
const filename = path.join(directory, 'consumers.mjs')
|
||||
await writeFile(filename, bundle.outputFiles[0].text)
|
||||
const consumers = await import(pathToFileURL(filename).href)
|
||||
for (const [name] of entries) {
|
||||
assert.equal(typeof consumers[name], 'function', `Missing actual consumer: ${name}`)
|
||||
}
|
||||
return consumers
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { LIMIT, SEED } from './check-job-log-performance-cases.mjs'
|
||||
|
||||
// No controls or CR: these cases retain the original full-output contract.
|
||||
export function* plainLogDifferentialCases() {
|
||||
const units = ['x', 'é', '界', '😀', '\ud83d', '\udc00', 'x\ud83d界\udc00']
|
||||
for (const unit of units) {
|
||||
for (let delta = -4; delta <= 4; delta++) {
|
||||
const text = unit.repeat(Math.floor(LIMIT / Buffer.byteLength(unit)) + delta)
|
||||
yield text
|
||||
yield `error: ${text}\n${'recent\n'.repeat(103)}`
|
||||
}
|
||||
}
|
||||
let seed = SEED
|
||||
const random = (max) => {
|
||||
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0
|
||||
return (seed >>> 8) % max
|
||||
}
|
||||
const tokens = [
|
||||
'plain text',
|
||||
'##[error]',
|
||||
'::error::',
|
||||
'error:',
|
||||
'FAILED',
|
||||
'exit code',
|
||||
'ENOENT',
|
||||
'EACCES',
|
||||
'panic:',
|
||||
'AssertionError',
|
||||
'emoji 😀',
|
||||
'\ud83d',
|
||||
'\udc00',
|
||||
'界',
|
||||
'é'
|
||||
]
|
||||
for (let iteration = 0; iteration < 5000; iteration++) {
|
||||
const rows = Array.from({ length: random(250) }, (_, index) => {
|
||||
const token = tokens[random(tokens.length)]
|
||||
const count = index === 0 && iteration % 20 === 0 ? LIMIT + random(4) : random(30)
|
||||
return `${token} ${index} ${units[random(units.length)].repeat(count)}`
|
||||
})
|
||||
yield rows.join('\n') + (iteration % 2 ? '\n' : '')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFile, writeFile } from 'node:fs/promises'
|
||||
import { createManifest } from './check-job-log-performance-cases.mjs'
|
||||
import { prepareBenchmark, runDescriptor, hash } from './check-job-log-byte-cap-benchmark.mjs'
|
||||
|
||||
assert(process.env.ORCA_CHECK_LOG_SMOKE_REPORT)
|
||||
const context = await prepareBenchmark()
|
||||
const manifest = createManifest({
|
||||
capturePath: process.env.ORCA_CHECK_LOG_CAPTURE
|
||||
}).filter(
|
||||
(item) => item.mode === 'cpu' && ['baseline', 'proposed'].includes(item.arm) && item.sample < 3
|
||||
)
|
||||
const records = []
|
||||
try {
|
||||
for (const descriptor of manifest) {
|
||||
records.push(await runDescriptor(context, descriptor))
|
||||
}
|
||||
await writeFile(
|
||||
process.env.ORCA_CHECK_LOG_SMOKE_REPORT,
|
||||
JSON.stringify(
|
||||
{
|
||||
purpose:
|
||||
'Selected three-sample CPU smoke; final prescribed medians and audit remain pending',
|
||||
identity: context.identity,
|
||||
selectionScriptHash: hash(await readFile(new URL(import.meta.url))),
|
||||
records
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
console.log(JSON.stringify({ completed: records.length }))
|
||||
} finally {
|
||||
await context.cleanup()
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { createCheckLogFixture } from './check-job-log-test-fixture.mjs'
|
||||
|
||||
export const BASELINE_REF = 'c745051d7d6885b86f27120b616f15396404ba3d'
|
||||
export const BASELINE_SOURCE_HASHES = {
|
||||
'ansi-escape-sequences.ts': '75c79a88fb91c8b7a128a1ccc133051cde9e3e09f553d4d4fbb8887f89b6d217',
|
||||
'check-job-log-tail-slice.ts': '24ccf1fc3a0a5b2da7eb3a218a7b3fb7e85e6100e8b60e0b7f6c9fe5665413e4',
|
||||
'utf8-byte-limits.ts': '01574b287b2d6ab0758112887e356c0d270b25db1b72651925ba8329eb4224e6',
|
||||
'own-retained-string.ts': 'addea88bc11c725192be2ff617b91315a875113844c95de03e5529cf3d7ef2e2',
|
||||
'owned-utf16-suffix.ts': '1ca0bb35dfff2d9f3c02caee3d05b0e0c97378df71c878622acf703524d2c475'
|
||||
}
|
||||
export const LIMIT = 16 * 1024
|
||||
export const SEED = 0xc0ffee16
|
||||
const mib = 1024 * 1024
|
||||
export const directSizes = [
|
||||
16 * 1024,
|
||||
32 * 1024,
|
||||
mib,
|
||||
2 * mib,
|
||||
8 * mib,
|
||||
16 * mib,
|
||||
31.5 * mib,
|
||||
63 * mib
|
||||
]
|
||||
export const extractionSizes = [16 * 1024, 32 * 1024, 2 * mib, 8 * mib, 63 * mib]
|
||||
export const families = [
|
||||
...[
|
||||
['osc', '\x1b]', '\x9d'],
|
||||
['dcs', '\x1bP', '\x90'],
|
||||
['apc', '\x1b_', '\x9f'],
|
||||
['pm', '\x1b^', '\x9e'],
|
||||
['sos', '\x1bX', '\x98'],
|
||||
['csi', '\x1b[', '\x9b']
|
||||
].flatMap(([family, seven, eight]) => [
|
||||
{ family, width: 7, pattern: `${seven}111`, survivor: '111', adverse: family !== 'csi' },
|
||||
{ family, width: 8, pattern: `${eight}111`, survivor: `${eight}111`, adverse: family !== 'csi' }
|
||||
]),
|
||||
{ family: 'partial-esc', width: 7, pattern: '\x1b ', survivor: '\x1b ', adverse: false },
|
||||
{ family: 'dense-osc', width: 7, pattern: '\x1b]body\x07x', survivor: 'x', adverse: false },
|
||||
{ family: 'dense-osc', width: 8, pattern: '\x9dbody\x9cx', survivor: 'x', adverse: false },
|
||||
{ family: 'dense-string', width: 7, pattern: '\x1bPbody\x1b\\x', survivor: 'x', adverse: false },
|
||||
{ family: 'dense-string', width: 8, pattern: '\x90body\x9cx', survivor: 'x', adverse: false },
|
||||
{
|
||||
family: 'mixed',
|
||||
width: 0,
|
||||
pattern: '\x1b]a\x07界\x90b\x9cx\x1b[31m😀\x1b[0m',
|
||||
survivor: '界x😀',
|
||||
adverse: false
|
||||
},
|
||||
{
|
||||
family: 'sparse',
|
||||
width: 7,
|
||||
pattern: `${'a'.repeat(4096)}\x1b]body\x07`,
|
||||
survivor: 'a'.repeat(4096),
|
||||
adverse: false
|
||||
},
|
||||
{
|
||||
family: 'plain',
|
||||
width: 0,
|
||||
pattern: 'ordinary 界😀 text ',
|
||||
survivor: 'ordinary 界😀 text ',
|
||||
adverse: false
|
||||
}
|
||||
]
|
||||
|
||||
export function repeatBytes(pattern, bytes) {
|
||||
const width = Buffer.byteLength(pattern)
|
||||
const copies = Math.floor(bytes / width)
|
||||
return {
|
||||
text: pattern.repeat(copies) + 'x'.repeat(bytes - copies * width),
|
||||
copies,
|
||||
padding: bytes - copies * width
|
||||
}
|
||||
}
|
||||
|
||||
export function workload(descriptor) {
|
||||
if (descriptor.mode === 'cpu') {
|
||||
if (descriptor.family === 'actual') {
|
||||
return { text: readFileSync(descriptor.capturePath, 'utf8') }
|
||||
}
|
||||
if (descriptor.family === 'fixture') {
|
||||
return { text: createCheckLogFixture().rawLog }
|
||||
}
|
||||
const line = '2026-10-10T05:41:30.4306569Z \x1b[32mordinary successful output 界\x1b[0m\n'
|
||||
const suffix = '\n\x1b[31merror: CPU_DIAGNOSTIC\x1b[0m\n at CPU_FRAME.ts:18:8\n'
|
||||
return { text: repeatBytes(line, descriptor.bytes - Buffer.byteLength(suffix)).text + suffix }
|
||||
}
|
||||
const family = families.find(
|
||||
(value) => value.family === descriptor.family && value.width === descriptor.width
|
||||
)
|
||||
assert(family, 'unknown family')
|
||||
const terminator =
|
||||
descriptor.variant === 'late'
|
||||
? family.family === 'csi'
|
||||
? 'm'
|
||||
: family.family === 'partial-esc'
|
||||
? '@'
|
||||
: descriptor.terminator
|
||||
: ''
|
||||
const prefix =
|
||||
descriptor.mode === 'coverage' && descriptor.position === 'earlier' ? 'error: EDGE ' : ''
|
||||
const generated = repeatBytes(
|
||||
family.pattern,
|
||||
descriptor.bytes - Buffer.byteLength(terminator) - Buffer.byteLength(prefix)
|
||||
)
|
||||
const text = prefix + generated.text + terminator
|
||||
let expected = family.survivor.repeat(generated.copies) + 'x'.repeat(generated.padding)
|
||||
if (family.family === 'csi' && generated.padding > 0) {
|
||||
expected =
|
||||
family.survivor.repeat(Math.max(0, generated.copies - 1)) + 'x'.repeat(generated.padding - 1)
|
||||
}
|
||||
if (terminator) {
|
||||
if (family.adverse) {
|
||||
expected = ''
|
||||
} else if (family.family === 'csi') {
|
||||
expected = family.survivor.repeat(Math.max(0, generated.copies - 1))
|
||||
} else if (family.family === 'partial-esc') {
|
||||
expected = family.survivor.repeat(Math.max(0, generated.copies - 1))
|
||||
}
|
||||
}
|
||||
if (descriptor.mode === 'coverage') {
|
||||
const tail = Array.from({ length: 210 }, (_, index) => `recent ${index}`).join('\n')
|
||||
return {
|
||||
text:
|
||||
descriptor.position === 'earlier'
|
||||
? `${text}\n${tail}`
|
||||
: descriptor.position === 'tail'
|
||||
? `${tail}\n${text}`
|
||||
: `${text}\n${tail}`,
|
||||
lineBytes: descriptor.bytes
|
||||
}
|
||||
}
|
||||
return { text, expected }
|
||||
}
|
||||
|
||||
function item(mode, arm, fields) {
|
||||
const value = {
|
||||
mode,
|
||||
arm,
|
||||
variant: 'unfinished',
|
||||
seed: SEED,
|
||||
generator: 'repeated-byte-pattern-v1',
|
||||
...fields
|
||||
}
|
||||
value.id = [
|
||||
mode,
|
||||
arm,
|
||||
value.family,
|
||||
value.width,
|
||||
value.bytes,
|
||||
value.position,
|
||||
value.variant,
|
||||
value.terminator === '\x1b\\'
|
||||
? 'st7'
|
||||
: value.terminator === '\x9c'
|
||||
? 'st8'
|
||||
: value.terminator === '\x07'
|
||||
? 'bel'
|
||||
: '',
|
||||
value.sample,
|
||||
value.noBuffer ? 'no-buffer' : ''
|
||||
]
|
||||
.filter((x) => x !== undefined)
|
||||
.join(':')
|
||||
return value
|
||||
}
|
||||
|
||||
export function createManifest({ capturePath } = {}) {
|
||||
const manifest = [
|
||||
item('control', 'proposed', { family: 'monitor', sample: 0 }),
|
||||
item('control', 'proposed', { family: 'cleanup', sample: 0 }),
|
||||
item('watchdog', 'proposed', { family: 'watchdog', sample: 0 })
|
||||
]
|
||||
for (const family of families.slice(0, 13)) {
|
||||
for (const bytes of extractionSizes) {
|
||||
for (const position of ['earlier', 'tail', 'fallback']) {
|
||||
for (const variant of ['unfinished', 'late']) {
|
||||
for (const terminator of variant === 'late' && family.adverse
|
||||
? ['\x1b\\', '\x9c', ...(family.family === 'osc' ? ['\x07'] : [])]
|
||||
: ['']) {
|
||||
manifest.push(
|
||||
item('coverage', 'proposed', {
|
||||
family: family.family,
|
||||
width: family.width,
|
||||
bytes,
|
||||
position,
|
||||
variant,
|
||||
terminator
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const arm of ['proposed', 'baseline']) {
|
||||
manifest.push(item('differential', arm, { family: 'plain-lf' }))
|
||||
}
|
||||
for (const family of families) {
|
||||
for (const bytes of [16 * 1024, 32 * 1024]) {
|
||||
manifest.push(
|
||||
item('counts', 'proposed', { family: family.family, width: family.width, bytes })
|
||||
)
|
||||
}
|
||||
}
|
||||
for (const family of families.filter((value) =>
|
||||
['dense-osc', 'dense-string', 'mixed', 'sparse'].includes(value.family)
|
||||
)) {
|
||||
for (const bytes of [16 * 1024, 32 * 1024]) {
|
||||
manifest.push(
|
||||
item('counts', 'proposed', {
|
||||
family: family.family,
|
||||
width: family.width,
|
||||
bytes,
|
||||
noBuffer: true
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
// Parent ordering alternates every direct pair; each child gets fresh three warmups.
|
||||
for (const family of families) {
|
||||
for (let pair = 0; pair < directSizes.length; pair += 2) {
|
||||
const sizes = directSizes.slice(pair, pair + 2)
|
||||
for (let sample = 0; sample < 7; sample++) {
|
||||
for (const arm of sample % 2 ? ['baseline', 'proposed'] : ['proposed', 'baseline']) {
|
||||
for (const bytes of sample % 2 ? sizes.toReversed() : sizes) {
|
||||
manifest.push(
|
||||
item('direct', arm, {
|
||||
family: family.family,
|
||||
width: family.width,
|
||||
bytes,
|
||||
sample,
|
||||
samples: 1,
|
||||
baselineTimeoutAllowed: arm === 'baseline' && family.adverse
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const cpuCases = [
|
||||
{ family: 'fixture', bytes: 0, samples: 15, warmups: 3 },
|
||||
...(capturePath ? [{ family: 'actual', capturePath, bytes: 0, samples: 15, warmups: 3 }] : []),
|
||||
...[8 * mib, 63 * mib].map((bytes) => ({
|
||||
family: 'short-lines',
|
||||
bytes,
|
||||
samples: 5,
|
||||
warmups: 1
|
||||
}))
|
||||
]
|
||||
for (const fields of cpuCases) {
|
||||
for (let sample = 0; sample < fields.samples; sample++) {
|
||||
for (const arm of sample % 2
|
||||
? ['full-log', 'baseline-bounded', 'proposed', 'baseline']
|
||||
: ['baseline', 'proposed', 'baseline-bounded', 'full-log']) {
|
||||
manifest.push(item('cpu', arm, { ...fields, samples: 1, sample }))
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const arm of ['pinned', 'baseline', 'proposed', 'no-buffer']) {
|
||||
manifest.push(item('retention', arm, { family: 'eight-parents' }))
|
||||
}
|
||||
for (const arm of ['proposed', 'no-buffer']) {
|
||||
manifest.push(item('retention', arm, { family: 'cache', count: 128 }))
|
||||
}
|
||||
manifest.push(item('retention', 'proposed', { family: 'near-cap', count: 3 }))
|
||||
return manifest
|
||||
}
|
||||
|
||||
export function selectRequiredDescriptors(manifest) {
|
||||
return manifest.filter((descriptor) => {
|
||||
if (descriptor.mode === 'coverage') {
|
||||
return (
|
||||
['osc', 'dcs', 'apc', 'pm', 'sos'].includes(descriptor.family) &&
|
||||
[16 * 1024, 32 * 1024, 2 * mib, 63 * mib].includes(descriptor.bytes) &&
|
||||
['earlier', 'tail'].includes(descriptor.position) &&
|
||||
descriptor.variant === 'unfinished'
|
||||
)
|
||||
}
|
||||
if (descriptor.mode === 'direct') {
|
||||
return (
|
||||
descriptor.bytes <= 32 * 1024 ||
|
||||
(descriptor.arm === 'proposed' &&
|
||||
descriptor.bytes >= 31.5 * mib &&
|
||||
['osc', 'dcs', 'dense-osc', 'dense-string', 'mixed', 'sparse'].includes(
|
||||
descriptor.family
|
||||
))
|
||||
)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
|
||||
import { plainLogDifferentialCases } from './check-job-log-differential-cases.mjs'
|
||||
import { LIMIT, workload } from './check-job-log-performance-cases.mjs'
|
||||
import { createCheckLogFixture } from './check-job-log-test-fixture.mjs'
|
||||
|
||||
assert(!process.versions.bun, 'Measurement child must use Node')
|
||||
const [bundlePath, serialized, identityJson] = process.argv.slice(2)
|
||||
const descriptor = JSON.parse(serialized)
|
||||
const identity = JSON.parse(identityJson)
|
||||
const digest = (value) => createHash('sha256').update(value).digest('hex')
|
||||
assert.equal(digest(readFileSync(bundlePath)), identity.bundleHash)
|
||||
for (const [name, expectedHash] of Object.entries(identity.measurementScriptHashes)) {
|
||||
assert.equal(
|
||||
digest(readFileSync(new URL(name, import.meta.url))),
|
||||
expectedHash,
|
||||
`Measurement script changed: ${name}`
|
||||
)
|
||||
}
|
||||
const module = await import(pathToFileURL(bundlePath).href)
|
||||
const runtime = {
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
bun: false
|
||||
}
|
||||
const emit = (record) => process.stdout.write(`${JSON.stringify(record)}\n`)
|
||||
const timers = new Set()
|
||||
const delay = (ms) =>
|
||||
new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
timers.delete(timer)
|
||||
resolve(performance.now())
|
||||
}, ms)
|
||||
timers.add(timer)
|
||||
})
|
||||
const boundaries = []
|
||||
const memory = (label) => {
|
||||
const value = process.memoryUsage()
|
||||
boundaries.push({ label, ...value })
|
||||
return value
|
||||
}
|
||||
const gc = () => {
|
||||
assert.equal(typeof globalThis.gc, 'function')
|
||||
void /probe/.test('probe')
|
||||
globalThis.gc()
|
||||
globalThis.gc()
|
||||
}
|
||||
|
||||
async function measure(operation, tiny = false, control = false) {
|
||||
const histogram = monitorEventLoopDelay({ resolution: 1 })
|
||||
try {
|
||||
histogram.enable()
|
||||
await delay(5)
|
||||
await delay(5)
|
||||
histogram.reset()
|
||||
await delay(5)
|
||||
assert(histogram.count > 0, 'Monitor has no real post-reset samples')
|
||||
const samplesBeforeWork = histogram.count
|
||||
const before = memory('before-timed-work')
|
||||
const utilizationBefore = performance.eventLoopUtilization()
|
||||
const cpuBefore = process.cpuUsage()
|
||||
const armed = performance.now()
|
||||
const primary = delay(5)
|
||||
const started = performance.now()
|
||||
let output
|
||||
let calls = 0
|
||||
do {
|
||||
output = operation()
|
||||
calls++
|
||||
} while (tiny && performance.now() - started < 30)
|
||||
const wallMs = performance.now() - started
|
||||
const cpu = process.cpuUsage(cpuBefore)
|
||||
const utilization = performance.eventLoopUtilization(utilizationBefore)
|
||||
const after = memory('after-timed-work')
|
||||
const fired = await primary
|
||||
const postWorkCount = histogram.count
|
||||
await delay(5)
|
||||
assert(histogram.count > postWorkCount, 'Monitor has no post-work sampling tick')
|
||||
const histogramMaxMs = histogram.max / 1e6
|
||||
const timerLatenessMs = Math.max(0, fired - armed - 5)
|
||||
for (const key of ['active', 'idle', 'utilization']) {
|
||||
assert(Number.isFinite(utilization[key]) && utilization[key] >= 0)
|
||||
}
|
||||
assert(utilization.active > 0 && utilization.utilization <= 1)
|
||||
if (control) {
|
||||
assert(timerLatenessMs >= 30)
|
||||
assert(histogramMaxMs >= 30)
|
||||
}
|
||||
return {
|
||||
output,
|
||||
sample: {
|
||||
calls,
|
||||
wallMs,
|
||||
cpu,
|
||||
utilization,
|
||||
timerLatenessMs,
|
||||
histogramMaxMs,
|
||||
samplesBeforeWork,
|
||||
postWorkCount,
|
||||
finalCount: histogram.count,
|
||||
before,
|
||||
after
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
assert(histogram.disable())
|
||||
assert(!histogram.disable())
|
||||
for (const timer of timers) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
timers.clear()
|
||||
}
|
||||
}
|
||||
|
||||
let inputIdentity
|
||||
function ready(input = '') {
|
||||
inputIdentity = {
|
||||
inputBytes: Buffer.byteLength(input),
|
||||
codeUnits: input.length,
|
||||
inputHash: digest(input)
|
||||
}
|
||||
emit({
|
||||
type: 'ready',
|
||||
identity,
|
||||
runtime,
|
||||
...inputIdentity
|
||||
})
|
||||
}
|
||||
function result(fields) {
|
||||
const maxima = (field) => Math.max(0, ...boundaries.map((item) => item[field]))
|
||||
emit({
|
||||
type: 'result',
|
||||
id: descriptor.id,
|
||||
mode: descriptor.mode,
|
||||
arm: descriptor.arm,
|
||||
family: descriptor.family,
|
||||
width: descriptor.width,
|
||||
bytes: descriptor.bytes,
|
||||
inputIdentity,
|
||||
variant: descriptor.variant,
|
||||
...fields,
|
||||
processLifetimeMaxRssKiB: process.resourceUsage().maxRSS,
|
||||
maxObservedBoundaryHeapUsedBytes: maxima('heapUsed'),
|
||||
maxObservedBoundaryExternalBytes: maxima('external'),
|
||||
maxObservedBoundaryArrayBuffersBytes: maxima('arrayBuffers'),
|
||||
boundaries
|
||||
})
|
||||
}
|
||||
|
||||
function assertSafe(output) {
|
||||
assert(Buffer.byteLength(output) <= LIMIT, 'Excerpt exceeded 16 KiB')
|
||||
assert(!module.TERMINAL_CONTROL_CHARACTER_PATTERN.test(output), 'Unsafe excerpt controls')
|
||||
assert(!output.includes('\r'), 'Unnormalized CR')
|
||||
}
|
||||
function legacySanitize(value) {
|
||||
return module.ownRetainedString(
|
||||
module
|
||||
.stripAnsiEscapeSequences(value)
|
||||
.replace(/\r\n?/g, '\n')
|
||||
.replace(module.TERMINAL_CONTROL_CHARACTER_PATTERN, '')
|
||||
)
|
||||
}
|
||||
|
||||
function retained() {
|
||||
if (descriptor.arm === 'no-buffer') {
|
||||
globalThis.Buffer = undefined
|
||||
module.resetOwnRetainedStringCopier()
|
||||
}
|
||||
const byteLength = new TextEncoder()
|
||||
const size = descriptor.family === 'near-cap' ? 63 * 1024 * 1024 : 2 * 1024 * 1024
|
||||
const count = descriptor.count ?? 8
|
||||
for (let index = 0; index < 3; index++) {
|
||||
module.ownRetainedString('warm owned string '.repeat(1000))
|
||||
}
|
||||
gc()
|
||||
const before = memory('warmed-retention-baseline')
|
||||
let excerpts = []
|
||||
let encodedTotal = 0
|
||||
for (let index = 0; index < count; index++) {
|
||||
const unit =
|
||||
descriptor.family === 'cache'
|
||||
? ['plain ', '界😀', '\x9dunfinished ', '\ud800a\udfff'][index % 4]
|
||||
: 'x'
|
||||
const chunk = unit.repeat(1024)
|
||||
let parent = [
|
||||
`${index}:`,
|
||||
...Array.from({ length: Math.ceil(size / chunk.length) }, () => chunk)
|
||||
].join('')
|
||||
// The flat parent is distinct for each retained result; source references die each iteration.
|
||||
const output =
|
||||
descriptor.arm === 'pinned'
|
||||
? parent.slice(-LIMIT)
|
||||
: descriptor.arm === 'baseline'
|
||||
? module.sliceCheckLogTail(parent)
|
||||
: module.sanitizeCheckJobLogExcerpt(parent)
|
||||
const bytes = byteLength.encode(output).length
|
||||
assert(bytes <= LIMIT)
|
||||
encodedTotal += bytes
|
||||
excerpts.push(output)
|
||||
parent = ''
|
||||
}
|
||||
gc()
|
||||
const retainedMemory = memory('retained-excerpts-after-double-gc')
|
||||
assert(encodedTotal <= 2 * 1024 * 1024)
|
||||
const retainedCount = excerpts.length
|
||||
excerpts = []
|
||||
gc()
|
||||
const released = memory('released-excerpts-after-double-gc')
|
||||
const delta = (value) =>
|
||||
Object.fromEntries(
|
||||
['heapUsed', 'external', 'arrayBuffers', 'rss'].map((field) => [
|
||||
field,
|
||||
value[field] - before[field]
|
||||
])
|
||||
)
|
||||
const growth = delta(retainedMemory)
|
||||
const release = delta(released)
|
||||
if (descriptor.family !== 'near-cap') {
|
||||
assert(release.heapUsed < 4 * 1024 * 1024, 'Excerpts did not release near the warmed heap')
|
||||
}
|
||||
result({ retainedCount, encodedTotal, growth, release, samples: [] })
|
||||
}
|
||||
|
||||
try {
|
||||
if (descriptor.mode === 'retention') {
|
||||
ready()
|
||||
retained()
|
||||
} else if (descriptor.mode === 'watchdog') {
|
||||
ready()
|
||||
emit({ type: 'entered', phase: 'watchdog' })
|
||||
while (true) {}
|
||||
} else if (descriptor.mode === 'control') {
|
||||
ready()
|
||||
memory('after-setup')
|
||||
if (descriptor.family === 'cleanup') {
|
||||
const monitor = monitorEventLoopDelay({ resolution: 1 })
|
||||
try {
|
||||
monitor.enable()
|
||||
void delay(1000)
|
||||
throw new Error('cleanup-sentinel')
|
||||
} catch (error) {
|
||||
assert.equal(error.message, 'cleanup-sentinel')
|
||||
} finally {
|
||||
assert(monitor.disable())
|
||||
assert(!monitor.disable())
|
||||
for (const timer of timers) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
timers.clear()
|
||||
}
|
||||
result({ samples: [], cleaned: true })
|
||||
} else {
|
||||
const measured = await measure(
|
||||
() => {
|
||||
const start = performance.now()
|
||||
while (performance.now() - start < 50) {}
|
||||
return ''
|
||||
},
|
||||
false,
|
||||
true
|
||||
)
|
||||
result({ samples: [measured.sample] })
|
||||
}
|
||||
} else if (descriptor.mode === 'differential') {
|
||||
ready()
|
||||
let cases = 0
|
||||
const outputs = createHash('sha256')
|
||||
memory('before-differential')
|
||||
for (const input of plainLogDifferentialCases()) {
|
||||
const output = module.sliceCheckLogTail(input)
|
||||
assert(Buffer.byteLength(output) <= LIMIT)
|
||||
outputs.update(JSON.stringify(output))
|
||||
cases++
|
||||
}
|
||||
memory('after-differential')
|
||||
result({ samples: [], cases, outputHash: outputs.digest('hex') })
|
||||
} else {
|
||||
const generated = workload(descriptor)
|
||||
let input = generated.text
|
||||
let expected = generated.expected
|
||||
const expectedHash = expected === undefined ? undefined : digest(expected)
|
||||
expected = undefined
|
||||
generated.expected = undefined
|
||||
generated.text = undefined
|
||||
ready(input)
|
||||
memory('after-setup')
|
||||
if (descriptor.mode === 'coverage') {
|
||||
module.resetObservations()
|
||||
const output = module.sliceCheckLogTail(input)
|
||||
if (descriptor.arm === 'proposed') {
|
||||
assertSafe(output)
|
||||
} else {
|
||||
assert(Buffer.byteLength(output) <= LIMIT)
|
||||
}
|
||||
const observations = module.observations()
|
||||
if (descriptor.arm === 'proposed') {
|
||||
assert((observations.strip?.maxBytes ?? 0) <= LIMIT)
|
||||
assert(
|
||||
(observations.candidate?.maxBytes ?? 0) <= (descriptor.bytes > LIMIT ? LIMIT / 2 : LIMIT)
|
||||
)
|
||||
assert(
|
||||
(observations.matcher?.maxBytes ?? 0) <= (descriptor.bytes > LIMIT ? LIMIT / 2 : LIMIT)
|
||||
)
|
||||
assert((observations.exact?.maxBytes ?? 0) <= LIMIT)
|
||||
}
|
||||
memory('after-coverage')
|
||||
result({
|
||||
samples: [],
|
||||
observations,
|
||||
outputBytes: Buffer.byteLength(output),
|
||||
outputHash: digest(output)
|
||||
})
|
||||
} else if (descriptor.mode === 'counts') {
|
||||
const original = String.prototype.charCodeAt
|
||||
let reads = 0
|
||||
const originalBuffer = globalThis.Buffer
|
||||
if (descriptor.noBuffer) {
|
||||
globalThis.Buffer = undefined
|
||||
module.resetOwnRetainedStringCopier()
|
||||
}
|
||||
try {
|
||||
String.prototype.charCodeAt = function (index) {
|
||||
reads++
|
||||
return original.call(this, index)
|
||||
}
|
||||
const output = module.stripAnsiEscapeSequences(input)
|
||||
globalThis.Buffer = originalBuffer
|
||||
assert.equal(digest(output), expectedHash)
|
||||
} finally {
|
||||
globalThis.Buffer = originalBuffer
|
||||
module.resetOwnRetainedStringCopier()
|
||||
String.prototype.charCodeAt = original
|
||||
}
|
||||
assert(reads >= 0 && reads <= 12 * input.length + 64)
|
||||
if (
|
||||
['osc', 'dcs', 'apc', 'pm', 'sos', 'dense-osc', 'dense-string', 'mixed', 'sparse'].includes(
|
||||
descriptor.family
|
||||
)
|
||||
) {
|
||||
assert(reads > 0)
|
||||
}
|
||||
result({ samples: [], reads, codeUnits: input.length })
|
||||
} else {
|
||||
const operation =
|
||||
descriptor.mode === 'direct'
|
||||
? () => module.stripAnsiEscapeSequences(input)
|
||||
: descriptor.arm === 'full-log'
|
||||
? () => module.sliceCheckLogTail(legacySanitize(input))
|
||||
: descriptor.arm === 'baseline-bounded'
|
||||
? () => legacySanitize(module.sliceCheckLogTail(input))
|
||||
: () => module.sliceCheckLogTail(input)
|
||||
const warmups = descriptor.warmups ?? 3
|
||||
memory('before-warmup')
|
||||
for (let warmup = 0; warmup < warmups; warmup++) {
|
||||
module.setPhase('warmup')
|
||||
emit({ type: 'entered', phase: 'warmup', warmup })
|
||||
operation()
|
||||
}
|
||||
memory('after-warmup')
|
||||
gc()
|
||||
const samples = []
|
||||
let outputBytes = 0
|
||||
let outputHash = ''
|
||||
let diagnosticRetained = false
|
||||
let stackRetained = false
|
||||
for (let sample = 0; sample < descriptor.samples; sample++) {
|
||||
module.setPhase('sample')
|
||||
emit({ type: 'entered', phase: 'sample', sample })
|
||||
const measured = await measure(
|
||||
operation,
|
||||
descriptor.mode === 'direct' && descriptor.bytes <= 32 * 1024
|
||||
)
|
||||
if (expectedHash) {
|
||||
assert.equal(digest(measured.output), expectedHash, 'Direct output changed')
|
||||
}
|
||||
if (descriptor.mode === 'cpu') {
|
||||
assert(Buffer.byteLength(measured.output) <= LIMIT)
|
||||
if (descriptor.arm !== 'baseline') {
|
||||
assertSafe(measured.output)
|
||||
}
|
||||
const plain =
|
||||
descriptor.arm === 'baseline' ? legacySanitize(measured.output) : measured.output
|
||||
diagnosticRetained = plain.includes(
|
||||
descriptor.family === 'short-lines'
|
||||
? 'error: CPU_DIAGNOSTIC'
|
||||
: descriptor.family === 'fixture'
|
||||
? createCheckLogFixture().expectedDiagnostic
|
||||
: 'No "Copy" export is defined on the "lucide-react" mock'
|
||||
)
|
||||
stackRetained = plain.includes(
|
||||
descriptor.family === 'short-lines'
|
||||
? 'CPU_FRAME.ts:18:8'
|
||||
: descriptor.family === 'fixture'
|
||||
? createCheckLogFixture().expectedStackFrame
|
||||
: 'CopyTabIdMenuItem.tsx:18:8'
|
||||
)
|
||||
assert(diagnosticRetained && stackRetained)
|
||||
}
|
||||
outputBytes = Buffer.byteLength(measured.output)
|
||||
outputHash = digest(measured.output)
|
||||
samples.push(measured.sample)
|
||||
}
|
||||
result({ samples, outputBytes, outputHash, diagnosticRetained, stackRetained })
|
||||
}
|
||||
input = ''
|
||||
}
|
||||
assert.equal(timers.size, 0)
|
||||
assert.equal(process.getActiveResourcesInfo().filter((name) => name === 'Timeout').length, 0)
|
||||
} finally {
|
||||
for (const timer of timers) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
timers.clear()
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
export function createCheckLogFixture() {
|
||||
const expectedDiagnostic = 'Error: expected 200, received 500'
|
||||
const expectedStackFrame = 'request.test.ts:18:8'
|
||||
const benignMarker = 'stderr | request test: retrying connection'
|
||||
const genericExitMarker = '##[error]Process completed with exit code 1.'
|
||||
return {
|
||||
expectedDiagnostic,
|
||||
expectedStackFrame,
|
||||
benignMarker,
|
||||
genericExitMarker,
|
||||
rawLog: [
|
||||
`\x1b[90m${benignMarker}\x1b[0m`,
|
||||
'\x1b[41m FAIL \x1b[0m request test',
|
||||
`\x1b[31mError\x1b[0m: expected 200, received 500`,
|
||||
` at handleRequest (${expectedStackFrame})`,
|
||||
genericExitMarker,
|
||||
''
|
||||
].join('\n')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { BASELINE_REF } from './check-job-log-performance-cases.mjs'
|
||||
|
||||
export function assertWatchdogExit(result) {
|
||||
const signalExit = result.code === null && ['SIGTERM', 'SIGKILL'].includes(result.signal)
|
||||
assert(result.timedOut === true && signalExit, 'Unverified watchdog termination')
|
||||
}
|
||||
|
||||
export function assertMeasurementRuntime(runtime) {
|
||||
assert(
|
||||
runtime?.bun === false &&
|
||||
typeof runtime.node === 'string' &&
|
||||
/^v\d+\.\d+\.\d+(?:[-+][\w.-]+)?$/.test(runtime.node) &&
|
||||
typeof runtime.platform === 'string' &&
|
||||
/^[a-z][a-z0-9]*$/.test(runtime.platform) &&
|
||||
typeof runtime.arch === 'string' &&
|
||||
/^[a-z][a-z0-9]*$/.test(runtime.arch),
|
||||
'Measurement requires verified Node'
|
||||
)
|
||||
}
|
||||
|
||||
export function validateChildProtocol(stdout, descriptor) {
|
||||
const protocol = stdout
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
assert(protocol.every((record) => record && typeof record === 'object'))
|
||||
const readyRecords = protocol.filter((record) => record.type === 'ready')
|
||||
const completed = protocol.filter((record) => record.type === 'result')
|
||||
assert(readyRecords.length === 1 && protocol[0] === readyRecords[0], 'Invalid ready framing')
|
||||
assert(
|
||||
completed.length <= 1 && (!completed.length || protocol.at(-1) === completed[0]),
|
||||
'Invalid result framing'
|
||||
)
|
||||
const ready = readyRecords[0]
|
||||
assertMeasurementRuntime(ready.runtime)
|
||||
if (completed.length) {
|
||||
for (const field of ['id', 'mode', 'arm']) {
|
||||
assert.equal(completed[0][field], descriptor[field], `Completion ${field} changed`)
|
||||
}
|
||||
assert.deepEqual(
|
||||
completed[0].inputIdentity,
|
||||
{ inputBytes: ready.inputBytes, codeUnits: ready.codeUnits, inputHash: ready.inputHash },
|
||||
'Completion input identity changed'
|
||||
)
|
||||
}
|
||||
const entries = protocol.slice(1, completed.length ? -1 : undefined)
|
||||
let entered
|
||||
let pendingStrip
|
||||
let warmups = 0
|
||||
let samples = 0
|
||||
for (const entry of entries) {
|
||||
if (descriptor.mode === 'watchdog') {
|
||||
assert(
|
||||
entries.length === 1 && entry.type === 'entered' && entry.phase === 'watchdog',
|
||||
'Invalid watchdog entry'
|
||||
)
|
||||
} else if (entry.type === 'entered-strip') {
|
||||
assert(
|
||||
descriptor.baselineTimeoutAllowed &&
|
||||
descriptor.mode === 'direct' &&
|
||||
descriptor.arm === 'baseline' &&
|
||||
['warmup', 'sample'].includes(entry.phase) &&
|
||||
((pendingStrip && pendingStrip.phase === entry.phase) || entries.length === 1),
|
||||
'Invalid strip entry'
|
||||
)
|
||||
pendingStrip = undefined
|
||||
} else {
|
||||
assert(
|
||||
['direct', 'cpu'].includes(descriptor.mode) && entry.type === 'entered' && !pendingStrip,
|
||||
'Invalid workload entry'
|
||||
)
|
||||
if (entry.phase === 'warmup') {
|
||||
assert(samples === 0 && warmups < (descriptor.warmups ?? 3))
|
||||
assert.equal(entry.warmup, warmups++, 'Invalid warmup order')
|
||||
} else {
|
||||
assert(entry.phase === 'sample' && warmups === (descriptor.warmups ?? 3))
|
||||
assert(samples < descriptor.samples)
|
||||
assert.equal(entry.sample, samples++, 'Invalid sample order')
|
||||
}
|
||||
if (descriptor.baselineTimeoutAllowed) {
|
||||
pendingStrip = entry
|
||||
}
|
||||
}
|
||||
entered =
|
||||
entry.type === (descriptor.baselineTimeoutAllowed ? 'entered-strip' : 'entered')
|
||||
? entry
|
||||
: undefined
|
||||
}
|
||||
assert(!completed.length || !pendingStrip, 'Completion preceded strip entry')
|
||||
return { ready, entered, completed, warmups }
|
||||
}
|
||||
|
||||
export const adverseCaseKey = (descriptor) =>
|
||||
JSON.stringify([
|
||||
descriptor.family,
|
||||
descriptor.width,
|
||||
descriptor.bytes,
|
||||
descriptor.variant,
|
||||
descriptor.terminator,
|
||||
descriptor.seed,
|
||||
descriptor.generator
|
||||
])
|
||||
|
||||
export function auditBaselineCensoring(manifest, byId) {
|
||||
for (const descriptor of manifest) {
|
||||
const record = byId.get(descriptor.id)
|
||||
if (descriptor.mode === 'watchdog' || record.status === 'watchdog_passed') {
|
||||
assert(
|
||||
descriptor.mode === 'watchdog' &&
|
||||
record.status === 'watchdog_passed' &&
|
||||
record.attempted === true
|
||||
)
|
||||
assert(record.settlementMs < 15000)
|
||||
assert.equal(record.identity.baselineRef, BASELINE_REF)
|
||||
assertMeasurementRuntime(record.runtime)
|
||||
assertWatchdogExit(record.watchdogTermination)
|
||||
assert.equal(record.code, record.watchdogTermination.code)
|
||||
assert.equal(record.signal, record.watchdogTermination.signal)
|
||||
}
|
||||
if (record.status === 'baseline_timeout') {
|
||||
assert(
|
||||
descriptor.baselineTimeoutAllowed &&
|
||||
descriptor.arm === 'baseline' &&
|
||||
descriptor.mode === 'direct'
|
||||
)
|
||||
assert(record.attempted === true && ['warmup', 'sample'].includes(record.phase))
|
||||
assert.equal(record.deadlineMs, 10000)
|
||||
assert(record.settlementMs < 15000)
|
||||
assert.equal(record.identity.baselineRef, BASELINE_REF)
|
||||
assertMeasurementRuntime(record.runtime)
|
||||
assertWatchdogExit(record.watchdogTermination)
|
||||
}
|
||||
if (record.status === 'baseline_censored') {
|
||||
const initial = byId.get(record.initialTimeoutId)
|
||||
assert(
|
||||
descriptor.baselineTimeoutAllowed && descriptor.sample > 0 && record.attempted === false
|
||||
)
|
||||
assert(
|
||||
initial?.status === 'baseline_timeout' &&
|
||||
initial.sample === 0 &&
|
||||
initial.attempted === true,
|
||||
'Missing verified initial adverse attempt'
|
||||
)
|
||||
assert.equal(
|
||||
adverseCaseKey(record),
|
||||
adverseCaseKey(initial),
|
||||
'Censoring crossed adverse cases'
|
||||
)
|
||||
assertWatchdogExit(initial.watchdogTermination)
|
||||
assert.deepEqual(record.identity, { ...initial.identity, id: descriptor.id })
|
||||
for (const field of ['inputBytes', 'codeUnits', 'inputHash']) {
|
||||
assert.equal(record[field], initial[field])
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const descriptor of manifest.filter(
|
||||
(item) => item.mode === 'direct' && item.baselineTimeoutAllowed && item.sample === 0
|
||||
)) {
|
||||
const slots = manifest.filter(
|
||||
(item) =>
|
||||
item.mode === 'direct' &&
|
||||
item.baselineTimeoutAllowed &&
|
||||
adverseCaseKey(item) === adverseCaseKey(descriptor)
|
||||
)
|
||||
assert.equal(slots.length, 7, 'Incomplete adverse baseline logical slots')
|
||||
assert.deepEqual(
|
||||
slots.map((item) => item.sample).toSorted((a, b) => a - b),
|
||||
[0, 1, 2, 3, 4, 5, 6]
|
||||
)
|
||||
for (const slot of slots) {
|
||||
const record = byId.get(slot.id)
|
||||
assert(
|
||||
record.attempted === true || record.status === 'baseline_censored',
|
||||
'Unattempted baseline sample'
|
||||
)
|
||||
assert.equal(
|
||||
record.status === 'completed' ? record.samples.length : record.unavailableSamples,
|
||||
1
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user