starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-21 16:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into attr-child-clock

Browse Source
This commit is contained in:
Brennan Benson
2026-09-20 22:58:41 -07:00
parent 933407ddc4 f492054bf0
commit c0ffb210cf
3019 changed files with 173217 additions and 14384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+43
View File
@@ -23,10 +23,15 @@
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
/config/patches/*.patch -text
# Same reason, and pnpm parses these too: a CRLF checkout makes the mobile
# patches unparseable, so Windows packaging dies on ERR_PNPM_INVALID_PATCH.
/mobile/patches/*.patch -text
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
# source patch under xterm-src/ instead. The sibling patches stay diffable.
/config/patches/@xterm__xterm@*.patch -diff
/config/patches/xterm-src/*.patch text eol=lf
# pnpm parses these unified diffs during Windows installs; keep checkout bytes stable.
/mobile/patches/*.patch -text
# Generated wrapper fixtures: collapse them in the PR diff so they stop drowning
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
@@ -48,3 +53,41 @@
/src/mobile-web/src/*.ts text eol=lf
/src/mobile-web/src/*.css text eol=lf
/src/mobile-web/src/*.png -text
# Mobile web page source. Same buildId hazard as src/mobile-web above: these bytes are
# hashed into the Phase C bundle, so a CRLF Windows checkout would ship a different
# buildId for identical source. web-entry/ does not exist yet; the pin lands ahead of it.
/mobile/src/** text eol=lf
/mobile/app/** text eol=lf
/mobile/web-entry/** text eol=lf
# The blanket pin above would mark a future binary as text; exempt the asset types an
# RN page actually carries, the same way src/mobile-web exempts its PNG.
/mobile/src/**/*.png -text
/mobile/src/**/*.jpg -text
/mobile/src/**/*.jpeg -text
/mobile/src/**/*.gif -text
/mobile/src/**/*.ico -text
/mobile/src/**/*.webp -text
/mobile/src/**/*.ttf -text
/mobile/src/**/*.otf -text
/mobile/src/**/*.woff -text
/mobile/src/**/*.woff2 -text
/mobile/app/**/*.png -text
/mobile/app/**/*.jpg -text
/mobile/app/**/*.jpeg -text
/mobile/app/**/*.gif -text
/mobile/app/**/*.ico -text
/mobile/app/**/*.webp -text
/mobile/app/**/*.ttf -text
/mobile/app/**/*.otf -text
/mobile/app/**/*.woff -text
/mobile/app/**/*.woff2 -text
/mobile/web-entry/**/*.png -text
/mobile/web-entry/**/*.jpg -text
/mobile/web-entry/**/*.jpeg -text
/mobile/web-entry/**/*.gif -text
/mobile/web-entry/**/*.ico -text
/mobile/web-entry/**/*.webp -text
/mobile/web-entry/**/*.ttf -text
/mobile/web-entry/**/*.otf -text
/mobile/web-entry/**/*.woff -text
/mobile/web-entry/**/*.woff2 -text
.github/actions/install-mobile-dependencies/action.yml
+22
View File
@@ -0,0 +1,22 @@
name: Install mobile dependencies
description: Frozen pnpm install for the mobile/ project, whose node_modules the mobile web bundle build and the mobile-aware lint passes resolve React Native and Expo from.
runs:
using: composite
steps:
# Why a separate install: mobile is its own pnpm project, so the root install leaves
# mobile/node_modules empty and every mobile import resolves to nothing.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the
# gitignored terminal/mermaid webview engine modules that tracked source imports.
# The drift guard mirrors the root install so a stale mobile lockfile fails by name --
# mobile's lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
shell: bash
working-directory: mobile
run: |
pnpm install --frozen-lockfile
# Job containers can run composite steps from a source mirror without .git.
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
.github/workflows/adhoc-mac-build.yml
+7
View File
@@ -184,6 +184,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -205,6 +208,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+96 -4
View File
@@ -78,7 +78,8 @@ jobs:
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
# The caller validated this too; re-check here so a malformed override
# can never reach a mutation through this reusable workflow.
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
@@ -537,9 +538,12 @@ jobs:
- name: Reversibly isolate and drain only the selected cell
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: drain
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
# Opens the window the report-only shadow health gate below judges this cell over.
echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
# A cell isolated by a failed canary is already migration-only, so
# isolate is a no-op there that does not advance the selector; the
@@ -590,11 +594,16 @@ jobs:
# Zero resource changes prove the prior run's apply completed and no
# restart will follow, keeping the incarnation check honest. Root
# outputs may lag a targeted apply, so judge resource_changes only.
# The backend service is targeted too, so its reviewed drain timeout
# and request logging can be the only thing left here; neither
# restarts an instance, so the validator below clears that on its
# own, without the template-and-MIG pair.
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_backend_service.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
if ! terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
@@ -617,7 +626,7 @@ jobs:
| select(.change.actions | any(. != "no-op" and . != "read"))
| .address] | join(","))'
echo 'requiring reviewed rollback-image drift'
terraform -chdir=infra/terraform show -json \
RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
@@ -629,14 +638,30 @@ jobs:
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}" \
| jq -e '.changes == 2' >/dev/null
"${POOL_ARGUMENTS[@]}")"
echo "${RESUME_REVIEW}"
jq -e '.changes == 2
or (.changes == 0 and ((.backendUpdate // []) | length) > 0)' \
<<< "${RESUME_REVIEW}" >/dev/null
# changes == 0 here means the template and MIG are converged and this cell's
# reviewed backend update is the only thing left, so the resume is not complete:
# apply it, or the cell silently keeps the 300-second drain and no request
# logging and the operator reads that as a finished roll. The plan holds nothing
# else (the validator bounded it to this cell's backend, and the template and MIG
# are no-ops in it), and neither attribute restarts an instance, so the
# incarnation check downstream stays honest. Template-and-MIG drift still applies
# nothing, which is what a resume means.
if test "$(jq -er '.changes' <<< "${RESUME_REVIEW}")" = 0; then
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
fi
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
- name: Apply only the selected same-cap template and MIG
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: apply
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
@@ -652,6 +677,7 @@ jobs:
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_backend_service.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
@@ -666,6 +692,10 @@ jobs:
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
# Stamped before the apply, not after it: the new container announces its listener while
# the MIG is still converging, so a bound taken at the end of this step is already past
# the announcement the shadow gate looks for.
echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -684,6 +714,9 @@ jobs:
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
# Recorded for the operator comparing verdicts; the gate's boot search opens at the
# apply-started-at stamp above, not here.
echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
@@ -697,6 +730,7 @@ jobs:
- name: Verify new incarnation, exact image, protocol, and durable safety
if: ${{ inputs.mode != 'verify' }}
id: verify-target
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
@@ -748,6 +782,7 @@ jobs:
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
@@ -794,6 +829,63 @@ jobs:
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
# Report only: this evaluates the oracles an operator reads by hand after a cell and records
# what it would have called, so its verdicts can be compared with the operator's over a full
# roll before it is ever allowed to block. Two independent guarantees keep it inert: the
# script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's
# outcome. The failsafe below therefore cannot fire on anything this step observes.
#
# It runs after the restore, not before it, for two reasons: the cell goes back into
# admission on exactly today's schedule rather than waiting out a minute of log reads, and
# the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it.
# These are fleet oracles anyway; when this does gate, what it gates is the next cell.
- name: Shadow health gate (report only)
id: shadow-gate
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# continue-on-error bounds this step's contribution to the job outcome, not its clock,
# and its reads are serialised. A timed-out step is a failed step, which continue-on-error
# absorbs; without this bound a Logging 429 storm or an expired credential makes every
# read cost its full retry budget and can push the job past timeout-minutes, and a
# cancelled job takes the whole wave with it. A healthy gate is already minutes of
# serial reads, so both bounds sit above that: the script settles at seven minutes and
# reaching this eight is the pathological case. Eight on top of a ~14-minute cell still
# leaves the job's 75 minutes intact.
timeout-minutes: 8
env:
DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }}
APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }}
APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }}
VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }}
SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate
SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
run: |
mkdir -p "${SHADOW_GATE_DIRECTORY}"
node dev/scripts/relay-same-cap-shadow-gate.mjs \
--cell-id "${TARGET_CELL_ID}" \
--cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" \
--director-service orca-cloud-relay \
--drain-started-at "${DRAIN_STARTED_AT}" \
--apply-started-at "${APPLY_STARTED_AT}" \
--apply-completed-at "${APPLY_COMPLETED_AT}" \
--verify-ended-at "${VERIFY_ENDED_AT}" \
--summary-file "${GITHUB_STEP_SUMMARY}" \
--output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}"
- name: Publish the shadow health gate verdict
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# One small JSON file; a retrying upload must not spend the wave's remaining minutes either.
timeout-minutes: 2
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
if-no-files-found: warn
retention-days: 14
overwrite: true
- id: cleanup-auth
if: ${{ failure() && inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+145 -1
View File
@@ -10,7 +10,7 @@ on:
type: choice
options: [verify, canary-apply, batch-apply, rollback]
cell-ids:
description: Ordered comma-separated serving cells; one canary or two to four batch cells
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
required: true
type: string
target-image-digest:
@@ -313,6 +313,144 @@ jobs:
wave-index: '3'
secrets: inherit
cell_5:
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
needs: [gate, cell_4]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '4'
secrets: inherit
cell_6:
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
needs: [gate, cell_5]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '5'
secrets: inherit
cell_7:
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
needs: [gate, cell_6]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '6'
secrets: inherit
cell_8:
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
needs: [gate, cell_7]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '7'
secrets: inherit
cell_9:
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
needs: [gate, cell_8]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '8'
secrets: inherit
cell_10:
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
needs: [gate, cell_9]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '9'
secrets: inherit
seal_canary:
if: ${{ inputs.mode == 'canary-apply' }}
needs: [gate, cell_1]
@@ -360,6 +498,12 @@ jobs:
- cell_2
- cell_3
- cell_4
- cell_5
- cell_6
- cell_7
- cell_8
- cell_9
- cell_10
- seal_canary
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
.github/workflows/cloud-operate-relay-production-rehome-job.yml
+1 -1
View File
@@ -320,7 +320,7 @@ jobs:
echo '### Regional rehome control'
jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
"${RUNNER_TEMP}/relay-rehome-control.json"
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`"' \
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \
"${RUNNER_TEMP}/relay-rehome-inventory.json"
} >> "${GITHUB_STEP_SUMMARY}"
.github/workflows/daemon-relocation-spike.yml
+22 -1
View File
@@ -57,7 +57,28 @@ jobs:
uses: actions/cache@v4
with:
path: dist/win-unpacked
key: win-unpacked-${{ hashFiles('src/**', 'config/**', 'package.json', 'pnpm-lock.yaml') }}
# mobile/ is in the key because beforePack requires out/mobile-web, whose bytes come from
# the mobile install and, once Phase C flips the bundle, from the page trees below; a
# mobile-only change must miss this cache, not reuse a stale installer. src/** and
# config/** already cover src/mobile-web and the two bundle builders.
key: >-
win-unpacked-${{ hashFiles(
'src/**',
'config/**',
'package.json',
'pnpm-lock.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-unpacked.outputs.cache-hit != 'true'
- name: Build unpacked app
if: steps.cache-unpacked.outputs.cache-hit != 'true'
.github/workflows/daily-mac-build.yml
+8
View File
@@ -156,6 +156,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
@@ -179,6 +182,11 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.freshness.outputs.should_build == 'true'
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
.github/workflows/dev-channel-win-build.yml
+7
View File
@@ -203,6 +203,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Caches the Electron binary and electron-builder's tool downloads (nsis,
# winCodeSign). Same key shape as release-cut's Windows leg.
@@ -229,6 +232,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why the packaging check runs before the 20-minute build: it only needs
# node_modules, and a stale config should cost seconds rather than a build.
- name: Verify dev-channel packaging identity
.github/workflows/hourly-mac-build.yml
+7
View File
@@ -164,6 +164,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -185,6 +188,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
.github/workflows/mobile.yml
+6
View File
@@ -103,6 +103,12 @@ jobs:
- name: Typecheck tests (ratchet)
run: pnpm run check:tests-typecheck
# This includes the bridged replay of the whole recording corpus, which used to be a second
# step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point:
# it fails when a divergence class grows, when a divergence lands in no class at all, or when
# one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is
# ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite,
# so folding it in costs a fraction of that in wall time and one step less to skip.
- name: Test
run: pnpm test
.github/workflows/pi-owner-runtime.yml
+4 -1
View File
@@ -2,8 +2,9 @@ name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/agent-status-handler-source.ts'
- 'src/main/pi/**'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- 'tests/tools/omp-completion-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
@@ -27,3 +28,5 @@ jobs:
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
- name: Verify OMP completion over native HTTP
run: node tests/tools/omp-completion-runtime-smoke.mjs
.github/workflows/pr.yml
+98 -17
View File
@@ -29,6 +29,7 @@ jobs:
should_run: ${{ steps.filter.outputs.should_run }}
native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
mobile_web_app: ${{ steps.filter.outputs.mobile_web_app }}
static_analysis: ${{ steps.filter.outputs.static_analysis }}
typecheck: ${{ steps.filter.outputs.typecheck }}
git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
@@ -142,24 +143,11 @@ jobs:
- name: Enforce type-aware code-quality baseline
run: pnpm run audit:code-quality:type-aware
# Why: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Mobile is a separate pnpm project,
# so the root install above leaves it empty and every mobile type degrades to
# an `error` type — reported as phantom findings against the changed lines.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
# the gitignored terminal/mermaid webview engine modules that tracked source imports,
# and skipping it degrades those very types the step exists to resolve. The drift
# guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
# lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
# Why here: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Without the install every mobile type
# degrades to an `error` type — reported as phantom findings against the changed lines.
- uses: ./.github/actions/install-mobile-dependencies
if: needs.code_paths.outputs.mobile_dependencies == 'true'
working-directory: mobile
run: |
pnpm install --frozen-lockfile
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
- name: Enforce changed-code quality
run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
@@ -661,6 +649,79 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts \
src/main/orcad/external-chromium-browser-process.integration.test.ts
# Why its own job: it needs mobile/node_modules and a real browser, and the sharded `test`
# matrix would pay for both on every shard to run two files. Dark through Phase C: this proves
# `build:mobile-web:app` on every PR that touches the page, and ships nothing -- packaging still
# builds the Phase A bootstrap via build:mobile-web.
mobile_web_app:
name: mobile web app bundle
needs: [code_paths]
if: needs.code_paths.outputs.mobile_web_app == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
# Why no native-runtime: the builder is esbuild and the render check is a browser. Nothing
# in this job loads node-pty.
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# The entry lives in mobile/ so one React resolves; without this every RN import is nothing.
- uses: ./.github/actions/install-mobile-dependencies
# Why the runner's Google Chrome and not a downloaded chromium: same reason as the orcad
# browser job -- Ubuntu 24.04 only ships an AppArmor userns profile for the Chrome .deb.
# Why fail instead of skip: a silently skipped render check is the failure this job exists
# to prevent.
- name: Resolve Chrome for the render check
run: |
set -euo pipefail
chrome="$(command -v google-chrome || command -v google-chrome-stable || true)"
if [ -z "$chrome" ]; then
echo "::error::No Google Chrome on the runner; the render check would silently skip."
exit 1
fi
"$chrome" --version
echo "ORCA_MOBILE_WEB_RENDER_BROWSER=$chrome" >> "$GITHUB_ENV"
# The drawer check runs on WebKit as well as Chrome, because the shell's iOS WebView is
# WebKit and the Chrome above cannot stand in for it. Downloaded rather than resolved from
# the runner: Ubuntu ships no WebKit build to point at.
- name: Install WebKit for the drawer check
run: pnpm exec playwright install --with-deps webkit
- name: Build and verify the app bundle
run: pnpm run build:mobile-web:app
# The bundling tests skip themselves where mobile dependencies are absent, which is how they
# stay green in the sharded `test` job. This is the job that installs them, so here a missing
# install has to fail rather than skip everything the job exists to run.
# Why a prefix and not a file list: the list this replaces had gone stale twice without
# anyone noticing, because a census whose closure block skips without the env flag below is
# green in the sharded `test` job whether or not it ever runs here. The prefix is the same
# one `pr-code-change-scope.mjs` fires this job on, so naming a test into the family is all
# it takes to have it run. Quoted because these are vitest filename filters, matched as
# substrings against the discovered files, and the shell must not touch them.
#
# Cost: 18 files in 25-30s wall, of which the frame-budget sweep is 2.5s. That sweep encodes
# 111 noise JPEGs in Chromium, so it is the one step here whose cost grows with its viewport
# set; adding rows to that set is a decision about this job's runtime.
- name: Builder, override census and render checks
env:
ORCA_MOBILE_WEB_APP_DEPS_REQUIRED: '1'
run: |
pnpm exec vitest run --config config/vitest.config.ts \
'config/scripts/mobile-web-app-' \
'config/scripts/build-mobile-web-app-bundle.test.mjs'
cross-version-wire:
name: cross-version wire compatibility
needs: [code_paths]
@@ -695,6 +756,8 @@ jobs:
tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts
tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts
tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts
tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts
tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts
managed_hook_node18:
name: managed hooks on Node 18
@@ -746,6 +809,13 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why --no-file-parallelism: every file here launches a full Electron stack twice, and each
# probe carries its own in-process deadline. Four at once on a 4-vCPU runner starve each other
@@ -859,6 +929,13 @@ jobs:
with:
native-runtime: node
persist-native-cache: 'false'
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Save compiled Node native modules
if: steps.deps.outputs.native-cache-hit != 'true'
@@ -1017,6 +1094,7 @@ jobs:
- shell_contracts
- test
- orcad_browser
- mobile_web_app
- cross-version-wire
- managed_hook_node18
- package
@@ -1053,6 +1131,8 @@ jobs:
TEST_SHOULD_RUN: ${{ needs.code_paths.outputs.test }}
ORCAD_BROWSER: ${{ needs.orcad_browser.result }}
ORCAD_BROWSER_SHOULD_RUN: ${{ needs.code_paths.outputs.orcad_browser }}
MOBILE_WEB_APP: ${{ needs.mobile_web_app.result }}
MOBILE_WEB_APP_SHOULD_RUN: ${{ needs.code_paths.outputs.mobile_web_app }}
CROSS_VERSION_WIRE: ${{ needs.cross-version-wire.result }}
CROSS_VERSION_WIRE_SHOULD_RUN: ${{ needs.code_paths.outputs.cross-version-wire }}
MANAGED_HOOK_NODE18: ${{ needs.managed_hook_node18.result }}
@@ -1095,6 +1175,7 @@ jobs:
check_job shell_contracts "$SHELL_CONTRACTS" "$SHELL_CONTRACTS_SHOULD_RUN"
check_job test "$TEST" "$TEST_SHOULD_RUN"
check_job orcad_browser "$ORCAD_BROWSER" "$ORCAD_BROWSER_SHOULD_RUN"
check_job mobile_web_app "$MOBILE_WEB_APP" "$MOBILE_WEB_APP_SHOULD_RUN"
check_job cross-version-wire "$CROSS_VERSION_WIRE" "$CROSS_VERSION_WIRE_SHOULD_RUN"
check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN"
check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN"
.github/workflows/release-cut.yml
+81 -36
View File
@@ -107,6 +107,10 @@ jobs:
with:
ref: ${{ github.event_name == 'schedule' && 'main' || inputs.ref }}
fetch-depth: 0
# Why: version math recovers unpublished tags; checkout's default
# fetch-tags:false hides them, so a patch cut recreates vX.Y.Z and
# `git push` overwrites the existing tag.
fetch-tags: true
- name: Setup Node.js
uses: actions/setup-node@v6
@@ -805,6 +809,16 @@ jobs:
with:
ref: refs/tags/${{ needs.cut.outputs.tag }}
- name: Restore draft-release scripts from the workflow ref
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- \
config/scripts/create-draft-release.mjs \
config/scripts/assert-github-release-is-draft.mjs
- name: Create draft release with bounded generated notes
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -846,7 +860,8 @@ jobs:
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- \
tests/e2e/golden-source-control-open-diff.spec.ts \
tests/e2e/golden-terminal-file-link.spec.ts
tests/e2e/golden-terminal-file-link.spec.ts \
tests/e2e/golden-worktree-create-switch.spec.ts
- name: Install native build tools
if: runner.os == 'Linux'
@@ -871,8 +886,17 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Why: this install runs lifecycle scripts, so node-gyp rebuilds
# native/windows-registry and fetches that Node version's headers from
# nodejs.org. One `read ECONNRESET` there failed this blocking gate and the
# whole cut. Retry like the release build's install below.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for platform golden
run: npx electron-vite build --mode e2e
@@ -1088,8 +1112,14 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Same node-gyp header fetch as the blocking golden gate above.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for terminal rendering evidence
run: npx electron-vite build --mode e2e
@@ -1161,14 +1191,14 @@ jobs:
~\AppData\Local\electron-builder\Cache
- os: ubuntu-latest
platform: linux-x64
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish always
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish always -c.publish.releaseType=draft
unpacked_dir: dist/linux-unpacked
eb_cache_path: |
~/.cache/electron
~/.cache/electron-builder
- os: ubuntu-24.04-arm
platform: linux-arm64
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_LINUX_ARM64_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --arm64 --publish always
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_LINUX_ARM64_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --arm64 --publish always -c.publish.releaseType=draft
unpacked_dir: dist/linux-arm64-unpacked
eb_cache_path: |
~/.cache/electron
@@ -1204,22 +1234,45 @@ jobs:
# ref, so cutting from an older/off-main ref whose tree predates a composite
# action would fail the step with "Can't find 'action.yml'". Restore the
# actions directory from the commit this workflow file itself came from.
- name: Restore composite actions from the workflow ref
if: matrix.platform == 'win' && github.run_attempt == 1
# Not Windows-only: every platform now consumes install-mobile-dependencies, so
# any of them can be the one whose cut ref predates the action.
- name: Restore draft-publish scripts from the workflow ref
# Why: this job checks out the release tag, so a cut from an older SHA
# still has electron-builder releaseType:release and no re-draft helper.
# The workflow YAML is from main; restore the scripts it invokes.
shell: bash
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
action_path=".github/actions/install-signpath-module/action.yml"
if [ -f "$action_path" ]; then
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs
- name: Restore composite actions from the workflow ref
shell: bash
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
PLATFORM: ${{ matrix.platform }}
run: |
set -euo pipefail
required=(.github/actions/install-mobile-dependencies/action.yml)
if [ "$PLATFORM" = win ] && [ "$GITHUB_RUN_ATTEMPT" = 1 ]; then
required+=(.github/actions/install-signpath-module/action.yml)
fi
missing=()
for action_path in "${required[@]}"; do
[ -f "$action_path" ] || missing+=("$action_path")
done
if [ "${#missing[@]}" -eq 0 ]; then
echo "Composite actions already present at the cut ref."
exit 0
fi
echo "Cut ref predates $action_path; restoring it from $WORKFLOW_SHA."
echo "Cut ref predates ${missing[*]}; restoring from $WORKFLOW_SHA."
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- .github/actions
test -f "$action_path"
for action_path in "${required[@]}"; do
test -f "$action_path"
done
# pnpm must be on PATH before setup-node so setup-node can locate the store for caching.
- name: Setup pnpm
@@ -1232,6 +1285,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: release builds hit the same native-module postinstall path as
# PR CI, so keep the pinned node-gyp override here too instead of
@@ -1272,6 +1328,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: `pnpm build:release` verifies the Linux computer-use provider by
# importing AT-SPI bindings, which are runtime package deps but are not
# present on stock GitHub Ubuntu release runners.
@@ -2160,36 +2220,21 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Verify release remains draft after artifact upload
# Why: the build matrix must never be the actor that exposes a partial
# release. If an uploader or GitHub transition flips draft early, fail
# this platform leg and leave the diagnostic monitor artifact behind.
# Why: electron-builder `--publish always` can create a public release
# as soon as this platform uploads. Re-draft immediately, then fail, so
# /releases/latest never keeps serving a missing Windows exe.
# Why bash: the Windows matrix defaults to pwsh, which does not expand
# "$TAG" into argv.
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ needs.cut.outputs.tag }}
run: |
set -euo pipefail
releases_json="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100")"
# Why: release upload must validate the draft before it is publicly visible.
draft="$(jq -e -r --arg tag "$TAG" '
map(select(.tag_name == $tag))
| if length == 1 and (.[0].draft | type) == "boolean" then (.[0].draft | tostring) else empty end
' <<<"$releases_json")" || {
echo "::error::Release $TAG was not found in the draft-aware releases list, or its draft state was missing."
exit 1
}
if [[ "$draft" != "true" ]]; then
echo "::error::Release $TAG was published during the ${{ matrix.platform }} artifact upload."
exit 1
fi
run: node config/scripts/assert-github-release-is-draft.mjs "${{ needs.cut.outputs.tag }}"
# Why post-publish for Linux: electron-builder packs and uploads in a
# single `--publish always` invocation, so there is no cheap insertion
# point between pack and upload without splitting those steps. Running
# verify last still blocks the bad release: the binary is uploaded to the
# draft, but a failed matrix job blocks `publish-release` from flipping
# the release from draft → published, so users never see it. A human then
# deletes the draft and re-cuts.
# Why post-pack for Linux: electron-builder packs and uploads in one
# `--publish always` invocation. The previous step re-drafts if that
# upload flipped the GitHub release public; this telemetry check still
# blocks `publish-release` from undrafting a bad binary.
#
# Why this guards against: a misconfigured CI run where
# `ORCA_POSTHOG_WRITE_KEY` is unset or the tag fails to classify
.github/workflows/release-mac-build.yml
+35 -19
View File
@@ -37,6 +37,14 @@ jobs:
with:
ref: refs/tags/${{ inputs.tag }}
- name: Restore draft-publish scripts from the workflow ref
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs
- name: Setup pnpm
uses: pnpm/setup@v2
with:
@@ -47,6 +55,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Cache the Electron binary + electron-builder tool downloads (notarytool,
# winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job, incl. mac.
@@ -74,6 +85,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Verify macOS signing environment
run: node config/scripts/verify-macos-release-env.mjs
env:
@@ -135,13 +150,28 @@ jobs:
# Kill only its child and require both PTY and watch recovery before packaging.
node config/scripts/relay-watcher-fault-harness.mjs
- name: Abort if the parent release-cut run was cancelled
# Why: this workflow is dispatched separately, so cancelling release-cut
# does not stop mac `--publish always`. A cancelled parent left v1.4.206
# public with only a partial mac upload.
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PARENT_RUN: ${{ inputs.release_run_id }}
run: |
set -euo pipefail
conclusion="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PARENT_RUN" --jq '.conclusion // empty')"
if [[ "$conclusion" == "cancelled" || "$conclusion" == "failure" || "$conclusion" == "timed_out" ]]; then
echo "::error::Parent release-cut run $PARENT_RUN is $conclusion; refusing to publish mac artifacts."
exit 1
fi
- name: Publish release artifacts (macOS)
uses: nick-fields/retry@v4
with:
timeout_minutes: 45
max_attempts: 3
retry_wait_seconds: 30
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always -c.publish.releaseType=draft
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -151,28 +181,14 @@ jobs:
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
- name: Verify release remains draft after artifact upload
# Why: the macOS build must never be the actor that exposes a partial
# release. If an uploader or GitHub transition flips draft early, fail
# this job so release-cut never publishes the release.
# Why: re-draft immediately if electron-builder flipped the GitHub
# release public, then fail. Checking without restoring leaves
# /releases/latest serving a missing Windows exe.
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
releases_json="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100")"
# Why: release upload must validate the draft before it is publicly visible.
draft="$(jq -e -r --arg tag "$TAG" '
map(select(.tag_name == $tag))
| if length == 1 and (.[0].draft | type) == "boolean" then (.[0].draft | tostring) else empty end
' <<<"$releases_json")" || {
echo "::error::Release $TAG was not found in the draft-aware releases list, or its draft state was missing."
exit 1
}
if [[ "$draft" != "true" ]]; then
echo "::error::Release $TAG was published during the mac artifact upload."
exit 1
fi
run: node config/scripts/assert-github-release-is-draft.mjs "${{ inputs.tag }}"
# Why post-publish for macOS: electron-builder packs and uploads in a
# single `--publish always` invocation, so there is no cheap insertion
.github/workflows/unit-tests.yml
+40 -7
View File
@@ -33,17 +33,13 @@ jobs:
native-runtime: node
node-version: ${{ matrix.node }}
cache-electron-package: 'true'
cache-dependency-path: |
pnpm-lock.yaml
cloud/pnpm-lock.yaml
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
# The real two-cell transport test imports cloud relay source and its contracts.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test shard
env:
ORCA_BALANCE_UNIT_SHARDS: '1'
@@ -68,6 +64,8 @@ jobs:
--exclude=src/shared/pty-reply-echo-shapes.node-pty.test.ts \
--exclude=src/shared/startup-shell-portability.live-shell.test.ts \
--exclude=src/shared/posix-command-path-lookup.test.ts \
--exclude=tests/e2e/relay-region-compatibility.unit.test.ts \
--exclude=tests/e2e/relay-region-correction.unit.test.ts \
--exclude=tests/e2e/cross-version-wire/** \
--shard=${{ matrix.shard }}/${{ matrix.shard_total }}
@@ -81,3 +79,38 @@ jobs:
path: ci-shards/
retention-days: 14
if-no-files-found: warn
relay_integration:
name: relay integration node ${{ fromJSON(inputs.node_versions)[0] }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: ${{ fromJSON(inputs.node_versions)[0] }}
cache-electron-package: 'true'
cache-dependency-path: |
pnpm-lock.yaml
cloud/pnpm-lock.yaml
# These two tests import the cloud relay workspace directly. Keeping them in one job
# avoids installing and building the same workspace once per unit-test shard.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test relay integration contracts
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
tests/e2e/relay-region-compatibility.unit.test.ts
tests/e2e/relay-region-correction.unit.test.ts
.github/workflows/win-crash-survival-e2e.yml
+18 -1
View File
@@ -55,6 +55,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -67,6 +70,9 @@ jobs:
uses: actions/cache@v4
with:
path: dist/orca-windows-setup.exe
# The mobile page trees are in the key because beforePack builds the mobile web bundle
# into the installer; src/** and config/** already cover src/mobile-web and the two
# bundle builders. A mobile-only change must miss this cache, not reuse a stale exe.
key: >-
crash-survival-installer-${{ hashFiles(
'src/**',
@@ -85,7 +91,12 @@ jobs:
'.npmrc',
'package.json',
'pnpm-lock.yaml',
'pnpm-workspace.yaml'
'pnpm-workspace.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why: production edits miss the installer cache by design, but Electron
@@ -101,6 +112,12 @@ jobs:
restore-keys: |
crash-survival-electron-builder-
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
.github/workflows/win-update-survival-e2e.yml
+6
View File
@@ -75,6 +75,12 @@ jobs:
path: dist/orca-windows-setup.exe
key: branch-installer-${{ hashFiles('src/**', 'config/**', 'native/**', 'resources/win32/**', 'package.json', 'pnpm-lock.yaml') }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
.github/workflows/windows-signing-rehearsal.yml
+7
View File
@@ -57,6 +57,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -78,6 +81,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: rehearsal builds are never published, so the official-build
# secrets (telemetry key, diagnostics URL) are intentionally omitted.
- name: Build app
.gitignore
+9
View File
@@ -98,11 +98,20 @@ docs/**
# The deployable docs app is source, not local engineering notes.
!docs/site/
!docs/site/**
!docs/audits/
!docs/audits/closed-editor-model-lifetime/
!docs/audits/closed-editor-model-lifetime/**
!docs/assets/
!docs/assets/**
!docs/audits/
!docs/audits/plugin-uninstall-log-retirement/
!docs/audits/plugin-uninstall-log-retirement/**
!docs/readme/
!docs/readme/**
!docs/STYLEGUIDE.md
!docs/audits/
!docs/audits/crashpad-read-limit/
!docs/audits/crashpad-read-limit/source-hashes.json
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
README.md
+2 -2
View File
@@ -36,7 +36,7 @@
Monitor and steer your agents from your phone — get notified when an agent finishes and send follow-ups from anywhere.
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.50](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.50/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
</td>
<td width="50%">
@@ -230,7 +230,7 @@ yay -S stably-orca-bin
Pair with your desktop app to monitor and steer your agents from your phone.
- **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) or [join TestFlight](https://testflight.apple.com/join/YjeGMQBA)
- **Android:** [Download APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
- **Android:** [Download APK 0.0.50](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.50/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
---
cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
+9 -2
View File
@@ -242,9 +242,16 @@ describe('relay incident live preflight', () => {
await expect(runIncidentLivePreflight(
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-3 argument.
// The last cell of a ten-cell same-cap batch: 10min + 9 * 75min exactly.
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', '4'], deps
['--state-file', agedState(685 * 60_000), '--wave-index', '9'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(685 * 60_000 + 1), '--wave-index', '9'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-9 argument.
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', '10'], deps
)).rejects.toThrow('usage:')
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', ''], deps
cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
+3 -1
View File
@@ -30,7 +30,9 @@ const FRESHNESS_RETRY_INTERVAL_MS = 15_000
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX_PATTERN = /^[0-3]$/
// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
// pins its own narrower range.
const WAVE_INDEX_PATTERN = /^[0-9]$/
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
cloud/apps/relay/src/app.ts
+41 -20
View File
@@ -2,7 +2,7 @@ import {
AssignmentRequestSchema,
IdleRegionalRehomeRequestSchema,
type IdleRegionalRehomeRequest,
type IdleRegionalRehomeOutcome,
type IdleRegionalRehomeResult,
type RegionCorrectionResponse,
isRelayCellConnectionHardCap,
RELAY_ADMISSION_BUDGETS,
@@ -27,10 +27,11 @@ import {
createRegionalRehomeTokenVerifier,
createRuntimeTokenVerifier
} from './admin-token-verifier.js'
import type {
CellFenceAttemptEvidence,
RelayAssignment,
RelayAssignmentStore
import {
RelayHomeCellUnavailableError,
type CellFenceAttemptEvidence,
type RelayAssignment,
type RelayAssignmentStore
} from './assignment-store.js'
import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
import { CELL_ADMISSION_STATES } from './cell-admission-selector.js'
@@ -78,7 +79,7 @@ export function createRelayApp(
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
}) => Promise<IdleRegionalRehomeResult>
drainHost?: (input: {
attemptId: string
userId: string
@@ -361,16 +362,17 @@ export function createRelayApp(
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'assign',
lane,
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error)
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
if (isRelayAssignmentCapacityError(error)) {
if (isRelayAssignmentUnavailableError(error)) {
if (lane === 'placement') {
operations.recordRegionSelection?.({ targetRegion, fallback: false })
}
@@ -389,11 +391,13 @@ export function createRelayApp(
fallback: lane === 'placement' && assignment.region !== targetRegion
})
// Grant-side counterpart of the rejection log: reconnect grants are rare
// enough to log and make "which cell is this host on" answerable.
if (lane === 'sticky') {
// enough to log and make "which cell is this host on" answerable. The
// placement-lane ones matter most — they are the only record that a host
// whose sticky lane failed verification landed anywhere at all.
if (body.data.reconnect) {
console.warn(
`[orca-relay] assignment granted lane=sticky host=${relayHostLogDigest(claims.relayHostId)}` +
` cell=${assignment.cellId}`
`[orca-relay] assignment granted lane=${lane} hinted=true` +
` host=${relayHostLogDigest(claims.relayHostId)} cell=${assignment.cellId}`
)
}
const lease = await new SignJWT({
@@ -466,16 +470,17 @@ export function createRelayApp(
leaseExpiresAt: assignment.leaseExpiresAt
})
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'resolve',
lane: 'none',
hinted: false,
relayHostId: body.data.relayHostId,
reason: operationError(error)
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
if (isRelayAssignmentCapacityError(error)) {
if (isRelayAssignmentUnavailableError(error)) {
return context.json({ error: operationError(error) }, 503)
}
if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context)
@@ -1948,25 +1953,41 @@ function logAssignmentRejection(input: {
hinted: boolean
relayHostId: string
reason: string
cause?: string
cell?: string
suppressed?: number
}): void {
console.warn(
`[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` +
` hinted=${input.hinted} reason=${input.reason}` +
` host=${relayHostLogDigest(input.relayHostId)}` +
(input.cause === undefined ? '' : ` cause=${input.cause}`) +
(input.cell === undefined ? '' : ` cell=${input.cell}`) +
(input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`)
)
}
function isRelayAssignmentCapacityError(error: unknown): boolean {
// The home-cell reason is not capacity, but it is the same answer to the client:
// retry, the director cannot place you right now.
function isRelayAssignmentUnavailableError(error: unknown): boolean {
return (
error instanceof Error &&
['relay_capacity_exhausted', 'relay_connection_headroom_exhausted'].includes(
error.message
)
[
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted',
'relay_home_cell_unavailable'
].includes(error.message)
)
}
function homeCellRejectionDetail(
error: unknown
): { cause: string; cell: string } | Record<string, never> {
return error instanceof RelayHomeCellUnavailableError
? { cause: error.unavailableCause, cell: error.cellId }
: {}
}
function isCanonicalRelayOrigin(value: string): boolean {
const url = new URL(value)
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
cloud/apps/relay/src/assignment-cleanup-steps.test.ts
+6 -4
View File
@@ -16,6 +16,7 @@ function stubStore(overrides: Partial<AssignmentCleanupStore> = {}) {
completeReadyEvacuations: method('completeReadyEvacuations'),
completeReadyRegionalRehomes: method('completeReadyRegionalRehomes'),
abortExpiredEvacuations: method('abortExpiredEvacuations'),
abortUnarrivedRegionalRehomes: method('abortUnarrivedRegionalRehomes'),
abortExpiredRegionalRehomes: method('abortExpiredRegionalRehomes'),
reapRegionalRehomeAttempts: method('reapRegionalRehomeAttempts'),
releaseExpiredActivityLeases: method('releaseExpiredActivityLeases'),
@@ -42,6 +43,7 @@ describe('assignment cleanup steps', () => {
'refreshRegionalRehomeLeases',
'completeReadyEvacuations',
'abortExpiredEvacuations',
'abortUnarrivedRegionalRehomes',
'abortExpiredRegionalRehomes',
'reapRegionalRehomeAttempts',
'releaseExpiredActivityLeases',
@@ -55,13 +57,13 @@ describe('assignment cleanup steps', () => {
)
})
it('covers all ten sweeps exactly once per run', async () => {
it('covers all eleven sweeps exactly once per run', async () => {
const { store, calls } = stubStore()
await runAssignmentCleanup(store)
expect(calls).toHaveLength(10)
expect(new Set(calls).size).toBe(10)
expect(assignmentCleanupSteps(store)).toHaveLength(10)
expect(calls).toHaveLength(11)
expect(new Set(calls).size).toBe(11)
expect(assignmentCleanupSteps(store)).toHaveLength(11)
})
})
cloud/apps/relay/src/assignment-cleanup-steps.ts
+4 -2
View File
@@ -1,9 +1,9 @@
import { runRelayBackgroundOperation } from './relay-background-operation.js'
// The ten periodic assignment sweeps the director runs every 30s. Each step
// The eleven periodic assignment sweeps the director runs every 30s. Each step
// re-derives its state from the database and is idempotent, so they carry no
// intra-tick ordering dependency — which is what makes per-step isolation
// sound: one failing sweep costs one tick of itself, never the other nine.
// sound: one failing sweep costs one tick of itself, never the other ten.
// (A single poisoned rehome row once silenced the whole chained form
// fleet-wide.) Sweep failures are logged, never fed into the rehome worker's
// dispatch-failure budget: a sweep exception is not a dispatch failure and
@@ -13,6 +13,7 @@ export type AssignmentCleanupStore = {
completeReadyEvacuations(): Promise<unknown>
completeReadyRegionalRehomes(): Promise<unknown>
abortExpiredEvacuations(): Promise<unknown>
abortUnarrivedRegionalRehomes(): Promise<unknown>
abortExpiredRegionalRehomes(): Promise<unknown>
reapRegionalRehomeAttempts(): Promise<unknown>
releaseExpiredActivityLeases(): Promise<unknown>
@@ -29,6 +30,7 @@ export function assignmentCleanupSteps(
['complete-ready-evacuations', () => assignments.completeReadyEvacuations()],
['complete-ready-regional-rehomes', () => assignments.completeReadyRegionalRehomes()],
['abort-expired-evacuations', () => assignments.abortExpiredEvacuations()],
['abort-unarrived-regional-rehomes', () => assignments.abortUnarrivedRegionalRehomes()],
['abort-expired-regional-rehomes', () => assignments.abortExpiredRegionalRehomes()],
['reap-regional-rehome-attempts', () => assignments.reapRegionalRehomeAttempts()],
['release-expired-activity-leases', () => assignments.releaseExpiredActivityLeases()],
cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts
+6
View File
@@ -101,6 +101,12 @@ describePostgres('PostgreSQL control supersession', () => {
cell.id
]
)
// The store reserves a unit per control lease, so a hand-written pair has to
// carry its own reservation or the fixture starts out of balance.
await databases[0]!.query(
`UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`,
[cell.id]
)
await stores[0]!.activateControl(identity, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
cloud/apps/relay/src/assignment-home-cell-unavailable.test.ts
+136
View File
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const HEARTBEAT_TTL_MS = 45_000
const START_MS = 100
const IDENTITY = { userId: 'user-1', relayHostId: 'host000000000001' }
// A connection-limited cell is what makes the committed fence mandatory, and
// that is the branch which used to answer "capacity exhausted".
const FENCED_CELL: RelayCellConfig = {
id: 'home',
url: 'https://home.example.com',
capacityRequests: 1_000,
connectionHardCap: 600,
connectionUnobservedBound: 50
}
const databases: RelayDatabase[] = []
afterEach(async () => {
for (const database of databases.splice(0)) await database.close()
})
interface Harness {
store: RelayAssignmentStore
heartbeat: (cell: RelayCellConfig, ready: boolean) => Promise<void>
setNow: (value: number) => void
}
async function setup(cells: RelayCellConfig[] = [FENCED_CELL]): Promise<Harness> {
const database = await openInMemoryRelayDatabase()
databases.push(database)
let now = START_MS
const store = new RelayAssignmentStore(database, () => now, {
requireLiveCells: true,
heartbeatTtlMs: HEARTBEAT_TTL_MS
})
await store.reconcileCells(cells, true)
const heartbeat = async (cell: RelayCellConfig, ready: boolean): Promise<void> => {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: `1111111${cells.indexOf(cell)}-1111-4111-8111-111111111111`,
startedAt: 50,
ready,
observedRequests: 0,
...(cell.connectionHardCap === undefined
? {}
: {
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionHardCap: cell.connectionHardCap,
connectionUnobservedBound: cell.connectionUnobservedBound
})
})
}
for (const cell of cells) await heartbeat(cell, true)
return { store, heartbeat, setNow: (value: number) => (now = value) }
}
async function assignFailure(store: RelayAssignmentStore): Promise<unknown> {
return await store.assign(IDENTITY).then(
() => new Error('assign unexpectedly succeeded'),
(error: unknown) => error
)
}
function homeCellError(error: unknown): RelayHomeCellUnavailableError {
expect(error).toBeInstanceOf(RelayHomeCellUnavailableError)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the assertion above.
return error as RelayHomeCellUnavailableError
}
describe('home cell unavailable', () => {
it('names a readiness failure rather than reporting fleet capacity', async () => {
const { store, heartbeat, setNow } = await setup()
await store.assign(IDENTITY)
setNow(START_MS + 1_000)
await heartbeat(FENCED_CELL, false)
const error = homeCellError(await assignFailure(store))
expect(error.message).toBe('relay_home_cell_unavailable')
expect(error.unavailableCause).toBe('not_ready')
expect(error.cellId).toBe(FENCED_CELL.id)
})
it('names a heartbeat gap as unheard even though the cell last reported ready', async () => {
const { store, setNow } = await setup()
await store.assign(IDENTITY)
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('unheard')
})
it('names a drained cell as draining ahead of its heartbeat gap', async () => {
const { store, setNow } = await setup()
await store.assign(IDENTITY)
await store.configureCell(FENCED_CELL, false)
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
})
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
await store.assign(IDENTITY)
await expect(
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
).rejects.toThrow('relay_capacity_exhausted')
})
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
const home: RelayCellConfig = {
id: 'home',
url: 'https://home.example.com',
capacityRequests: 1_000
}
const spare: RelayCellConfig = {
id: 'spare',
url: 'https://spare.example.com',
capacityRequests: 1_000
}
const { store, heartbeat, setNow } = await setup([home, spare])
expect((await store.assign(IDENTITY)).cellId).toBe(home.id)
setNow(START_MS + 1_000)
await heartbeat(home, false)
expect((await store.assign(IDENTITY)).cellId).toBe(spare.id)
})
})
cloud/apps/relay/src/assignment-inventory-snapshot.test.ts
+1
View File
@@ -71,6 +71,7 @@ describe('assignment inventory snapshot', () => {
targetRegistered: 0,
completedLast24Hours: 0,
abortedLast24Hours: 0,
hostNotArrivedLast24Hours: 0,
oldestActiveAgeMs: null
})
cloud/apps/relay/src/assignment-inventory-snapshot.ts
+16 -1
View File
@@ -1,5 +1,6 @@
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS } from './regional-rehome-abort-reason.js'
export type CellInventorySnapshotRow = {
cellId: string
@@ -22,6 +23,7 @@ export type AssignmentInventorySnapshot = {
targetRegistered: number
completedLast24Hours: number
abortedLast24Hours: number
hostNotArrivedLast24Hours: number
oldestActiveAgeMs: number | null
}
}
@@ -79,6 +81,10 @@ export async function readAssignmentInventorySnapshot(
AS completed_last_24_hours,
COALESCE(SUM(CASE WHEN attempt.aborted_at >= ? THEN 1 ELSE 0 END), 0)
AS aborted_last_24_hours,
COALESCE(SUM(CASE WHEN attempt.aborted_at >= ?
AND attempt.abort_reason = 'host_not_arrived'
THEN 1 ELSE 0 END), 0)
AS host_not_arrived_last_24_hours,
MIN(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN attempt.created_at END) AS oldest_active_at
FROM relay_region_rehome_attempts attempt
@@ -86,7 +92,11 @@ export async function readAssignmentInventorySnapshot(
ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id
AND migration.assignment_epoch = attempt.assignment_epoch`,
[now - 24 * 60 * 60_000, now - 24 * 60 * 60_000]
[
now - REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS,
now - REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS,
now - REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS
]
)
)[0]
const oldestActiveAt = optionalInteger(regionalRehomeRow, 'oldest_active_at')
@@ -117,6 +127,10 @@ export async function readAssignmentInventorySnapshot(
targetRegistered: asInteger(regionalRehomeRow, 'target_registered'),
completedLast24Hours: asInteger(regionalRehomeRow, 'completed_last_24_hours'),
abortedLast24Hours: asInteger(regionalRehomeRow, 'aborted_last_24_hours'),
hostNotArrivedLast24Hours: asInteger(
regionalRehomeRow,
'host_not_arrived_last_24_hours'
),
oldestActiveAgeMs: oldestActiveAt === null ? null : now - oldestActiveAt
}
}
@@ -147,6 +161,7 @@ export function formatAssignmentInventorySnapshot(
` targetRegistered=${snapshot.regionalRehomes.targetRegistered}` +
` completedLast24Hours=${snapshot.regionalRehomes.completedLast24Hours}` +
` abortedLast24Hours=${snapshot.regionalRehomes.abortedLast24Hours}` +
` hostNotArrivedLast24Hours=${snapshot.regionalRehomes.hostNotArrivedLast24Hours}` +
` oldestActiveAgeMs=${snapshot.regionalRehomes.oldestActiveAgeMs ?? 'none'}`
)
return lines
cloud/apps/relay/src/assignment-rejection-logging.test.ts
+59 -1
View File
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignment } from './assignment-store.js'
import { RelayHomeCellUnavailableError, type RelayAssignment } from './assignment-store.js'
import type { RelayConfig } from './config.js'
const fakes = vi.hoisted(() => ({
@@ -52,6 +52,39 @@ describe('assignment rejection logging', () => {
expect(line).not.toContain(host)
})
it('separates an unavailable home cell from capacity and names its cause', async () => {
const host = 'cccccccccccccccc'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => {
throw new RelayHomeCellUnavailableError('cell-asia-1', 'not_ready')
}),
// The sticky lane refuses a host whose home cell is not live, so this
// arrives hinted on the placement lane.
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(503)
expect(await response.json()).toEqual({ error: 'relay_home_cell_unavailable' })
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment rejected')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('reason=relay_home_cell_unavailable')
expect(line).toContain('cause=not_ready')
expect(line).toContain('cell=cell-asia-1')
expect(line).not.toContain('relay_capacity_exhausted')
expect(line).not.toContain(host)
})
it('logs an unhinted placement rejection without the raw host id', async () => {
const host = 'gggggggggggggggg'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
@@ -221,6 +254,31 @@ describe('assignment grant logging', () => {
expect(line).not.toContain(host)
})
it('logs a hinted grant served by the placement lane', async () => {
const host = 'rrrrrrrrrrrrrrrr'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-new', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(200)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment granted')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('cell=cell-new')
expect(line).not.toContain(host)
})
it('does not log unhinted placement grants', async () => {
const host = 'pppppppppppppppp'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
cloud/apps/relay/src/assignment-row-lock-order-postgres.test.ts
+138
View File
@@ -0,0 +1,138 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const cells = [
{
id: 'row-lock-order-a',
url: 'https://row-lock-order-a.example.com',
capacityRequests: 200,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
},
{
id: 'row-lock-order-b',
url: 'https://row-lock-order-b.example.com',
capacityRequests: 200,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
]
const userId = 'row-lock-order-user'
const hosts = ['rowlockhost00001', 'rowlockhost00002', 'rowlockhost00003', 'rowlockhost00004'].map(
(relayHostId) => ({ userId, relayHostId })
)
// Why a deadlock counter and not "it eventually succeeded": the transaction
// wrapper retries 40P01 three times, so a cycle that fires on every wave still
// reports success to the caller while burning the retry budget that turns into
// a 503 under load. PostgreSQL counts every detected cycle in pg_stat_database,
// which sees through the retry.
describePostgres('PostgreSQL row lock order', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
for (let index = 0; index < 4; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
[userId]
)
for (const table of [
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [userId])
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
async function deadlockCount(): Promise<number> {
const rows = await databases[0]!.query(
`SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()`
)
return Number(rows[0]!.deadlocks)
}
it('runs the cell accept and the placement retry concurrently without a cycle', async () => {
await removeTestRows(databases[0]!)
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await stores[0]!.reconcileCells(cells)
for (const cell of cells) {
await stores[0]!.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600,
connectionUnobservedBound: 50
})
}
const placements = new Map<string, { cellId: string; assignmentEpoch: number }>()
for (const identity of hosts) {
const assignment = await stores[0]!.assign(identity)
placements.set(identity.relayHostId, assignment)
}
const before = await deadlockCount()
// The accept path (host rows, then the cell row last) against the paths
// that must read the inventory first: placement and evacuation.
for (let round = 0; round < 12; round++) {
await Promise.allSettled(
hosts.flatMap((identity, index) => {
const placement = placements.get(identity.relayHostId)!
const store = stores[index % stores.length]!
const other = stores[(index + 1) % stores.length]!
return [
store.activateControl(identity, {
cellId: placement.cellId,
assignmentEpoch: placement.assignmentEpoch,
generation: round + 2
}),
other.assign(identity),
other.startEvacuation(
identity,
placement.cellId === cells[0]!.id ? cells[1]!.id : cells[0]!.id
)
]
})
)
}
const after = await deadlockCount()
expect(after - before).toBe(0)
}, 120_000)
})
cloud/apps/relay/src/assignment-store.test.ts
+5
View File
@@ -1886,6 +1886,11 @@ describe('RelayAssignmentStore', () => {
'control:cell-b:3'
]
)
// The store reserves a unit per control lease, so a hand-written pair has to
// carry its own reservation or the fixture starts out of balance.
await database!.query(
`UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = 'cell-b'`
)
const latest = await store.activateControl(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch,
cloud/apps/relay/src/assignment-store.ts
+262 -61
View File
@@ -1,5 +1,13 @@
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import { IDLE_REHOME_PAGE_SIZE, selectIdleRegionalRehomes } from './idle-regional-rehome-selection.js'
import {
selectIdleRegionalRehomes,
type IdleRegionalRehomeCandidate,
type IdleRehomeHostCursor
} from './idle-regional-rehome-selection.js'
import {
RegionalRehomePollTelemetry,
type RegionalRehomePollGate
} from './regional-rehome-poll-telemetry.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import {
previewRegionalRehomeEligibility,
@@ -22,6 +30,7 @@ import {
type RelayRegion,
type RegionCorrectionRequest,
type RegionCorrectionResponse,
type IdleRegionalRehomeCommit,
type IdleRegionalRehomeRequest,
} from '@orca-cloud/relay-contract'
import {
@@ -72,6 +81,10 @@ import {
regionalRehomePoolPressure,
regionalRehomeSafetyFailure
} from './regional-rehome-safety.js'
import {
REGIONAL_REHOME_ARRIVAL_WINDOW_MS,
type RegionalRehomeAbortReason
} from './regional-rehome-abort-reason.js'
import {
ABANDONED_REGISTERED_MIGRATION,
DURABLY_FENCED_MIGRATION_SOURCE,
@@ -334,6 +347,26 @@ const ACTIVITY_REQUEST_UNITS: Record<AssignmentActivityKind, number> = {
}
const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000
// THE ROW LOCK ORDER. Every transaction that takes more than one of these
// takes them in this order, whichever role it runs on:
//
// 1. relay_assignments (the host's row)
// 2. relay_assignment_migrations
// relay_assignment_activity_leases
// 3. relay_control_connection_reservations (lockControlConnectionReservations)
// 4. relay_cells (lockCellInventory / lockCellRows / the
// conditional reservation UPDATE)
//
// relay_cells is last because it is the only row shared by every host on a
// cell: a transaction that takes it early holds it across every round trip
// that follows, and on a cell far from PostgreSQL that is what turns accepts
// into a queue. Everything above it is per-host, so holding it longer costs
// only that host. Paths that read the inventory to make a placement decision
// cannot defer relay_cells, so they lock the host's rows from tiers 1-3 up
// front instead, before the inventory, and re-check what they read afterwards.
// Tier 1 is what serialises two transactions on the same host; the tiers below
// it keep transactions on *different* hosts from cycling through relay_cells.
// Why: one global FOR UPDATE over a 23-row table serialises every director and
// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled
// client for a full second, so the queue converts contention into pool
@@ -391,6 +424,25 @@ class AssignmentInventoryScopeChanged extends Error {
}
}
// Why a reason of its own: a host whose home cell is fenced-but-unattested is
// refused regardless of fleet headroom, so reporting it as capacity sends
// operators after capacity that was never short. Every cell boot and every
// readiness dip produces these.
export type RelayHomeCellUnavailableCause =
| 'draining'
| 'booting'
| 'unheard'
| 'not_ready'
export class RelayHomeCellUnavailableError extends Error {
constructor(
readonly cellId: string,
readonly unavailableCause: RelayHomeCellUnavailableCause
) {
super('relay_home_cell_unavailable')
}
}
// Debt holds connection headroom for a control that may still arrive shortly
// after its director-side timeout. Nothing legitimately arrives minutes late
// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that
@@ -827,6 +879,9 @@ export class RelayAssignmentStore {
let retryScope: RetriedAssignmentInventoryScope =
inventoryScope === 'all' ? 'all' : 'general'
return await this.database.transaction(async (transaction) => {
// The retry paths below open with the inventory, so this path takes its
// host rows before any of them rather than where the others do.
await this.lockControlConnectionReservations(transaction, identity, lockMode)
let lockedCells =
inventoryScope === 'all'
? await this.lockCellInventory(transaction, lockMode)
@@ -921,7 +976,10 @@ export class RelayAssignmentStore {
)) &&
!(await this.cellHasCommittedFence(transaction, current.cellId, now))
) {
throw new Error('relay_capacity_exhausted')
throw new RelayHomeCellUnavailableError(
current.cellId,
await this.homeCellUnavailableCause(transaction, current.cellId, now)
)
}
forcedDeadReassignment = true
}
@@ -2673,7 +2731,13 @@ export class RelayAssignmentStore {
)
if (assignment.cellId !== text(row, 'cell_id')) moved++
} catch (error) {
if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error
// One unplaceable host must not end the sweep for the rest.
if (
!(error instanceof RelayHomeCellUnavailableError) &&
!(error instanceof Error && error.message === 'relay_capacity_exhausted')
) {
throw error
}
}
}
return moved
@@ -3329,49 +3393,83 @@ export class RelayAssignmentStore {
return previewRegionCorrection(this.database, this.now())
}
private idleRegionalCandidateOffset = 0
private idleRegionalCandidateCursor: IdleRehomeHostCursor = null
private readonly regionalRehomePollTelemetry = new RegionalRehomePollTelemetry()
async selectIdleRegionalRehomeCandidates(
processSafety?: RegionalRehomeSafetySnapshot
): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
): Promise<IdleRegionalRehomeCandidate[]> {
const now = this.now()
if (!processSafety || this.regionalRehomeCohortPercent === 0) return []
const gated = (gate: RegionalRehomePollGate): IdleRegionalRehomeCandidate[] => {
this.regionalRehomePollTelemetry.record({ now, gate, candidates: 0 })
return []
}
if (!processSafety) return gated('process-safety-unavailable')
if (this.regionalRehomeCohortPercent === 0) return gated('cohort-zero')
const control = (await this.database.query(
"SELECT enabled, not_before FROM relay_region_rehome_control WHERE control_id = 'global'"
`SELECT enabled, not_before, preference_max_age_ms, host_cooldown_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
))[0]
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) return []
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) {
return gated('control-closed')
}
// The dispatch budget is durable and global, but until now only
// `commitIdleRegionalRehome` consulted it -- after the join had already run and
// the worker had already POSTed every candidate to its source cell. An absent
// row means the budget has never been spent, so it opens the gate.
const worker = (await this.database.query(
`SELECT paused_until, next_dispatch_at FROM relay_region_rehome_worker_state
WHERE worker_id = 'global'`
))[0]
if (worker && (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now)) {
return gated('budget-closed')
}
const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now)
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return []
const candidates = await selectIdleRegionalRehomes({
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return gated('fleet-safety')
const startedAt = performance.now()
const selection = await selectIdleRegionalRehomes({
database: this.database, now, heartbeatTtlMs: this.heartbeatTtlMs,
cohortPercent: this.regionalRehomeCohortPercent, offset: this.idleRegionalCandidateOffset,
cohortPercent: this.regionalRehomeCohortPercent,
preferenceMaxAgeMs: Number(control.preference_max_age_ms),
hostCooldownMs: Number(control.host_cooldown_ms),
cursor: this.idleRegionalCandidateCursor,
connectionHeadroom: await this.connectionHeadroomByCell(this.database),
cellIsClean: regionalRehomeCellSafetyIsClean
})
this.idleRegionalCandidateOffset = candidates.length < IDLE_REHOME_PAGE_SIZE
? 0 : this.idleRegionalCandidateOffset + candidates.length
return candidates
this.idleRegionalCandidateCursor = selection.cursor
this.regionalRehomePollTelemetry.record({
now,
gate: 'open',
candidates: selection.candidates.length,
selectionMs: performance.now() - startedAt
})
return selection.candidates
}
async commitIdleRegionalRehome(
request: IdleRegionalRehomeRequest,
processSafety?: RegionalRehomeSafetySnapshot,
cohortPercent = this.regionalRehomeCohortPercent
): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> {
): Promise<IdleRegionalRehomeCommit> {
const prior = await this.reconcileIdleRegionalRehome(request)
if (prior !== 'not-committed') return { outcome: prior }
if (!processSafety || !Number.isInteger(cohortPercent) || cohortPercent <= 0 || cohortPercent > 100) {
return { outcome: 'deferred' }
if (!processSafety) return { outcome: 'deferred', reason: 'director-safety-stale' }
if (!Number.isInteger(cohortPercent) || cohortPercent <= 0 || cohortPercent > 100) {
return { outcome: 'deferred', reason: 'cohort-closed' }
}
let safetyDisable: Record<string, string | number> | null = null
const result = await this.database.transaction(async (transaction): Promise<{ outcome: 'committed' | 'deferred' | 'stale' }> => {
// Set on every fleet-safety failure, disable or not: the pause is durable
// and global either way, so no later candidate in this poll can get past it.
let safetyPaused = false
const result = await this.database.transaction(async (transaction): Promise<IdleRegionalRehomeCommit> => {
safetyDisable = null
safetyPaused = false
const now = this.now()
const control = (await transaction.queryLocked(
`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`
))[0]
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) {
return { outcome: 'deferred' }
return { outcome: 'deferred', reason: 'control-closed' }
}
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
@@ -3382,13 +3480,15 @@ export class RelayAssignmentStore {
`SELECT * FROM relay_region_rehome_worker_state WHERE worker_id = 'global'`
))[0]!
if (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now) {
return { outcome: 'deferred' }
return { outcome: 'deferred', reason: 'budget-closed' }
}
const open = (await transaction.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE completed_at IS NULL AND aborted_at IS NULL`
))[0]
if (Number(open?.count ?? 0) >= REGIONAL_REHOME_CONCURRENT_LIMIT) return { outcome: 'deferred' }
if (Number(open?.count ?? 0) >= REGIONAL_REHOME_CONCURRENT_LIMIT) {
return { outcome: 'deferred', reason: 'concurrency-limit' }
}
const attempt = await this.startRegionalRehomeCandidate(transaction, {
identity: request,
sourceCellId: request.sourceCellId,
@@ -3402,9 +3502,14 @@ export class RelayAssignmentStore {
skips: [],
idleRequest: request,
cohortPercent,
onSafetyDisabled: (event) => { safetyDisable = event }
onSafetyDisabled: (event) => {
safetyDisable = event
safetyPaused = true
}
})
if (!attempt) return { outcome: 'deferred' }
if (!attempt) {
return { outcome: 'deferred', reason: safetyPaused ? 'fleet-safety' : 'candidate-ineligible' }
}
await this.markRegionalRehomeDispatchClaimed(
transaction, request.attemptId, now, Math.ceil(60_000 / Number(control.rate_per_minute))
)
@@ -3756,14 +3861,16 @@ export class RelayAssignmentStore {
? activityId
: pendingId
: activityId
await this.removeSupersededSameCellControls(
// Accumulated, not applied: every cell-row write on this path is folded
// into one conditional statement issued last, below.
let reservationDelta = -(await this.removeSupersededSameCellControls(
transaction,
identity,
activityLeases,
input.cellId,
retainedActivityId,
now
)
))
if (existing) {
await transaction.query(
`UPDATE relay_assignment_activity_leases SET expires_at = ?, updated_at = ?
@@ -3781,7 +3888,7 @@ export class RelayAssignmentStore {
)
await this.touchAssignment(transaction, identity, expiresAt, now)
} else {
await this.adjustCellReservationAtomically(transaction, input.cellId, 1)
reservationDelta += ACTIVITY_REQUEST_UNITS.control
await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
@@ -3837,6 +3944,17 @@ export class RelayAssignmentStore {
input.idleRegionalRehome && input.cellIncarnation ? 1 : 0
]
)
// Last, and only if the count actually moved: the cell row is shared by
// every host on the cell, and this transaction spans a dozen round
// trips. Holding its write lock from the first of them capped a
// far-from-Postgres cell at a couple of accepts a second.
if (reservationDelta !== 0) {
await this.adjustCellReservationAtomically(
transaction,
input.cellId,
reservationDelta
)
}
return activityId
})
})
@@ -3881,6 +3999,7 @@ export class RelayAssignmentStore {
}
if (sourceCellId === targetCellId) throw new Error('target_matches_source')
await this.lockAssignmentActivities(transaction, identity)
await this.lockControlConnectionReservations(transaction, identity)
const cells = await this.lockCellInventory(transaction, 'request')
const target = cells.find((row) => text(row, 'cell_id') === targetCellId)
if (!target || integer(target, 'enabled') !== 1) throw new Error('target_cell_unavailable')
@@ -4086,6 +4205,7 @@ export class RelayAssignmentStore {
): Promise<DeadSourceCompletionResult> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.lockControlConnectionReservations(transaction, identity)
let lockedCells: SqlRow[] | undefined
if (inventoryFirst) {
try {
@@ -4233,6 +4353,7 @@ export class RelayAssignmentStore {
): Promise<RelayAssignmentMigration> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
await this.lockControlConnectionReservations(transaction, identity)
const lockedCells = inventoryFirst
? await this.lockCellInventory(transaction, 'request')
: undefined
@@ -4725,7 +4846,10 @@ export class RelayAssignmentStore {
throw new Error('migration_activity_topology_mismatch')
}
}
if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'request')
if (obsoleteLeases.length > 0) {
await this.lockControlConnectionReservations(transaction, identity)
await this.lockCellInventory(transaction, 'request')
}
for (const lease of obsoleteLeases) {
await this.removeActivityLease(transaction, identity, lease, now)
}
@@ -5068,6 +5192,7 @@ export class RelayAssignmentStore {
const sourceCellId = text(assignment, 'cell_id')
if (sourceCellId === targetCellId) throw new Error('target_matches_source')
await this.lockAssignmentActivities(transaction, identity)
await this.lockControlConnectionReservations(transaction, identity)
const cells = await this.lockCellInventory(transaction, 'request')
const admission = await cellAdmissionStates(transaction)
const targetRow = cells.find(
@@ -5399,6 +5524,7 @@ export class RelayAssignmentStore {
}
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
assertAssignmentActivityCounts(assignment, activityLeases, 0)
await this.lockControlConnectionReservations(transaction, input.identity, 'nowait')
const cells = await this.lockCellInventory(transaction, 'nowait')
const admission = await cellAdmissionStates(transaction)
const regions = new Map(
@@ -6245,8 +6371,43 @@ export class RelayAssignmentStore {
return integer(completed[0]!, 'changes') + integer(aborted[0]!, 'changes')
}
// A move the host never finished: it holds no activity on the source and is
// not present at the target, so the registered migration row can do nothing
// but occupy one of the eight concurrent slots until something clears it.
// Rolling it back leaves the durable assignment on the source, so the host
// lands where it started whenever it next reconnects.
async abortUnarrivedRegionalRehomes(limit = 100): Promise<number> {
return await this.rollBackStalledRegionalRehomes({
sweep: 'abort-unarrived-regional-rehomes',
minimumAttemptAgeMs: REGIONAL_REHOME_ARRIVAL_WINDOW_MS,
abortReason: 'host_not_arrived',
disableControl: false,
limit
})
}
// The last-resort latch, and the only sweep that disables the switch. With
// the arrival sweep above running it should never reach a row; one that
// survives a day past dispatch means the rollback path itself is broken.
async abortExpiredRegionalRehomes(limit = 100): Promise<number> {
return await this.rollBackStalledRegionalRehomes({
sweep: 'abort-expired-regional-rehomes',
minimumAttemptAgeMs: REGIONAL_REHOME_MAX_REFRESH_MS,
abortReason: 'max_refresh_expired',
disableControl: true,
limit
})
}
private async rollBackStalledRegionalRehomes(input: {
sweep: string
minimumAttemptAgeMs: number
abortReason: RegionalRehomeAbortReason
disableControl: boolean
limit: number
}): Promise<number> {
const now = this.now()
const dispatchedBefore = now - input.minimumAttemptAgeMs
const quarantined = this.quarantinedRegionalRehomeAttemptIds(now)
const exclusion = quarantined.length
? ` AND attempt_id NOT IN (${quarantined.map(() => '?').join(', ')})`
@@ -6257,7 +6418,7 @@ export class RelayAssignmentStore {
WHERE completed_at IS NULL AND aborted_at IS NULL
AND created_at <= ?${exclusion}
ORDER BY created_at, attempt_id LIMIT ?`,
[now - REGIONAL_REHOME_MAX_REFRESH_MS, ...quarantined, limit]
[dispatchedBefore, ...quarantined, input.limit]
)
let aborted = 0
let inventoryBusy = 0
@@ -6294,7 +6455,7 @@ export class RelayAssignmentStore {
!migration ||
optionalInteger(attempt, 'completed_at') !== undefined ||
optionalInteger(attempt, 'aborted_at') !== undefined ||
integer(attempt, 'created_at') > now - REGIONAL_REHOME_MAX_REFRESH_MS ||
integer(attempt, 'created_at') > dispatchedBefore ||
optionalInteger(migration, 'completed_at') !== undefined ||
optionalInteger(migration, 'aborted_at') !== undefined
) {
@@ -6318,6 +6479,7 @@ export class RelayAssignmentStore {
integer(lease, 'expires_at') > now
)
if (targetActive) return false
await this.lockControlConnectionReservations(transaction, identity, 'nowait')
const cells = await this.lockCellInventory(transaction, 'nowait')
const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId)
const admission = await cellAdmissionStates(transaction)
@@ -6362,16 +6524,22 @@ export class RelayAssignmentStore {
[now, now, identity.userId, identity.relayHostId, assignmentEpoch]
)
await transaction.query(
`UPDATE relay_region_rehome_attempts SET aborted_at = ?, updated_at = ?
`UPDATE relay_region_rehome_attempts
SET aborted_at = ?, abort_reason = ?, updated_at = ?
WHERE attempt_id = ?`,
[now, now, text(attempt, 'attempt_id')]
)
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1`,
[now]
[now, input.abortReason, now, text(attempt, 'attempt_id')]
)
// Only the last-resort latch turns the feature off. A host that closed
// its laptop mid-move says nothing about whether rehoming is safe, and
// one such row a day would otherwise disable the switch every day.
if (input.disableControl) {
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = 0, updated_at = ?
WHERE control_id = 'global' AND enabled = 1`,
[now]
)
}
return true
})
this.regionalRehomeCandidateQuarantine.delete(attemptId)
@@ -6381,7 +6549,7 @@ export class RelayAssignmentStore {
}
if (changed) aborted++
}
warnSweepCellInventoryBusy('abort-expired-regional-rehomes', inventoryBusy)
warnSweepCellInventoryBusy(input.sweep, inventoryBusy)
return aborted
}
@@ -6484,7 +6652,10 @@ export class RelayAssignmentStore {
]
.map((activityId) => activityLeaseById(activityLeases, activityId))
.filter((lease): lease is SqlRow => lease !== undefined)
if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait')
if (obsoleteLeases.length > 0) {
await this.lockControlConnectionReservations(transaction, identity, 'nowait')
await this.lockCellInventory(transaction, 'nowait')
}
for (const lease of obsoleteLeases) {
await this.removeActivityLease(transaction, identity, lease, now)
}
@@ -6502,6 +6673,7 @@ export class RelayAssignmentStore {
)
return true
}
await this.lockControlConnectionReservations(transaction, identity, 'nowait')
const cells = await this.lockCellInventory(transaction, 'nowait')
const sourceCellId = text(row, 'source_cell_id')
const admissionRows = await transaction.query(
@@ -6921,6 +7093,24 @@ export class RelayAssignmentStore {
)
}
// Tier 3 of the row lock order: a path that will take relay_cells and also
// touch this host's reservations takes them here, before the cell rows. The
// set is the host's own rows, so it is small and known before any placement
// decision is read.
private async lockControlConnectionReservations(
database: RelayDatabase,
identity: AssignmentIdentity,
mode: CellInventoryLockMode = 'request'
): Promise<void> {
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
await database.queryLocked(
`SELECT reservation_id FROM relay_control_connection_reservations
WHERE user_id = ? AND relay_host_id = ? ORDER BY reservation_id ASC`,
[identity.userId, identity.relayHostId],
wait
)
}
// Unlocked on purpose: this only names the row to lock next, and the caller
// re-checks the pin once the assignment row is held.
private async pinnedCellId(
@@ -7124,6 +7314,31 @@ export class RelayAssignmentStore {
return rows.length === 1
}
// Reports which of `cellIsLive`'s conditions failed, so the rejection log
// separates an expected drain or boot from a cell whose readiness went out
// from under its hosts.
private async homeCellUnavailableCause(
database: RelayDatabase,
cellId: string,
now: number
): Promise<RelayHomeCellUnavailableCause> {
const row = (
await database.query(
`SELECT cell.enabled, runtime.last_heartbeat_at
FROM relay_cells cell
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
WHERE cell.cell_id = ?`,
[cellId]
)
)[0]
if (!row) return 'booting'
if (integer(row, 'enabled') === 0) return 'draining'
const heartbeatAt = optionalInteger(row, 'last_heartbeat_at')
if (heartbeatAt === undefined) return 'booting'
// Readiness is all that is left: `cellIsLive` already refused this cell.
return heartbeatAt <= now - this.heartbeatTtlMs ? 'unheard' : 'not_ready'
}
private async cellHasActiveFence(cellId: string): Promise<boolean> {
const rows = await this.database.query(
`SELECT fence.cell_id FROM relay_cell_fences fence
@@ -7551,6 +7766,9 @@ export class RelayAssignmentStore {
await this.adjustActivityCount(database, identity, kind, -1, now, now)
}
// Returns the request units this removal frees on `cellId`. The caller folds
// them into the one conditional cell-row write it makes just before COMMIT,
// so no statement here touches the fleet's most contended row.
private async removeSupersededSameCellControls(
database: RelayDatabase,
identity: AssignmentIdentity,
@@ -7558,14 +7776,14 @@ export class RelayAssignmentStore {
cellId: string,
retainedActivityId: string,
now: number
): Promise<void> {
): Promise<number> {
const superseded = leases.filter(
(lease) =>
activityKind(lease) === 'control' &&
text(lease, 'cell_id') === cellId &&
text(lease, 'activity_id') !== retainedActivityId
)
if (superseded.length === 0) return
if (superseded.length === 0) return 0
if (
superseded.some(
(lease) => integer(lease, 'request_units') !== ACTIVITY_REQUEST_UNITS.control
@@ -7573,10 +7791,6 @@ export class RelayAssignmentStore {
) {
throw new Error('activity_lease_shape_mismatch')
}
// Why: this recomputes one cell's reservation from its leases, so only that
// row needs to be held; the 23-row inventory lock here serialised every
// desktop control rebind in the fleet behind every other one.
const cellRow = (await this.lockCellRows(database, [cellId]))[0]
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'
@@ -7590,22 +7804,9 @@ export class RelayAssignmentStore {
WHERE user_id = ? AND relay_host_id = ?`,
[remainingControls, now, identity.userId, identity.relayHostId]
)
const cellUnitsRow = (
await database.query(
`SELECT COALESCE(SUM(request_units), 0) AS request_units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[cellId]
)
)[0]!
const cellUnits = integer(cellUnitsRow, 'request_units')
if (!cellRow) throw new Error('assigned_cell_missing')
if (cellUnits > integer(cellRow, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
await database.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[cellUnits, now, cellId]
)
// The shape check above proved every superseded lease holds exactly the
// control unit, so the freed units are exact without re-summing the cell.
return superseded.length * ACTIVITY_REQUEST_UNITS.control
}
private async adjustActivityCount(
cloud/apps/relay/src/boot-database-open.test.ts
+126
View File
@@ -0,0 +1,126 @@
import { afterEach, describe, expect, it, vi, type MockInstance } from 'vitest'
import { openRelayDatabaseAtBoot } from './boot-database-open.js'
import type { RelayDatabase } from './database.js'
const input = { dataDir: '/tmp/orca-relay-boot', databaseUrl: 'postgres://relay@localhost/relay' }
// The message the fleet actually saw: pg-pool reports the connect timeout with
// no SQLSTATE, so the classifier has only this text to go on.
const connectTimeout = (): Error => new Error('Connection terminated due to connection timeout')
const database = {} as RelayDatabase
function loggedEvents(warn: MockInstance<typeof console.warn>): string[] {
return warn.mock.calls.map((call) => String(JSON.parse(String(call[0])).event))
}
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
describe('relay boot database open', () => {
it('waits out a cold proxy instead of failing the boot', async () => {
vi.useFakeTimers()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const open = vi
.fn<() => Promise<RelayDatabase>>()
.mockRejectedValueOnce(connectTimeout())
.mockRejectedValueOnce(connectTimeout())
.mockResolvedValue(database)
const opening = openRelayDatabaseAtBoot(input, open)
await vi.runAllTimersAsync()
expect(await opening).toBe(database)
expect(open).toHaveBeenCalledTimes(3)
expect(open).toHaveBeenCalledWith(input)
expect(loggedEvents(warn)).toEqual([
'orca_relay_boot_database_retry',
'orca_relay_boot_database_retry',
'orca_relay_boot_database_recovered'
])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempt: 1,
delayMs: expect.any(Number),
code: 'unknown',
connectionTimeout: true
})
})
it('fails the boot immediately when the database rejects the relay', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const denied = Object.assign(new Error('password authentication failed'), { code: '28P01' })
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(denied)
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(denied)
expect(open).toHaveBeenCalledTimes(1)
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempts: 1,
retryable: false,
code: 'unknown'
})
})
// A retry re-runs the schema apply, which must never re-queue a boot DDL
// behind the writers that beat it; the request path treats these as transient.
it.each(['55P03', '57014', '53300'])(
'refuses to re-queue the schema apply after SQLSTATE %s',
async (code) => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const contention = Object.assign(new Error('lock unavailable'), { code })
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(contention)
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(contention)
expect(open).toHaveBeenCalledTimes(1)
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempts: 1,
retryable: false,
code
})
}
)
it('waits out a connection failure the driver does report a SQLSTATE for', async () => {
vi.useFakeTimers()
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const unreachable = Object.assign(new Error('connection refused'), { code: '08006' })
const open = vi
.fn<() => Promise<RelayDatabase>>()
.mockRejectedValueOnce(unreachable)
.mockResolvedValue(database)
const opening = openRelayDatabaseAtBoot(input, open)
await vi.runAllTimersAsync()
expect(await opening).toBe(database)
expect(open).toHaveBeenCalledTimes(2)
})
it('gives up once the retry budget is spent', async () => {
vi.useFakeTimers()
vi.setSystemTime(0)
vi.spyOn(Math, 'random').mockReturnValue(0)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const failure = connectTimeout()
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(failure)
const opening = openRelayDatabaseAtBoot(input, open)
const rejection = expect(opening).rejects.toBe(failure)
await vi.runAllTimersAsync()
await rejection
expect(Date.now()).toBeLessThanOrEqual(45_000)
expect(open.mock.calls.length).toBeGreaterThan(1)
const events = loggedEvents(warn)
expect(events.at(-1)).toBe('orca_relay_boot_database_failed')
expect(events.filter((event) => event === 'orca_relay_boot_database_retry')).toHaveLength(
open.mock.calls.length - 1
)
expect(JSON.parse(String(warn.mock.calls.at(-1)?.[0]))).toMatchObject({
attempts: open.mock.calls.length,
retryable: true,
connectionTimeout: true
})
})
})
cloud/apps/relay/src/boot-database-open.ts
+71
View File
@@ -0,0 +1,71 @@
import { openRelayDatabase, type RelayDatabase, type RelayDatabaseOpenInput } from './database.js'
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
import {
isPostgresPoolConnectFailure,
isPostgresPoolConnectTimeout
} from './postgres-pool-pressure.js'
import { postgresErrorCodeCategory } from './postgres-query-failure.js'
// Only a failure to reach Postgres at all. A retry here re-runs the schema
// apply, and applyPostgresSchema refuses to repeat a DDL lock timeout on
// purpose: relation locks are granted in queue order, so a repeat parks every
// writer behind the same statement again. 55P03, 57014 and 53300 therefore stay
// terminal at boot even though the request path calls them transient.
function isBootDatabaseUnreachable(error: unknown): boolean {
const code = postgresErrorCodeCategory(error)
return isPostgresPoolConnectFailure(error) || code === '08001' || code === '08006'
}
// A cell boots beside a cloud-sql-proxy that is itself still dialling, so the
// first pool acquire can outrun the 2s connect timeout that protects the
// request path. The window is longer than a proxy cold start and shorter than
// the restart loop it replaces.
const BOOT_OPEN_RETRY = {
attempts: 20,
windowMs: 45_000,
baseDelayMs: 250,
maxDelayMs: 4_000,
jitterMs: 250,
isRetryable: isBootDatabaseUnreachable
}
function bootDatabaseErrorFields(error: unknown): Record<string, unknown> {
return {
code: postgresErrorCodeCategory(error),
connectionTimeout: isPostgresPoolConnectTimeout(error)
}
}
export async function openRelayDatabaseAtBoot(
input: RelayDatabaseOpenInput,
open: (input: RelayDatabaseOpenInput) => Promise<RelayDatabase> = openRelayDatabase
): Promise<RelayDatabase> {
return await retryTransientDatabaseStartup(
async () => await open(input),
BOOT_OPEN_RETRY,
{
onRetry: ({ attempt, delayMs, error }) =>
console.warn(
JSON.stringify({
event: 'orca_relay_boot_database_retry',
attempt,
delayMs,
...bootDatabaseErrorFields(error)
})
),
onRecovered: ({ attempts }) =>
console.warn(
JSON.stringify({ event: 'orca_relay_boot_database_recovered', attempts })
),
onGaveUp: ({ attempts, error, retryable }) =>
console.warn(
JSON.stringify({
event: 'orca_relay_boot_database_failed',
attempts,
retryable,
...bootDatabaseErrorFields(error)
})
)
}
)
}
cloud/apps/relay/src/cell-admission-startup.ts
+21 -29
View File
@@ -1,14 +1,19 @@
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { isRelayDatabaseTransientError } from './database.js'
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
type CellAdmissionStartupConfig = Pick<RelayConfig, 'role' | 'cells'>
type CellAdmissionStore = Pick<RelayAssignmentStore, 'reconcileCellsAtStartup'>
const STARTUP_RECONCILE_ATTEMPTS = 20
const STARTUP_RECONCILE_RETRY_WINDOW_MS = 45_000
const STARTUP_RECONCILE_RETRY_BASE_MS = 250
const STARTUP_RECONCILE_RETRY_JITTER_MS = 250
const STARTUP_RECONCILE_RETRY = {
attempts: 20,
windowMs: 45_000,
// Flat: the contention this waits out is another director's schema lock, which
// clears on its own schedule rather than easing as the wait grows.
baseDelayMs: 250,
maxDelayMs: 250,
jitterMs: 250
}
export function roleOwnsAssignmentMaintenance(role: RelayConfig['role']): boolean {
// Cell workers share the database but the director is the sole authority
@@ -23,34 +28,21 @@ export async function reconcileCellAdmissionAtStartup(
// Admission is operator/director state. A new worker must not enable itself
// before its distinct candidate has passed production preflight.
if (config.role === 'cell') return
const retryDeadline = Date.now() + STARTUP_RECONCILE_RETRY_WINDOW_MS
for (let attempt = 1; attempt <= STARTUP_RECONCILE_ATTEMPTS; attempt += 1) {
try {
await assignments.reconcileCellsAtStartup(config.cells)
if (attempt > 1) {
await retryTransientDatabaseStartup(
async () => await assignments.reconcileCellsAtStartup(config.cells),
STARTUP_RECONCILE_RETRY,
{
onRecovered: ({ attempts }) =>
console.warn(
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts: attempt })
)
}
return
} catch (error) {
const remainingMs = retryDeadline - Date.now()
if (
attempt === STARTUP_RECONCILE_ATTEMPTS ||
remainingMs <= 0 ||
!isRelayDatabaseTransientError(error)
) {
if (isRelayDatabaseTransientError(error)) {
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts })
),
onGaveUp: ({ attempts, retryable }) => {
if (retryable) {
console.warn(
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts: attempt })
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts })
)
}
throw error
}
const delayMs =
STARTUP_RECONCILE_RETRY_BASE_MS +
Math.floor(Math.random() * (STARTUP_RECONCILE_RETRY_JITTER_MS + 1))
await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs)))
}
}
)
}
cloud/apps/relay/src/cell-inventory-lock-census.test.ts
+67 -2
View File
@@ -32,7 +32,9 @@ const CENSUS: CensusEntry[] = [
// only the one or two cell rows they touch, in cell_id order (lockCellRows),
// so they cannot cycle with placement's ordered inventory lock, and the
// 23-row lock there had serialised every reconnect in the fleet behind every
// other one.
// other one. The control accept path went one step further and takes no cell
// read lock at all: its single conditional write is the last statement before
// COMMIT.
{ method: 'startEvacuation', mode: 'request', reach: 'request' },
{ method: 'completeEvacuationFromDeadSourceOnce', mode: 'request', reach: 'request' },
{ method: 'completeEvacuationFromDeadSourceOnce', mode: 'nowait', reach: 'request' },
@@ -45,7 +47,9 @@ const CENSUS: CensusEntry[] = [
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
// Both regional-rehome abort sweeps share this rollback; only the 24-hour
// one also disables the durable switch.
{ method: 'rollBackStalledRegionalRehomes', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
@@ -172,6 +176,63 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
return sites
}
// Tier 3 and tier 4 of the row lock order documented in assignment-store.ts. A
// transaction that takes relay_cells before this host's reservation rows can
// cycle with one that takes them the other way round, and PostgreSQL resolves
// that as a 40P01 during exactly the drain and rehome waves these paths exist
// to run. The cell row is the one every host on a cell shares, so it is the
// lock that must be taken last, which fixes the direction for everyone else.
const CELL_LOCK_CALL =
/this\.(?:lockCellInventory|lockGeneralCellInventory|lockCellRows|adjustCellReservationAtomically|adjustCellReservation)\(|UPDATE relay_cells/
const RESERVATION_LOCK_CALL =
/this\.(?:lockControlConnectionReservations|insertControlConnectionReservation|claimControlConnectionReservation|releaseSupersededControlConnectionReservations)\(|(?:UPDATE|INTO|DELETE FROM)\s+relay_control_connection_reservations/
// The lock helpers themselves, plus the one reporting query that reads both
// tables without locking either.
const ROW_LOCK_ORDER_EXEMPT = [
'lockCellInventory',
'lockGeneralCellInventory',
'lockCellRows',
'lockControlConnectionReservations',
'adjustCellReservation',
'adjustCellReservationAtomically',
'insertControlConnectionReservation',
'claimControlConnectionReservation',
'releaseSupersededControlConnectionReservations',
'cellDeploymentStatus'
]
function methodSpans(lines: string[]): { name: string; start: number; end: number }[] {
const starts: { name: string; start: number }[] = []
lines.forEach((line, index) => {
const declaration = DECLARATION.exec(line)
if (declaration) starts.push({ name: declaration[1]!, start: index })
})
return starts.map((entry, index) => ({
...entry,
end: starts[index + 1]?.start ?? lines.length
}))
}
function pathsTakingCellsBeforeReservations(lines: string[]): string[] {
const offending: string[] = []
for (const span of methodSpans(lines)) {
if (ROW_LOCK_ORDER_EXEMPT.includes(span.name)) continue
let cell = Number.POSITIVE_INFINITY
let reservation = Number.POSITIVE_INFINITY
for (let index = span.start; index < span.end; index++) {
const line = lines[index]!
if (CELL_LOCK_CALL.test(line)) cell = Math.min(cell, index)
if (RESERVATION_LOCK_CALL.test(line)) reservation = Math.min(reservation, index)
}
if (cell < reservation && reservation !== Number.POSITIVE_INFINITY) {
offending.push(span.name)
}
}
return offending
}
describe('cell inventory lock call-site census', () => {
it('classifies every call site exactly as recorded', () => {
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
@@ -213,6 +274,10 @@ describe('cell inventory lock call-site census', () => {
expect(rawSites).toEqual(INLINE_CELL_LOCK_SITES)
})
it('takes the host reservation rows before the shared cell row everywhere', () => {
expect(pathsTakingCellsBeforeReservations(storeSource())).toEqual([])
})
it('leaves no call site taking the inventory without naming a mode', () => {
const source = readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8')
const unclassified = source
cloud/apps/relay/src/cell-inventory-lock-contention.test.ts
+9 -1
View File
@@ -8,6 +8,14 @@ const fakes = vi.hoisted(() => ({
return { rows: [], rowCount: 0 }
}),
release: vi.fn(),
// A real pooled client is an EventEmitter, and the acquire path attaches an
// `error` listener to it before handing it to the caller.
client: () => ({
query: fakes.query,
release: fakes.release,
on: vi.fn(),
removeListener: vi.fn()
}),
end: vi.fn(async () => undefined)
}))
@@ -19,7 +27,7 @@ vi.mock('pg', () => ({
waitingCount = 0
end = fakes.end
on = vi.fn()
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
connect = vi.fn(async () => fakes.client())
}
}
}))
cloud/apps/relay/src/control-accept-cell-row-lock-postgres.test.ts
+199
View File
@@ -0,0 +1,199 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const cell = {
id: 'accept-lock-postgres',
url: 'https://accept-lock-postgres.example.com',
capacityRequests: 2,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
const userId = 'accept-lock-postgres-user'
const first = { userId, relayHostId: 'acceptlockhost01' }
const second = { userId, relayHostId: 'acceptlockhost02' }
// Why: the accept path now enforces the capacity ceiling inside its single
// conditional cell-row write instead of behind a SELECT ... FOR UPDATE it held
// for the rest of the transaction. Two accepts reaching for the same last slot
// are what would expose a lost update if that check were no longer atomic.
describePostgres('PostgreSQL control accept without a held cell row', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
databases.push(
await openRelayDatabase({ databaseUrl, dataDir: '' }),
await openRelayDatabase({ databaseUrl, dataDir: '' })
)
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
[userId]
)
for (const table of [
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [userId])
}
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
it('lets exactly one of two racing accepts take the last capacity slot', async () => {
await removeTestRows(databases[0]!)
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await prepareCell(stores[0]!)
// Both hosts hold a grant, then drop the control the grant reserved, so the
// cell has exactly one free slot and two accepts that each want it.
const epochs = new Map<string, number>()
for (const identity of [first, second]) {
const assignment = await stores[0]!.assign(identity)
epochs.set(identity.relayHostId, assignment.assignmentEpoch)
const control = await stores[0]!.activateControl(identity, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await stores[0]!.releaseActivity(identity, control)
}
await databases[0]!.query(
`UPDATE relay_cells SET reserved_requests = ? WHERE cell_id = ?`,
[cell.capacityRequests - 1, cell.id]
)
await databases[0]!.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, 'splice:ballast', 'splice', ?, 1, 90100, 100)`,
[userId, 'acceptlockhost03', cell.id]
)
const outcomes = await Promise.allSettled([
stores[0]!.activateControl(first, {
cellId: cell.id,
assignmentEpoch: epochs.get(first.relayHostId)!,
generation: 2
}),
stores[1]!.activateControl(second, {
cellId: cell.id,
assignmentEpoch: epochs.get(second.relayHostId)!,
generation: 2
})
])
expect(outcomes.filter((outcome) => outcome.status === 'fulfilled')).toHaveLength(1)
const rejection = outcomes.find((outcome) => outcome.status === 'rejected')
expect(String((rejection as PromiseRejectedResult).reason)).toContain(
'relay_capacity_exhausted'
)
const cells = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cell.id]
)
expect(Number(cells[0]!.reserved_requests)).toBe(cell.capacityRequests)
const units = await databases[0]!.query(
`SELECT COALESCE(SUM(request_units), 0) AS units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[cell.id]
)
expect(Number(units[0]!.units)).toBe(cell.capacityRequests)
}, 20_000)
it('rebinds a control while another connection holds the cell row', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await prepareCell(store)
const assignment = await store.assign(first)
await store.activateControl(first, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
let release!: () => void
const released = new Promise<void>((resolve) => {
release = resolve
})
let held!: () => void
const heldPromise = new Promise<void>((resolve) => {
held = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id])
held()
await released
})
await heldPromise
// Retiring generation 1 and installing generation 2 leaves the cell's
// reservation where it was, so the accept has no reason to wait on the row
// at all. Reading it up front is what used to make it wait, and then fail
// at the request-path lock bound.
try {
await expect(
store.activateControl(first, {
cellId: cell.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 2
})
).resolves.toBe(`control:${cell.id}:2`)
} finally {
release()
await holder
}
const cells = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cell.id]
)
const units = await databases[0]!.query(
`SELECT COALESCE(SUM(request_units), 0) AS units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[cell.id]
)
expect(Number(cells[0]!.reserved_requests)).toBe(Number(units[0]!.units))
}, 20_000)
async function prepareCell(store: RelayAssignmentStore): Promise<void> {
await store.reconcileCells([cell])
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600,
connectionUnobservedBound: 50
})
}
})
cloud/apps/relay/src/control-accept-cell-row-lock.test.ts
+148
View File
@@ -0,0 +1,148 @@
import { beforeEach, describe, expect, it } from 'vitest'
import type { RelayCellConfig } from './config.js'
import { RelayAssignmentStore } from './assignment-store.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type RelayLockOptions,
type RelayTransactionOptions,
type SqlRow
} from './database.js'
const CELL: RelayCellConfig = {
id: 'accept-cell-a',
url: 'https://accept-a.example.com',
capacityRequests: 2
}
const host = { userId: 'accept-user', relayHostId: 'acceptho00000001' }
const second = { userId: 'accept-user', relayHostId: 'acceptho00000002' }
const third = { userId: 'accept-user', relayHostId: 'acceptho00000003' }
type Statement = { sql: string; locked: boolean }
// Records the statements a transaction issues, in order, so what the accept
// path does with the shared cell row can be asserted rather than described.
class RecordingDatabase implements RelayDatabase {
constructor(
private readonly inner: RelayDatabase,
readonly statements: Statement[] = []
) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push({ sql, locked: false })
return await this.inner.query(sql, params)
}
async queryLocked(
sql: string,
params: unknown[] = [],
options: RelayLockOptions = {}
): Promise<SqlRow[]> {
this.statements.push({ sql, locked: true })
return await this.inner.queryLocked(sql, params, options)
}
async transaction<T>(
operation: (transaction: RelayDatabase) => Promise<T>,
options: RelayTransactionOptions = {}
): Promise<T> {
return await this.inner.transaction(
async (transaction) =>
await operation(new RecordingDatabase(transaction, this.statements)),
options
)
}
async close(): Promise<void> {
await this.inner.close()
}
}
describe('control accept cell-row lock span', () => {
let recorder: RecordingDatabase
let store: RelayAssignmentStore
let assignmentEpoch: number
beforeEach(async () => {
recorder = new RecordingDatabase(await openInMemoryRelayDatabase())
store = new RelayAssignmentStore(recorder, () => 100)
await store.reconcileCells([CELL])
assignmentEpoch = (await store.assign(host)).assignmentEpoch
})
async function accept(generation: number): Promise<string> {
recorder.statements.length = 0
return await store.activateControl(host, {
cellId: CELL.id,
assignmentEpoch,
generation
})
}
function cellStatements(): Statement[] {
return recorder.statements.filter((statement) => /relay_cells/.test(statement.sql))
}
async function reservedRequests(): Promise<number> {
const row = (
await recorder.query(`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`, [
CELL.id
])
)[0]!
return Number(row.reserved_requests)
}
async function cellLeaseUnits(): Promise<number> {
const row = (
await recorder.query(
`SELECT COALESCE(SUM(request_units), 0) AS units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[CELL.id]
)
)[0]!
return Number(row.units)
}
it('writes the shared cell row once, last, and never takes it as a read lock', async () => {
const control = await accept(1)
await store.releaseActivity(host, control)
await accept(2)
const cells = cellStatements()
expect(cells.map((statement) => statement.locked)).toEqual([false])
expect(cells[0]!.sql).toContain('RETURNING cell_id')
// The contended row is written by the last statement of the transaction, so
// its write lock is held across the commit alone, not the whole accept.
expect(recorder.statements.at(-1)).toBe(cells[0])
})
it('leaves the cell row untouched when a rebind retires and installs one control', async () => {
await accept(1)
await accept(2)
expect(cellStatements()).toEqual([])
expect(await reservedRequests()).toBe(1)
expect(await cellLeaseUnits()).toBe(1)
})
it('keeps the reservation equal to the cell lease units across repeated rebinds', async () => {
for (const generation of [1, 2, 3, 4, 5]) await accept(generation)
expect(await reservedRequests()).toBe(await cellLeaseUnits())
expect(await reservedRequests()).toBe(1)
})
it('still refuses an accept that would exceed the cell capacity', async () => {
const control = await accept(1)
await store.assign(second)
await store.releaseActivity(host, control)
await store.assign(third)
expect(await reservedRequests()).toBe(CELL.capacityRequests)
await expect(accept(2)).rejects.toThrow('relay_capacity_exhausted')
expect(await reservedRequests()).toBe(CELL.capacityRequests)
expect(await cellLeaseUnits()).toBe(CELL.capacityRequests)
})
})
cloud/apps/relay/src/database-postgres-timeout.test.ts
+9 -1
View File
@@ -8,6 +8,14 @@ const fakes = vi.hoisted(() => ({
lifecycle: [] as string[],
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
release: vi.fn(),
// A real pooled client is an EventEmitter, and the acquire path attaches an
// `error` listener to it before handing it to the caller.
client: () => ({
query: fakes.query,
release: fakes.release,
on: vi.fn(),
removeListener: vi.fn()
}),
end: vi.fn(async () => undefined)
}))
@@ -18,7 +26,7 @@ vi.mock('pg', () => ({
idleCount = 1
waitingCount = 0
on = vi.fn()
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
connect = vi.fn(async () => fakes.client())
private readonly label: string
constructor(config: Record<string, unknown>) {
cloud/apps/relay/src/database-startup-retry.ts
+52
View File
@@ -0,0 +1,52 @@
import { isRelayDatabaseTransientError } from './database.js'
export type DatabaseStartupRetryPolicy = {
attempts: number
windowMs: number
baseDelayMs: number
maxDelayMs: number
jitterMs: number
// Which failures this particular startup step may repeat. Not every caller can
// repeat everything the request path calls transient: what the retry re-runs
// decides that, so the call site owns it.
isRetryable?: (error: unknown) => boolean
}
export type DatabaseStartupRetryObserver = {
onRetry?: (event: { attempt: number; delayMs: number; error: unknown }) => void
onRecovered?: (event: { attempts: number }) => void
onGaveUp?: (event: { attempts: number; error: unknown; retryable: boolean }) => void
}
function retryDelayMs(policy: DatabaseStartupRetryPolicy, attempt: number): number {
const backoffMs = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs)
return backoffMs + Math.floor(Math.random() * (policy.jitterMs + 1))
}
// Startup work that a cold dependency - a proxy sidecar that just started, a
// database still accepting the fleet back - can fail once and serve a moment
// later. The wall-clock window, not the attempt count, is the real bound.
export async function retryTransientDatabaseStartup<T>(
operation: () => Promise<T>,
policy: DatabaseStartupRetryPolicy,
observer: DatabaseStartupRetryObserver = {}
): Promise<T> {
const retryDeadline = Date.now() + policy.windowMs
for (let attempt = 1; ; attempt += 1) {
try {
const result = await operation()
if (attempt > 1) observer.onRecovered?.({ attempts: attempt })
return result
} catch (error) {
const remainingMs = retryDeadline - Date.now()
const retryable = (policy.isRetryable ?? isRelayDatabaseTransientError)(error)
if (attempt === policy.attempts || remainingMs <= 0 || !retryable) {
observer.onGaveUp?.({ attempts: attempt, error, retryable })
throw error
}
const delayMs = Math.min(retryDelayMs(policy, attempt), remainingMs)
observer.onRetry?.({ attempt, delayMs, error })
await new Promise((resolve) => setTimeout(resolve, delayMs))
}
}
}
cloud/apps/relay/src/database.ts
+9 -3
View File
@@ -297,6 +297,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
),
completed_at BIGINT,
aborted_at BIGINT,
abort_reason TEXT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
UNIQUE (user_id, relay_host_id, assignment_epoch)
@@ -676,6 +677,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
// Nullable with no default, so the rewrite is catalog-only; every row
// aborted before this column existed reads as an unattributed abort.
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS abort_reason TEXT`,
// Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because
// this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write,
// and all 28 directors reach it at once; a lock timeout here must not restart the instance, which
@@ -988,7 +992,7 @@ async function waitForPostgresRetry(random: () => number = Math.random): Promise
await new Promise((resolve) => setTimeout(resolve, delayMs))
}
class PostgresDatabase implements RelayDatabase {
export class PostgresDatabase implements RelayDatabase {
readonly dialect = 'postgres' as const
private readonly pressure: PostgresPoolPressure
private readonly holds = new CellInventoryHoldSamples()
@@ -1208,13 +1212,15 @@ async function backfillRelayCellRegions(database: RelayDatabase): Promise<void>
)
}
export async function openRelayDatabase(input: {
export type RelayDatabaseOpenInput = {
databaseUrl?: string
dataDir: string
poolMax?: number
applicationName?: string
statementTimeoutMs?: number
}): Promise<RelayDatabase> {
}
export async function openRelayDatabase(input: RelayDatabaseOpenInput): Promise<RelayDatabase> {
let database: RelayDatabase
if (input.databaseUrl) {
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
cloud/apps/relay/src/host-session-registry.ts
+17 -7
View File
@@ -18,6 +18,9 @@ import {
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type IdleRegionalRehomeCommit,
type IdleRegionalRehomeDeferReason,
type IdleRegionalRehomeResult,
type RelayHostCloseReason,
type RelayRegion
} from '@orca-cloud/relay-contract'
@@ -191,7 +194,7 @@ export class HostSessionRegistry {
{
attemptId: string
authorityKey: string
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
promise: Promise<IdleRegionalRehomeResult>
}
>()
@@ -205,9 +208,9 @@ export class HostSessionRegistry {
sourceCellIncarnation: string
targetCellId: string
},
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
commit: () => Promise<IdleRegionalRehomeCommit>,
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
): Promise<IdleRegionalRehomeResult> {
const authorityKey = JSON.stringify([
input.userId,
input.sourceAssignmentEpoch,
@@ -236,7 +239,7 @@ export class HostSessionRegistry {
!session.socket ||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
)
return { outcome: 'deferred' }
return { outcome: 'deferred', reason: 'host-unsupported' }
if (
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
session.activeConnIds.size !== 0 ||
@@ -246,12 +249,18 @@ export class HostSessionRegistry {
return { outcome: 'busy' }
const revision = session.authorityRevision
const promise = Promise.resolve().then(async () => {
let outcome: 'committed' | 'deferred' | 'stale'
let outcome: IdleRegionalRehomeCommit['outcome']
// The commit's reason survives only while the outcome stays deferred;
// a reconcile that finds a durable outcome answers with that instead.
let reason: IdleRegionalRehomeDeferReason | undefined
try {
outcome = (await commit()).outcome
const commitResult = await commit()
outcome = commitResult.outcome
reason = commitResult.reason
if (outcome === 'deferred') {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
if (outcome !== 'deferred') reason = undefined
}
} catch {
let delay = 100
@@ -259,6 +268,7 @@ export class HostSessionRegistry {
try {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
reason = undefined
break
} catch {
await new Promise<void>((resolve) => {
@@ -276,7 +286,7 @@ export class HostSessionRegistry {
}
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
this.idleAttempts.delete(input.relayHostId)
return { outcome }
return reason === undefined ? { outcome } : { outcome, reason }
})
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
return promise
cloud/apps/relay/src/idle-regional-rehome-selection.ts
+277 -95
View File
@@ -4,114 +4,296 @@ import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
// How many decision rows one poll is allowed to look at. The poll runs about
// fifty times a minute across the directors, so its cost has to be set by this
// number and not by the size of the fleet or the width of the cohort.
export const IDLE_REHOME_DECISION_WINDOW = 500
// Where the last window ended. A keyset beats OFFSET: `OFFSET n` still has to
// produce and throw away n rows, and n grew by a page on every poll that
// dispatched, so the scan got more expensive the longer the rollout ran.
export type IdleRehomeHostCursor = { userId: string; relayHostId: string } | null
export type IdleRegionalRehomeCandidate = IdleRegionalRehomeRequest & { sourceCellUrl: string }
export type IdleRegionalRehomeSelection = {
candidates: IdleRegionalRehomeCandidate[]
cursor: IdleRehomeHostCursor
}
type SourceCell = {
cellId: string
region: string
cellIncarnation: string
startedAt: number
cellUrl: string
}
type TargetCell = { cellId: string; capacityRequests: number; reservedRequests: number }
type SelectionInput = {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
preferenceMaxAgeMs: number
hostCooldownMs: number
cursor: IdleRehomeHostCursor
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
}
const CELL_INVENTORY_QUERY = `SELECT cell.cell_id, cell.cell_url, cell.enabled,
cell.capacity_requests, cell.reserved_requests, region.region,
admission.admission_state, capability.cell_incarnation AS capability_incarnation,
capability.regional_rehome_protocol
FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
LEFT JOIN relay_cell_capabilities capability ON capability.cell_id = cell.cell_id`
export async function selectIdleRegionalRehomes(
input: SelectionInput
): Promise<IdleRegionalRehomeSelection> {
const cells = await readCellInventory(input)
if (!cells.sources.size || !cells.targetsByRegion.size) return { candidates: [], cursor: null }
const sourceRegions = [...new Set([...cells.sources.values()].map((cell) => cell.region))]
const targetRegions = [...cells.targetsByRegion.keys()]
const decisionFilter = `outcome = 'conclusive' AND policy_version = 1
AND preferred_region IN (${placeholders(targetRegions.length)})
AND incumbent_region IN (${placeholders(sourceRegions.length)})
AND preferred_region <> incumbent_region
AND expires_at > ? AND observed_at >= ? AND cohort_bucket < ?`
const decisionParams = [
...targetRegions,
...sourceRegions,
input.now,
input.now - input.preferenceMaxAgeMs,
input.cohortPercent
]
const after = input.cursor ? [input.cursor.userId, input.cursor.relayHostId] : []
const afterFilter = input.cursor ? 'AND (user_id, relay_host_id) > (?, ?)' : ''
// The window is taken first and on its own so the poll knows where it stopped
// reading, not just where it stopped emitting. Every gate below this point can
// reject a host, and a cursor that only advanced past emitted rows would park
// on a rejected host forever.
const window = await input.database.query(
`SELECT user_id, relay_host_id FROM relay_region_decisions
WHERE ${decisionFilter} ${afterFilter}
ORDER BY user_id, relay_host_id LIMIT ?`,
[...decisionParams, ...after, IDLE_REHOME_DECISION_WINDOW]
)
if (!window.length) return { candidates: [], cursor: null }
const windowEnd = window[window.length - 1]!
const windowWasFull = window.length === IDLE_REHOME_DECISION_WINDOW
const sourceList = [...cells.sources.values()]
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
// The verification names the window's keys rather than repeating its LIMIT:
// the two reads take separate snapshots, and a decision that turned eligible
// between them would otherwise shift the second LIMIT and push the last host
// out of it while the cursor still advanced past it.
`SELECT d.user_id, d.relay_host_id, d.preferred_region, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation
FROM (SELECT user_id, relay_host_id, preferred_region, incumbent_region, assignment_epoch
FROM relay_region_decisions
WHERE ${decisionFilter}
AND (user_id, relay_host_id) IN (${Array.from({ length: window.length }, () => '(?,?)').join(',')})
-- The LIMIT cannot truncate a key set this size; it is here because without
-- it Postgres flattens the subquery, estimates one row out of the join, and
-- drives the whole plan from a sequential scan of the capability table.
ORDER BY user_id, relay_host_id LIMIT ?) d
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND a.assignment_epoch = d.assignment_epoch
JOIN (${inlineRows(SOURCE_CELL_COLUMNS, sourceList.length)}) source
ON source.cell_id = a.cell_id AND source.region = d.incumbent_region
JOIN relay_control_capabilities host ON host.user_id = d.user_id
AND host.relay_host_id = d.relay_host_id AND host.cell_id = a.cell_id
AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = source.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = d.user_id
AND lease.relay_host_id = d.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND lease.expires_at > ? AND lease.updated_at >= source.started_at
WHERE NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = d.user_id AND migration.relay_host_id = d.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
WHERE attempt.user_id = d.user_id AND attempt.relay_host_id = d.relay_host_id
AND attempt.created_at > ?)
ORDER BY d.user_id, d.relay_host_id, host.generation DESC
-- Counted in hosts, because a host with one eligible target has to be able
-- to fill a page on its own. A host with many leaves part of this page
-- unread, and the cursor stops where the page stopped, so it is re-read
-- next poll rather than skipped.
LIMIT ?`,
[
...decisionParams,
...window.flatMap((row) => [row.user_id, row.relay_host_id]),
IDLE_REHOME_DECISION_WINDOW,
...sourceList.flatMap((cell) => [cell.cellId, cell.region, cell.cellIncarnation, cell.startedAt]),
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
input.now - input.hostCooldownMs,
IDLE_REHOME_PAGE_SIZE
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
const units = rows.length ? await sourceRequestUnits(input.database, rows) : new Map<string, number>()
const candidates: IdleRegionalRehomeCandidate[] = []
let stoppedAt: IdleRehomeHostCursor = null
for (const row of rows) {
// Whole hosts only: the lower-priority targets are a host's fallbacks when
// the first one defers, and splitting them across pages loses them.
if (candidates.length >= IDLE_REHOME_PAGE_SIZE) {
return { candidates, cursor: stoppedAt }
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
const source = cells.sources.get(String(row.source_cell_id))!
const sourceUnits = units.get(hostKey(row)) ?? 0
for (const target of cells.targetsByRegion.get(String(row.preferred_region)) ?? []) {
if (target.reservedRequests + 1 + sourceUnits > target.capacityRequests) continue
candidates.push(idleRehomeCandidate(row, source, target.cellId))
}
stoppedAt = { userId: String(row.user_id), relayHostId: String(row.relay_host_id) }
}
// A full verification page may have been cut short of the window's end, so only
// a page that ran the window out may wrap to the head of the keyspace.
if (rows.length === IDLE_REHOME_PAGE_SIZE) return { candidates, cursor: stoppedAt }
return {
candidates,
cursor: windowWasFull
? { userId: String(windowEnd.user_id), relayHostId: String(windowEnd.relay_host_id) }
: null
}
}
// Every cell predicate the candidate join used to re-evaluate per (host, cell)
// pair. There are tens of cells and tens of thousands of hosts, so this is
// resolved once per poll against the four small inventory tables.
async function readCellInventory(
input: SelectionInput
): Promise<{ sources: Map<string, SourceCell>; targetsByRegion: Map<string, TargetCell[]> }> {
const { database, now } = input
const [runtimeRows, safetyRows, inventory] = await Promise.all([
database.query('SELECT * FROM relay_cell_runtime'),
database.query('SELECT * FROM relay_cell_rehome_safety'),
database.query(CELL_INVENTORY_QUERY)
])
const runtimes = new Map(runtimeRows.map((row) => [String(row.cell_id), row]))
const safety = new Map(safetyRows.map((row) => [String(row.cell_id), row]))
const sources = new Map<string, SourceCell>()
const targetsByRegion = new Map<string, TargetCell[]>()
const load = new Map<string, number>()
for (const cell of inventory) {
const cellId = String(cell.cell_id)
const runtime = runtimes.get(cellId)
if (!runtime || !input.cellIsClean(safety.get(cellId), runtime, now)) continue
if (
Number(cell.enabled) !== 1 ||
cell.admission_state !== 'general' ||
cell.region == null ||
Number(runtime.ready) !== 1 ||
Number(runtime.last_heartbeat_at) <= now - input.heartbeatTtlMs ||
cell.capability_incarnation == null ||
String(cell.capability_incarnation) !== String(runtime.cell_incarnation) ||
Number(cell.regional_rehome_protocol) < 3
) {
continue
}
const region = String(cell.region)
sources.set(cellId, {
cellId,
region,
cellIncarnation: String(runtime.cell_incarnation),
startedAt: Number(runtime.started_at),
cellUrl: String(cell.cell_url)
})
if (input.connectionHeadroom.get(cellId) === false) continue
const capacityRequests = Number(cell.capacity_requests)
const reservedRequests = Number(cell.reserved_requests)
const targets = targetsByRegion.get(region) ?? []
targets.push({ cellId, capacityRequests, reservedRequests })
targetsByRegion.set(region, targets)
load.set(cellId, (reservedRequests + Number(runtime.observed_requests)) / capacityRequests)
}
for (const targets of targetsByRegion.values()) {
targets.sort(
(left, right) =>
load.get(left.cellId)! - load.get(right.cellId)! || (left.cellId < right.cellId ? -1 : 1)
)
}
return { sources, targetsByRegion }
}
// One grouped read for the page instead of a correlated aggregate per (host, cell) pair.
async function sourceRequestUnits(
database: RelayDatabase,
rows: SqlRow[]
): Promise<Map<string, number>> {
const seen = new Set<string>()
const params: unknown[] = []
for (const row of rows) {
if (seen.has(hostKey(row))) continue
seen.add(hostKey(row))
params.push(row.user_id, row.relay_host_id, row.source_cell_id)
}
const sums = await database.query(
`SELECT user_id, relay_host_id, COALESCE(SUM(request_units), 0) AS request_units
FROM relay_assignment_activity_leases
WHERE (user_id, relay_host_id, cell_id) IN (${Array.from({ length: seen.size }, () => '(?,?,?)').join(',')})
GROUP BY user_id, relay_host_id`,
params
)
return new Map(sums.map((row) => [hostKey(row), Number(row.request_units)]))
}
const SOURCE_CELL_COLUMNS = [
['cell_id', 'TEXT'],
['region', 'TEXT'],
['cell_incarnation', 'TEXT'],
['started_at', 'BIGINT']
] as const
function placeholders(count: number): string {
return Array.from({ length: count }, () => '?').join(',')
}
// A derived table the planner can hash, in the one syntax both Postgres and the
// SQLite test engine accept (`VALUES ... AS t(col)` and LATERAL are not common to
// both). Only the first branch is cast; both engines take the union's types from it.
function inlineRows(columns: ReadonlyArray<readonly [string, string]>, rows: number): string {
const first = columns.map(([name, type]) => `CAST(? AS ${type}) AS ${name}`)
const rest = Array.from({ length: rows - 1 }, () => `UNION ALL SELECT ${placeholders(columns.length)}`)
return `SELECT ${first.join(', ')} ${rest.join(' ')}`
}
function hostKey(row: SqlRow): string {
return `${String(row.user_id)}${String(row.relay_host_id)}`
}
function idleRehomeCandidate(
row: SqlRow,
source: SourceCell,
targetCellId: string
): IdleRegionalRehomeCandidate {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: source.cellId,
sourceCellIncarnation: source.cellIncarnation,
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: source.cellUrl }
}
cloud/apps/relay/src/idle-regional-rehome-store.test.ts
+60 -7
View File
@@ -171,9 +171,11 @@ describe('constrained idle regional assignment transaction', () => {
)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
outcome: capacity === 11 ? 'committed' : 'deferred'
})
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual(
capacity === 11
? { outcome: 'committed' }
: { outcome: 'deferred', reason: 'candidate-ineligible' }
)
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
expect(await store.resolve(identity)).toMatchObject({
@@ -182,6 +184,53 @@ describe('constrained idle regional assignment transaction', () => {
})
})
it.each(['next_dispatch_at', 'paused_until'] as const)(
'skips the candidate join while %s holds the durable dispatch budget closed',
async (column) => {
const { store, database, safety } = await setup()
const query = vi.spyOn(database, 'query')
// One assignment only: setup leaves both fields at 0, and naming the other
// one too would assign this column twice, which Postgres rejects.
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt + 1]
)
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(2)
expect(query.mock.calls[1]![0]).toMatch(/FROM relay_region_rehome_worker_state/s)
}
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt]
)
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(2)
}
)
it('polls when the worker state row has never been written', async () => {
const { store, database, safety } = await setup()
await database.query('DELETE FROM relay_region_rehome_worker_state')
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
})
it('leaves the candidate page offset untouched across a closed dispatch budget', async () => {
const { store, database, safety } = await setup()
const first = await store.selectIdleRegionalRehomeCandidates(safety)
await database.query(
`UPDATE relay_region_rehome_worker_state SET next_dispatch_at = ? WHERE worker_id = 'global'`,
[safety.observedAt + 1]
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
await database.query(
`UPDATE relay_region_rehome_worker_state SET next_dispatch_at = 0 WHERE worker_id = 'global'`
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
@@ -245,7 +294,7 @@ describe('constrained idle regional assignment transaction', () => {
} finally {
held.release()
}
expect(await commit).toEqual({ outcome: 'deferred' })
expect(await commit).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
}
@@ -321,7 +370,7 @@ describe('constrained idle regional assignment transaction', () => {
const { store, safety, request } = await setup()
expect(
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
).toEqual({ outcome: 'deferred' })
).toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
@@ -335,9 +384,13 @@ describe('constrained idle regional assignment transaction', () => {
it('does not commit without process safety or cohort authorization', async () => {
const { store, safety, request, database } = await setup()
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
expect(await store.commitIdleRegionalRehome(request)).toEqual({
outcome: 'deferred',
reason: 'director-safety-stale'
})
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
outcome: 'deferred'
outcome: 'deferred',
reason: 'cohort-closed'
})
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
cloud/apps/relay/src/idle-regional-rehome-window.test.ts
+310
View File
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { IDLE_REHOME_DECISION_WINDOW } from './idle-regional-rehome-selection.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
// One source cell and three targets, so a poll has to rank targets per host
// rather than take the single one the two-cell fixture leaves it.
const cells = [
{ id: 'us', url: 'https://us.example.test', region: 'us-central1' as const, capacityRequests: 100 },
{ id: 'asia-busy', url: 'https://asia-busy.example.test', region: 'asia-east2' as const, capacityRequests: 100 },
{ id: 'asia-idle', url: 'https://asia-idle.example.test', region: 'asia-east2' as const, capacityRequests: 100 },
{ id: 'asia-mid', url: 'https://asia-mid.example.test', region: 'asia-east2' as const, capacityRequests: 100 }
]
const incarnations = cells.map((_, index) => `${index + 1}${'1'.repeat(7)}-1111-4111-8111-111111111111`)
const observed = [0, 60, 10, 30]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: observed[index]!
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
return { store, database, safety, now }
}
async function seedHost(
store: RelayAssignmentStore,
identity: { userId: string; relayHostId: string }
): Promise<void> {
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: 'us',
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
// Clone one seeded host's rows under new identities, which is far cheaper than
// driving the full activation path thousands of times.
async function cloneHosts(
database: RelayDatabase,
template: { userId: string; relayHostId: string },
count: number
): Promise<void> {
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const row = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
template.userId,
template.relayHostId
])
)[0]!
const columns = Object.keys(row)
const projection = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
for (let index = 0; index < count; index++) {
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${projection.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`clone-${String(index).padStart(5, '0')}`,
`clonehost${String(index).padStart(7, '0')}`,
template.userId,
template.relayHostId
]
)
}
}
}
describe('idle regional rehome candidate window', () => {
const identity = { userId: 'window-test', relayHostId: 'abcdefghijklmnop' }
it('offers every eligible target for a host, least loaded first', async () => {
const { store, safety } = await setup()
await seedHost(store, identity)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates.map((candidate) => candidate.targetCellId)).toEqual([
'asia-idle',
'asia-mid',
'asia-busy'
])
expect(new Set(candidates.map((candidate) => candidate.sourceCellUrl))).toEqual(
new Set(['https://us.example.test'])
)
// Every candidate is the same move to a different target, so the attempt ids differ.
expect(new Set(candidates.map((candidate) => candidate.attemptId)).size).toBe(3)
})
it('drops only the targets without room for the host plus its source activity', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 4 WHERE user_id = ?',
[identity.userId]
)
// Five units needed: four source units plus the assignment the move reserves.
await database.query("UPDATE relay_cells SET capacity_requests = 4 WHERE cell_id = 'asia-idle'")
const short = await store.selectIdleRegionalRehomeCandidates(safety)
expect(short.map((candidate) => candidate.targetCellId)).toEqual(['asia-mid', 'asia-busy'])
// Exactly enough room is enough; it ranks last because the ratio is per capacity.
await database.query("UPDATE relay_cells SET capacity_requests = 5 WHERE cell_id = 'asia-idle'")
const exact = await store.selectIdleRegionalRehomeCandidates(safety)
expect(exact.map((candidate) => candidate.targetCellId)).toEqual([
'asia-mid',
'asia-busy',
'asia-idle'
])
})
it('reads a bounded window of decisions however many hosts are eligible', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, IDLE_REHOME_DECISION_WINDOW + 200)
const query = vi.spyOn(database, 'query')
await store.selectIdleRegionalRehomeCandidates(safety)
const calls = query.mock.calls.map((call) => call[0])
const window = calls.findIndex((sql) => /FROM relay_region_decisions\s*$/m.test(sql))
expect(window).toBeGreaterThanOrEqual(0)
expect(query.mock.calls[window]![1]!.at(-1)).toBe(IDLE_REHOME_DECISION_WINDOW)
// No statement pages by OFFSET any more: that was the cost that grew with the rollout.
expect(calls.some((sql) => /OFFSET/i.test(sql))).toBe(false)
})
it('keeps the window\'s last host when a decision turns eligible between the two reads', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
// Exactly one full window, whose last key in sort order is the seeded host.
await cloneHosts(database, identity, IDLE_REHOME_DECISION_WINDOW - 1)
await database.query(
"UPDATE relay_assignment_activity_leases SET expires_at = ? WHERE user_id LIKE 'clone-%'",
[now - 1]
)
const template = (
await database.query('SELECT * FROM relay_region_decisions WHERE user_id = ?', [
identity.userId
])
)[0]!
const columns = Object.keys(template)
const query = database.query.bind(database)
let inserted = false
vi.spyOn(database, 'query').mockImplementation(async (sql, params) => {
const rows = await query(sql, params)
// A decision that becomes eligible after the window is read and sorts
// inside it: a second LIMIT would push the window's last host out.
if (!inserted && /^SELECT user_id, relay_host_id FROM relay_region_decisions/.test(sql)) {
inserted = true
await query(
`INSERT INTO relay_region_decisions (${columns.join(', ')})
VALUES (${columns.map(() => '?').join(', ')})`,
columns.map((column) =>
column === 'user_id'
? 'clone-99999'
: column === 'relay_host_id'
? 'latehost99999999'
: template[column]
)
)
}
return rows
})
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates.map((candidate) => candidate.userId)).toEqual([
identity.userId,
identity.userId,
identity.userId
])
})
it('walks the whole population in bounded pages and wraps only at the end', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, 120)
const seen = new Set<string>()
let pages = 0
let wrapped = false
// 121 hosts x 3 targets is 363 candidates, so the page cap has to be hit
// several times before the window runs out and the cursor wraps.
for (let poll = 0; poll < 20 && !wrapped; poll++) {
const page = await store.selectIdleRegionalRehomeCandidates(safety)
pages += 1
const before = seen.size
for (const candidate of page) seen.add(`${candidate.userId}/${candidate.targetCellId}`)
if (seen.size === before && page.length > 0) wrapped = true
if (page.length < 3) wrapped = true
}
expect(pages).toBeGreaterThan(1)
expect(seen.size).toBe(121 * 3)
})
it('does not stall on a host the window found but the join rejected', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, 2)
// The first host in key order loses its control lease, so it can never be a
// candidate; an emitted-rows cursor would sit on it forever.
await database.query('UPDATE relay_assignment_activity_leases SET expires_at = ? WHERE user_id = ?', [
now - 1,
'clone-00000'
])
const first = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first.map((candidate) => candidate.userId)).not.toContain('clone-00000')
expect(new Set(first.map((candidate) => candidate.userId))).toEqual(
new Set(['clone-00001', identity.userId])
)
})
it('excludes a host inside its rehome cooldown and takes it back after', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
await database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id, source_cell_incarnation,
target_cell_id, target_cell_incarnation, previous_epoch, assignment_epoch, drain_grace_ms,
send_attempts, created_at, updated_at)
VALUES ('cooled', ?, ?, 'asia-east2', 'us', ?, 'asia-idle', ?, 0, 9, 0, 0, ?, ?)`,
[identity.userId, identity.relayHostId, incarnations[0], incarnations[2], now - 1_000, now]
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
await database.query('UPDATE relay_region_rehome_attempts SET created_at = ?', [
now - 604_800_000 - 1
])
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
it('excludes a host outside the cohort', async () => {
const { store, database } = await setup()
await seedHost(store, identity)
const now = 100_000_000 + 86_400_000
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
await database.query('UPDATE relay_region_decisions SET cohort_bucket = 40')
const narrow = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 40 })
expect(await narrow.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
const wide = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 41 })
expect(await wide.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
})
cloud/apps/relay/src/index.ts
+2 -2
View File
@@ -10,10 +10,10 @@ import {
reconcileCellAdmissionAtStartup,
roleOwnsAssignmentMaintenance
} from './cell-admission-startup.js'
import { openRelayDatabaseAtBoot } from './boot-database-open.js'
import {
consumeRelayCellInventoryHold,
consumeRelayDatabasePoolPressure,
openRelayDatabase,
readRelayDatabasePoolPressure
} from './database.js'
import { runAssignmentCleanup } from './assignment-cleanup-steps.js'
@@ -28,7 +28,7 @@ import {
} from './registered-migration-inventory.js'
const config = loadRelayConfig()
const database = await openRelayDatabase({
const database = await openRelayDatabaseAtBoot({
databaseUrl: config.databaseUrl,
dataDir: config.dataDir,
poolMax: config.databasePoolMax,
cloud/apps/relay/src/postgres-checked-out-client-error.test.ts
+80
View File
@@ -0,0 +1,80 @@
import { EventEmitter } from 'node:events'
import { describe, expect, it, vi } from 'vitest'
import { PostgresDatabase } from './database.js'
// Stands in for a pg client between acquire and release. pg-pool assigns
// `release` per checkout, which is the property the guard wraps.
class FakePoolClient extends EventEmitter {
readonly released: Array<Error | boolean | undefined> = []
readonly statements: string[] = []
constructor(private readonly respond: (sql: string) => { rows: unknown[]; rowCount: number }) {
super()
}
query = vi.fn((sql: string) => {
this.statements.push(sql)
return Promise.resolve(this.respond(sql))
})
release = (error?: Error | boolean): void => {
this.released.push(error)
}
}
function poolOf(client: FakePoolClient) {
return { totalCount: 1, idleCount: 0, waitingCount: 0, connect: async () => client }
}
describe('checked-out PostgreSQL client failure handling', () => {
it('crashes the process when nothing listens, which is the bug being fixed', () => {
// Node's own contract: this is what killed cell c28 on 2026-09-20 20:18Z.
const unguarded = new EventEmitter()
expect(() => unguarded.emit('error', new Error('Connection terminated unexpectedly'))).toThrow(
'Connection terminated unexpectedly'
)
})
it('absorbs the error, rejects the transaction, and releases the client as failed', async () => {
const terminated = Object.assign(new Error('Connection terminated unexpectedly'), {
code: '57P01'
})
let listenersWhileCheckedOut = 0
const client: FakePoolClient = new FakePoolClient((sql) => {
if (sql !== 'SELECT 1') return { rows: [], rowCount: 0 }
listenersWhileCheckedOut = client.listenerCount('error')
// Cloud SQL terminating the session: the client emits `error` and the
// in-flight statement rejects with the same failure.
expect(() => client.emit('error', terminated)).not.toThrow()
throw terminated
})
const warning = vi.spyOn(console, 'warn').mockImplementation(() => {})
const database = new PostgresDatabase(poolOf(client) as never)
await expect(
database.transaction(async (transaction) => await transaction.query('SELECT 1'))
).rejects.toBe(terminated)
expect(listenersWhileCheckedOut).toBe(1)
expect(client.listenerCount('error')).toBe(0)
expect(client.released).toEqual([terminated])
expect(client.statements).toEqual(['BEGIN', 'SELECT 1', 'ROLLBACK'])
expect(warning).toHaveBeenCalledWith(
'[orca-relay] checked-out PostgreSQL client failed: 57P01 Connection terminated unexpectedly'
)
warning.mockRestore()
})
it('releases a healthy client back to the pool with no error', async () => {
const client = new FakePoolClient(() => ({ rows: [{ one: 1 }], rowCount: 1 }))
const database = new PostgresDatabase(poolOf(client) as never)
await expect(
database.transaction(async (transaction) => await transaction.query('SELECT 1'))
).resolves.toEqual([{ one: 1 }])
expect(client.released).toEqual([undefined])
expect(client.listenerCount('error')).toBe(0)
})
})
cloud/apps/relay/src/postgres-pool-pressure.test.ts
+3 -1
View File
@@ -1,3 +1,4 @@
import { EventEmitter } from 'node:events'
import { describe, expect, it, vi } from 'vitest'
import { PostgresPoolPressure } from './postgres-pool-pressure.js'
@@ -32,7 +33,8 @@ describe('PostgreSQL pool pressure', () => {
now = 2_250
pool.waitingCount--
resolveConnection({ query: vi.fn(), release: vi.fn() })
// An EventEmitter because the acquire path now attaches an `error` listener.
resolveConnection(Object.assign(new EventEmitter(), { query: vi.fn(), release: vi.fn() }))
await pending
expect(pressure.consumeCounts()).toMatchObject({
databasePoolWaiting: 0,
cloud/apps/relay/src/postgres-pool-pressure.ts
+38 -1
View File
@@ -30,6 +30,10 @@ function errorMessage(error: unknown): string {
return String((error as { message?: unknown } | null)?.message)
}
function errorCode(error: unknown): string {
return String((error as { code?: unknown } | null)?.code)
}
function isPostgresPoolAcquireFailure(error: unknown): boolean {
return typeof error === 'object' && error !== null && poolAcquireFailures.has(error)
}
@@ -131,12 +135,45 @@ export class PostgresPoolPressure {
}
async function markedAcquire(connection: Promise<pg.PoolClient>): Promise<pg.PoolClient> {
let client: pg.PoolClient
try {
return await connection
client = await connection
} catch (error) {
if (typeof error === 'object' && error !== null) poolAcquireFailures.add(error)
throw error
}
return guardCheckedOutClient(client)
}
// pg-pool strips its own `error` listener when it hands a client out
// (pg-pool@3.14.0 index.js:344) and only reattaches it in `_release`
// (index.js:385), so a checked-out client has no `error` listener at all. A
// backend that terminates that session mid-statement therefore emits `error`
// with nothing listening, which is an unhandled 'error' event and kills the
// process. `pool.on('error')` cannot cover this: pg-pool routes there only from
// the idle listener. Every relay checkout awaits this function, so it is the
// one seam that sees them all.
function guardCheckedOutClient(client: pg.PoolClient): pg.PoolClient {
let failure: Error | undefined
const onError = (error: Error) => {
failure ??= error
// Printable unlike the idle path: a checked-out client is past the
// handshake, so its error carries no connection string.
console.warn(
`[orca-relay] checked-out PostgreSQL client failed: ${errorCode(error)} ${errorMessage(error)}`
)
}
client.on('error', onError)
// pg-pool assigns a fresh `release` on every acquire, so this never stacks.
const release = client.release.bind(client)
client.release = (releaseError?: Error | boolean) => {
client.removeListener('error', onError)
// Passing the error makes pg-pool destroy the client instead of returning a
// dead connection to the pool for the next caller to trip over.
release(releaseError ?? failure)
}
return client
}
export function emptyPostgresPoolPressureCounts(): PostgresPoolPressureCounts {
cloud/apps/relay/src/postgres-query-failure.test.ts
+10 -2
View File
@@ -3,7 +3,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const fakes = vi.hoisted(() => ({
connectError: undefined as unknown,
query: vi.fn(async (_sql: string, _params?: unknown[]) => ({ rows: [], rowCount: 0 })),
release: vi.fn()
release: vi.fn(),
// A real pooled client is an EventEmitter, and the acquire path attaches an
// `error` listener to it before handing it to the caller.
client: () => ({
query: fakes.query,
release: fakes.release,
on: vi.fn(),
removeListener: vi.fn()
})
}))
vi.mock('pg', () => ({
@@ -15,7 +23,7 @@ vi.mock('pg', () => ({
on = vi.fn()
async connect() {
if (fakes.connectError) throw fakes.connectError
return { query: fakes.query, release: fakes.release }
return fakes.client()
}
async end() {}
}
cloud/apps/relay/src/postgres-query-failure.ts
+10 -4
View File
@@ -21,6 +21,14 @@ const ERROR_CODES = new Set([
'EPIPE'
])
// A recognised SQLSTATE or errno, or 'unknown': whatever else a driver attached
// to `code` is not a bounded log category.
export function postgresErrorCodeCategory(error: unknown): string {
const code =
typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined
return typeof code === 'string' && ERROR_CODES.has(code) ? code : 'unknown'
}
export function reportPostgresQueryFailure(input: {
error: unknown
phase: QueryFailurePhase
@@ -32,10 +40,8 @@ export function reportPostgresQueryFailure(input: {
}): void {
// Emit only bounded categories: error messages and SQL can contain credentials or identities.
try {
const error = input.error as { code?: unknown; message?: unknown } | null
const code =
typeof error?.code === 'string' && ERROR_CODES.has(error.code) ? error.code : 'unknown'
const connectionTimeout = isPostgresPoolConnectTimeout(error)
const code = postgresErrorCodeCategory(input.error)
const connectionTimeout = isPostgresPoolConnectTimeout(input.error)
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_query_failed',
cloud/apps/relay/src/region-correction-preview.ts
+17 -2
View File
@@ -1,5 +1,6 @@
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
import { REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS } from './regional-rehome-abort-reason.js'
import {
REGIONAL_REHOME_CONCURRENT_LIMIT,
REGION_DECISION_TTL_MS
@@ -13,6 +14,11 @@ export type RegionCorrectionPreview = {
availableMigrationSlots: number
globalSafetyFailure: string | null
counts: Record<string, number>
// Rehomes rolled back to their source in the last day, by the reason the
// sweep recorded. A rising `host_not_arrived` is what a leak looks like
// before it fills the concurrency budget; `unattributed` covers the abort
// paths that settle an attempt without naming one.
abortedLast24Hours: Record<string, number>
}
export async function previewRegionalRehomeEligibility(input: {
@@ -25,7 +31,7 @@ export async function previewRegionalRehomeEligibility(input: {
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<RegionCorrectionPreview> {
const { database, now } = input
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations, aborts] =
await Promise.all([
database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
@@ -63,6 +69,12 @@ export async function previewRegionalRehomeEligibility(input: {
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
),
database.query(
`SELECT COALESCE(abort_reason, 'unattributed') AS reason, COUNT(*) AS count
FROM relay_region_rehome_attempts WHERE aborted_at >= ?
GROUP BY COALESCE(abort_reason, 'unattributed')`,
[now - REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS]
)
])
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
@@ -152,6 +164,9 @@ export async function previewRegionalRehomeEligibility(input: {
openMigrations,
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
globalSafetyFailure: input.globalSafetyFailure,
counts
counts,
abortedLast24Hours: Object.fromEntries(
aborts.map((row) => [String(row.reason), Number(row.count)])
)
}
}
cloud/apps/relay/src/regional-rehome-abort-reason.ts
+18
View File
@@ -0,0 +1,18 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
// Why an attempt was rolled back to its source. Attempts settled by the
// migration-side abort paths, and every row aborted before the column existed,
// leave it null.
export const REGIONAL_REHOME_ABORT_REASONS = ['host_not_arrived', 'max_refresh_expired'] as const
export type RegionalRehomeAbortReason = (typeof REGIONAL_REHOME_ABORT_REASONS)[number]
// One whole migration lease with the host absent from the target and owning
// nothing on the source. Nothing legitimately takes that long: the attach
// deadline is seconds, and a host that is still moving holds a lease at one end
// or the other. The control's `drain_grace_ms` does not fit — idle rehome
// commits with a grace of zero, so these attempts never carry one.
export const REGIONAL_REHOME_ARRIVAL_WINDOW_MS = ASSIGNMENT_LIMITS.migrationLeaseMs
// The window the inventory line and the preview both report aborts over.
export const REGIONAL_REHOME_ABORT_REPORT_WINDOW_MS = 24 * 60 * 60_000
cloud/apps/relay/src/regional-rehome-inventory-line-census.test.ts
+163
View File
@@ -0,0 +1,163 @@
import { describe, expect, it } from 'vitest'
import {
formatAssignmentInventorySnapshot,
type AssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
// The enable workflow reads the director's rehome inventory line out of Cloud
// Logging with a parser that lives in another language, in another package, and
// is never exercised against the formatter that writes the line. When
// `hostNotArrivedLast24Hours` shipped, the parser read a healthy line as no
// evidence at all and the run failed closed, disabling the durable switch. This
// test is the missing edge: the real formatter's output, through the real
// parser, so a future field fails here instead of in an operator's run.
type InventoryEvidence = {
active: number
awaitingReceipt: number
targetRegistered: number
completedLast24Hours: number
abortedLast24Hours: number
hostNotArrivedLast24Hours: number | null
oldestActiveAgeMs: number | null
}
// Imported through a computed URL on purpose: the script is plain ESM outside
// this package's compile scope, so a static import would not resolve.
async function loadParser(): Promise<(entries: unknown[], options: unknown) => InventoryEvidence> {
const source = new URL(
'../../../dev/scripts/relay-rehome-aggregate-evidence.mjs',
import.meta.url
).href
const loaded: unknown = await import(/* @vite-ignore */ source)
if (!(loaded !== null && typeof loaded === 'object' && 'parseRegionalRehomeInventory' in loaded)) {
throw new Error('relay-rehome-aggregate-evidence.mjs no longer exports its parser')
}
const parse = loaded.parseRegionalRehomeInventory
if (typeof parse !== 'function') throw new Error('parseRegionalRehomeInventory is not callable')
return (entries, options) => readEvidence(parse(entries, options))
}
function readEvidence(value: unknown): InventoryEvidence {
if (value === null || typeof value !== 'object') throw new Error('parser returned no evidence')
const counts = ['active', 'awaitingReceipt', 'targetRegistered', 'completedLast24Hours', 'abortedLast24Hours'] as const
const evidence: Record<string, number | null> = {}
for (const key of [...counts, 'hostNotArrivedLast24Hours', 'oldestActiveAgeMs'] as const) {
if (!(key in value)) throw new Error(`parser dropped ${key}`)
const read: unknown = Reflect.get(value, key)
if (read !== null && typeof read !== 'number') throw new Error(`${key} is not a count`)
evidence[key] = read
}
for (const key of counts) {
if (evidence[key] === null) throw new Error(`${key} must be a number`)
}
return {
active: Number(evidence['active']),
awaitingReceipt: Number(evidence['awaitingReceipt']),
targetRegistered: Number(evidence['targetRegistered']),
completedLast24Hours: Number(evidence['completedLast24Hours']),
abortedLast24Hours: Number(evidence['abortedLast24Hours']),
hostNotArrivedLast24Hours: evidence['hostNotArrivedLast24Hours'] ?? null,
oldestActiveAgeMs: evidence['oldestActiveAgeMs'] ?? null
}
}
function snapshot(
regionalRehomes: AssignmentInventorySnapshot['regionalRehomes']
): AssignmentInventorySnapshot {
return {
cells: [],
activityLeases: { total: 0, expired: 0, requestUnits: 0 },
connectionReservations: { outstanding: 0, lateArrivalDebt: 0 },
regionalRehomes
}
}
function inventoryLine(snapshotValue: AssignmentInventorySnapshot): string {
const line = formatAssignmentInventorySnapshot(snapshotValue).find((candidate) =>
candidate.startsWith('[orca-relay] regional rehome inventory ')
)
if (!line) throw new Error('the formatter no longer emits a rehome inventory line')
return line
}
describe('regional rehome inventory line census', () => {
it('parses what the director actually prints, field for field', async () => {
const parse = await loadParser()
const regionalRehomes = {
active: 3,
awaitingReceipt: 1,
targetRegistered: 2,
completedLast24Hours: 41,
abortedLast24Hours: 12,
hostNotArrivedLast24Hours: 5,
oldestActiveAgeMs: 77_731_209
}
const now = Date.parse('2026-09-20T12:00:00Z')
const evidence = parse(
[{ timestamp: '2026-09-20T11:59:00Z', textPayload: inventoryLine(snapshot(regionalRehomes)) }],
{ now, maxAgeMs: 5 * 60_000 }
)
expect(evidence).toEqual({ ...regionalRehomes })
})
it('parses the line an idle fleet prints, with no oldest active age', async () => {
const parse = await loadParser()
const now = Date.parse('2026-09-20T12:00:00Z')
const evidence = parse(
[
{
timestamp: '2026-09-20T11:59:00Z',
textPayload: inventoryLine(
snapshot({
active: 0,
awaitingReceipt: 0,
targetRegistered: 0,
completedLast24Hours: 0,
abortedLast24Hours: 0,
hostNotArrivedLast24Hours: 0,
oldestActiveAgeMs: null
})
)
}
],
{ now, maxAgeMs: 5 * 60_000 }
)
expect(evidence.oldestActiveAgeMs).toBeNull()
expect(evidence.hostNotArrivedLast24Hours).toBe(0)
})
it('covers every counter the formatter puts on the line', async () => {
const parse = await loadParser()
// A field the parser ignores is a field the operator never sees, so the
// census fails when the formatter gains one and this test is not updated.
const line = inventoryLine(
snapshot({
active: 1,
awaitingReceipt: 1,
targetRegistered: 1,
completedLast24Hours: 1,
abortedLast24Hours: 1,
hostNotArrivedLast24Hours: 1,
oldestActiveAgeMs: 1
})
)
const printed = line
.slice('[orca-relay] regional rehome inventory '.length)
.split(' ')
.map((field) => field.split('=')[0])
const surfaced = Object.keys(
parse([{ timestamp: '2026-09-20T11:59:00Z', textPayload: line }], {
now: Date.parse('2026-09-20T12:00:00Z'),
maxAgeMs: 5 * 60_000
})
)
expect([...printed].sort()).toEqual([...surfaced].sort())
})
})
cloud/apps/relay/src/regional-rehome-poll-telemetry.test.ts
+58
View File
@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest'
import {
REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
RegionalRehomePollTelemetry
} from './regional-rehome-poll-telemetry.js'
describe('regional rehome poll telemetry', () => {
it('names the gate that stopped the poll, not just the empty result', () => {
const lines: string[] = []
const telemetry = new RegionalRehomePollTelemetry((line) => lines.push(line))
let now = 1_000
for (let poll = 0; poll < 3; poll++) {
telemetry.record({ now: (now += 6_000), gate: 'budget-closed', candidates: 0 })
}
telemetry.record({ now: (now += 6_000), gate: 'open', candidates: 0, selectionMs: 12 })
expect(lines).toEqual([])
telemetry.record({
now: now + REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
gate: 'open',
candidates: 7,
selectionMs: 30
})
expect(lines).toHaveLength(1)
expect(JSON.parse(lines[0]!)).toMatchObject({
event: 'orca_relay_regional_rehome_poll_summary',
polls: 5,
'budget-closed': 3,
open: 2,
candidates: 7,
selectionMsMax: 30
})
})
it('starts a fresh window after each summary', () => {
const lines: string[] = []
const telemetry = new RegionalRehomePollTelemetry((line) => lines.push(line))
telemetry.record({ now: 0, gate: 'control-closed', candidates: 0 })
telemetry.record({
now: REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
gate: 'control-closed',
candidates: 0
})
telemetry.record({
now: REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS * 2,
gate: 'fleet-safety',
candidates: 0
})
expect(lines).toHaveLength(2)
expect(JSON.parse(lines[1]!)).toMatchObject({
polls: 1,
'control-closed': 0,
'fleet-safety': 1,
candidates: 0,
selectionMsMax: 0,
selectionMsP95: 0
})
})
})
cloud/apps/relay/src/regional-rehome-poll-telemetry.ts
+69
View File
@@ -0,0 +1,69 @@
// A gated poll and a poll that simply found nobody to move both produce zero
// candidates and no attempt row, so an operator watching a stalled rollout
// cannot tell them apart. One aggregated line a minute per director names the
// gate and prices the selection, at a rate a 50-polls-a-minute worker can afford.
export type RegionalRehomePollGate =
| 'open'
| 'cohort-zero'
| 'process-safety-unavailable'
| 'control-closed'
| 'budget-closed'
| 'fleet-safety'
export const REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS = 60_000
const EMPTY_GATES: Record<RegionalRehomePollGate, number> = {
open: 0,
'cohort-zero': 0,
'process-safety-unavailable': 0,
'control-closed': 0,
'budget-closed': 0,
'fleet-safety': 0
}
export class RegionalRehomePollTelemetry {
private windowStartedAt: number | null = null
private gates = { ...EMPTY_GATES }
private candidates = 0
private selectionSamplesMs: number[] = []
constructor(private readonly write: (line: string) => void = (line) => console.warn(line)) {}
record(input: {
now: number
gate: RegionalRehomePollGate
candidates: number
selectionMs?: number
}): void {
if (this.windowStartedAt === null) this.windowStartedAt = input.now
this.gates[input.gate] += 1
this.candidates += input.candidates
if (input.selectionMs !== undefined) this.selectionSamplesMs.push(input.selectionMs)
if (input.now - this.windowStartedAt < REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS) return
this.write(
JSON.stringify({
event: 'orca_relay_regional_rehome_poll_summary',
windowMs: input.now - this.windowStartedAt,
polls: Object.values(this.gates).reduce((total, count) => total + count, 0),
...this.gates,
candidates: this.candidates,
selectionMsMax: round(Math.max(0, ...this.selectionSamplesMs)),
selectionMsP95: round(percentile(this.selectionSamplesMs, 0.95))
})
)
this.windowStartedAt = input.now
this.gates = { ...EMPTY_GATES }
this.candidates = 0
this.selectionSamplesMs = []
}
}
function percentile(samples: number[], fraction: number): number {
if (!samples.length) return 0
const sorted = [...samples].sort((a, b) => a - b)
return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))]!
}
function round(value: number): number {
return Math.round(value * 100) / 100
}
cloud/apps/relay/src/regional-rehome-postgres.test.ts
+2 -2
View File
@@ -426,7 +426,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
unlock()
await held
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
await expect(claim).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
@@ -450,7 +450,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred' })
).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
cloud/apps/relay/src/regional-rehome-store.test.ts
+116 -4
View File
@@ -1,4 +1,6 @@
import { describe, expect, it } from 'vitest'
import { readAssignmentInventorySnapshot } from './assignment-inventory-snapshot.js'
import { REGIONAL_REHOME_ARRIVAL_WINDOW_MS } from './regional-rehome-abort-reason.js'
import {
RelayAssignmentStore as BaseRelayAssignmentStore,
type RegionalRehomeAttempt,
@@ -413,7 +415,8 @@ describe('regional rehome assignment state', () => {
const warnings = collectDisableWarnings()
try {
expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({
outcome: 'deferred'
outcome: 'deferred',
reason: 'fleet-safety'
})
} finally {
warnings.restore()
@@ -452,8 +455,10 @@ describe('regional rehome assignment state', () => {
const warnings = collectDisableWarnings()
try {
// Per-cell, so it excludes this target rather than stopping the poll.
expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({
outcome: 'deferred'
outcome: 'deferred',
reason: 'candidate-ineligible'
})
} finally {
warnings.restore()
@@ -778,7 +783,8 @@ describe('regional rehome assignment state', () => {
)
expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({
outcome: 'deferred'
outcome: 'deferred',
reason: 'fleet-safety'
})
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
@@ -1085,6 +1091,112 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('rolls a host that never reached its target back to the source at the arrival window', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const sourceControl = await activatePreferredSource(context, identity)
expect(await context.store.tryIdleRehome()).not.toBeNull()
await context.store.releaseActivity(identity, sourceControl)
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: 2
})
context.advance(REGIONAL_REHOME_ARRIVAL_WINDOW_MS - 1)
await freshHeartbeats(context)
expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(0)
context.advance(1)
await freshHeartbeats(context)
expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(1)
// Where the host was, so its next reconnect lands on the source it left.
expect(await context.store.resolve(identity)).toMatchObject({
cellId: source.id,
assignmentEpoch: 3
})
expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(0)
await context.database.close()
})
it('leaves the durable switch alone when it rolls back an unarrived host', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const sourceControl = await activatePreferredSource(context, identity)
expect(await context.store.tryIdleRehome()).not.toBeNull()
await context.store.releaseActivity(identity, sourceControl)
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: 2
})
const before = await context.store.inspectRegionalRehomeControl()
context.advance(REGIONAL_REHOME_ARRIVAL_WINDOW_MS)
await freshHeartbeats(context)
expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(1)
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: before.generation,
enabled: true
})
const [attempt] = await context.database.query(
'SELECT abort_reason FROM relay_region_rehome_attempts'
)
expect(attempt!.abort_reason).toBe('host_not_arrived')
// What the rollout tracker reads to see a leak before it fills the budget.
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.abortedLast24Hours).toEqual({ host_not_arrived: 1 })
expect(
(await readAssignmentInventorySnapshot(context.database, context.now())).regionalRehomes
).toMatchObject({ abortedLast24Hours: 1, hostNotArrivedLast24Hours: 1 })
await context.database.close()
})
it('leaves a host that did reach its target for the completion sweep', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const sourceControl = await activatePreferredSource(context, identity)
expect(await context.store.tryIdleRehome()).not.toBeNull()
// Past the window, but present at the target: the sweep reads live
// ownership, not the attempt's age alone.
context.advance(REGIONAL_REHOME_ARRIVAL_WINDOW_MS)
await freshHeartbeats(context)
await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: 2,
generation: 1,
cellIncarnation: targetIncarnation
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: 2
})
await context.store.releaseActivity(identity, sourceControl)
expect(await context.store.abortUnarrivedRegionalRehomes()).toBe(0)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
await context.database.close()
})
it('names the 24-hour latch in its own abort reason', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const sourceControl = await activatePreferredSource(context, identity)
expect(await context.store.tryIdleRehome()).not.toBeNull()
await context.store.releaseActivity(identity, sourceControl)
context.advance(24 * 60 * 60_000)
await heartbeat(context.store, source, sourceIncarnation, 3, 2)
expect(await context.store.abortExpiredRegionalRehomes()).toBe(1)
const [attempt] = await context.database.query(
'SELECT abort_reason FROM relay_region_rehome_attempts'
)
expect(attempt!.abort_reason).toBe('max_refresh_expired')
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({ enabled: false })
await context.database.close()
})
it('rolls back an inactive registered target only after the 24-hour bound', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
@@ -1760,7 +1872,7 @@ function hookAfterCandidateScan(
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
query: async (sql, params) => {
const rows = await delegate.query(sql, params)
if (!fired && sql.includes('SELECT a.user_id, a.relay_host_id')) {
if (!fired && sql.includes('SELECT d.user_id, d.relay_host_id')) {
fired = true
await hook(delegate)
}
cloud/apps/relay/src/regional-rehome-worker.test.ts
+140
View File
@@ -86,6 +86,76 @@ describe('regional rehome worker', () => {
).toBeNull()
})
it('stops walking the page when the source names a deferral no candidate can pass', async () => {
const fetchImpl = respondWith([{ outcome: 'deferred', reason: 'concurrency-limit' }])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 3, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(1)
expect(summaries.entries).toEqual([
{
event: 'orca_relay_idle_rehome_dispatch_summary',
candidates: 3,
dispatched: 1,
stoppedBy: 'concurrency-limit',
outcomes: { 'deferred:concurrency-limit': 1 }
}
])
})
it('keeps its whole-page walk when the source sends no reason at all', async () => {
const fetchImpl = respondWith([{ outcome: 'deferred' }])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 3, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(3)
expect(summaries.entries[0]).toMatchObject({
stoppedBy: null,
outcomes: { deferred: 3 }
})
})
it('walks past a deferral that only concerns the one candidate', async () => {
const fetchImpl = respondWith([
{ outcome: 'deferred', reason: 'host-unsupported' },
{ outcome: 'busy' },
{ outcome: 'committed' }
])
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 4, summaries)
} finally {
summaries.restore()
}
expect(fetchImpl).toHaveBeenCalledTimes(3)
expect(summaries.entries[0]).toMatchObject({
dispatched: 3,
stoppedBy: 'committed',
outcomes: { 'deferred:host-unsupported': 1, busy: 1, committed: 1 }
})
})
it('counts a source that answers with an error in the same summary', async () => {
const fetchImpl = vi.fn(async () => new Response('nope', { status: 503 }))
const summaries = collectSummaries()
try {
await runOnePoll(fetchImpl, 2, summaries)
} finally {
summaries.restore()
}
expect(summaries.entries[0]).toMatchObject({ dispatched: 2, outcomes: { failed: 2 } })
})
it('treats the reconnect threshold as per-cell and excludes the director', () => {
const cells = 2
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
@@ -108,6 +178,76 @@ describe('regional rehome worker', () => {
})
})
// Answers each POST with the next scripted body, repeating the last one.
function respondWith(bodies: { outcome: string; reason?: string }[]) {
let index = 0
return vi.fn(async () => {
const body = bodies[Math.min(index++, bodies.length - 1)]!
return new Response(JSON.stringify({ v: 1, ...body }), {
headers: { 'content-type': 'application/json' }
})
})
}
function collectSummaries() {
const entries: Record<string, unknown>[] = []
let arrived: (() => void) | undefined
// The worker polls once the moment it is constructed, so the poll under test
// is that one; `first` is how a test waits for it rather than for a tick.
const first = new Promise<void>((resolve) => {
arrived = resolve
})
const original = console.warn
console.warn = (line: unknown, ...rest: unknown[]) => {
try {
const parsed = JSON.parse(line as string) as Record<string, unknown>
if (parsed.event === 'orca_relay_idle_rehome_dispatch_summary') {
entries.push(parsed)
arrived?.()
return
}
} catch {
// Not a JSON log line; fall through to the original writer.
}
original(line as string, ...rest)
}
return { entries, first, restore: () => (console.warn = original) }
}
async function runOnePoll(
fetchImpl: typeof fetch,
candidates: number,
summaries: { first: Promise<void> }
): Promise<void> {
const assignments = {
selectIdleRegionalRehomeCandidates: vi.fn(async () =>
Array.from({ length: candidates }, (_, index) => ({
v: 1 as const,
attemptId: `00000000-0000-4000-8000-00000000000${index}`,
userId: `user-${index}`,
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'us-c1',
sourceCellUrl: 'https://us-c1.relay.example.test',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
targetCellId: 'asia-c1'
}))
)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
fetch: fetchImpl,
identityToken: async () => 'token',
safetySnapshot: () => safety(Date.now()),
intervalMs: 60_000
})!
try {
await summaries.first
} finally {
worker.stop()
}
}
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
role: 'director',
cloud/apps/relay/src/regional-rehome-worker.ts
+32 -3
View File
@@ -1,4 +1,7 @@
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
import {
IdleRegionalRehomeResponseSchema,
isGlobalIdleRegionalRehomeDeferral
} from '@orca-cloud/relay-contract'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
@@ -48,8 +51,13 @@ export function startRegionalRehomeWorker(
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
if (candidates.length === 0) return
const token = await tokenProvider(audience)
const outcomes: Record<string, number> = {}
const tally = (key: string) => {
outcomes[key] = (outcomes[key] ?? 0) + 1
}
let stoppedBy: string | null = null
for (const candidate of candidates) {
if (stopped) return
if (stopped) break
const { sourceCellUrl, ...request } = candidate
try {
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
@@ -64,6 +72,7 @@ export function startRegionalRehomeWorker(
})
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
tally(body.reason ? `${body.outcome}:${body.reason}` : body.outcome)
if (body.outcome === 'committed') {
console.warn(
JSON.stringify({
@@ -72,9 +81,18 @@ export function startRegionalRehomeWorker(
targetCellId: candidate.targetCellId
})
)
return
stoppedBy = 'committed'
break
}
// Every remaining candidate would re-read the same durable row and
// answer the same way, so the rest of this page is wasted POSTs.
// A source on an older image sends no reason and keeps the old walk.
if (body.outcome === 'deferred' && isGlobalIdleRegionalRehomeDeferral(body.reason)) {
stoppedBy = body.reason
break
}
} catch (error) {
tally('failed')
// The source may have committed; its durable outcome owns recovery.
console.warn(
JSON.stringify({
@@ -84,6 +102,17 @@ export function startRegionalRehomeWorker(
)
}
}
// One line per poll that dispatched: silence used to be the only signal
// that 100+ candidates all came back deferred.
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_dispatch_summary',
candidates: candidates.length,
dispatched: Object.values(outcomes).reduce((total, count) => total + count, 0),
stoppedBy,
outcomes
})
)
} catch (error) {
console.warn(
JSON.stringify({
cloud/apps/relay/src/relay-schema-lock-targets.test.ts
+1
View File
@@ -126,6 +126,7 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
{ kind: 'column', table: 'relay_region_rehome_control', name: 'host_cooldown_ms', skipWhen: 'present' },
{ kind: 'column', table: 'relay_control_capabilities', name: 'idle_regional_rehome', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'abort_reason', skipWhen: 'present' },
{ kind: 'index-by-name', name: 'relay_assignment_activity_expiry', skipWhen: 'absent' },
{
kind: 'reloption',
cloud/apps/relay/src/relay-server.ts
+1 -1
View File
@@ -141,7 +141,7 @@ export function createRelayServer(
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
return Promise.resolve({ outcome: 'deferred' })
return Promise.resolve({ outcome: 'deferred', reason: 'director-safety-stale' })
}
return sessions.idleRehome(input,
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
cloud/dev/scripts/relay-load-connection-failure.mjs
+12 -4
View File
@@ -1,12 +1,20 @@
// A refused home cell is not fleet capacity, so it gets its own bucket rather
// than inflating the capacity count a run is read for.
const ASSIGNMENT_REJECTION_BUCKETS = {
relay_capacity_exhausted: 'assignment_capacity_exhausted',
relay_connection_headroom_exhausted: 'assignment_capacity_exhausted',
relay_home_cell_unavailable: 'assignment_home_cell_unavailable'
}
export function relayLoadFailureReason(error) {
const message = error instanceof Error ? error.message : String(error)
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
if (tokenExchange) return `token_http_${tokenExchange[1]}`
const assignment =
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_capacity_exhausted|relay_connection_headroom_exhausted))?$/.exec(
message
)
if (assignment?.[1] === '503' && assignment[2]) return 'assignment_capacity_exhausted'
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_[a-z_]+))?$/.exec(message)
if (assignment?.[1] === '503' && assignment[2]) {
return ASSIGNMENT_REJECTION_BUCKETS[assignment[2]] ?? `assignment_http_${assignment[1]}`
}
if (assignment) return `assignment_http_${assignment[1]}`
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
if (closed) return `control_close_${closed[1]}`
cloud/dev/scripts/relay-load-control-peer.mjs
+4 -3
View File
@@ -11,9 +11,10 @@ const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import
requireFromRelay.resolve('@orca-cloud/relay-contract')
)
const CAPACITY_ASSIGNMENT_ERRORS = [
const REPORTABLE_ASSIGNMENT_ERRORS = [
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted'
'relay_connection_headroom_exhausted',
'relay_home_cell_unavailable'
]
function waitForOpen(socket, timeoutMs = 10_000) {
@@ -744,7 +745,7 @@ export class RelayLoadControlPeer {
'relay assignment timeout',
(status, errorCode) =>
`relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`,
CAPACITY_ASSIGNMENT_ERRORS
REPORTABLE_ASSIGNMENT_ERRORS
)
if (
typeof body.cellUrl !== 'string' ||
cloud/dev/scripts/relay-monitor-evidence.mjs
+3 -1
View File
@@ -10,7 +10,9 @@ const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
const EVIDENCE_MAX_AGE_MS = 5 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX = /^[0-3]$/
// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
// pins its own narrower range.
const WAVE_INDEX = /^[0-9]$/
const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000
const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000
const MIGRATION_POLICIES = new Set([
cloud/dev/scripts/relay-monitor-evidence.test.mjs
+6 -1
View File
@@ -242,7 +242,7 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
await ageState(17 * 60_000)
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
await assert.doesNotReject(authorityAt('1'))
await assert.rejects(authorityAt('4'), /wave index is invalid/)
await assert.rejects(authorityAt('10'), /wave index is invalid/)
await assert.rejects(authorityAt('x'), /wave index is invalid/)
// Both edges of one predecessor job timeout: 5min + 75min exactly.
await ageState(80 * 60_000)
@@ -259,6 +259,11 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
await assert.doesNotReject(authorityAt('3'))
await ageState(230 * 60_000 + 1)
await assert.rejects(authorityAt('3'), /authority is incomplete or stale/)
// The last cell of a ten-cell same-cap batch: 5min + 9 * 75min exactly.
await ageState(680 * 60_000)
await assert.doesNotReject(authorityAt('9'))
await ageState(680 * 60_000 + 1)
await assert.rejects(authorityAt('9'), /authority is incomplete or stale/)
} finally {
await rm(directory, { recursive: true, force: true })
}
cloud/dev/scripts/relay-production-same-cap-wave.mjs
+4 -3
View File
@@ -34,7 +34,7 @@ function cells(value) {
const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
if (
parsed.length < 1 ||
parsed.length > 4 ||
parsed.length > 10 ||
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
@@ -76,8 +76,9 @@ export function validateSameCapWave(input) {
if (input.mode === 'canary-apply' && selected.length !== 1) {
throw new Error('canary mode requires exactly one cell')
}
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
throw new Error('batch mode requires two to four cells')
// Ten is the wave workflow's statically declared serial cell-job chain, cell_1..cell_10.
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 10)) {
throw new Error('batch mode requires two to ten cells')
}
// Later waves expect the selector to advance by exactly 2 per predecessor,
// which a resumed rollback cell (isolate skipped, +1) violates.
cloud/dev/scripts/relay-production-same-cap-wave.test.mjs
+42
View File
@@ -57,6 +57,48 @@ test('requires one canary or a bounded reviewed batch', () => {
}), /cells/)
})
// The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the
// chain would silently drop its tail cells, so it is refused before any mutation.
test('a batch fills the serial cell chain and never overflows it', () => {
const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general')
const batch = (count) => {
const cellIds = general.slice(0, count).join(',')
return validateSameCapWave({
mode: 'batch-apply',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
canaryRunId: '42'
})
}
assert.equal(batch(10).cells.length, 10)
assert.throws(() => batch(11), /same-cap wave cells are invalid/)
assert.throws(() => batch(1), /batch mode requires two to ten cells/)
})
// The validator's ten-cell bound is only true if the workflow really declares ten strictly
// serial cell jobs and frees the lease after all of them.
test('the wave workflow chains exactly ten serial cell jobs', () => {
const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
for (let index = 0; index < 10; index += 1) {
const job = index + 1
assert.match(dispatch, new RegExp(`\n cell_${job}:\n`), `cell_${job} is missing`)
assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`))
assert.match(dispatch, new RegExp(`wave-index: '${index}'`))
if (index > 0) {
assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`))
assert.match(
dispatch,
new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` +
`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`)
)
}
assert.match(dispatch, new RegExp(`\n - cell_${job}\n`), `release_lease must need cell_${job}`)
}
assert.doesNotMatch(dispatch, /\n cell_11:/)
})
test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)
cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs
+20 -5
View File
@@ -56,7 +56,9 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
)
// The relaxation is only safe if the reviewed validator actually runs on
// the NON-converged branch, in same-cap-cell mode, with the trust config
// the validator requires, restricted to the template-and-MIG change pair.
// the validator requires, restricted to the template-and-MIG change pair or,
// when only the reviewed backend attributes are left, to those alone — and
// that last case then has to be applied, not waved through as converged.
assert.match(
job,
/if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n then\n/
@@ -75,8 +77,20 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
)
assert.match(
job,
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}"\)"\n {12}echo "\$\{RESUME_REVIEW\}"\n {12}jq -e '\.changes == 2\n {16}or \(\.changes == 0 and \(\(\.backendUpdate \/\/ \[\]\) \| length\) > 0\)' \\\n {14}<<< "\$\{RESUME_REVIEW\}" >\/dev\/null/
)
// A resume whose only unapplied change is the reviewed backend update must apply it. Leaving
// it is how a cell keeps the 300-second drain and no request logging behind a green resume.
assert.match(
job,
/if test "\$\(jq -er '\.changes' <<< "\$\{RESUME_REVIEW\}"\)" = 0; then\n {14}terraform -chdir=infra\/terraform apply -auto-approve \\\n {16}"\$\{RUNNER_TEMP\}\/relay-same-cap-resume\.tfplan"\n {12}fi\n/
)
// Template-and-MIG drift still applies nothing on resume, which is what a resume means.
const resumeStep = job.slice(
job.indexOf('- name: Require converged Terraform state and a stable MIG on resume'),
job.indexOf('- name: Apply only the selected same-cap template and MIG')
)
assert.equal(resumeStep.split('terraform -chdir=infra/terraform apply').length, 2)
assert.match(job, /resume requires the isolated migration-only cell/)
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
@@ -139,9 +153,10 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
// The wrapper validates the override before anything runs, passes it to every
// cell, seals it into the canary artifact, and prints it in the run summary.
assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
// One per cell job in the serial cell_1..cell_10 chain.
assert.equal(
wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
4
10
)
assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
@@ -163,12 +178,12 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
]) {
const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
assert.match(body, /\^\[0-3\]\$/)
assert.match(body, /\^\[0-9\]\$/)
}
// Aged-evidence replay via job re-runs is fenced: mutations are
// single-dispatch, so a failed cell needs a fresh gate and monitor run.
assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
for (const index of [0, 1, 2, 3]) {
for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) {
assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
}
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
cloud/dev/scripts/relay-rehome-aggregate-evidence.mjs
+50 -9
View File
@@ -1,6 +1,24 @@
import { pathToFileURL } from 'node:url'
const INVENTORY = /^\[orca-relay\] regional rehome inventory active=(\d+) awaitingReceipt=(\d+) targetRegistered=(\d+) completedLast24Hours=(\d+) abortedLast24Hours=(\d+) oldestActiveAgeMs=(none|\d+)$/
const INVENTORY_PREFIX = '[orca-relay] regional rehome inventory '
// A counters line, so every field is a bare name and a non-negative integer or
// `none`. Pinning the whole line instead is what broke the enable workflow when
// `hostNotArrivedLast24Hours` shipped: the director grew a field and the parser
// read a healthy line as no evidence at all. Tolerating extra fields is safe
// only because the value shape stays fenced — `hostId=someone` is still not a
// counter, so an identity-bearing lookalike cannot slip through as an extra.
const FIELD = /^([A-Za-z][A-Za-z0-9]*)=(none|\d{1,15})$/
// `oldestActiveAgeMs` is the one required field the director can report as
// `none`; a count that reads `none` is a line this parser does not recognise,
// not evidence worth failing the run over.
const REQUIRED_COUNTS = [
'active',
'awaitingReceipt',
'targetRegistered',
'completedLast24Hours',
'abortedLast24Hours'
]
const REQUIRED_FIELDS = [...REQUIRED_COUNTS, 'oldestActiveAgeMs']
function count(value, name) {
const parsed = Number(value)
@@ -8,20 +26,43 @@ function count(value, name) {
return parsed
}
// Returns the field map, or null for anything that is not this line.
export function readRegionalRehomeInventoryFields(textPayload) {
if (typeof textPayload !== 'string' || !textPayload.startsWith(INVENTORY_PREFIX)) return null
const fields = new Map()
for (const token of textPayload.slice(INVENTORY_PREFIX.length).split(' ')) {
const field = FIELD.exec(token)
if (!field || fields.has(field[1])) return null
fields.set(field[1], field[2])
}
if (!REQUIRED_FIELDS.every((name) => fields.has(name))) return null
if (REQUIRED_COUNTS.some((name) => fields.get(name) === 'none')) return null
return fields
}
// Absent is not zero: a director on an older image emits no such field, and
// reporting 0 would read as "no leaks" rather than "not measured".
function optionalCount(fields, name) {
const value = fields.get(name)
if (value === undefined || value === 'none') return null
return count(value, name)
}
export function parseRegionalRehomeInventory(entries, options = {}) {
if (!Array.isArray(entries)) throw new Error('logging response must be an array')
const parsed = entries.flatMap((entry) => {
const match = INVENTORY.exec(entry?.textPayload ?? '')
const fields = readRegionalRehomeInventoryFields(entry?.textPayload ?? '')
const timestamp = Date.parse(entry?.timestamp ?? '')
if (!match || !Number.isFinite(timestamp)) return []
if (!fields || !Number.isFinite(timestamp)) return []
return [{
timestamp,
active: count(match[1], 'active'),
awaitingReceipt: count(match[2], 'awaiting receipt'),
targetRegistered: count(match[3], 'target registered'),
completedLast24Hours: count(match[4], 'completed'),
abortedLast24Hours: count(match[5], 'aborted'),
oldestActiveAgeMs: match[6] === 'none' ? null : count(match[6], 'oldest active age')
active: count(fields.get('active'), 'active'),
awaitingReceipt: count(fields.get('awaitingReceipt'), 'awaiting receipt'),
targetRegistered: count(fields.get('targetRegistered'), 'target registered'),
completedLast24Hours: count(fields.get('completedLast24Hours'), 'completed'),
abortedLast24Hours: count(fields.get('abortedLast24Hours'), 'aborted'),
hostNotArrivedLast24Hours: optionalCount(fields, 'hostNotArrivedLast24Hours'),
oldestActiveAgeMs: optionalCount(fields, 'oldestActiveAgeMs')
}]
}).sort((left, right) => right.timestamp - left.timestamp)
if (parsed.length === 0) throw new Error('no aggregate regional rehome inventory evidence')
cloud/dev/scripts/relay-rehome-aggregate-evidence.test.mjs
+97
View File
@@ -1,6 +1,7 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { parseRegionalRehomeInventory } from './relay-rehome-aggregate-evidence.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
const now = Date.parse('2026-08-14T12:00:00Z')
@@ -22,10 +23,38 @@ test('selects the newest fresh aggregate-only regional rehome inventory', () =>
targetRegistered: 1,
completedLast24Hours: 9,
abortedLast24Hours: 0,
hostNotArrivedLast24Hours: null,
oldestActiveAgeMs: 30_000
})
})
test('reads a line the director grew a field on, wherever the field sits', () => {
const result = parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:58:00Z',
textPayload:
'[orca-relay] regional rehome inventory hostNotArrivedLast24Hours=4 active=2' +
' awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=7' +
' oldestActiveAgeMs=30000 someFieldFromALaterRelease=11'
}], { now, maxAgeMs: 5 * 60_000 })
assert.equal(result.hostNotArrivedLast24Hours, 4)
assert.equal(result.abortedLast24Hours, 7)
assert.equal(result.oldestActiveAgeMs, 30_000)
})
test('reports an unmeasured host-not-arrived count as absent, not as zero', () => {
const [withField, withoutField] = ['4', null].map((value) =>
parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:58:00Z',
textPayload:
'[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' +
' completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none' +
(value === null ? '' : ` hostNotArrivedLast24Hours=${value}`)
}], { now, maxAgeMs: 5 * 60_000 })
)
assert.equal(withField.hostNotArrivedLast24Hours, 4)
assert.equal(withoutField.hostNotArrivedLast24Hours, null)
})
test('rejects stale, malformed, and identity-bearing lookalikes', () => {
assert.throws(() => parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:00:00Z',
@@ -36,3 +65,71 @@ test('rejects stale, malformed, and identity-bearing lookalikes', () => {
textPayload: '[orca-relay] regional rehome inventory active=0 hostId=secret'
}], { now }), /no aggregate/)
})
test('keeps out an identity-bearing field riding along on a complete line', () => {
const complete =
'[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' +
' completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none'
for (const extra of [' hostId=secret', ' userId=someone@example.test', ' note=a b']) {
assert.throws(
() => parseRegionalRehomeInventory(
[{ timestamp: '2026-08-14T11:59:00Z', textPayload: complete + extra }],
{ now }
),
/no aggregate/,
extra
)
}
})
test('refuses a line missing a required field, or repeating one', () => {
const missing =
'[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' +
' completedLast24Hours=0 oldestActiveAgeMs=none'
assert.throws(
() => parseRegionalRehomeInventory(
[{ timestamp: '2026-08-14T11:59:00Z', textPayload: missing }],
{ now }
),
/no aggregate/
)
assert.throws(
() => parseRegionalRehomeInventory(
[{ timestamp: '2026-08-14T11:59:00Z', textPayload: `${missing} abortedLast24Hours=0 abortedLast24Hours=1` }],
{ now }
),
/no aggregate/
)
assert.throws(
() => parseRegionalRehomeInventory(
[{
timestamp: '2026-08-14T11:59:00Z',
textPayload: missing.replace('active=0', 'active=none') + ' abortedLast24Hours=0'
}],
{ now }
),
/no aggregate/
)
})
// The third edge of the chain the enable workflow depends on. The formatter is
// pinned against this parser in the relay package's inventory-line census; this
// pins the parser against the summary an operator reads, so a field that
// reaches the evidence JSON and stops there fails here.
test('publishes every parsed counter in the operator step summary', () => {
const job = readRelayWorkflow('operate-relay-production-rehome-job.yml')
// The jq program and the file it reads sit on separate continuation lines, so
// match the whole render rather than one line of it.
const summary = /jq -r '([^']*)' \\\n\s*"\$\{RUNNER_TEMP\}\/relay-rehome-inventory\.json"/.exec(job)?.[1]
assert.ok(summary, 'the rehome job no longer renders the inventory evidence')
const evidence = parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:58:00Z',
textPayload:
'[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0' +
' completedLast24Hours=0 abortedLast24Hours=0 hostNotArrivedLast24Hours=0 oldestActiveAgeMs=none'
}], { now, maxAgeMs: 5 * 60_000 })
for (const key of Object.keys(evidence)) {
if (key === 'timestamp') continue
assert.ok(summary.includes(`.${key}`), `${key} is missing from the step summary`)
}
})
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+94
View File
@@ -336,6 +336,29 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.equal(workflow.split('POOL_ARGUMENTS=()').length, 3)
})
// One cell's whole serving path and nothing else: the template, the MIG bound to it, and the
// backend service, whose reviewed drain timeout would otherwise need a fleet-wide root apply.
it('targets exactly this cell template, MIG, and backend on every plan the job runs', () => {
const plans = workflow.split('terraform -chdir=infra/terraform plan').slice(1)
assert.equal(plans.length, 2)
for (const plan of plans) {
const lines = plan.split('\n')
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
const call = lines.slice(0, end + 1).join('\n')
assert.deepEqual(
[...call.matchAll(/-target=([\w.]+)\[\\"\$\{TARGET_CELL_ID\}\\"\]/g)]
.map(([, resource]) => resource),
[
'google_compute_instance_template.relay_gce_cell',
'google_compute_instance_group_manager.relay_gce_cell',
'google_compute_backend_service.relay_gce_cell'
]
)
// Any target that is not one of those three, or not scoped to this cell, fails here.
assert.equal(call.split('-target=').length, 4)
}
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
@@ -695,6 +718,77 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3)
})
// One validator verdict decides three different outcomes. Run the predicates the job ships
// rather than restating them, because restating them is how the two drift apart.
it('decides refuse, apply, or skip on resume from the shipped predicate', () => {
const step = workflow.slice(
workflow.indexOf('- name: Require converged Terraform state and a stable MIG on resume'),
workflow.indexOf('- name: Apply only the selected same-cap template and MIG')
)
const accept = /jq -e '(\.changes == 2\n[\s\S]*?)' \\\n\s+<<< "\$\{RESUME_REVIEW\}"/.exec(step)
assert.notEqual(accept, null, 'the resume step no longer gates on a validator verdict')
assert.match(step, /if test "\$\(jq -er '\.changes' <<< "\$\{RESUME_REVIEW\}"\)" = 0; then/)
const outcome = (review) => {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', [
`RESUME_REVIEW=${JSON.stringify(JSON.stringify(review))}`,
`jq -e '${accept[1]}' <<< "\${RESUME_REVIEW}" >/dev/null || { echo refuse; exit 0; }`,
'if test "$(jq -er \'.changes\' <<< "${RESUME_REVIEW}")" = 0',
'then echo apply; else echo skip; fi'
].join('\n')], { encoding: 'utf8' })
assert.equal(resolved.status, 0, resolved.stderr)
return resolved.stdout.trim()
}
// Template and MIG converged, this cell's reviewed backend not: apply it here, or the cell
// keeps the 300-second drain and no request logging behind a green resume.
assert.equal(
outcome({ changes: 0, backendUpdate: ['connection_draining_timeout_sec', 'log_config.0'] }),
'apply'
)
assert.equal(outcome({ changes: 0, backendUpdate: ['log_config.0'] }), 'apply')
// Template-and-MIG drift still applies nothing, which is what a resume means.
assert.equal(outcome({ changes: 2 }), 'skip')
assert.equal(
outcome({ changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }),
'skip'
)
// Anything the validator did not bound to this cell's reviewed change set fails the step.
assert.equal(outcome({ changes: 0 }), 'refuse')
assert.equal(outcome({ changes: 0, backendUpdate: [] }), 'refuse')
assert.equal(outcome({ changes: 1, backendUpdate: ['log_config.0'] }), 'refuse')
assert.equal(outcome({ changes: 3 }), 'refuse')
})
// The stranded path is the other reader of `changes`, and a pending backend update must not
// suppress the explicit MIG roll that is the only thing clearing a stranded cell's drain flag.
it('rolls a stranded MIG on the shipped predicate, backend update or not', () => {
const apply = workflow
.split('name: Apply only the selected same-cap template and MIG')[1]
.split('\n - id:')[0]
const condition =
/if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0); then/
.exec(apply)
assert.notEqual(condition, null, 'the stranded roll no longer gates on the plan review')
const rolls = (stage, review) => {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', [
`ROLLBACK_STAGE=${stage}`,
`PLAN_REVIEW=${JSON.stringify(JSON.stringify(review))}`,
`if test "\${ROLLBACK_STAGE}" = stranded \\\n ${condition[1]}; then`,
'echo replace; else echo no-replace; fi'
].join('\n')], { encoding: 'utf8' })
assert.equal(resolved.status, 0, resolved.stderr)
return resolved.stdout.trim()
}
assert.equal(rolls('stranded', { changes: 0 }), 'replace')
assert.equal(
rolls('stranded', { changes: 0, backendUpdate: ['connection_draining_timeout_sec'] }),
'replace'
)
// A real template replacement already restarts the instance; rolling again would be a second.
assert.equal(rolls('stranded', { changes: 2 }), 'no-replace')
assert.equal(rolls('resume', { changes: 0 }), 'no-replace')
assert.equal(rolls('none', { changes: 0 }), 'no-replace')
})
it('waits on the image a stranded cell actually serves', () => {
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs
+254
View File
@@ -0,0 +1,254 @@
// Windowing, thresholds, and the verdict for the same-cap post-wave shadow health gate. Pure: it
// takes already-read log samples and returns a judgement, so every rule here is unit-testable
// without touching production. The reader lives in relay-same-cap-shadow-gate.mjs.
// Status vocabulary, worst-first. 'unverified' is a read that did not complete or that hit the
// entry limit; it can never settle to 'pass', because a truncated count is not evidence of calm.
export const CHECK_STATUSES = ['would-block', 'unverified', 'warn', 'pass']
export const VERDICTS = { PASS: 'PASS', WARN: 'WARN', WOULD_BLOCK: 'WOULD_BLOCK' }
// Cloud Logging silently returns only `--limit` entries, so every read is split into sub-windows
// this long and a sub-window that comes back exactly at the limit is reported as truncated.
export const SUB_WINDOW_MINUTES = 10
export const ENTRY_LIMIT = 20000
// Clock-hour-aligned comparisons: the same wall-clock minutes one and two days earlier.
export const BASELINE_OFFSET_HOURS = [24, 48]
// The asia-east2 cells share a 16-connection pool at 176 ms RTT, which is where pool pressure
// shows up first for the whole fleet.
export const FLEET_POOL_CELL_IDS = [
'production-gce-c27',
'production-gce-c28',
'production-gce-c29'
]
export const SHADOW_GATE_THRESHOLDS = {
// A US ramp legitimately lifts director 503s far above a quiet baseline (61-71/min against a
// 20-60/min baseline was healthy), so this is a multiple of the busier baseline with an
// absolute floor underneath it, never a fixed rate.
director503: { blockMultiple: 10, blockFloor: 200, warnMultiple: 3, warnFloor: 100 },
// One sample at 71 waiters is a burst that drains; three in a row is a pool that does not.
pool: { waitersMax: 50, waitersConsecutiveSamples: 3, sqlFailuresDelta: 200 },
cloudSqlFatal: { warnAbove: 0, blockAbove: 20 },
// With no drain timestamp (a resumed rollback skips the drain) the window still has to start
// somewhere; this is how far back of the verify end it reaches instead.
fallbackWindowMinutes: 30,
// A read that stalls must not be allowed to spend the job's remaining minutes.
readTimeoutMs: 60_000,
// Reads are serialised, so a failure mode that makes every read cost its full retry budget
// (an expired credential, a Logging 429 storm) scales with the window, not with one read.
// Past this the gate stops reading and reports the rest unverified, which is a verdict; the
// step's own timeout-minutes sits above it and exists only for a hung process. Set well clear
// of a healthy gate's own serial read time, or ordinary days report unverified tails and the
// shadow roll stops measuring the thing it exists to measure. Raise both bounds together.
overallDeadlineMs: 420_000
}
const MINUTE_MS = 60_000
const HOUR_MS = 3_600_000
export function parseTimestamp(value, label) {
const parsed = typeof value === 'string' ? Date.parse(value) : Number.NaN
if (Number.isNaN(parsed)) throw new Error(`${label} is not an RFC 3339 timestamp: ${value}`)
return new Date(parsed)
}
export function formatTimestamp(date) {
return `${date.toISOString().slice(0, 19)}Z`
}
/**
* The window a cell's roll is judged over: its drain start to its verify end. A resumed rollback
* never drains, so the apply start, then a fixed lookback, stands in for it.
*/
export function resolveWindow({
drainStartedAt,
applyStartedAt,
verifyEndedAt,
fallbackMinutes = SHADOW_GATE_THRESHOLDS.fallbackWindowMinutes
}) {
const endedAt = parseTimestamp(verifyEndedAt, 'verify end')
const start = drainStartedAt || applyStartedAt
const startedAt = start
? parseTimestamp(start, 'window start')
: new Date(endedAt.getTime() - fallbackMinutes * MINUTE_MS)
if (startedAt >= endedAt) throw new Error('shadow gate window starts at or after it ends')
return { startedAt, endedAt, startedFrom: drainStartedAt ? 'drain' : start ? 'apply' : 'fallback' }
}
export function splitWindow({ startedAt, endedAt }, minutes = SUB_WINDOW_MINUTES) {
const step = minutes * MINUTE_MS
const windows = []
for (let cursor = startedAt.getTime(); cursor < endedAt.getTime(); cursor += step) {
windows.push({
startedAt: new Date(cursor),
endedAt: new Date(Math.min(cursor + step, endedAt.getTime()))
})
}
return windows
}
export function shiftWindow({ startedAt, endedAt }, hours) {
return {
startedAt: new Date(startedAt.getTime() - hours * HOUR_MS),
endedAt: new Date(endedAt.getTime() - hours * HOUR_MS)
}
}
/**
* Counts per clock minute across sub-window reads. A sub-window that returned exactly the entry
* limit is truncated, so its minutes are floors, not counts, and the whole read is unverified.
*/
export function countByMinute(reads, limit = ENTRY_LIMIT) {
const perMinute = new Map()
let truncated = false
for (const read of reads) {
if (read.failed || read.timestamps.length >= limit) truncated = true
for (const timestamp of read.timestamps) {
const minute = timestamp.slice(0, 16)
perMinute.set(minute, (perMinute.get(minute) ?? 0) + 1)
}
}
let peak = 0
let peakMinute = null
let total = 0
for (const [minute, count] of perMinute) {
total += count
if (count > peak) {
peak = count
peakMinute = minute
}
}
return { perMinute: Object.fromEntries(perMinute), total, peak, peakMinute, truncated }
}
// Longest run of consecutive samples at or above the threshold.
export function longestRunAtOrAbove(values, threshold) {
let longest = 0
let run = 0
for (const value of values) {
run = value > threshold ? run + 1 : 0
if (run > longest) longest = run
}
return longest
}
export function judgeDirector503({ observed, baselines }) {
const { blockMultiple, blockFloor, warnMultiple, warnFloor } = SHADOW_GATE_THRESHOLDS.director503
const baselinePeak = Math.max(0, ...baselines.map((baseline) => baseline.peak))
const baselineTruncated = baselines.some((baseline) => baseline.truncated)
const detail = {
peakPerMinute: observed.peak,
peakMinute: observed.peakMinute,
total: observed.total,
baselinePeakPerMinute: baselinePeak,
baselines: baselines.map(({ label, peak, total, truncated }) => ({
label,
peakPerMinute: peak,
total,
truncated
})),
blockAbove: Math.max(baselinePeak * blockMultiple, blockFloor),
warnAbove: Math.max(baselinePeak * warnMultiple, warnFloor)
}
if (observed.truncated || baselineTruncated) return { status: 'unverified', ...detail }
if (observed.peak > detail.blockAbove) return { status: 'would-block', ...detail }
if (observed.peak > detail.warnAbove) return { status: 'warn', ...detail }
return { status: 'pass', ...detail }
}
/**
* The cell's own container: it has to have announced its listener since the apply began, and it
* must not have crashed anywhere in that span. Counting crashes only after the *last* listener
* would erase a crash-restart loop, whose later announcement looks like a clean boot; the MIG
* recreates the instance, so everything on this instance id since the apply belongs to this roll.
*
* A missing announcement only means a failure where a restart was expected. A resumed rollback
* deliberately restarts nothing, so there is no boot for this oracle to observe and its silence
* says nothing either way.
*/
export function judgeCellServing({ listeningAt, crashesSinceApply, read, expectBoot = true }) {
const detail = {
listeningAt: listeningAt ?? null,
crashesSinceApply: crashesSinceApply ?? 0,
expectBoot
}
if (read?.failed) return { status: 'unverified', ...detail }
if (!listeningAt) return { status: expectBoot ? 'would-block' : 'unverified', ...detail }
if (detail.crashesSinceApply > 0) return { status: 'would-block', ...detail }
return { status: 'pass', ...detail }
}
/**
* Pool pressure. A single spike is a burst the pool absorbs; the block rule needs the pressure to
* persist across consecutive samples, which is what separates it from the one-sample false
* positives a literal rule produced this week.
*/
export function judgePool({ label, samples, failed = false, truncated = false }) {
const { waitersMax, waitersConsecutiveSamples, sqlFailuresDelta } = SHADOW_GATE_THRESHOLDS.pool
const waiters = samples.map((sample) => sample.databasePoolWaitersMax ?? 0)
const failures = samples.map((sample) => sample.sqlFailuresDelta ?? 0)
const detail = {
label,
samples: samples.length,
waitersMax: Math.max(0, ...waiters),
consecutiveSamplesOverWaitersThreshold: longestRunAtOrAbove(waiters, waitersMax),
sqlFailuresDeltaMax: Math.max(0, ...failures),
reconnectsDeltaMax: Math.max(0, ...samples.map((sample) => sample.reconnectsDelta ?? 0)),
totalConnectionsMax: Math.max(0, ...samples.map((sample) => sample.totalConnections ?? 0)),
databasePoolWaitingMax: Math.max(0, ...samples.map((sample) => sample.databasePoolWaiting ?? 0)),
waitersThreshold: waitersMax,
consecutiveSamplesThreshold: waitersConsecutiveSamples,
sqlFailuresDeltaThreshold: sqlFailuresDelta,
truncated
}
// A truncated sample run has holes, and the consecutive-sample rule reads a hole as a recovery.
if (failed || truncated || samples.length === 0) return { status: 'unverified', ...detail }
if (
detail.consecutiveSamplesOverWaitersThreshold >= waitersConsecutiveSamples
|| detail.sqlFailuresDeltaMax > sqlFailuresDelta
) return { status: 'would-block', ...detail }
if (detail.waitersMax > waitersMax) return { status: 'warn', ...detail }
return { status: 'pass', ...detail }
}
export function judgeCloudSqlFatal({ count, truncated = false, failed = false }) {
const { warnAbove, blockAbove } = SHADOW_GATE_THRESHOLDS.cloudSqlFatal
const detail = { count, warnAbove, blockAbove }
if (failed || truncated) return { status: 'unverified', ...detail }
if (count > blockAbove) return { status: 'would-block', ...detail }
if (count > warnAbove) return { status: 'warn', ...detail }
return { status: 'pass', ...detail }
}
export function combineVerdict(checks) {
const statuses = Object.values(checks).map((check) => check.status)
if (statuses.includes('would-block')) return VERDICTS.WOULD_BLOCK
if (statuses.includes('unverified') || statuses.includes('warn')) return VERDICTS.WARN
return VERDICTS.PASS
}
export function renderStepSummary(report) {
const rows = Object.entries(report.checks).map(([name, check]) => {
const numbers = Object.entries(check)
.filter(([key, value]) => key !== 'status' && value !== null && typeof value !== 'object')
.map(([key, value]) => `${key}=${value}`)
.join(', ')
return `| ${name} | ${check.status} | ${numbers} |`
})
return [
`## Shadow health gate (report only): ${report.verdict}`,
'',
`Cell \`${report.cellId}\`, window ${report.window.startedAt} to ${report.window.endedAt}`,
`(start taken from: ${report.window.startedFrom}).`,
'This gate never fails the job. Compare its verdict with the operator call for this cell.',
'',
'| check | status | numbers |',
'| --- | --- | --- |',
...rows,
''
].join('\n')
}
cloud/dev/scripts/relay-same-cap-shadow-gate.mjs
+325
View File
@@ -0,0 +1,325 @@
#!/usr/bin/env node
// Post-wave shadow health gate for a same-cap cell roll. Reads exactly the oracles an operator
// reads by hand today, writes a PASS / WARN / WOULD_BLOCK verdict with its numbers to a JSON
// artifact and the step summary, and always exits 0 on a verdict: this runs in report-only mode so
// its calls can be compared with the operator's over a full roll before it is allowed to block.
//
// Every filter is built from validated, pattern-pinned inputs and handed to gcloud as argv, never
// through a shell.
import { execFile } from 'node:child_process'
import { appendFile, writeFile } from 'node:fs/promises'
import { pathToFileURL } from 'node:url'
import { promisify } from 'node:util'
import {
BASELINE_OFFSET_HOURS,
ENTRY_LIMIT,
FLEET_POOL_CELL_IDS,
SHADOW_GATE_THRESHOLDS,
combineVerdict,
countByMinute,
formatTimestamp,
judgeCellServing,
judgeCloudSqlFatal,
judgeDirector503,
judgePool,
renderStepSummary,
resolveWindow,
shiftWindow,
splitWindow
} from './relay-same-cap-shadow-gate-verdict.mjs'
const execFileAsync = promisify(execFile)
const CELL_ID = /^production-gce-c[1-9][0-9]*$/
const CELL_HOST = /^c[1-9][0-9]*\.relay\.onorca\.dev$/
const PROJECT_ID = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/
const SERVICE_NAME = /^[a-z][a-z0-9-]{0,62}$/
export const READ_ATTEMPTS = 3
const READ_RETRY_DELAY_MS = 5000
const READ_TIMEOUT_MS = SHADOW_GATE_THRESHOLDS.readTimeoutMs
const OVERALL_DEADLINE_MS = SHADOW_GATE_THRESHOLDS.overallDeadlineMs
// json(timestamp) over a busy minute is a few hundred KB; leave room for the widest sub-window.
const READ_MAX_BUFFER_BYTES = 256 * 1024 * 1024
export function parseShadowGateArguments(argv) {
const values = new Map()
for (let index = 0; index < argv.length; index += 2) {
if (!argv[index].startsWith('--')) throw new Error(`expected a flag, got ${argv[index]}`)
values.set(argv[index].slice(2), argv[index + 1])
}
const required = (name, pattern) => {
const value = values.get(name) ?? ''
if (!pattern.test(value)) throw new Error(`--${name} is not acceptable: ${value}`)
return value
}
const config = {
cellId: required('cell-id', CELL_ID),
cellHost: required('cell-host', CELL_HOST),
projectId: required('project-id', PROJECT_ID),
directorService: required('director-service', SERVICE_NAME),
drainStartedAt: values.get('drain-started-at') || '',
// The listener lands while the MIG is still converging, so the boot search has to open at the
// apply's start; a bound taken at its completion is already past the announcement it looks for.
applyStartedAt: values.get('apply-started-at') || '',
applyCompletedAt: values.get('apply-completed-at') || '',
verifyEndedAt: values.get('verify-ended-at') || '',
outputFile: values.get('output-file') || '',
summaryFile: values.get('summary-file') || ''
}
if (!config.cellHost.startsWith(`${config.cellId.replace('production-gce-', '')}.`)) {
throw new Error(`--cell-host ${config.cellHost} is not the host of ${config.cellId}`)
}
if (!config.outputFile) throw new Error('--output-file is required')
return config
}
function timestampBounds({ startedAt, endedAt }) {
return `timestamp>="${formatTimestamp(startedAt)}" AND timestamp<"${formatTimestamp(endedAt)}"`
}
/**
* One bounded `gcloud logging read`. A read that cannot complete is reported as failed rather than
* thrown: a missing oracle must surface as an unverified check, not as a crashed gate.
*/
async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) {
const args = [
'logging', 'read', filter,
'--project', reader.projectId,
'--format', projection,
'--limit', String(limit),
'--order', 'desc'
]
let lastError
for (let attempt = 1; attempt <= READ_ATTEMPTS; attempt += 1) {
// Every remaining read short-circuits once the budget is gone, so the gate always reaches a
// verdict instead of being killed part-way through with nothing written.
const remainingMs = reader.deadlineAt - reader.now()
if (remainingMs <= 0) {
return { entries: [], failed: true, error: 'shadow gate read deadline exceeded' }
}
try {
const timeoutMs = Math.min(reader.readTimeoutMs, remainingMs)
const { stdout } = await reader.runGcloud(args, { timeoutMs })
return { entries: JSON.parse(stdout || '[]'), failed: false }
} catch (error) {
lastError = error
if (attempt < READ_ATTEMPTS) {
await new Promise((resolve) => setTimeout(resolve, reader.retryDelayMs))
}
}
}
return { entries: [], failed: true, error: String(lastError?.message ?? lastError) }
}
async function readTimestampsOverWindow(reader, { filter, window }) {
const reads = []
for (const subWindow of splitWindow(window)) {
const read = await readLogEntries(reader, {
filter: `${filter} AND ${timestampBounds(subWindow)}`,
projection: 'json(timestamp)'
})
reads.push({
failed: read.failed,
timestamps: read.entries.map((entry) => entry.timestamp)
})
}
return countByMinute(reads)
}
function directorFilter({ directorService }) {
return `resource.type="cloud_run_revision"`
+ ` AND resource.labels.service_name="${directorService}"`
+ ` AND httpRequest.status=503`
}
// Cells log through the COS container agent, so the text lives in jsonPayload.message; a
// textPayload filter matches nothing here and returns zero without saying so.
const CELL_LOG_SCOPE = 'resource.type="gce_instance" AND logName:"cos_containers"'
async function readDirector503(reader, { config, window }) {
const filter = directorFilter(config)
const observed = await readTimestampsOverWindow(reader, { filter, window })
const baselines = []
for (const hours of BASELINE_OFFSET_HOURS) {
const counts = await readTimestampsOverWindow(reader, {
filter,
window: shiftWindow(window, hours)
})
baselines.push({ label: `${hours}h-earlier`, ...counts })
}
return judgeDirector503({ observed, baselines })
}
/**
* The cell's new container. The listener announcement after the apply identifies both that the
* cell is serving and which instance it is serving on; crashes are then scoped to that instance,
* because instance_id is stable across a container restart and is the only cell label these
* entries carry.
*/
async function readCellServing(reader, { config, window, searchFrom, expectBoot }) {
const listening = await readLogEntries(reader, {
filter: `${CELL_LOG_SCOPE}`
+ ` AND jsonPayload.message:"listening on https://${config.cellHost}"`
+ ` AND ${timestampBounds({ startedAt: searchFrom, endedAt: window.endedAt })}`,
projection: 'json(timestamp,resource.labels.instance_id)',
limit: 50
})
// Newest first: the most recent announcement is the boot this wave produced.
const boot = listening.entries[0]
if (listening.failed || !boot) {
return {
serving: judgeCellServing({ listeningAt: null, read: listening, expectBoot }),
instanceId: null
}
}
const crashes = await readLogEntries(reader, {
filter: `${CELL_LOG_SCOPE}`
+ ` AND jsonPayload.message:"throw er"`
+ ` AND resource.labels.instance_id="${boot.resource.labels.instance_id}"`
+ ` AND ${timestampBounds({ startedAt: searchFrom, endedAt: window.endedAt })}`,
projection: 'json(timestamp)',
limit: 100
})
return {
serving: judgeCellServing({
listeningAt: boot.timestamp,
crashesSinceApply: crashes.entries.length,
read: crashes,
expectBoot
}),
instanceId: boot.resource.labels.instance_id
}
}
const RUNTIME_METRIC_FIELDS = [
'totalConnections',
'databasePoolWaitersMax',
'databasePoolWaiting',
'sqlFailuresDelta',
'reconnectsDelta'
]
async function readRuntimeMetrics(reader, { cellId, window }) {
const projection = `json(timestamp,${RUNTIME_METRIC_FIELDS
.map((field) => `jsonPayload.${field}`)
.join(',')})`
const samples = []
let failed = false
let truncated = false
// Samples land every 30 s, so a 10-minute sub-window holds ~20. A read that comes back at this
// many is not a calm sub-window, it is a truncated one, and its gaps read as recoveries.
const limit = 500
for (const subWindow of splitWindow(window)) {
const read = await readLogEntries(reader, {
filter: `${CELL_LOG_SCOPE}`
+ ` AND jsonPayload.event="orca_relay_runtime_metrics"`
+ ` AND jsonPayload.cellId="${cellId}"`
+ ` AND ${timestampBounds(subWindow)}`,
projection,
limit
})
if (read.failed) failed = true
if (read.entries.length >= limit) truncated = true
for (const entry of read.entries) {
samples.push({ timestamp: entry.timestamp, ...entry.jsonPayload })
}
}
return { samples, failed, truncated }
}
async function readCloudSqlFatal(reader, { window }) {
const counts = await readTimestampsOverWindow(reader, {
filter: `resource.type="cloudsql_database" AND "FATAL"`,
window
})
return judgeCloudSqlFatal({ count: counts.total, truncated: counts.truncated })
}
export async function evaluateShadowGate(config, {
runGcloud,
retryDelayMs = READ_RETRY_DELAY_MS,
readTimeoutMs = READ_TIMEOUT_MS,
overallDeadlineMs = OVERALL_DEADLINE_MS,
now = Date.now
}) {
const reader = {
runGcloud,
retryDelayMs,
readTimeoutMs,
now,
deadlineAt: now() + overallDeadlineMs,
projectId: config.projectId
}
const window = resolveWindow(config)
// Everything this roll's instance logged, from the moment the apply could first restart it.
const searchFrom = config.applyStartedAt
? new Date(Date.parse(config.applyStartedAt))
: window.startedAt
// Serialised on purpose: a burst of concurrent reads is what earns a Logging 429, and a 429 is
// the one failure that comes back as a short answer rather than an error.
const director503 = await readDirector503(reader, { config, window })
// A fallback window start means neither the drain nor the apply ran, which is the resumed
// rollback that restarts nothing; there is then no boot to find.
const cell = await readCellServing(reader, {
config,
window,
searchFrom,
expectBoot: window.startedFrom !== 'fallback'
})
const cloudSql = await readCloudSqlFatal(reader, { window })
const cellMetrics = await readRuntimeMetrics(reader, { cellId: config.cellId, window })
const checks = {
director503,
cellServing: cell.serving,
cellPool: judgePool({ label: config.cellId, ...cellMetrics }),
cloudSqlFatal: cloudSql
}
for (const fleetCellId of FLEET_POOL_CELL_IDS) {
if (fleetCellId === config.cellId) continue
const metrics = await readRuntimeMetrics(reader, { cellId: fleetCellId, window })
checks[`fleetPool:${fleetCellId}`] = judgePool({ label: fleetCellId, ...metrics })
}
return {
reportOnly: true,
cellId: config.cellId,
cellHost: config.cellHost,
cellInstanceId: cell.instanceId,
window: {
startedAt: formatTimestamp(window.startedAt),
endedAt: formatTimestamp(window.endedAt),
startedFrom: window.startedFrom,
// Recorded, not judged: an operator comparing verdicts needs to see how long the apply took
// next to when the cell actually came back.
applyCompletedAt: config.applyCompletedAt || null
},
verdict: combineVerdict(checks),
checks
}
}
async function main() {
const config = parseShadowGateArguments(process.argv.slice(2))
const report = await evaluateShadowGate(config, {
// `timeout` makes Node kill the child itself; continue-on-error bounds the job's outcome but
// not its clock, and a stalled read would otherwise spend the rollout's remaining minutes.
runGcloud: (args, { timeoutMs }) => execFileAsync('gcloud', args, {
maxBuffer: READ_MAX_BUFFER_BYTES,
timeout: timeoutMs,
killSignal: 'SIGKILL'
})
})
await writeFile(config.outputFile, `${JSON.stringify(report, null, 2)}\n`)
if (config.summaryFile) await appendFile(config.summaryFile, renderStepSummary(report))
console.log(JSON.stringify(report, null, 2))
}
// Report only: a verdict, including WOULD_BLOCK, is a successful run. Only a crash exits non-zero,
// and the job still runs this step under continue-on-error.
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs
+538
View File
@@ -0,0 +1,538 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { readRelayWorkflow } from './relay-repository.mjs'
import {
READ_ATTEMPTS,
evaluateShadowGate,
parseShadowGateArguments
} from './relay-same-cap-shadow-gate.mjs'
import {
ENTRY_LIMIT,
SHADOW_GATE_THRESHOLDS,
SUB_WINDOW_MINUTES,
combineVerdict,
countByMinute,
formatTimestamp,
judgeCellServing,
judgeCloudSqlFatal,
judgeDirector503,
judgePool,
longestRunAtOrAbove,
renderStepSummary,
resolveWindow,
shiftWindow,
splitWindow
} from './relay-same-cap-shadow-gate-verdict.mjs'
const ARGV = [
'--cell-id', 'production-gce-c28',
'--cell-host', 'c28.relay.onorca.dev',
'--project-id', 'onorca-cloud',
'--director-service', 'orca-cloud-relay',
'--drain-started-at', '2026-09-20T20:00:00Z',
'--apply-started-at', '2026-09-20T20:15:00Z',
'--apply-completed-at', '2026-09-20T20:19:30Z',
'--verify-ended-at', '2026-09-20T20:30:00Z',
'--output-file', '/tmp/shadow.json'
]
function minuteOfTimestamps(minute, count) {
return Array.from(
{ length: count },
(_, index) => `${minute}:${String(index % 60).padStart(2, '0')}Z`
)
}
test('binds every gcloud input to a pinned pattern and to one cell', () => {
assert.equal(parseShadowGateArguments(ARGV).cellId, 'production-gce-c28')
// A filter is a string; anything that could steer one has to be refused before it is built.
assert.throws(() => parseShadowGateArguments(ARGV.with(1, 'production-gce-c28" OR "x')))
assert.throws(() => parseShadowGateArguments(ARGV.with(3, 'evil.example.test')))
assert.throws(() => parseShadowGateArguments(ARGV.with(5, 'Onorca Cloud')))
assert.throws(() => parseShadowGateArguments(ARGV.with(7, 'orca cloud relay')))
// Host and cell id must name the same cell, or the serving check reads a neighbour.
assert.throws(() => parseShadowGateArguments(ARGV.with(3, 'c29.relay.onorca.dev')))
// A run with nowhere to write its verdict is not a report-only run, it is a silent one.
assert.throws(() => parseShadowGateArguments(ARGV.slice(0, 16)))
})
test('the window runs from drain start to verify end, with named fallbacks', () => {
const full = resolveWindow({
drainStartedAt: '2026-09-20T20:00:00Z',
applyStartedAt: '2026-09-20T20:15:00Z',
verifyEndedAt: '2026-09-20T20:30:00Z'
})
assert.equal(formatTimestamp(full.startedAt), '2026-09-20T20:00:00Z')
assert.equal(full.startedFrom, 'drain')
// A resumed rollback never drains, so the apply stands in for the start.
assert.equal(resolveWindow({
applyStartedAt: '2026-09-20T20:15:00Z',
verifyEndedAt: '2026-09-20T20:30:00Z'
}).startedFrom, 'apply')
assert.equal(formatTimestamp(resolveWindow({
verifyEndedAt: '2026-09-20T20:30:00Z'
}).startedAt), '2026-09-20T20:00:00Z')
assert.throws(() => resolveWindow({
drainStartedAt: '2026-09-20T20:30:00Z',
verifyEndedAt: '2026-09-20T20:30:00Z'
}))
assert.throws(() => resolveWindow({ verifyEndedAt: 'not-a-time' }))
})
test('reads are split into sub-windows no longer than the truncation bound', () => {
const windows = splitWindow(resolveWindow({
drainStartedAt: '2026-09-20T20:00:00Z',
verifyEndedAt: '2026-09-20T20:47:00Z'
}))
assert.equal(windows.length, 5)
for (const window of windows) {
const minutes = (window.endedAt - window.startedAt) / 60_000
assert.ok(minutes > 0 && minutes <= SUB_WINDOW_MINUTES, `${minutes} minutes`)
}
assert.equal(formatTimestamp(windows.at(-1).endedAt), '2026-09-20T20:47:00Z')
const baseline = shiftWindow(windows[0], 24)
assert.equal(formatTimestamp(baseline.startedAt), '2026-09-19T20:00:00Z')
})
test('a sub-window that came back at the entry limit is truncated, never a count', () => {
const truncated = countByMinute([
{ timestamps: minuteOfTimestamps('2026-09-19T15:34', ENTRY_LIMIT) }
])
assert.equal(truncated.truncated, true)
const failed = countByMinute([{ failed: true, timestamps: [] }])
assert.equal(failed.truncated, true)
const counted = countByMinute([
{ timestamps: minuteOfTimestamps('2026-09-19T15:34', 4722) },
{ timestamps: minuteOfTimestamps('2026-09-19T15:33', 278) }
])
assert.deepEqual(
{
peak: counted.peak,
peakMinute: counted.peakMinute,
total: counted.total,
truncated: counted.truncated
},
{ peak: 4722, peakMinute: '2026-09-19T15:34', total: 5000, truncated: false }
)
})
test('director 503s are judged against the busier baseline, not a fixed rate', () => {
const baselines = [
{ label: '24h-earlier', peak: 48, total: 80, truncated: false },
{ label: '48h-earlier', peak: 69, total: 100, truncated: false }
]
// The 2026-09-19 c28 wave: 4722/min against 48 and 69/min baselines.
assert.equal(judgeDirector503({
observed: { peak: 4722, peakMinute: '2026-09-19T15:34', total: 5000, truncated: false },
baselines
}).status, 'would-block')
// The false positive a literal rule produced: a US ramp at 71/min over a 20-60/min baseline.
assert.equal(judgeDirector503({
observed: { peak: 71, peakMinute: '2026-09-18T01:10', total: 300, truncated: false },
baselines: [
{ label: '24h-earlier', peak: 60, total: 400, truncated: false },
{ label: '48h-earlier', peak: 20, total: 90, truncated: false }
]
}).status, 'pass')
// A truncated read cannot settle to pass, however calm its visible counts are.
assert.equal(judgeDirector503({
observed: { peak: 3, total: 3, truncated: true },
baselines
}).status, 'unverified')
assert.equal(judgeDirector503({
observed: { peak: 3, total: 3, truncated: false },
baselines: [baselines[0], { ...baselines[1], truncated: true }]
}).status, 'unverified')
})
test('the cell has to announce its listener and stay up across the whole apply', () => {
assert.equal(judgeCellServing({
listeningAt: '2026-09-20T20:18:27Z',
crashesSinceApply: 0
}).status, 'pass')
assert.equal(judgeCellServing({ listeningAt: null }).status, 'would-block')
// A resumed rollback restarts nothing, so there is no boot to find and silence proves nothing.
assert.equal(judgeCellServing({ listeningAt: null, expectBoot: false }).status, 'unverified')
assert.equal(judgeCellServing({
listeningAt: '2026-09-20T20:18:27Z',
crashesSinceApply: 1
}).status, 'would-block')
assert.equal(judgeCellServing({
listeningAt: null,
read: { failed: true }
}).status, 'unverified')
})
test('pool pressure blocks only when it persists across consecutive samples', () => {
assert.equal(longestRunAtOrAbove([10, 60, 10, 60, 60, 60, 10], 50), 3)
const burst = judgePool({
label: 'production-gce-c27',
// The single-sample waiters=71 that a literal rule called an outage.
samples: [
{ databasePoolWaitersMax: 12 },
{ databasePoolWaitersMax: 71 },
{ databasePoolWaitersMax: 9 }
]
})
assert.equal(burst.status, 'warn')
assert.equal(burst.consecutiveSamplesOverWaitersThreshold, 1)
assert.equal(judgePool({
label: 'production-gce-c28',
samples: [
{ databasePoolWaitersMax: 148 },
{ databasePoolWaitersMax: 125 },
{ databasePoolWaitersMax: 154 }
]
}).status, 'would-block')
assert.equal(judgePool({
label: 'production-gce-c28',
samples: [{ databasePoolWaitersMax: 2, sqlFailuresDelta: 489 }]
}).status, 'would-block')
assert.equal(judgePool({
label: 'production-gce-c29',
samples: [{ databasePoolWaitersMax: 3, sqlFailuresDelta: 0, totalConnections: 500 }]
}).status, 'pass')
// No samples at all is silence, not health.
assert.equal(judgePool({ label: 'production-gce-c29', samples: [] }).status, 'unverified')
assert.equal(judgePool({
label: 'production-gce-c29',
samples: [{ databasePoolWaitersMax: 1 }],
failed: true
}).status, 'unverified')
// A truncated sample run has holes, and a hole reads to the run rule as a recovery.
assert.equal(judgePool({
label: 'production-gce-c29',
samples: [{ databasePoolWaitersMax: 1 }],
truncated: true
}).status, 'unverified')
})
test('Cloud SQL FATALs warn from the first one and block on a run of them', () => {
assert.equal(judgeCloudSqlFatal({ count: 0 }).status, 'pass')
assert.equal(judgeCloudSqlFatal({ count: 1 }).status, 'warn')
assert.equal(judgeCloudSqlFatal({ count: 21 }).status, 'would-block')
assert.equal(judgeCloudSqlFatal({ count: 0, truncated: true }).status, 'unverified')
})
test('the verdict is the worst check, and an unverified read never reads as PASS', () => {
assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'pass' } }), 'PASS')
assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'warn' } }), 'WARN')
assert.equal(combineVerdict({ a: { status: 'pass' }, b: { status: 'unverified' } }), 'WARN')
assert.equal(
combineVerdict({ a: { status: 'would-block' }, b: { status: 'unverified' } }),
'WOULD_BLOCK'
)
})
// The step that owns each stamp, so a stamp's presence is judged where it has to be written.
const STAMP_STEPS = {
drain: '- name: Reversibly isolate and drain only the selected cell',
apply: '- name: Apply only the selected same-cap template and MIG',
'verify-target': '- name: Verify new incarnation, exact image, protocol, and durable safety'
}
// One step's own lines: from its marker to the next sibling step at the same indent.
function stepBody(workflow, marker) {
const start = workflow.indexOf(marker)
assert.notEqual(start, -1, `the job no longer has a step named ${marker}`)
const next = workflow.indexOf('\n - ', start + marker.length)
return workflow.slice(start, next === -1 ? undefined : next)
}
const C28_INSTANCE = '5031087219978409220'
// Runtime-metrics samples at the 30 s cadence production emits them at, unless a case needs
// enough of them inside one sub-window to reach the read's limit.
function metricSamples({ cellId, from, count, payload = {}, intervalMs = 30_000 }) {
return Array.from({ length: count }, (_, index) => ({
matches: ['orca_relay_runtime_metrics', `jsonPayload.cellId="${cellId}"`],
timestamp: new Date(Date.parse(from) + index * intervalMs).toISOString(),
payload: {
totalConnections: 857,
databasePoolWaitersMax: 4,
databasePoolWaiting: 1,
sqlFailuresDelta: 0,
reconnectsDelta: 0,
...payload
}
}))
}
// The exact entry shapes production returned for c28 on 2026-09-20: the crash at 20:18:10Z and
// the listener at 20:18:27Z, both on instance 5031087219978409220.
function productionLikeEntries() {
return [
{
matches: ['listening on https://c28.relay.onorca.dev'],
timestamp: '2026-09-20T20:18:27.470301969Z',
instanceId: C28_INSTANCE
},
...metricSamples({ cellId: 'production-gce-c28', from: '2026-09-20T20:20:00Z', count: 20 }),
...metricSamples({ cellId: 'production-gce-c27', from: '2026-09-20T20:20:00Z', count: 20 }),
...metricSamples({ cellId: 'production-gce-c29', from: '2026-09-20T20:20:00Z', count: 20 })
]
}
/**
* A gcloud seam that honours the filter it is given: its timestamp bounds, its instance-id scope,
* the `--limit`, and the newest-first order. A fake that ignored the bounds would let a
* wrongly-bounded query pass, which is exactly the bug class these tests exist to catch.
*/
function gcloudSeam(entries = productionLikeEntries()) {
const calls = []
return {
calls,
retryDelayMs: 0,
runGcloud: async (args, options) => {
const filter = args[2]
const limit = Number(args[args.indexOf('--limit') + 1])
calls.push({ filter, limit, options })
const startedAt = Date.parse(/timestamp>="([^"]+)"/.exec(filter)[1])
const endedAt = Date.parse(/timestamp<"([^"]+)"/.exec(filter)[1])
const instanceId = /resource\.labels\.instance_id="([^"]+)"/.exec(filter)?.[1]
const matched = entries.filter((entry) => {
const at = Date.parse(entry.timestamp)
if (at < startedAt || at >= endedAt) return false
if (instanceId && entry.instanceId !== instanceId) return false
return entry.matches.every((needle) => filter.includes(needle))
})
matched.sort((left, right) => Date.parse(right.timestamp) - Date.parse(left.timestamp))
return {
stdout: JSON.stringify(matched.slice(0, limit).map((entry) => ({
timestamp: entry.timestamp,
...(entry.instanceId ? { resource: { labels: { instance_id: entry.instanceId } } } : {}),
...(entry.payload ? { jsonPayload: entry.payload } : {})
})))
}
}
}
}
test('a healthy roll reads as PASS and names the instance it proved serving', async () => {
const seam = gcloudSeam()
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.equal(report.verdict, 'PASS')
assert.equal(report.reportOnly, true)
assert.equal(report.cellInstanceId, C28_INSTANCE)
assert.equal(report.window.startedFrom, 'drain')
assert.equal(report.window.applyCompletedAt, '2026-09-20T20:19:30Z')
assert.deepEqual(Object.keys(report.checks).sort(), [
'cellPool',
'cellServing',
'cloudSqlFatal',
'director503',
'fleetPool:production-gce-c27',
'fleetPool:production-gce-c29'
])
// Every read carries explicit bounds: --freshness does not bind on these logs.
for (const { filter } of seam.calls) {
assert.match(filter, /timestamp>="[^"]+" AND timestamp<"[^"]+"/)
}
// Cell text lives in jsonPayload.message; a textPayload filter matches nothing and says so.
assert.equal(seam.calls.some(({ filter }) => filter.includes('textPayload')), false)
assert.match(renderStepSummary(report), /Shadow health gate \(report only\): PASS/)
})
// The listener lands while the MIG is still converging, so a boot search opening at the apply's
// completion finds nothing and calls a healthy roll a failure.
test('the boot search opens at the apply start, not at its completion', async () => {
const seam = gcloudSeam()
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.equal(report.checks.cellServing.status, 'pass')
assert.equal(report.checks.cellServing.listeningAt, '2026-09-20T20:18:27.470301969Z')
const listenerRead = seam.calls.find(({ filter }) => filter.includes('listening on https://'))
assert.match(listenerRead.filter, /timestamp>="2026-09-20T20:15:00Z"/)
// The listener at 20:18:27 sits after the apply start and before its completion at 20:19:30,
// so a completion-bounded search would have missed it entirely.
assert.ok(Date.parse('2026-09-20T20:18:27.470301969Z') < Date.parse('2026-09-20T20:19:30Z'))
})
// A crash-restart loop ends with a listener announcement that looks like a clean boot. Counting
// crashes only after the last announcement erases the loop that produced it.
test('a crash before the final listener still counts against the roll', async () => {
const seam = gcloudSeam([
...productionLikeEntries(),
{
matches: ['throw er'],
timestamp: '2026-09-20T20:18:10.651702662Z',
instanceId: C28_INSTANCE
}
])
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.equal(report.checks.cellServing.crashesSinceApply, 1)
assert.equal(report.checks.cellServing.status, 'would-block')
assert.equal(report.verdict, 'WOULD_BLOCK')
const crashRead = seam.calls.find(({ filter }) => filter.includes('throw er'))
// Bounded at the apply start, and still scoped to the instance the listener identified.
assert.match(crashRead.filter, /timestamp>="2026-09-20T20:15:00Z"/)
assert.match(crashRead.filter, new RegExp(`resource\\.labels\\.instance_id="${C28_INSTANCE}"`))
})
test('a crash on a neighbouring instance is not charged to this cell', async () => {
const seam = gcloudSeam([
...productionLikeEntries(),
{ matches: ['throw er'], timestamp: '2026-09-20T20:18:10Z', instanceId: '9999999999999999999' }
])
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.equal(report.checks.cellServing.crashesSinceApply, 0)
assert.equal(report.checks.cellServing.status, 'pass')
})
// A sample run returned at the read's limit has holes, and the consecutive-sample rule reads a
// hole as a recovery, so it must not be judged as though it were complete.
test('a runtime-metrics read at its limit is unverified, not a calm cell', async () => {
const seam = gcloudSeam([
...productionLikeEntries(),
// 600 samples packed into the first sub-window, past the 500-entry read limit.
...metricSamples({
cellId: 'production-gce-c28',
from: '2026-09-20T20:00:00Z',
count: 600,
intervalMs: 500,
payload: { databasePoolWaitersMax: 1 }
})
])
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.equal(report.checks.cellPool.status, 'unverified')
assert.equal(report.checks.cellPool.truncated, true)
// The neighbours were read normally, so only the truncated cell is unverified.
assert.equal(report.checks['fleetPool:production-gce-c27'].status, 'pass')
assert.equal(report.verdict, 'WARN')
})
test('a resume, which restarts nothing, does not read a missing boot as a failure', async () => {
const resumed = ARGV.with(9, '').with(11, '').with(13, '')
const seam = gcloudSeam(productionLikeEntries().filter(
(entry) => !entry.matches[0].startsWith('listening')
))
const report = await evaluateShadowGate(parseShadowGateArguments(resumed), seam)
assert.equal(report.window.startedFrom, 'fallback')
assert.equal(report.checks.cellServing.status, 'unverified')
assert.equal(report.verdict, 'WARN')
})
test('a gcloud read that never completes is unverified, not a crashed gate', async () => {
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), {
retryDelayMs: 0,
runGcloud: async () => { throw new Error('PERMISSION_DENIED') }
})
assert.equal(report.verdict, 'WARN')
assert.equal(report.checks.director503.status, 'unverified')
assert.equal(report.checks.cellServing.status, 'unverified')
})
// continue-on-error bounds the job's outcome but not its clock; an unbounded read could spend the
// rollout's remaining minutes before the job's own timeout noticed.
test('every read is given a bounded timeout, and a timed-out read is just a failed read', async () => {
const seam = gcloudSeam()
await evaluateShadowGate(parseShadowGateArguments(ARGV), seam)
assert.ok(seam.calls.length > 0)
for (const { options } of seam.calls) {
assert.equal(options.timeoutMs, SHADOW_GATE_THRESHOLDS.readTimeoutMs)
assert.ok(options.timeoutMs > 0 && options.timeoutMs <= 120_000)
}
const timedOut = await evaluateShadowGate(parseShadowGateArguments(ARGV), {
retryDelayMs: 0,
runGcloud: async () => { throw Object.assign(new Error('ETIMEDOUT'), { killed: true }) }
})
assert.equal(timedOut.checks.director503.status, 'unverified')
assert.equal(timedOut.verdict, 'WARN')
})
// The reads are serialised, so the cost of a failure that makes every one of them spend its full
// retry budget scales with the window. The deadline is what turns that into a verdict rather than
// a cancelled job, which would take every later cell in the wave with it.
test('the gate stops reading at its own deadline and still reports a verdict', async () => {
const seam = gcloudSeam()
// A clock where every read costs its whole retry budget, which is the case the deadline exists
// for: an expired credential or a Logging 429 storm answers nothing, slowly, every time.
let elapsedMs = 0
const report = await evaluateShadowGate(parseShadowGateArguments(ARGV), {
...seam,
now: () => {
elapsedMs += SHADOW_GATE_THRESHOLDS.readTimeoutMs * READ_ATTEMPTS
return elapsedMs
}
})
// Everything past the deadline is skipped rather than attempted, so the gate cannot outlive it.
assert.ok(seam.calls.length > 0, 'the gate must still attempt reads inside its budget')
assert.ok(
seam.calls.length * SHADOW_GATE_THRESHOLDS.readTimeoutMs * READ_ATTEMPTS <=
SHADOW_GATE_THRESHOLDS.overallDeadlineMs,
'the gate read past its own deadline'
)
// A verdict, not a crash: a skipped read is an unverified check, which can never read as PASS.
assert.equal(report.reportOnly, true)
assert.equal(report.verdict, 'WARN')
assert.equal(report.checks.cellServing.status, 'unverified')
// No read is ever given more time than the budget still has left.
for (const { options } of seam.calls) {
assert.ok(options.timeoutMs > 0)
assert.ok(options.timeoutMs <= SHADOW_GATE_THRESHOLDS.readTimeoutMs)
}
})
test('the job runs the gate report-only, after verification, and uploads its artifact', () => {
const workflow = readRelayWorkflow('deploy-relay-production-same-cap-job.yml')
const gate = workflow.slice(workflow.indexOf('- name: Shadow health gate (report only)'))
assert.notEqual(gate, '')
// Two independent guarantees that no verdict can fail a cell: the step's own exit code and this.
assert.match(gate.slice(0, gate.indexOf('run:')), /continue-on-error: true/)
assert.match(gate, /relay-same-cap-shadow-gate\.mjs/)
// The gate and its upload must be bounded in time as well as in outcome: a step that runs past
// the job's timeout-minutes gets the job cancelled, and cancellation stops the whole wave.
const gateHeader = gate.slice(0, gate.indexOf('run:'))
assert.match(gateHeader, /timeout-minutes: (\d+)/)
const stepTimeoutMinutes = Number(/timeout-minutes: (\d+)/.exec(gateHeader)[1])
assert.equal(stepTimeoutMinutes, 8)
// The script has to settle on its own before the runner kills it, or the artifact is never
// written and the step reports nothing at all.
assert.ok(
SHADOW_GATE_THRESHOLDS.overallDeadlineMs < stepTimeoutMinutes * 60_000,
'the gate deadline must leave the step time to write its verdict'
)
const upload = workflow.slice(workflow.indexOf('- name: Publish the shadow health gate verdict'))
assert.match(upload.slice(0, upload.indexOf('uses:')), /timeout-minutes: 2/)
assert.match(
workflow,
/name: relay-same-cap-shadow-gate-\$\{\{ inputs\.target-cell-id \}\}-\$\{\{ github\.run_id \}\}\.json/
)
// The gate is judged over the wave it just ran, so the job has to stamp its own steps, and the
// stamps reach the script through the environment rather than being expanded into its shell.
for (const [step, output] of [
['drain', 'drain-started-at'],
['apply', 'apply-started-at'],
['apply', 'apply-completed-at'],
['verify-target', 'verify-ended-at']
]) {
// Scoped to the step that owns the stamp: a stamp written anywhere else in the job would
// still satisfy a whole-file match while recording the wrong instant.
assert.match(
stepBody(workflow, STAMP_STEPS[step]),
new RegExp(`${output}=\\$\\(date -u \\+%FT%TZ\\)`),
`${output} must be stamped inside the ${step} step`
)
assert.match(gate, new RegExp(`\\$\\{\\{ steps\\.${step}\\.outputs\\.${output} \\}\\}`))
assert.match(gate, new RegExp(`--${output} "\\$\\{[A-Z_]+\\}"`))
}
// The apply-start stamp has to precede the operation that can restart the instance, or the
// listener it bounds the search by has already happened. Presence is asserted before order,
// because indexOf answers -1 for an absent stamp and -1 precedes everything.
const applyStep = stepBody(workflow, STAMP_STEPS.apply)
const stampedAt = applyStep.indexOf('apply-started-at=')
const appliedAt = applyStep.indexOf('terraform -chdir=infra/terraform apply')
assert.notEqual(stampedAt, -1, 'the apply step does not stamp apply-started-at at all')
assert.notEqual(appliedAt, -1, 'the apply step no longer runs terraform apply')
assert.ok(stampedAt < appliedAt, 'apply-started-at must be stamped before terraform apply')
// Verification has to have happened first, or the gate judges a cell nothing checked, and the
// restore too, so reading logs never holds the cell out of admission for longer than today.
for (const earlier of [
'- name: Verify new incarnation, exact image, protocol, and durable safety',
'- name: Restore only the verified selected cell to its entry admission'
]) {
assert.ok(
workflow.indexOf(earlier) < workflow.indexOf('- name: Shadow health gate (report only)'),
earlier
)
}
})
cloud/dev/scripts/validate-relay-asia-topology-plan.mjs
+10 -2
View File
@@ -2,6 +2,14 @@ import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
const REGION = 'asia-east2'
// Mirrors local.relay_gce_topology in infra/terraform/relay-gce-cells.tf, which Terraform
// cannot export to JS; the census test below the validator equates the two by reading the
// .tf source, so this pair and the topology `check` assert cannot drift apart.
export const RELAY_CELL_BACKEND_TIMEOUT_SECONDS = 86_400
export const RELAY_CELL_CONNECTION_DRAIN_SECONDS = 60
// Not a topology local: the default of var.relay_gce_cell_log_sample_rate, which no
// environment overrides. The same census test equates it with variables.tf.
export const RELAY_CELL_LOG_SAMPLE_RATE = 1
const CELL_SHAPES = {
production: {
domain: 'relay.onorca.dev',
@@ -117,8 +125,8 @@ function requireCellBackend(change, config, cellId) {
const hostname = cellId.split('-').at(-1)
const name = `${relayGceName(config.environment)}-${hostname}`
if (
after?.timeout_sec !== 86_400 ||
after?.connection_draining_timeout_sec !== 300 ||
after?.timeout_sec !== RELAY_CELL_BACKEND_TIMEOUT_SECONDS ||
after?.connection_draining_timeout_sec !== RELAY_CELL_CONNECTION_DRAIN_SECONDS ||
after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' ||
after?.protocol !== 'HTTP' ||
after?.port_name !== 'relay' ||
cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs
+58 -2
View File
@@ -1,6 +1,12 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { validateRelayAsiaTopologyPlan } from './validate-relay-asia-topology-plan.mjs'
import {
RELAY_CELL_BACKEND_TIMEOUT_SECONDS,
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
RELAY_CELL_LOG_SAMPLE_RATE,
validateRelayAsiaTopologyPlan
} from './validate-relay-asia-topology-plan.mjs'
const image = `us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:${'a'.repeat(64)}`
const config = { environment: 'staging', cells: ['staging-gce-c4'], image }
@@ -49,7 +55,8 @@ const resources = [
update_policy: [{ replacement_method: 'RECREATE', max_surge_fixed: 0, max_unavailable_fixed: 1 }]
}),
create('google_compute_backend_service.relay_gce_cell["staging-gce-c4"]', {
timeout_sec: 86_400, connection_draining_timeout_sec: 300,
timeout_sec: RELAY_CELL_BACKEND_TIMEOUT_SECONDS,
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
load_balancing_scheme: 'EXTERNAL_MANAGED', protocol: 'HTTP', port_name: 'relay',
session_affinity: 'NONE',
health_checks: ['projects/p/global/healthChecks/orca-cloud-staging-relay-gce-ready'],
@@ -221,3 +228,52 @@ test('rejects incomplete NAT, backend, and URL routing shapes', () => {
/no exact backend route/
)
})
// Terraform cannot export a local to JS, so this validator restates two topology values that
// the Asia workflow applies. Read the .tf source and equate all three statements of each: the
// local, the topology `check` assert that pins it, and the constant above.
test('the reviewed backend constants match the Terraform topology they validate', () => {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const local = (name) => {
const found = new RegExp(`\\n\\s*${name}\\s*=\\s*(\\d+)\\n`).exec(terraform)
assert.notEqual(found, null, `relay_gce_topology has no ${name}`)
return Number(found[1])
}
const asserted = (name) => {
const found =
new RegExp(`local\\.relay_gce_topology\\.${name}\\s*==\\s*(\\d+)`).exec(terraform)
assert.notEqual(found, null, `the topology check does not pin ${name}`)
return Number(found[1])
}
for (const [name, constant] of [
['backend_timeout_seconds', RELAY_CELL_BACKEND_TIMEOUT_SECONDS],
['connection_drain_seconds', RELAY_CELL_CONNECTION_DRAIN_SECONDS]
]) {
assert.equal(local(name), constant, `${name} local differs from the validator constant`)
assert.equal(asserted(name), constant, `${name} check assert differs from the validator`)
}
// The sample rate is a variable, not a local, and no environment file overrides it, so the
// declared default is what every cell backend gets. Equate the default and that absence.
const variables = readFileSync(
new URL('../../infra/terraform/variables.tf', import.meta.url),
'utf8'
)
const declared =
/variable "relay_gce_cell_log_sample_rate" \{[\s\S]*?\n {2}default {5}= (\d+)\n/.exec(variables)
assert.notEqual(declared, null, 'relay_gce_cell_log_sample_rate declares no default')
assert.equal(Number(declared[1]), RELAY_CELL_LOG_SAMPLE_RATE)
assert.match(terraform, /sample_rate = var\.relay_gce_cell_log_sample_rate/)
for (const environment of ['production', 'staging']) {
assert.doesNotMatch(
readFileSync(
new URL(`../../infra/terraform/environments/${environment}.tfvars`, import.meta.url),
'utf8'
),
/relay_gce_cell_log_sample_rate/,
`${environment} overrides the reviewed log sample rate`
)
}
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+63 -1
View File
@@ -1,5 +1,14 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import {
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
RELAY_CELL_LOG_SAMPLE_RATE
} from './validate-relay-asia-topology-plan.mjs'
const CELL_BACKEND_RESOURCE = 'google_compute_backend_service.relay_gce_cell'
const CONNECTION_DRAIN_PATH = 'connection_draining_timeout_sec'
// A backend with no logging has `log_config: []`, so gaining the block moves this one path.
const CELL_LOG_CONFIG_PATH = 'log_config.0'
const SERVICE_ACCOUNT_EMAIL =
/^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com$/
@@ -292,6 +301,56 @@ function requireDesiredStartupScript(script, config) {
}
}
// The same-cap job targets this cell's backend service so the two declared-but-unapplied
// settings land one cell at a time: an unindexed root plan pulls the whole MIG and template
// resources in as dependencies, which standing image drift turns into a 29-cell roll. Each
// attribute is optional because a cell that already has it plans no change for it.
// Splitting the backend out here keeps `changes` the template-and-MIG count both callers read.
function takeCellBackendUpdate(changes, config) {
const backends = changes.filter(
({ address }) => typeof address === 'string' && address.startsWith(`${CELL_BACKEND_RESOURCE}[`)
)
if (config.mode !== 'same-cap-cell' || backends.length === 0) {
return { rest: changes, backendUpdate: [] }
}
const [backend] = backends
if (
backends.length !== 1 ||
backend.address !== `${CELL_BACKEND_RESOURCE}[${JSON.stringify(config.cellId)}]` ||
backend.deposed !== undefined ||
!sameActions(backend, ['update'])
) {
throw new Error('cell plan may change only this cell backend drain and request logging')
}
const after = backend.change?.after
const moved = changedPaths(backend.change?.before, after)
const logging = after?.log_config?.[0]
const accepted = [CONNECTION_DRAIN_PATH, CELL_LOG_CONFIG_PATH].filter((path) =>
moved.includes(path))
if (
accepted.length === 0 ||
(moved.includes(CONNECTION_DRAIN_PATH) &&
after?.[CONNECTION_DRAIN_PATH] !== RELAY_CELL_CONNECTION_DRAIN_SECONDS) ||
(moved.includes(CELL_LOG_CONFIG_PATH) &&
(after?.log_config?.length !== 1 ||
logging?.enable !== true ||
logging?.sample_rate !== RELAY_CELL_LOG_SAMPLE_RATE))
) {
throw new Error('cell plan may change only this cell backend drain and request logging')
}
const backendComputed = new Set(['fingerprint', 'generated_id'])
requireOnlyPaths(
backend,
new Set([
...accepted,
...unknownPaths(backend.change.after_unknown).filter((path) => backendComputed.has(path))
]),
accepted,
backendComputed
)
return { rest: changes.filter((change) => change !== backend), backendUpdate: accepted }
}
function plannedResources(module) {
if (!module) return []
return [
@@ -530,11 +589,13 @@ export function validateCapacityPlan(plan, config) {
config.mode === 'same-cap-image' &&
!/^.+@sha256:[a-f0-9]{64}$/.test(config.rollbackImage ?? '')
) throw new Error('same-cap image Terraform plan has an invalid rollback image')
const changes = mutations(plan)
const { rest: changes, backendUpdate } = takeCellBackendUpdate(mutations(plan), config)
const backend = backendUpdate.length > 0 ? { backendUpdate } : {}
if (changes.length === 0) {
return {
mode: config.mode,
changes: 0,
...backend,
...(config.mode === 'same-cap-image' ? { changeKind: 'none' } : {})
}
}
@@ -555,6 +616,7 @@ export function validateCapacityPlan(plan, config) {
return {
mode: config.mode,
changes: changes.length,
...backend,
...(config.mode === 'same-cap-image'
? {
changeKind: replacement
cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
+172
View File
@@ -1,5 +1,9 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
RELAY_CELL_CONNECTION_DRAIN_SECONDS,
RELAY_CELL_LOG_SAMPLE_RATE
} from './validate-relay-asia-topology-plan.mjs'
import {
parseCapacityPlanArguments,
validateCapacityPlan as validateCapacityPlanRaw
@@ -1054,3 +1058,171 @@ test('the database pool argument is accepted by same-cap-cell mode alone', () =>
/applies only to same-cap-cell validation/
)
})
test('a same-cap roll may carry only this cell backend drain and request logging', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const startup = (selectedImage) => [
" printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '1000'",
" printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'",
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup(rollbackImage) },
after: { metadata_startup_script: startup(image), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
// The exact shape a live US cell's backend plans: 300 s drain and no log_config at all.
const loggingAfter = [{
enable: true,
optional_fields: null,
optional_mode: null,
sample_rate: RELAY_CELL_LOG_SAMPLE_RATE
}]
const backendChange = ({ drain = true, logging = true }) => ({
address: 'google_compute_backend_service.relay_gce_cell["staging-gce-c3"]',
change: {
actions: ['update'],
before: {
connection_draining_timeout_sec: drain ? 300 : RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: logging ? [] : loggingAfter,
timeout_sec: 86_400,
fingerprint: 'before'
},
after: {
connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
log_config: loggingAfter,
timeout_sec: 86_400,
fingerprint: null
},
after_unknown: { fingerprint: true }
}
})
const backend = backendChange({})
const drainOnly = backendChange({ logging: false })
const loggingOnly = backendChange({ drain: false })
const sameCapConfig = {
...config,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: 'relay-director@project.iam.gserviceaccount.com',
rehomeAudience: 'https://relay.onorca.dev/v1/admin/host-drain',
regionalRehomeProtocol: '0'
}
const refused = /only this cell backend drain and request logging/
// Both attributes ride along with the roll without inflating the template-and-MIG count
// the apply's stranded branch and the resume's drift branch both read.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 2,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
// Each is independently optional: a cell that already has one plans no change for it.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, drainOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['connection_draining_timeout_sec'] }
)
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager, loggingOnly] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2, backendUpdate: ['log_config.0'] }
)
// Once both are applied the backend is simply absent from the plan.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
// A cell whose template and MIG have converged but whose backend has not is still clean.
assert.deepEqual(
validateCapacityPlan({ resource_changes: [backend] }, sameCapConfig),
{
mode: 'same-cap-cell',
changes: 0,
backendUpdate: ['connection_draining_timeout_sec', 'log_config.0']
}
)
assert.deepEqual(validateCapacityPlan({ resource_changes: [] }, sameCapConfig), {
mode: 'same-cap-cell',
changes: 0
})
const extraAttribute = structuredClone(backend)
extraAttribute.change.after.timeout_sec = 3_600
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, extraAttribute] },
sameCapConfig
),
/changes outside the reviewed capacity fields/
)
const otherCell = structuredClone(backend)
otherCell.address = 'google_compute_backend_service.relay_gce_cell["production-gce-c27"]'
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, otherCell] }, sameCapConfig),
refused
)
const unreviewedDrain = structuredClone(backend)
unreviewedDrain.change.after.connection_draining_timeout_sec =
RELAY_CELL_CONNECTION_DRAIN_SECONDS + 1
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, unreviewedDrain] },
sameCapConfig
),
refused
)
const sampledLogging = structuredClone(backend)
sampledLogging.change.after.log_config[0].sample_rate = RELAY_CELL_LOG_SAMPLE_RATE / 2
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, sampledLogging] },
sameCapConfig
),
refused
)
const disabledLogging = structuredClone(backend)
disabledLogging.change.after.log_config[0].enable = false
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, disabledLogging] },
sameCapConfig
),
refused
)
const replaced = structuredClone(backend)
replaced.change.actions = ['create', 'delete']
assert.throws(
() => validateCapacityPlan({ resource_changes: [template, manager, replaced] }, sameCapConfig),
refused
)
// Only the same-cap wave targets a backend service; every other mode still refuses one.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager, backend] },
{ ...config, mode: 'cell', image }
),
/only the exact instance template and MIG/
)
})
cloud/infra/terraform/README.md
+3 -2
View File
@@ -311,8 +311,9 @@ Every `relay_gce_cells` entry is one durable cell generation and must pin both i
image and its Artifact Registry relay image. Terraform creates one private COS instance template, one size-one zonal
MIG, and one backend service for that exact host. The MIG uses `RECREATE`, zero surge, and one
unavailable worker; `/health` alone drives autoheal while SQL/JWKS-backed `/ready` controls LB
admission. The backend timeout is 86,400 seconds with connection draining, and the URL map aborts
unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata
admission. The backend timeout is 86,400 seconds and connection draining is 60 seconds, which
covers only a host still mid-handshake because the rollout drains a cell before Terraform runs.
The URL map aborts unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata
credentials, fetches the two relay secrets without logging them, and runs a digest-pinned Cloud SQL
Auth Proxy beside the digest-pinned relay image.
cloud/infra/terraform/relay-gce-cells.tf
+6 -3
View File
@@ -16,7 +16,7 @@ locals {
backend_group_count = 1
public_access_config_count = 0
backend_timeout_seconds = 86400
connection_drain_seconds = 300
connection_drain_seconds = 60
}
relay_gce_cell_urls = {
for cell_id, cell in var.relay_gce_cells :
@@ -108,9 +108,12 @@ check "relay_gce_fixed_one_topology" {
local.relay_gce_topology.max_unavailable == 1 &&
local.relay_gce_topology.backend_group_count == 1 &&
local.relay_gce_topology.public_access_config_count == 0 &&
local.relay_gce_topology.backend_timeout_seconds == 86400
local.relay_gce_topology.backend_timeout_seconds == 86400 &&
# A rollout drains every host off the cell before Terraform runs, so this covers only a
# host still mid-handshake; pinned so a raise cannot re-add rollout wall clock unseen.
local.relay_gce_topology.connection_drain_seconds == 60
)
error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, and the 86,400-second WebSocket timeout."
error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, the 86,400-second WebSocket timeout, and the 60-second connection drain."
}
}
cloud/package.json
+1 -1
View File
@@ -21,7 +21,7 @@
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/push-validation-workflow.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/push-validation-workflow.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"typecheck": "pnpm -r typecheck"
},
"devDependencies": {
cloud/packages/relay-contract/src/idle-regional-rehome.test.ts
+51
View File
@@ -0,0 +1,51 @@
import { describe, expect, it } from 'vitest'
import {
GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS,
IDLE_REGIONAL_REHOME_DEFER_REASONS,
IdleRegionalRehomeResponseSchema,
isGlobalIdleRegionalRehomeDeferral
} from './idle-regional-rehome.js'
describe('idle regional rehome response', () => {
it('accepts a source cell that has never heard of the reason field', () => {
expect(IdleRegionalRehomeResponseSchema.parse({ v: 1, outcome: 'deferred' })).toEqual({
v: 1,
outcome: 'deferred'
})
})
it('carries every reason the source can send', () => {
for (const reason of IDLE_REGIONAL_REHOME_DEFER_REASONS) {
expect(
IdleRegionalRehomeResponseSchema.parse({ v: 1, outcome: 'deferred', reason })
).toEqual({ v: 1, outcome: 'deferred', reason })
}
})
it('reads a reason it does not know as absent instead of failing the response', () => {
expect(
IdleRegionalRehomeResponseSchema.parse({ v: 1, outcome: 'deferred', reason: 'from-a-newer-cell' })
).toEqual({ v: 1, outcome: 'deferred' })
})
it('drops a field added after this decoder shipped', () => {
expect(
IdleRegionalRehomeResponseSchema.parse({ v: 1, outcome: 'committed', movedAt: 17 })
).toEqual({ v: 1, outcome: 'committed' })
})
it('still rejects an outcome it cannot act on', () => {
expect(() => IdleRegionalRehomeResponseSchema.parse({ v: 1, outcome: 'moved' })).toThrow()
})
it('classifies only the poll-wide deferrals as global', () => {
for (const reason of IDLE_REGIONAL_REHOME_DEFER_REASONS) {
expect(isGlobalIdleRegionalRehomeDeferral(reason)).toBe(
GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS.some((global) => global === reason)
)
}
expect(isGlobalIdleRegionalRehomeDeferral(undefined)).toBe(false)
expect(GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS).toContain('concurrency-limit')
expect(isGlobalIdleRegionalRehomeDeferral('candidate-ineligible')).toBe(false)
})
})
cloud/packages/relay-contract/src/idle-regional-rehome.ts
+48 -2
View File
@@ -15,12 +15,58 @@ export const IdleRegionalRehomeRequestSchema = z
})
.strict()
// Deferrals no other candidate in the same poll can get past: the source
// re-reads the same durable row for every request, so the next POST takes the
// same branch. The director stops walking its list on one of these.
export const GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS = [
'control-closed',
'budget-closed',
'concurrency-limit',
'cohort-closed',
'fleet-safety'
] as const
export const IDLE_REGIONAL_REHOME_DEFER_REASONS = [
...GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS,
'candidate-ineligible',
'host-unsupported',
'director-safety-stale'
] as const
export const IdleRegionalRehomeResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['busy', 'committed', 'deferred', 'stale'])
outcome: z.enum(['busy', 'committed', 'deferred', 'stale']),
// Optional both ways: a source cell on an older image omits it and the
// director keeps its walk-the-whole-list behaviour, and a reason a newer
// cell adds later reads as absent instead of failing the whole response.
reason: z.enum(IDLE_REGIONAL_REHOME_DEFER_REASONS).optional().catch(undefined)
})
.strict()
// Unknown keys are dropped rather than rejected, so the next optional field
// on this response does not have to wait for every director to redeploy.
.strip()
export type IdleRegionalRehomeRequest = z.infer<typeof IdleRegionalRehomeRequestSchema>
export type IdleRegionalRehomeOutcome = z.infer<typeof IdleRegionalRehomeResponseSchema>['outcome']
export type IdleRegionalRehomeDeferReason = (typeof IDLE_REGIONAL_REHOME_DEFER_REASONS)[number]
// What the source cell answers: `busy` and `stale` come from the host session,
// the rest from the durable commit, and `reason` is set only for a deferral.
export type IdleRegionalRehomeResult = {
outcome: IdleRegionalRehomeOutcome
reason?: IdleRegionalRehomeDeferReason
}
export type IdleRegionalRehomeCommit = {
outcome: Exclude<IdleRegionalRehomeOutcome, 'busy'>
reason?: IdleRegionalRehomeDeferReason
}
export type GlobalIdleRegionalRehomeDeferReason =
(typeof GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS)[number]
export function isGlobalIdleRegionalRehomeDeferral(
reason: IdleRegionalRehomeDeferReason | undefined
): reason is GlobalIdleRegionalRehomeDeferReason {
return GLOBAL_IDLE_REGIONAL_REHOME_DEFER_REASONS.some((global) => global === reason)
}
config/build-plugins/pdfjs-viewer-assets.ts
+100
View File
@@ -0,0 +1,100 @@
import { createReadStream, cpSync, existsSync, mkdirSync, statSync } from 'node:fs'
import { createRequire } from 'node:module'
import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path'
import type { Plugin } from 'vite'
export const PDFJS_VIEWER_ASSET_DIRS = ['cmaps', 'standard_fonts', 'wasm'] as const
function isAssetDirectory(
value: string | undefined
): value is (typeof PDFJS_VIEWER_ASSET_DIRS)[number] {
return value !== undefined && PDFJS_VIEWER_ASSET_DIRS.some((directory) => directory === value)
}
function pdfjsRoot(): string {
return dirname(createRequire(import.meta.url).resolve('pdfjs-dist/package.json'))
}
function assetPath(root: string, pathname: string): string | undefined {
let decoded: string
try {
decoded = decodeURIComponent(pathname)
} catch {
return undefined
}
if (decoded.includes('\0') || decoded.includes('\\')) {
return undefined
}
const parts = decoded.split('/').filter(Boolean)
const directory = parts[0]
if (parts.length < 2 || !isAssetDirectory(directory)) {
return undefined
}
if (parts.some((part) => part === '.' || part === '..' || part.includes('\\'))) {
return undefined
}
const candidate = resolve(root, ...parts)
const base = resolve(root, directory)
const withinBase = relative(base, candidate)
if (withinBase.length === 0 || withinBase.startsWith('..') || isAbsolute(withinBase)) {
return undefined
}
return candidate
}
function copyAssets(root: string, outputDir: string): void {
for (const directory of PDFJS_VIEWER_ASSET_DIRS) {
const source = join(root, directory)
if (!existsSync(source)) {
throw new Error(`[pdfjs-viewer-assets] missing ${source}`)
}
cpSync(source, join(outputDir, directory), { recursive: true })
}
}
export function createPdfjsViewerAssetsPlugin(root = pdfjsRoot()): Plugin {
return {
name: 'pdfjs-viewer-assets',
configureServer(server) {
server.middlewares.use((request, response, next) => {
let pathname: string
try {
pathname = new URL(request.url ?? '/', 'http://localhost').pathname
} catch {
next()
return
}
const filePath = assetPath(root, pathname)
if (!filePath || (request.method !== 'GET' && request.method !== 'HEAD')) {
next()
return
}
let size: number
try {
size = statSync(filePath).size
} catch {
next()
return
}
response.statusCode = 200
response.setHeader('Content-Length', size)
response.setHeader(
'Content-Type',
extname(filePath) === '.wasm' ? 'application/wasm' : 'application/octet-stream'
)
if (request.method === 'HEAD') {
response.end()
return
}
createReadStream(filePath).pipe(response)
})
},
writeBundle(options) {
if (!options.dir) {
throw new Error('[pdfjs-viewer-assets] output directory is required')
}
mkdirSync(options.dir, { recursive: true })
copyAssets(root, options.dir)
}
}
}
config/electron-builder.config.cjs
+5 -1
View File
@@ -665,7 +665,11 @@ module.exports = {
provider: 'github',
owner: 'stablyai',
repo: devChannelRepo ?? 'orca',
releaseType: devChannelRepo ? 'prerelease' : 'release'
// Why draft on the main repo: `--publish always` otherwise creates a
// public GitHub release as soon as the first platform uploads, and
// /releases/latest serves a missing Windows exe. release-cut undrafts
// only after every required asset exists.
releaseType: devChannelRepo ? 'prerelease' : 'draft'
}
}
config/oxlint-anti-slop.json
+12 -1
View File
@@ -82,7 +82,18 @@
}
},
{
"files": ["**/browser-pane/annotate/**"],
"files": [
"**/browser-pane/annotate/**",
"**/browser-pane/ClientHostedBrowserPagePane.markup.test.tsx"
],
"rules": {
"anti-slop/no-shape-in-symbol-names": "off"
}
},
// The markup tests outside annotate/ drive its API, whose payload key is `shapes`
// (`MarkupOverlay.onComplete({ imageElement, shapes })`): the domain term, not a structure.
{
"files": ["**/browser-pane/*.markup.test.tsx"],
"rules": {
"anti-slop/no-shape-in-symbol-names": "off"
}
config/patches/@xterm__addon-image@0.10.0-beta.300.patch
+367
View File
File diff suppressed because one or more lines are too long
config/patches/@xterm__xterm@6.1.0-beta.303.patch
+212 -68
View File
File diff suppressed because one or more lines are too long
config/patches/xterm-src/@xterm__addon-image@0.10.0-beta.300.src.patch
+322
View File
@@ -0,0 +1,322 @@
diff --git a/src/IIPHandler.ts b/src/IIPHandler.ts
index 559b907416eb38318f439d060d7f89311ed34c7e..8541b4b0ea0d6b451aaae49007d69df64c5bd088 100644
--- a/src/IIPHandler.ts
+++ b/src/IIPHandler.ts
@@ -34,6 +34,7 @@ const DEFAULT_HEADER: IHeaderFields = {
export class IIPHandler implements IOscHandler, IResetHandler {
+ private _generation = 0;
private _aborted = false;
private _hp = new HeaderParser();
private _header: IHeaderFields = DEFAULT_HEADER;
@@ -55,6 +56,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
}
public reset(): void {
+ this._generation++;
this._hp.reset();
this._dec.release();
this._qoiDec.release();
@@ -198,8 +200,13 @@ export class IIPHandler implements IOscHandler, IResetHandler {
blob = new Blob([this._dec.data8], { type: metrics.mime });
}
this._dec.release();
+ const generation = this._generation;
return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
.then(bm => {
+ if (generation !== this._generation) {
+ bm.close();
+ return true;
+ }
this._storage.addImage(bm);
return true;
})
diff --git a/src/ImageAddon.ts b/src/ImageAddon.ts
index 8fd39543118cd420e36c1614c1af370b6c7bbfbb..0c44d2a81642113417bf8dc10a4faa76d7cc5864 100644
--- a/src/ImageAddon.ts
+++ b/src/ImageAddon.ts
@@ -113,6 +113,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
}
public dispose(): void {
+ for (const handler of this._handlers.values()) handler.reset();
for (const obj of this._disposables) {
obj.dispose();
}
diff --git a/src/ImageRenderer.ts b/src/ImageRenderer.ts
index 5854efaec1fdf9dfcb886023542998a563b6d2f2..3afaf9bd63ffd7a4cdf32bf0ac24cf33a8aa814f 100644
--- a/src/ImageRenderer.ts
+++ b/src/ImageRenderer.ts
@@ -186,16 +186,17 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
const dx = col * width;
const dy = row * height;
// safari bug: never access image source out of bounds
- const finalWidth = count * width + sx > img.width ? img.width - sx : count * width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
+ const finalWidth = count * sourceWidth + sx > img.width ? img.width - sx : count * sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
// Floor all pixel offsets to get stable tile mapping without any overflows.
// Note: For not pixel perfect aligned cells like in the DOM renderer
@@ -204,7 +205,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
- Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight)
+ Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight)
);
}
@@ -219,19 +220,20 @@ export class ImageRenderer extends Disposable implements IDisposable {
}
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
- const finalWidth = width + sx > img.width ? img.width - sx : width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
-
- const canvas = ImageRenderer.createCanvas(this.document, finalWidth, finalHeight);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
+ const finalWidth = sourceWidth + sx > img.width ? img.width - sx : sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
+
+ const canvas = ImageRenderer.createCanvas(this.document, Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight));
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight),
- 0, 0, Math.floor(finalWidth), Math.floor(finalHeight)
+ 0, 0, canvas.width, canvas.height
);
return canvas;
}
@@ -299,11 +301,16 @@ export class ImageRenderer extends Disposable implements IDisposable {
spec.actualCellSize.height = originalHeight;
return;
}
- const canvas = ImageRenderer.createCanvas(
- this.document,
- Math.ceil(spec.orig!.width * currentWidth / originalWidth),
- Math.ceil(spec.orig!.height * currentHeight / originalHeight)
- );
+ const scaledWidth = Math.ceil(spec.orig!.width * currentWidth / originalWidth);
+ const scaledHeight = Math.ceil(spec.orig!.height * currentHeight / originalHeight);
+ // Upscale visible tiles directly; a full zoomed copy can dwarf the image budget.
+ if (scaledWidth * scaledHeight > spec.orig!.width * spec.orig!.height) {
+ spec.actual = spec.orig;
+ spec.actualCellSize.width = originalWidth;
+ spec.actualCellSize.height = originalHeight;
+ return;
+ }
+ const canvas = ImageRenderer.createCanvas(this.document, scaledWidth, scaledHeight);
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height);
@@ -415,7 +422,11 @@ export class ImageRenderer extends Disposable implements IDisposable {
for (let i = 0; i < width; i += bWidth) {
ctx2.drawImage(blueprint, i, 0);
}
- ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap);
+ const placeholder = this._placeholder;
+ ImageRenderer.createImageBitmap(placeholder).then(bitmap => {
+ if (this._placeholder !== placeholder) bitmap?.close();
+ else this._placeholderBitmap = bitmap;
+ }).catch(() => {});
}
public get document(): Document | undefined {
diff --git a/src/kitty/KittyGraphicsHandler.ts b/src/kitty/KittyGraphicsHandler.ts
index de889dfff75d9ecc8ab47a025e6989ffe75bb202..54ebea9c061e5bb92b187cab7a53bc1fa320c4f8 100644
--- a/src/kitty/KittyGraphicsHandler.ts
+++ b/src/kitty/KittyGraphicsHandler.ts
@@ -7,6 +7,7 @@ import { IDisposable } from '@xterm/xterm';
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { CELL_SIZE_DEFAULT } from '../ImageStorage';
+import { imageType } from '../IIPMetrics';
import { KittyImageStorage } from './KittyImageStorage';
import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
import {
@@ -37,6 +38,7 @@ const DECODER_OK = Constants.DECODER_OK as unknown as DecodeStatus.OK;
// Kitty graphics protocol handler with streaming base64 decoding.
export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDisposable {
private _aborted = false;
+ private _generation = 0;
private _decodeError = false;
private _activeDecoder: Base64Decoder | null = null;
@@ -80,6 +82,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
public reset(): void {
+ this._generation++;
this._cleanupAllPending();
if (this._activeDecoder) {
this._activeDecoder.release();
@@ -200,6 +203,25 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
this._activeDecoder = pending.decoder;
}
if (!this._activeDecoder) {
+ // Budget WASM capacity, including one page of decoder state and rounding.
+ const decoderCapacity = this._maxEncodedBytes + 131072;
+ if (decoderCapacity > this._opts.storageLimit * 1000000) {
+ this._aborted = true;
+ if (this._parsedCommand?.id !== undefined) {
+ this._sendResponse(this._parsedCommand.id, 'ENOMEM:pending image budget exceeded', this._parsedCommand.quiet ?? 0);
+ }
+ return;
+ }
+ const maxPending = Math.max(1, Math.floor(this._opts.storageLimit * 1000000 / decoderCapacity));
+ while (this._pendingTransmissions.size >= maxPending) {
+ const oldest = this._pendingTransmissions.entries().next().value;
+ if (!oldest) break;
+ oldest[1].decoder.release();
+ this._removePendingEntry(oldest[0]);
+ if (oldest[1].cmd.id !== undefined) {
+ this._sendResponse(oldest[1].cmd.id, 'ENOMEM:pending image budget exceeded', oldest[1].cmd.quiet ?? 0);
+ }
+ }
this._activeDecoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
this._activeDecoder.init();
}
@@ -550,9 +572,11 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
+ const generation = this._generation;
let bitmap: ImageBitmap | undefined = await this._createBitmap(image);
try {
+ if (generation !== this._generation) throw new Error('image decode canceled');
const cropX = Math.max(0, cmd.x ?? 0);
const cropY = Math.max(0, cmd.y ?? 0);
const cropW = cmd.sourceWidth || (bitmap.width - cropX);
@@ -660,6 +684,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
}
+ if (generation !== this._generation) throw new Error('image decode canceled');
const zIndex = cmd.zIndex ?? 0;
this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex);
bitmap = undefined; // ownership transferred to storage
@@ -693,6 +718,12 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
if (image.format === KittyFormat.PNG) {
+ const metrics = imageType(bytes);
+ // IHDR dimensions are parsed with signed shifts, so a value >= 0x80000000 comes
+ // back negative and a bare `>` pixel-limit test passes it; require positive.
+ if (metrics.mime !== 'image/png' || !(metrics.width > 0) || !(metrics.height > 0) || metrics.width * metrics.height > this._opts.pixelLimit) {
+ throw new RangeError('PNG exceeds pixel limit or has invalid dimensions');
+ }
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
@@ -775,27 +806,45 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
- } catch {
+ } catch (error) {
+ if (error instanceof RangeError) throw error;
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
- const ds = new DecompressionStream(format);
- const writer = ds.writable.getWriter();
- writer.write(compressed as BufferSource);
- writer.close();
-
+ const limit = Math.min(this._opts.kittySizeLimit, this._opts.pixelLimit * 4, this._opts.storageLimit * 1000000);
+ let offsetIn = 0;
+ // Bound inflation within one native transform before its output is budgeted.
+ const source = new ReadableStream<BufferSource>({
+ pull(controller) {
+ if (offsetIn >= compressed.length) {
+ controller.close();
+ return;
+ }
+ const end = Math.min(offsetIn + 4096, compressed.length);
+ controller.enqueue(new Uint8Array(compressed.subarray(offsetIn, end)));
+ offsetIn = end;
+ }
+ });
+ const reader = source.pipeThrough(new DecompressionStream(format)).getReader();
const chunks: Uint8Array[] = [];
- const reader = ds.readable.getReader();
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- chunks.push(value);
+ let totalLength = 0;
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ totalLength += value.byteLength;
+ if (totalLength > limit) {
+ await reader.cancel().catch(() => {});
+ throw new RangeError('decompressed image exceeds byte limit');
+ }
+ chunks.push(value);
+ }
+ } finally {
+ reader.releaseLock();
}
- const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
diff --git a/src/kitty/KittyImageStorage.ts b/src/kitty/KittyImageStorage.ts
index 1f5c09ec9e2700f8f6dbd8436a1802217dfc99ef..016943a77c7e8e27da5899d54cd48d82761a87c8 100644
--- a/src/kitty/KittyImageStorage.ts
+++ b/src/kitty/KittyImageStorage.ts
@@ -83,6 +83,25 @@ export class KittyImageStorage implements IDisposable {
this._evictUndisplayedImages();
}
+ // Encoded images awaiting placement are outside ImageStorage's pixel budget.
+ // Unplaced payloads are evicted first so a new upload cannot erase a visible
+ // image while abandoned blobs still hold budget; placed ones go only when
+ // that is not enough, because the byte cap is a hard bound. The new image is
+ // always stored, so an oversized one overshoots by at most one payload
+ // (itself bounded by kittySizeLimit) rather than being dropped after an OK ack.
+ const byteLimit = this._storage.getLimit() * 1000000;
+ this._images.delete(imageId);
+ let retainedBytes = 0;
+ for (const image of this._images.values()) retainedBytes += image.data.size;
+ for (const evictPlaced of [false, true]) {
+ for (const [oldestId, image] of this._images) {
+ if (retainedBytes + imageData.data.size <= byteLimit) break;
+ if (this._kittyIdToStorageId.has(oldestId) !== evictPlaced) continue;
+ retainedBytes -= image.data.size;
+ this.deleteById(oldestId);
+ }
+ }
+
this._images.set(imageId, {
...imageData,
id: imageId
config/patches/xterm-src/@xterm__xterm@6.1.0-beta.303.src.patch
+194 -50
View File
@@ -88,21 +88,28 @@ index 497afcf535f3eaca00889525a77e15eb633ccd96..96d499b34605f860608382114c3fbdc0
export interface IBrowser {
diff --git a/src/browser/input/CompositionHelper.ts b/src/browser/input/CompositionHelper.ts
index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe0c516b06 100644
index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..4d62369d0005f465318ec93e221fd3ee3acfddfa 100644
--- a/src/browser/input/CompositionHelper.ts
+++ b/src/browser/input/CompositionHelper.ts
@@ -3,8 +3,9 @@
@@ -1,10 +1,15 @@
+/// <reference lib="es2022.intl" />
/**
* Copyright (c) 2016 The xterm.js authors. All rights reserved.
* @license MIT
*/
-import { IRenderService } from '../services/Services';
-import { IBufferService, ICoreService, IOptionsService } from '../../common/services/Services';
+import { IRenderService, IThemeService } from '../services/Services';
import { IBufferService, ICoreService, IOptionsService } from '../../common/services/Services';
+import { IBufferService, ICoreService, IOptionsService, IUnicodeService } from '../../common/services/Services';
+import { WidthCacheFontVariantCanvas } from '../renderer/dom/WidthCache';
+import { addDisposableListener } from '../Dom';
+import { Disposable } from '../../common/Lifecycle';
+import { color } from '../../common/Color';
import { C0 } from '../../common/data/EscapeSequences';
interface IPosition {
@@ -12,6 +13,27 @@ interface IPosition {
@@ -12,6 +17,27 @@ interface IPosition {
end: number;
}
@@ -130,7 +137,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
/**
* Encapsulates the logic for handling compositionstart, compositionupdate and compositionend
* events, displaying the in-progress composition to the UI and forwarding the final composition
@@ -24,6 +46,15 @@ export class CompositionHelper {
@@ -24,6 +50,15 @@ export class CompositionHelper {
*/
private _isComposing: boolean;
public get isComposing(): boolean { return this._isComposing; }
@@ -146,7 +153,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
/**
* The position within the input textarea's value of the current composition.
@@ -36,52 +67,144 @@ export class CompositionHelper {
@@ -36,52 +71,144 @@ export class CompositionHelper {
*/
private _compositionSuffix: string;
@@ -204,6 +211,8 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+
+ private _compositionEndTimer?: ReturnType<typeof setTimeout>;
+
+ private _preeditRenderer?: CompositionPreeditRenderer;
+
+ /** The preedit's own span, used to anchor the native candidate window. */
+ private _compositionPreedit?: HTMLElement;
+
@@ -213,9 +222,6 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ /** The insertion caret painted above the renderer cursor the composition view covers. */
+ private _compositionCaret?: HTMLElement;
+
+ /** The last preedit rendered, so a row repaint can re-render without a composition event. */
+ private _compositionViewData?: string;
+
+ // Keep the six-argument form callable: upstream's own CompositionHelper.test.ts constructs
+ // this class directly, and `.npmignore` strips src/**/*.test.ts from the published tarball, so
+ // the shipped patch has no hunk that could update that call. Dropping this overload fails the
@@ -237,7 +243,8 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
@ICoreService private readonly _coreService: ICoreService,
- @IRenderService private readonly _renderService: IRenderService
+ @IRenderService private readonly _renderService: IRenderService,
+ @IThemeService private readonly _themeService?: IThemeService
+ @IThemeService private readonly _themeService?: IThemeService,
+ @IUnicodeService private readonly _unicodeService?: IUnicodeService
) {
this._isComposing = false;
- this._isSendingComposition = false;
@@ -301,7 +308,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
}
/**
@@ -89,22 +212,93 @@ export class CompositionHelper {
@@ -89,22 +216,95 @@ export class CompositionHelper {
* @param ev The event.
*/
public compositionupdate(ev: Pick<CompositionEvent, 'data'>): void {
@@ -401,10 +408,12 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ this._compositionTransactionId++;
+ this._compositionView.classList.remove('active');
+ this._resetCompositionView();
+ this._preeditRenderer?.dispose();
+ this._preeditRenderer = undefined;
}
/**
@@ -113,7 +307,19 @@ export class CompositionHelper {
@@ -113,7 +313,19 @@ export class CompositionHelper {
* @returns Whether the Terminal should continue processing the keydown event.
*/
public keydown(ev: KeyboardEvent): boolean {
@@ -425,7 +434,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
if (ev.keyCode === 20 || ev.keyCode === 229) {
// 20 is CapsLock, 229 is Enter
// Continue composing if the keyCode is the "composition character"
@@ -128,6 +334,10 @@ export class CompositionHelper {
@@ -128,6 +340,10 @@ export class CompositionHelper {
this._finalizeComposition(false);
}
@@ -436,7 +445,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
if (ev.keyCode === 229) {
// If the "composition character" is used but gets to this point it means a non-composition
// character (eg. numbers and punctuation) was pressed when the IME was active.
@@ -138,6 +348,74 @@ export class CompositionHelper {
@@ -138,6 +354,74 @@ export class CompositionHelper {
return true;
}
@@ -511,7 +520,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
/**
* Finalizes the composition, resuming regular input actions. This is called when a composition
* is ending.
@@ -146,23 +424,52 @@ export class CompositionHelper {
@@ -146,23 +430,52 @@ export class CompositionHelper {
* compositionend event is triggered, such as enter, so that the composition is sent before
* the command is executed.
*/
@@ -575,7 +584,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
// Since composition* events happen before the changes take place in the textarea on most
// browsers, use a setTimeout with 0ms time to allow the native compositionend event to
@@ -172,37 +479,315 @@ export class CompositionHelper {
@@ -172,37 +485,315 @@ export class CompositionHelper {
// - The last compositionupdate event's data property does not always accurately describe
// the character, a counter example being Korean where an ending consonsant can move to
// the following character if the following input is a vowel.
@@ -609,15 +618,14 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ pending.finalizerTimer = undefined;
+ if (this._compositionTransactionId === pending.transactionId) {
+ this._isAwaitingCompositionEnd = false;
}
- }, 0);
+ }
+ if (this._pendingComposition === pending) {
+ this._sendPendingComposition(pending, true);
+ }
+ });
}
}
+ }
+ }
+
+ private _sendPendingComposition(
+ pending: IPendingComposition,
+ includeFollowingInput: boolean = false
@@ -805,7 +813,8 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ id: pending.transactionId,
+ data: input,
+ dataPendingReconciliation: true
+ }
}
- }, 0);
+ }
+ ));
+ }
@@ -813,9 +822,9 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ private _dispatchCompositionSessionEvent(event: CustomEvent): void {
+ if (typeof this._textarea.dispatchEvent === 'function') {
+ this._textarea.dispatchEvent(event);
+ }
+ }
+
}
}
+ private _dispatchCompositionTransactionSettled(): void {
+ this._dispatchCompositionSessionEvent(new CustomEvent(
+ 'xterm-composition-transaction-settled',
@@ -918,7 +927,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
/**
* Apply any changes made to the textarea after the current event chain is allowed to complete.
* This should be called when not currently composing but a keydown event with the "composition
@@ -222,6 +807,9 @@ export class CompositionHelper {
@@ -222,6 +813,9 @@ export class CompositionHelper {
const diff = newValue.replace(oldValue, '');
@@ -928,7 +937,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
this._dataAlreadySent = diff;
if (newValue.length > oldValue.length) {
@@ -236,6 +824,101 @@ export class CompositionHelper {
@@ -236,6 +830,105 @@ export class CompositionHelper {
}, 0);
}
@@ -938,42 +947,46 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ * box hiding the character under the cursor. Nothing reaches the pty while composing, so those
+ * cells still hold their characters; only what the overlay shows changes.
+ */
+ private _renderCompositionView(data: string, rowRemainder = this._getRowRemainderText()): void {
+ private _renderCompositionView(data: string): void {
+ if (!data) {
+ this._resetCompositionView();
+ return;
+ }
+ // Keep DOM order LTR so the insertion caret follows the preedit.
+ const preeditText = `‎${data}‎`;
+ this._compositionViewData = data;
+ const doc = this._compositionView.ownerDocument;
+ const preedit = doc.createElement('span');
+ preedit.className = 'xterm-composition-preedit';
+ // Underlined so the composing text stays distinguishable from the tail it pushed right.
+ preedit.style.flexShrink = '0';
+ preedit.style.textDecoration = 'underline';
+ preedit.textContent = preeditText;
+ preedit.textContent = `‎${data}‎`;
+ const caret = doc.createElement('span');
+ caret.className = 'xterm-composition-caret';
+ caret.setAttribute('aria-hidden', 'true');
+ const children = [preedit, caret];
+ let remainder: HTMLElement | undefined;
+ if (rowRemainder) {
+ remainder = doc.createElement('span');
+ remainder.className = 'xterm-composition-remainder';
+ // Why: the view is nowrap, which collapses runs of spaces, so committed padding would draw
+ // its trailing glyph cells to the left of where the grid has them.
+ remainder.style.whiteSpace = 'pre';
+ remainder.textContent = rowRemainder;
+ children.push(remainder);
+ }
+ this._compositionView.replaceChildren(...children);
+ this._compositionView.replaceChildren(preedit, caret);
+ this._compositionPreedit = preedit;
+ this._compositionCaret = caret;
+ this._compositionRemainder = remainder;
+ this._compositionRemainder = undefined;
+ this._renderCompositionRemainder(this._getRowRemainderText());
+ this._styleCompositionCaret();
+ }
+
+ private _renderCompositionRemainder(data: string): void {
+ if (!data) {
+ this._compositionRemainder?.remove();
+ this._compositionRemainder = undefined;
+ return;
+ }
+ if (!this._compositionRemainder) {
+ const remainder = this._compositionView.ownerDocument.createElement('span');
+ remainder.className = 'xterm-composition-remainder';
+ // Preserve padding cells before the trailing glyphs.
+ remainder.style.whiteSpace = 'pre';
+ this._compositionView.appendChild(remainder);
+ this._compositionRemainder = remainder;
+ }
+ this._compositionRemainder.textContent = data;
+ }
+
+ /** The committed row text from the cursor rightwards — what a mid-line preedit would cover. */
+ private _getRowRemainderText(): string {
+ const buffer = this._bufferService.buffer;
@@ -1009,11 +1022,11 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ }
+
+ private _resetCompositionView(): void {
+ this._preeditRenderer?.clear();
+ this._compositionView.textContent = '';
+ this._compositionPreedit = undefined;
+ this._compositionRemainder = undefined;
+ this._compositionCaret = undefined;
+ this._compositionViewData = '';
+ this._compositionView.style.display = '';
+ this._compositionView.style.justifyContent = '';
+ }
@@ -1030,7 +1043,7 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
/**
* Positions the composition view on top of the cursor and the textarea just below it (so the
* IME helper dialog is positioned correctly).
@@ -243,10 +926,23 @@ export class CompositionHelper {
@@ -243,10 +936,23 @@ export class CompositionHelper {
* necessary as the IME events across browsers are not consistently triggered.
*/
public updateCompositionElements(dontRecurse?: boolean): void {
@@ -1045,17 +1058,35 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
+ // compare adds no layout read.
+ const rowRemainder = this._getRowRemainderText();
+ if (
+ this._compositionViewData &&
+ this._compositionPreedit &&
+ rowRemainder !== (this._compositionRemainder?.textContent ?? '')
+ ) {
+ this._renderCompositionView(this._compositionViewData, rowRemainder);
+ this._renderCompositionRemainder(rowRemainder);
+ }
+ this._styleCompositionCaret();
+
if (this._bufferService.buffer.isCursorInViewport) {
const cursorX = Math.min(this._bufferService.buffer.x, this._bufferService.cols - 1);
@@ -265,20 +961,38 @@ export class CompositionHelper {
@@ -254,31 +960,156 @@ export class CompositionHelper {
const cursorTop = this._bufferService.buffer.y * this._renderService.dimensions.css.cell.height;
const cursorLeft = cursorX * this._renderService.dimensions.css.cell.width;
+ this._compositionView.style.setProperty('--xterm-composition-cell-width', this._renderService.dimensions.css.cell.width + 'px');
this._compositionView.style.left = cursorLeft + 'px';
this._compositionView.style.top = cursorTop + 'px';
this._compositionView.style.height = cellHeight + 'px';
this._compositionView.style.lineHeight = cellHeight + 'px';
this._compositionView.style.fontFamily = this._optionsService.rawOptions.fontFamily;
this._compositionView.style.fontSize = this._optionsService.rawOptions.fontSize + 'px';
+ if (this._compositionPreedit && this._unicodeService && typeof Intl.Segmenter !== 'undefined') {
+ this._preeditRenderer ??= new CompositionPreeditRenderer(
+ this._unicodeService, this._compositionView.ownerDocument, () => this.updateCompositionElements(true)
+ );
+ this._preeditRenderer.render(this._compositionPreedit);
+ }
// Limit the composition view width to the space between the cursor and
// the terminal's right edge, preventing it from overflowing the terminal.
const maxWidth = this._bufferService.cols * this._renderService.dimensions.css.cell.width - cursorLeft;
this._compositionView.style.maxWidth = maxWidth + 'px';
this._compositionView.style.overflow = 'hidden';
@@ -1100,9 +1131,122 @@ index c9ec396ab66cb966d49aa63bed09cdf9cd6c4246..d4d5106d8dd723eceed87310c193ccfe
- setTimeout(() => this.updateCompositionElements(true), 0);
+ this._cancelDeferredTimer(this._compositionViewTimer);
+ this._compositionViewTimer = this._defer(() => this.updateCompositionElements(true));
+ }
+ }
+}
+
+const CJK = /^[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\uac00-\ud7a3\u3131-\u318e\uff01-\uff60]$/u;
+const NATIVE = /[\p{Mark}\p{Extended_Pictographic}]/u;
+const MAX_PREEDIT_LAYOUT_WORK = 128;
+
+class CompositionPreeditRenderer extends Disposable {
+ private readonly _segmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+ private _measure?: WidthCacheFontVariantCanvas;
+ private _preedit?: HTMLElement;
+ private _font = '';
+
+ constructor(private readonly _unicodeService: IUnicodeService, doc: Document, refresh: () => void) {
+ super();
+ if (doc.fonts) {
+ this._register(addDisposableListener(doc.fonts, 'loadingdone', () => {
+ this._font = '';
+ if (this._preedit) {
+ refresh();
+ }
+ }));
+ }
+ }
+
+ public clear(): void {
+ this._preedit = undefined;
+ this._font = '';
+ }
+
+ public override dispose(): void {
+ this.clear();
+ this._measure = undefined;
+ super.dispose();
+ }
+
+ public render(preedit: HTMLElement): void {
+ const doc = preedit.ownerDocument;
+ const style = doc.defaultView?.getComputedStyle(preedit);
+ if (!style) {
+ return;
+ }
+ const font = [style.fontFamily, style.fontSize, style.fontWeight, style.fontStyle, doc.defaultView?.devicePixelRatio].join('|');
+ if (this._preedit === preedit && this._font === font) {
+ return;
+ }
+ this._preedit = preedit;
+ this._font = font;
+ const data = (preedit.textContent ?? '').slice(1, -1);
+ const widths = new Map<string, number>();
+ const nodes: Node[] = [doc.createTextNode('‎')];
+ let start = 0;
+ let spacing: string | undefined;
+ let work = 0;
+ const emit = (end: number): void => {
+ if (end === start) {
+ return;
+ }
+ const text = data.slice(start, end);
+ if (spacing === undefined) {
+ nodes.push(doc.createTextNode(text));
+ } else {
+ const span = doc.createElement('span');
+ span.style.letterSpacing = spacing;
+ span.style.fontKerning = 'none';
+ span.style.textDecoration = 'inherit';
+ span.textContent = text;
+ nodes.push(span);
+ work++;
+ }
+ start = end;
+ };
+ for (const { segment, index } of this._segmenter.segment(data)) {
+ // Bound cold measurements and styled runs without changing an already-corrected prefix.
+ if (work >= MAX_PREEDIT_LAYOUT_WORK) {
+ emit(index);
+ spacing = undefined;
+ break;
+ }
+ let nextSpacing: string | undefined;
+ const columns = this._unicodeService.wcwidth(segment.codePointAt(0)!);
+ if (CJK.test(segment) && !NATIVE.test(segment) && (columns === 2 || /^[\uff61-\uff9f]$/u.test(segment))) {
+ let width = widths.get(segment);
+ if (width === undefined) {
+ this._measure ??= new WidthCacheFontVariantCanvas();
+ this._measure.setFont(style.fontFamily, parseFloat(style.fontSize), parseFloat(style.fontWeight) || 400, style.fontStyle === 'italic');
+ width = this._measure.measure(segment);
+ widths.set(segment, width);
+ work++;
+ }
+ nextSpacing = `calc(var(--xterm-composition-cell-width) * ${columns} - ${width}px)`;
+ }
+ if (nextSpacing !== spacing) {
+ emit(index);
+ spacing = nextSpacing;
+ }
}
+ emit(data.length);
+ nodes.push(doc.createTextNode('‎'));
+ preedit.replaceChildren(...nodes);
}
}
diff --git a/src/browser/renderer/dom/WidthCache.ts b/src/browser/renderer/dom/WidthCache.ts
index d45938515e7a9903de0da0d61716fd42e4deea0c..17d6577e388eb2e4397338093e35373864b822a4 100644
--- a/src/browser/renderer/dom/WidthCache.ts
+++ b/src/browser/renderer/dom/WidthCache.ts
@@ -142,7 +142,7 @@ export class WidthCache implements IDisposable {
}
}
-class WidthCacheFontVariantCanvas implements IWidthCacheFontVariantCanvas {
+export class WidthCacheFontVariantCanvas implements IWidthCacheFontVariantCanvas {
private _canvas: OffscreenCanvas | HTMLCanvasElement;
private _ctx: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
diff --git a/src/common/SortedList.ts b/src/common/SortedList.ts
index 8a10076e3963e33b4a7d1e4602333eb3f4772dc9..c6dcf18b762e3c56fe22e9c2d49b8e550d96f915 100644
--- a/src/common/SortedList.ts
config/patches/xterm-upstream.json
+25
View File
@@ -111,6 +111,31 @@
"args": ["run", "esbuild-package"]
}
]
},
{
"name": "@xterm/addon-image",
"version": "0.10.0-beta.300",
"packageDir": "addons/addon-image",
"sourcePatch": "config/patches/xterm-src/@xterm__addon-image@0.10.0-beta.300.src.patch",
"patch": "config/patches/@xterm__addon-image@0.10.0-beta.300.patch",
"generatedPaths": ["lib/"],
"build": [
{
"cwd": "../..",
"command": "npm",
"args": ["run", "build"]
},
{
"cwd": ".",
"command": "npm",
"args": ["run", "package"]
},
{
"cwd": "../..",
"command": "npm",
"args": ["run", "esbuild-package"]
}
]
}
],
"forbiddenBuildScripts": {
config/scripts/assert-github-release-is-draft.mjs
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env node
import { pathToFileURL } from 'node:url'
const API_VERSION = '2022-11-28'
function githubHeaders(token) {
return {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${token}`,
'X-GitHub-Api-Version': API_VERSION
}
}
async function githubJson(fetchImpl, url, token, options = {}) {
const res = await fetchImpl(url, {
...options,
headers: {
...githubHeaders(token),
...options.headers
}
})
if (!res.ok) {
const body = await res.text().catch(() => '')
throw new Error(`GitHub request failed ${res.status} ${res.statusText}: ${body.slice(0, 300)}`)
}
return res.json()
}
export function matchingDesktopReleases(releases, tag) {
const version = tag.startsWith('v') ? tag.slice(1) : tag
return (releases ?? []).filter((release) => {
const tagName = release?.tag_name
const name = release?.name
return tagName === tag || tagName === version || name === tag || name === version
})
}
export async function restorePublishedDesktopReleasesToDraft({
repo,
tag,
token,
fetchImpl = fetch,
log = console.log
}) {
if (!repo) {
throw new Error('repo is required')
}
if (!tag) {
throw new Error('tag is required')
}
if (!token) {
throw new Error('token is required')
}
const releases = await githubJson(
fetchImpl,
`https://api.github.com/repos/${repo}/releases?per_page=100`,
token
)
if (!Array.isArray(releases)) {
throw new Error(`GitHub releases response for ${repo} was not an array`)
}
const matches = matchingDesktopReleases(releases, tag)
if (matches.length === 0) {
throw new Error(`No GitHub release named ${tag} was found after artifact upload`)
}
const restored = []
for (const release of matches) {
if (release?.draft === true) {
continue
}
if (!Number.isInteger(release.id)) {
throw new Error(`Release ${tag} is missing a GitHub release id`)
}
const patched = await githubJson(
fetchImpl,
`https://api.github.com/repos/${repo}/releases/${release.id}`,
token,
{
method: 'PATCH',
body: JSON.stringify({ draft: true, make_latest: 'false' })
}
)
log(`Restored GitHub release ${release.id} (${release.tag_name}) to draft.`)
restored.push(patched)
}
return restored
}
async function main() {
// Why env TAG: the Windows release-cut matrix uses pwsh, which does not
// expand bash-style "$TAG" in argv. The step still exports TAG.
const tag = process.argv[2] || process.env.TAG
const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN
const repo = process.env.GITHUB_REPOSITORY || 'stablyai/orca'
const restored = await restorePublishedDesktopReleasesToDraft({
repo,
tag,
token,
log: (message) => console.error(message)
})
if (restored.length > 0) {
console.error(
`::error::Release ${tag} was published during artifact upload. Restored ${restored.length} release(s) to draft so /releases/latest does not serve partial assets.`
)
process.exit(1)
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
console.error(error.message)
process.exit(1)
})
}
config/scripts/assert-github-release-is-draft.test.mjs
+165
View File
@@ -0,0 +1,165 @@
import { readFileSync } from 'node:fs'
import { createRequire } from 'node:module'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
import { parse } from 'yaml'
import {
matchingDesktopReleases,
restorePublishedDesktopReleasesToDraft
} from './assert-github-release-is-draft.mjs'
const require = createRequire(import.meta.url)
const repoRoot = join(import.meta.dirname, '../..')
function jsonResponse(body, init = {}) {
return {
ok: init.ok ?? true,
status: init.status ?? 200,
statusText: init.statusText ?? 'OK',
json: vi.fn(async () => body),
text: vi.fn(async () => JSON.stringify(body))
}
}
describe('matchingDesktopReleases', () => {
it('matches tagged, untagged-name, and version-name releases', () => {
const releases = [
{ id: 1, tag_name: 'v1.4.206', name: 'v1.4.206', draft: true },
{ id: 2, tag_name: 'untagged-abc', name: '1.4.206', draft: false },
{ id: 3, tag_name: 'v1.4.205', name: 'v1.4.205', draft: false }
]
expect(matchingDesktopReleases(releases, 'v1.4.206').map((release) => release.id)).toEqual([
1, 2
])
})
})
describe('restorePublishedDesktopReleasesToDraft', () => {
it('leaves drafts alone', async () => {
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(
jsonResponse([{ id: 1, tag_name: 'v1.4.206', name: 'v1.4.206', draft: true }])
)
await expect(
restorePublishedDesktopReleasesToDraft({
repo: 'stablyai/orca',
tag: 'v1.4.206',
token: 'token',
fetchImpl,
log: vi.fn()
})
).resolves.toEqual([])
expect(fetchImpl).toHaveBeenCalledTimes(1)
})
it('re-drafts a published match immediately', async () => {
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(
jsonResponse([{ id: 9, tag_name: 'v1.4.206', name: '1.4.206', draft: false }])
)
.mockResolvedValueOnce(jsonResponse({ id: 9, tag_name: 'v1.4.206', draft: true }))
const log = vi.fn()
await expect(
restorePublishedDesktopReleasesToDraft({
repo: 'stablyai/orca',
tag: 'v1.4.206',
token: 'token',
fetchImpl,
log
})
).resolves.toEqual([{ id: 9, tag_name: 'v1.4.206', draft: true }])
expect(fetchImpl).toHaveBeenNthCalledWith(
2,
'https://api.github.com/repos/stablyai/orca/releases/9',
expect.objectContaining({
method: 'PATCH',
body: JSON.stringify({ draft: true, make_latest: 'false' })
})
)
expect(log).toHaveBeenCalledWith('Restored GitHub release 9 (v1.4.206) to draft.')
})
it('fails closed when no matching release exists', async () => {
const fetchImpl = vi.fn().mockResolvedValueOnce(jsonResponse([]))
await expect(
restorePublishedDesktopReleasesToDraft({
repo: 'stablyai/orca',
tag: 'v1.4.206',
token: 'token',
fetchImpl
})
).rejects.toThrow('No GitHub release named v1.4.206 was found after artifact upload')
})
})
describe('release draft workflow contract', () => {
it('keeps GitHub releases draft until publish-release undrafts complete assets', () => {
const releaseWorkflow = parse(
readFileSync(join(repoRoot, '.github/workflows/release-cut.yml'), 'utf8')
)
const macWorkflow = parse(
readFileSync(join(repoRoot, '.github/workflows/release-mac-build.yml'), 'utf8')
)
const electronBuilderConfig = require('../electron-builder.config.cjs')
const cutCheckout = releaseWorkflow.jobs.cut.steps.find((step) => step.name === 'Checkout ref')
const linuxDraftStep = releaseWorkflow.jobs.build.steps.find(
(step) => step.name === 'Verify release remains draft after artifact upload'
)
const publishRelease = releaseWorkflow.jobs['publish-release'].steps.find(
(step) => step.name === 'Publish release'
)
const macSteps = macWorkflow.jobs['build-mac'].steps
const abortParentStep = macSteps.find(
(step) => step.name === 'Abort if the parent release-cut run was cancelled'
)
const macPublishStep = macSteps.find(
(step) => step.name === 'Publish release artifacts (macOS)'
)
const macDraftStep = macSteps.find(
(step) => step.name === 'Verify release remains draft after artifact upload'
)
expect(electronBuilderConfig.publish.releaseType).toBe('draft')
expect(cutCheckout.with['fetch-tags']).toBe(true)
expect(linuxDraftStep.shell).toBe('bash')
expect(linuxDraftStep.run).toContain('assert-github-release-is-draft.mjs')
expect(linuxDraftStep.run).toContain('needs.cut.outputs.tag')
expect(publishRelease.run).toContain('gh release edit')
expect(publishRelease.run).toContain('--draft=false')
expect(macSteps.indexOf(abortParentStep)).toBeLessThan(macSteps.indexOf(macPublishStep))
expect(abortParentStep.env.PARENT_RUN).toBe('${{ inputs.release_run_id }}')
expect(abortParentStep.run).toContain('refusing to publish mac artifacts')
expect(macDraftStep.shell).toBe('bash')
expect(macDraftStep.run).toContain('assert-github-release-is-draft.mjs')
expect(macDraftStep.run).toContain('inputs.tag')
expect(macPublishStep.with.command).toContain('-c.publish.releaseType=draft')
const linuxCommands = releaseWorkflow.jobs.build.strategy.matrix.include
.filter((entry) => String(entry.platform).startsWith('linux'))
.map((entry) => entry.release_command)
expect(linuxCommands.length).toBe(2)
for (const command of linuxCommands) {
expect(command).toContain('-c.publish.releaseType=draft')
}
const createRestore = releaseWorkflow.jobs['create-release'].steps.find(
(step) => step.name === 'Restore draft-release scripts from the workflow ref'
)
const buildRestore = releaseWorkflow.jobs.build.steps.find(
(step) => step.name === 'Restore draft-publish scripts from the workflow ref'
)
const macRestore = macSteps.find(
(step) => step.name === 'Restore draft-publish scripts from the workflow ref'
)
expect(createRestore.run).toContain('create-draft-release.mjs')
expect(buildRestore.run).toContain('assert-github-release-is-draft.mjs')
expect(macRestore.run).toContain('assert-github-release-is-draft.mjs')
})
})
config/scripts/build-mobile-web-app-bundle.mjs
+560
View File
@@ -0,0 +1,560 @@
import { readFile } from 'node:fs/promises'
import { realpathSync } from 'node:fs'
import { basename, extname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import * as esbuild from 'esbuild'
import {
MOBILE_WEB_BUNDLE_ENTRYPOINT,
hashedAsset,
isDirectInvocation,
readDesktopVersion,
readProtocolWindow,
sha256Hex,
writeMobileWebBundleTree,
contentTypeForExtension
} from './build-mobile-web-bundle.mjs'
import {
ROUTE_SOURCE_LOADERS,
assertRoutesCarryNoSynchronousExports,
collectMobileWebAppRoutes,
renderMobileWebAppRouteManifest,
routePathnameFromKey
} from './mobile-web-app-route-manifest.mjs'
import { MOBILE_WEB_PAGE_ROUTES } from './mobile-web-page-routes.mjs'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const mobileDir = join(projectDir, 'mobile')
const defaultAppDir = join(mobileDir, 'app')
const entryPoint = join(mobileDir, 'web-entry', 'index.tsx')
const defaultOutDir = join(projectDir, 'out', 'mobile-web-app')
/**
* Every shim the app bundle needs, each one a documented Metro/RN-Web gap. `appliesTo` reads the
* esbuild option that implements the shim, so the list cannot claim a shim the build does not
* apply and a dropped option fails the named shim rather than the whole build.
*/
export const MOBILE_WEB_APP_SHIMS = [
{
// react-native has no browser build; react-native-web is the whole point of Route A.
name: 'react-native-web-alias',
appliesTo: (options) => options.alias?.['react-native'] === 'react-native-web'
},
{
// RN ships untranspiled JSX inside .js files (expo-router's own build/ included).
name: 'js-as-jsx',
appliesTo: (options) => options.loader?.['.js'] === 'jsx'
},
{
// RN code assumes a Hermes/Metro `global`; the browser only has `globalThis`.
name: 'global-as-globalthis',
appliesTo: (options) => options.define?.global === 'globalThis'
},
{
// RN and Expo modules read process.env at module scope, before any of our code runs.
name: 'process-banner',
appliesTo: (options) => options.banner?.js?.includes('globalThis.process ??=') === true
},
{
// Zod probes for a usable JIT with `new Function('')`, which the shell's CSP reports even
// though Zod catches the throw and runs interpreted. Turned off before any module, because a
// schema constructed at module scope reaches the probe before our own code can run.
name: 'zod-jitless-banner',
appliesTo: (options) => options.banner?.js?.includes('__zod_globalConfig') === true
},
{
// lucide-react-native@1.14.0's barrel re-exports LucideProvider from a context.mjs that does
// not export it. Metro's loose CJS interop tolerates it; esbuild's strict ESM does not.
// Web-build only: patching the package would change what the shipped native app consumes.
name: 'lucide-barrel-provider',
appliesTo: (options) =>
options.plugins?.some((plugin) => plugin.name === LUCIDE_PLUGIN_NAME) === true
},
{
// AsyncStorage's web build is window.localStorage, which the shell's page does not have:
// Android turns DOM storage off and on iOS the origin host is the session id, so anything
// written there is gone on the next remount. The page module holds the app's own values,
// primed by `init` and written back over the `storage` grant.
name: 'async-storage-over-the-bridge',
appliesTo: (options) =>
options.alias?.['@react-native-async-storage/async-storage'] === PAGE_ASYNC_STORAGE_MODULE
},
{
// esbuild has no require.context, so the route tree is generated and injected.
name: 'route-manifest',
appliesTo: (options) =>
options.plugins?.some((plugin) => plugin.name === ROUTE_MANIFEST_PLUGIN_NAME) === true
}
]
/**
* react-native-web's own root reset: the same declaration set and `id="expo-reset"` as Expo's web
* template (`@expo/cli/static/template/index.html`), minified — the template's own block is
* pretty-printed with comments, so this is 112 bytes against its 410. Nothing generates it for a
* document built here.
*
* Every box below the mount is `flex: 1` against its parent, so with no definite height on all
* three the root measures 0 and the collapse is silent: the screen still lays out, still reaches
* the accessibility tree at the right offsets, and never paints or hit-tests below the header.
* A phone showed the header over a blank list with every row readable to VoiceOver and no row
* tappable (lane C1.7, both platforms).
*
* Inline, because the shell's CSP already allows `style-src 'unsafe-inline'` for the sheet
* react-native-web injects at runtime; a linked asset would need a second round trip before the
* first frame and would paint the collapsed layout until it landed.
*
* Height, `overflow` and the root's flex box and nothing else, which is what the template carries:
* react-native-web emits `body{margin:0}` in that runtime sheet, so a copy here would only cover
* the frames before it lands and would make this string something to keep in step with two sources.
*/
export const MOBILE_WEB_APP_ROOT_RESET =
'<style id="expo-reset">html,body{height:100%}body{overflow:hidden}' +
'#root{display:flex;height:100%;flex:1}</style>'
const PAGE_ASYNC_STORAGE_MODULE = join(
mobileDir,
'src',
'mobile-web-shell',
'bridge',
'page-async-storage.ts'
)
/**
* Zod's compiled path, off before any module runs.
*
* Zod decides whether it may compile by constructing `new Function('')` and reading the throw as
* "no JIT here". Under the shell's `script-src 'self'` that throw is exactly what happens, Zod
* catches it and takes the interpreted path — but the browser files a `securitypolicyviolation`
* report first, and it does so on every page load. Zod's own source gates the probe on `jitless`
* for this case, so nothing here is a workaround.
*
* In the banner rather than a module that calls `z.config`, because a module cannot win the race.
* `$ZodObject` reads `allowsEval` when a schema is *constructed*, not parsed, so the first
* module-scope `z.object(...)` in the bundle fires the probe — and esbuild evaluates the chunk
* holding zod and its callers before the chunk holding any module of ours that imports zod. An
* entry import placed first was measured losing that race; the banner runs before every module.
*
* `globalConfig` is `globalThis.__zod_globalConfig`, which zod adopts with `??=` rather than
* replacing, so setting the flag on it here is what zod itself reads.
*/
const ZOD_JITLESS_BANNER =
'globalThis.__zod_globalConfig ??= {}; globalThis.__zod_globalConfig.jitless = true;'
const ROUTE_MANIFEST_PLUGIN_NAME = 'orca-route-manifest'
const LUCIDE_PLUGIN_NAME = 'orca-lucide-barrel-provider'
/** The entry output's name, so classifying the outputs never has to guess which one it is. */
const ENTRY_CHUNK_NAME = 'entry'
// mobile/web-entry/route-manifest.ts is a real typed file rather than a virtual specifier, so the
// entry typechecks and Metro can still resolve it; only its body is replaced here.
function routeManifestPlugin(manifestSource) {
return {
name: ROUTE_MANIFEST_PLUGIN_NAME,
setup(build) {
build.onLoad({ filter: /web-entry[\\/]route-manifest\.ts$/ }, () => ({
contents: manifestSource,
loader: 'js',
resolveDir: mobileDir
}))
}
}
}
const lucideBarrelPlugin = {
name: LUCIDE_PLUGIN_NAME,
setup(build) {
build.onLoad({ filter: /lucide-react-native[\\/].*[\\/]context\.mjs$/ }, async (args) => ({
contents: `${await readFile(args.path, 'utf8')}\nexport const LucideProvider = ({ children }) => children;\n`,
loader: 'js'
}))
}
}
/** Split out so a test can read the options MOBILE_WEB_APP_SHIMS claims, without a build. */
export function mobileWebAppBuildOptions(routes) {
return {
// Fixed so no absolute path of this checkout can reach the output.
absWorkingDir: mobileDir,
entryPoints: [entryPoint],
bundle: true,
minify: true,
// Virtual: write is false, so outdir only names the emitted files esbuild hands back.
outdir: 'dist',
write: false,
// esm, because `splitting` requires it and a per-route chunk is the point: with iife and
// static imports esbuild emitted one 8.16 MB script for all 14 routes.
format: 'esm',
splitting: true,
// esbuild's `[hash]` is over the metafile's input keys, which are paths relative to
// absWorkingDir, so this name is not a function of the bytes and differs between two
// checkouts of one commit. It is a placeholder: renameOutputsByContent replaces it below.
chunkNames: '[hash]',
// Pinned rather than defaulted, so the entry is found by name and not by elimination.
entryNames: ENTRY_CHUNK_NAME,
target: ['es2022'],
charset: 'utf8',
legalComments: 'none',
// No sourcemap: it is an emitted file and would carry this checkout's absolute paths into the
// bundle. The metafile carries them too but is never written and never hashed; it is the only
// thing that says which output is the entry, which of its imports are static, and which
// outputs each one names.
sourcemap: false,
metafile: true,
logLevel: 'silent',
jsx: 'automatic',
// One React: resolve everything from mobile/node_modules, which is where the entry lives.
nodePaths: [join(mobileDir, 'node_modules')],
alias: {
'react-native': 'react-native-web',
'@react-native-async-storage/async-storage': PAGE_ASYNC_STORAGE_MODULE
},
plugins: [routeManifestPlugin(renderMobileWebAppRouteManifest(routes)), lucideBarrelPlugin],
resolveExtensions: [
'.web.tsx',
'.web.ts',
'.web.jsx',
'.web.js',
'.tsx',
'.ts',
'.jsx',
'.js',
'.json'
],
// Images are emitted as same-origin assets, not data: URLs, so their content-hashed names keep
// the buildId reproducible and the bytes out of every chunk that imports one. The policy now
// admits data: for images, but that is for a preview the page composes at runtime, not for a
// bundled asset. A font would fail the build here rather than silently ship under font-src 'none'.
loader: {
...ROUTE_SOURCE_LOADERS,
'.png': 'file',
'.jpg': 'file',
'.jpeg': 'file',
'.gif': 'file',
'.webp': 'file',
'.svg': 'file'
},
assetNames: '[hash]',
// Absolute, because the document is served at every route depth and a path relative to the
// script would resolve against the route instead.
publicPath: '/assets',
banner: {
js: `globalThis.process ??= { env: { NODE_ENV: 'production', EXPO_OS: 'web' }, platform: 'web', version: '', nextTick: (fn) => setTimeout(fn, 0) };${ZOD_JITLESS_BANNER}`
},
define: {
global: 'globalThis',
__DEV__: 'false',
'process.env.NODE_ENV': '"production"',
'process.env.EXPO_OS': '"web"',
'process.env.EXPO_ROUTER_IMPORT_MODE': '"sync"'
}
}
}
/**
* What the browser must have before the first route can paint: the entry plus every chunk it
* reaches by static import, transitively. A dynamic import is what the split exists to defer, so
* it is where this stops.
*
* The bound the verifier holds is this number and not the entry file alone, because esbuild puts
* the code shared by entry and routes in a chunk the entry imports statically: budgeting the entry
* file on its own would fall as the shared chunk grew.
*/
export function entryStaticClosure(metafile, entryOutputPath) {
const reached = new Set([entryOutputPath])
const queue = [entryOutputPath]
while (queue.length > 0) {
const current = queue.shift()
for (const imported of metafile.outputs[current]?.imports ?? []) {
if (imported.kind !== 'import-statement' || reached.has(imported.path)) {
continue
}
reached.add(imported.path)
queue.push(imported.path)
}
}
return reached
}
/**
* Every emitted output, renamed to the sha256 of its own final bytes.
*
* esbuild's `[hash]` is computed over the metafile's input keys, and those keys are paths
* relative to absWorkingDir. A tree whose mobile/node_modules is a symlink keys most of its
* inputs as `../../<somewhere>/...`, a tree that holds a real directory keys them as
* `node_modules/...`, and a byte-identical chunk comes out under a different name in each. The
* name is embedded in every importer, so the difference cascades into a different buildId for one
* commit -- and every phone re-downloads a bundle whose bytes never changed.
*
* Renaming here is what removes the path from the output. Leaves first, so an importer is hashed
* only once the names written inside it are final: an image before the chunk that loads it, a
* chunk before the chunk that imports it, the entry last. The result is what `hashedAsset` would
* name each of these anyway, which is how the name inside the bytes and the manifest's own sha256
* stay the same string.
*/
export function renameOutputsByContent(metafile, outputFiles) {
const emitted = new Map(
outputFiles.map((file) => [basename(file.path), Buffer.from(file.contents)])
)
const importsOf = new Map(
Object.entries(metafile.outputs).map(([output, { imports }]) => [
basename(output),
(imports ?? []).map((entry) => basename(entry.path)).filter((name) => emitted.has(name))
])
)
const renamed = new Map()
const open = new Set()
function rename(name) {
const done = renamed.get(name)
if (done) {
return done
}
if (open.has(name)) {
// Two outputs naming each other have no content hash at all, so this is a hard stop rather
// than a fallback. esbuild's splitting emits a DAG; nothing in the tree has produced one.
throw new Error(
`[build-mobile-web-app-bundle] ${name} is in an output cycle and cannot be content-named`
)
}
open.add(name)
let bytes = emitted.get(name)
for (const child of importsOf.get(name) ?? []) {
const { name: childName } = rename(child)
// publicPath already rewrote the specifier to this exact shape, and an esbuild output name
// is a token that appears nowhere else.
bytes = Buffer.from(
bytes.toString('utf8').split(`/assets/${child}`).join(`/assets/${childName}`),
'utf8'
)
}
open.delete(name)
const result = { name: `${sha256Hex(bytes)}${extname(name)}`, bytes }
renamed.set(name, result)
return result
}
for (const name of [...emitted.keys()].sort()) {
rename(name)
}
return renamed
}
/**
* Which emitted chunk each route key's `import()` lands in. esbuild puts a route module in exactly
* one output, so the metafile's own inputs answer it; nothing downstream can, because by then
* every name is a hash of bytes and the route's source path is gone from the bundle.
*/
export function routeChunkNames(metafile, routes, renamed) {
const owner = new Map()
for (const [output, { inputs }] of Object.entries(metafile.outputs)) {
for (const input of Object.keys(inputs ?? {})) {
// Absolute, and through realpath on the lookup side below: esbuild writes its input keys
// relative to absWorkingDir after resolving symlinks, so a route reached through one (every
// scratch tree under /var on macOS) is keyed by a path the caller never spelled.
owner.set(resolve(mobileDir, input), basename(output))
}
}
return Object.fromEntries(
routes.map(({ key, module }) => {
const emittedName = owner.get(realpathSync(module))
if (!emittedName) {
throw new Error(`[build-mobile-web-app-bundle] ${key} reached no output`)
}
return [key, renamed.get(emittedName).name]
})
)
}
const isScriptOutput = (path) => path.endsWith('.js')
// appDir is a seam for the tests, which bundle a scratch route tree; production always uses mobile/app.
/**
* Every source module one page route reaches, as the builder itself resolves them.
*
* Both entry points are needed: `app/h/_layout.tsx` wraps every route under it, and its imports are
* part of the page as surely as the route module's.
*/
export async function mobileWebAppRouteClosure(routeModule) {
return await mobileWebAppModuleClosure(['app/h/_layout', routeModule])
}
/**
* The same closure for any entry modules, which a route plus the layout is one case of.
*
* One definition of "what a page contains", read from `metafile.inputs` — the modules the entries
* pull in — rather than from `entryStaticClosure`, which walks emitted chunks and answers what a
* browser must download.
*
* A component a route mounts rather than one the router registers — `MobileBrowserPane` is the
* first with a pin of its own — has a closure to certify and no route to name it by. Pass it alone
* to read what it reaches on its own, or beside `app/h/_layout` to read what it adds to a page.
*
* `splitting: false` and a per-name output are required for a multi-entry build; with the defaults
* esbuild fails on two outputs claiming `dist/entry.js`.
*
* Note for anyone comparing this with a parity pin: `c1-page-closure.ts`, and the closures C2.6,
* C5.2 and C3.2 generate, derive theirs by the C1.6 method inside the mobile suite. The two are
* not the same computation, and a divergence between them is a finding rather than noise.
*/
export async function mobileWebAppModuleClosure(entryModules, { absWorkingDir } = {}) {
const base = mobileWebAppBuildOptions(MOBILE_WEB_PAGE_ROUTES)
const result = await esbuild.build({
...base,
// A census that plants a module to show the walk would report it needs a tree of its own; the
// real ones never pass this and keep measuring `mobile/`.
...(absWorkingDir ? { absWorkingDir } : {}),
// Extensionless, so `resolveExtensions` picks the same file the bundle ships: a route with a
// `.web.tsx` sibling resolves to that one, and naming the `.tsx` path explicitly would measure
// the native switch no browser ever loads.
entryPoints: entryModules.map((entry) => entry.replace(/\.tsx?$/, '')),
splitting: false,
entryNames: '[name]',
plugins: base.plugins.filter((plugin) => plugin.name !== ROUTE_MANIFEST_PLUGIN_NAME),
write: false,
metafile: true,
logLevel: 'silent'
})
const inputs = Object.keys(result.metafile.inputs)
return {
modules: inputs,
/** Everything outside `node_modules`: this repository's own source, which a census reads. */
local: inputs.filter((input) => !input.includes('node_modules'))
}
}
export async function bundleMobileWebApp({ appDir = defaultAppDir } = {}) {
const routes = await collectMobileWebAppRoutes(appDir)
await assertRoutesCarryNoSynchronousExports(routes)
const result = await esbuild.build(mobileWebAppBuildOptions(routes))
const entryOutputPath = Object.keys(result.metafile.outputs).find(
(path) => basename(path) === `${ENTRY_CHUNK_NAME}.js`
)
if (!entryOutputPath) {
throw new Error('[build-mobile-web-app-bundle] esbuild emitted no entry script')
}
const renamed = renameOutputsByContent(result.metafile, result.outputFiles)
const entry = renamed.get(basename(entryOutputPath))
const byName = (left, right) => (left.name < right.name ? -1 : 1)
const others = [...renamed.entries()]
.filter(([emittedName]) => emittedName !== basename(entryOutputPath))
.map(([emittedName, output]) => ({ emittedName, ...output }))
// Chunks keep their new name into the served path: the entry imports them by it, and
// publicPath has already made that specifier /assets/<name>.
const chunks = others.filter(({ emittedName }) => isScriptOutput(emittedName)).sort(byName)
const images = others.filter(({ emittedName }) => !isScriptOutput(emittedName)).sort(byName)
const closure = entryStaticClosure(result.metafile, entryOutputPath)
return {
script: entry.bytes,
chunks,
images,
// Counted off the renamed bytes rather than the metafile's own sizes, which are from before
// the names inside each output grew. Only the metafile knows which import is static; see
// entryStaticClosure.
entryStaticBytes: [...closure].reduce(
(total, path) => total + (renamed.get(basename(path))?.bytes.byteLength ?? 0),
0
),
routeKeys: routes.map((route) => route.key),
routeChunks: routeChunkNames(result.metafile, routes, renamed)
}
}
/**
* The declared page routes, checked against the tree that was actually bundled.
*
* A declaration naming a screen this bundle has no module for would reach a phone as a route the
* shell opens the page for and the page then paints as Unmatched. Failing the build is the only
* place that mismatch is visible to whoever wrote the declaration.
*/
export function resolveMobileWebPageRoutes(routeKeys, declared = MOBILE_WEB_PAGE_ROUTES) {
const bundled = new Set(routeKeys.map(routePathnameFromKey).filter((path) => path !== null))
for (const route of declared) {
if (!bundled.has(route.pathname)) {
throw new Error(
`[build-mobile-web-app-bundle] declared page route ${route.pathname} has no module in the bundle`
)
}
}
return declared.map((route) => ({ pathname: route.pathname, grants: [...route.grants] }))
}
/**
* `pageRoutes` rides with `appDir`: the declarations name screens in the real route tree, so a
* caller bundling some other tree has none to check against and says so by passing its own.
*/
export async function buildMobileWebAppBundle({
appDir,
outDir = defaultOutDir,
pageRoutes = MOBILE_WEB_PAGE_ROUTES
} = {}) {
const [
desktopVersion,
protocolWindow,
{ script, chunks, images, entryStaticBytes, routeChunks, routeKeys }
] = await Promise.all([
readDesktopVersion(),
readProtocolWindow(),
bundleMobileWebApp({ appDir })
])
// Every output is already named by its own bytes, and a name is written inside whatever imports
// it, so hashedAsset here reproduces the name rather than choosing one.
const scriptAsset = hashedAsset(script, 'js')
const written = [
scriptAsset,
...[...chunks, ...images].map(({ name, bytes }) => hashedAsset(bytes, extname(name).slice(1)))
]
// Root-absolute, unlike the Phase A bootstrap's bare relative src: this document is served at
// every route depth (/h/<hostId>/tasks), where a relative href resolves against the route and
// 404s. A <base> tag would be the other fix, but the shell's CSP sets base-uri 'none'.
// type="module", because the entry is esm and reaches its routes through import(). Same-origin
// module and chunk both load under the shell's script-src 'self'; the policy is unchanged.
const html =
'<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8" />\n' +
'<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />\n' +
`<title>Orca</title>\n${MOBILE_WEB_APP_ROOT_RESET}\n</head>\n<body>\n<div id="root"></div>\n` +
`<script type="module" src="/${scriptAsset.path}"></script>\n</body>\n</html>\n`
const indexBytes = Buffer.from(html, 'utf8')
const indexAsset = {
bytes: indexBytes,
path: MOBILE_WEB_BUNDLE_ENTRYPOINT,
sha256: sha256Hex(indexBytes),
byteLength: indexBytes.byteLength,
contentType: contentTypeForExtension('html')
}
const { manifest } = await writeMobileWebBundleTree({
outDir,
written: [indexAsset, ...written],
desktopVersion,
protocolWindow,
routes: resolveMobileWebPageRoutes(routeKeys, pageRoutes)
})
return {
manifest,
outDir,
routeChunks,
routeKeys,
entryStaticBytes,
// The entry counts: it is a chunk the browser fetches, and the budget is about how many.
chunkCount: chunks.length + 1,
// Everything the routes import that is not a script, which is the rest of the asset budget.
imageCount: images.length
}
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
try {
const { manifest, outDir, routeKeys, entryStaticBytes, chunkCount } =
await buildMobileWebAppBundle()
console.log(
`[build-mobile-web-app-bundle] OK — ${String(routeKeys.length)} route(s), ` +
`${String(chunkCount)} chunk(s), ${String(entryStaticBytes)} bytes before the first route, ` +
`${String(manifest.assets.length)} asset(s), ${String(manifest.totalBytes)} bytes, ` +
`buildId ${manifest.buildId} -> ${outDir}`
)
} catch (error) {
// The route guards fail here by design, and every throw on this path already names its
// source, so a stack only buries which route and which export.
console.error(error.message)
process.exit(1)
}
}
config/scripts/build-mobile-web-app-bundle.test.mjs
+755
View File
@@ -0,0 +1,755 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, relative } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import {
MOBILE_WEB_APP_ROOT_RESET,
MOBILE_WEB_APP_SHIMS,
bundleMobileWebApp,
buildMobileWebAppBundle,
entryStaticClosure,
mobileWebAppBuildOptions,
renameOutputsByContent,
resolveMobileWebPageRoutes,
routeChunkNames
} from './build-mobile-web-app-bundle.mjs'
import {
MOBILE_WEB_APP_ROUTE_ROOT,
ROUTE_SOURCE_LOADERS,
collectMobileWebAppRouteKeys,
collectMobileWebAppRoutes,
routePathnameFromKey
} from './mobile-web-app-route-manifest.mjs'
import {
MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES,
MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES,
MOBILE_WEB_APP_SOURCE_DIRS,
assertAssetCeilingFitsShell,
mobileWebAppBundleMaxAssets,
mobileWebAppBundleMaxChunks,
readMobileWebBundleMaxAssets,
verifyMobileWebAppBundle
} from './verify-mobile-web-app-bundle.mjs'
import {
BINARY_SOURCE_EXTENSIONS,
assertNoCarriageReturnsInSource
} from './verify-mobile-web-bundle.mjs'
import {
computeMobileWebBundleBuildId,
hashedAsset,
readDesktopVersion,
readProtocolWindow,
sha256Hex,
writeMobileWebBundleTree
} from './build-mobile-web-bundle.mjs'
import {
MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES,
MOBILE_WEB_BUNDLE_MAX_ASSETS
} from '../../src/shared/mobile-web-bundle/manifest-contract.js'
import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const appDir = join(projectDir, 'mobile', 'app')
// The sharded `test` job does not install mobile dependencies, so anything that runs esbuild over
// the route tree is skipped there and run for real in pr.yml's mobile_web_app job.
const bundles = mobileWebAppDependenciesPresent()
const describeBundling = bundles ? describe : describe.skip
const itBundling = bundles ? it : it.skip
/** Every script the page loads. A route's code is in a chunk now, not in the entry. */
function allScriptSource({ script, chunks }) {
return [script, ...chunks.map((chunk) => chunk.bytes)].map((bytes) => bytes.toString('utf8'))
}
async function withScratch(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-app-test-'))
try {
return await run(scratch)
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
/**
* Every page route this bundle declares, written out rather than read from the source that
* produces it: the point is to pin the list, and comparing the manifest to its own input would
* pass whatever that input became. Shared by the two assertions below, which is also what keeps
* this file under the 600-line cap.
*/
const EXPECTED_PAGE_ROUTES = [
{ pathname: '/h/[hostId]', grants: ['navigate', 'storage', 'haptics'] },
{
pathname: '/h/[hostId]/agent-history/[worktreeId]',
grants: ['navigate', 'storage', 'haptics']
},
{
pathname: '/h/[hostId]/tasks',
grants: ['navigate', 'storage', 'externalLink', 'haptics', 'native.clipboard.write']
},
{
pathname: '/h/[hostId]/files/[worktreeId]',
grants: ['navigate', 'storage', 'externalLink', 'haptics']
},
{
pathname: '/h/[hostId]/files/preview/[worktreeId]',
grants: ['navigate', 'storage', 'externalLink', 'haptics']
}
]
describe('the page routes the manifest declares', () => {
it('turns a route key into the URL pattern expo-router gives it', () => {
expect(routePathnameFromKey('./h/[hostId]/index.tsx')).toBe('/h/[hostId]')
expect(routePathnameFromKey('./h/[hostId]/tasks.tsx')).toBe('/h/[hostId]/tasks')
expect(routePathnameFromKey('./h/[hostId]/session/[worktreeId].tsx')).toBe(
'/h/[hostId]/session/[worktreeId]'
)
})
it('answers null for a layout, which is not a screen anyone navigates to', () => {
expect(routePathnameFromKey('./h/_layout.tsx')).toBeNull()
expect(routePathnameFromKey('./h/[hostId]/_layout.tsx')).toBeNull()
})
it('declares only routes the bundle has a module for', async () => {
const keys = await collectMobileWebAppRouteKeys(appDir)
expect(resolveMobileWebPageRoutes(keys)).toEqual(EXPECTED_PAGE_ROUTES)
})
it('fails the build on a declaration the bundle cannot render', () => {
// The mismatch reaches a phone as a route the shell opens the page for and the page then
// paints as Unmatched. This is the only place whoever wrote the declaration can see it.
expect(() =>
resolveMobileWebPageRoutes(
['./h/[hostId]/index.tsx'],
[{ pathname: '/h/[hostId]/gone', grants: [] }]
)
).toThrow('has no module in the bundle')
})
itBundling(
'reaches the built manifest, where the build id does not move for it',
async () => {
await withScratch(async (scratch) => {
const { manifest } = await buildMobileWebAppBundle({ outDir: join(scratch, 'bundle') })
expect(manifest.routes).toEqual(EXPECTED_PAGE_ROUTES)
// The routes are derived from the same tree the script is built from, so the assets
// already decide them and the id has no reason to carry them as well.
expect(manifest.buildId).toBe(computeMobileWebBundleBuildId(manifest.assets))
})
},
240_000
)
})
describe('the CRLF pin', () => {
it('exempts the same extensions in .gitattributes as the CRLF scan skips', async () => {
const attributes = await readFile(join(projectDir, '.gitattributes'), 'utf8')
for (const tree of MOBILE_WEB_APP_SOURCE_DIRS) {
const pattern = `/${relative(projectDir, tree).split('\\').join('/')}/**`
for (const extension of BINARY_SOURCE_EXTENSIONS) {
// Without the exemption the blanket `text eol=lf` pin above it rewrites the binary and
// every asset hash with it.
expect(attributes, `${pattern}/*${extension} is not exempt`).toContain(
`${pattern}/*${extension} -text`
)
}
}
})
})
describeBundling('the app bundle', () => {
it('resolves react-native to react-native-web and leaves no require.context', async () => {
const sources = allScriptSource(await bundleMobileWebApp())
for (const source of sources) {
expect(source).not.toContain('require.context')
}
// react-native-web's touch responder is proof the alias resolved rather than the native stub.
expect(sources.some((source) => source.includes('ResponderTouchHistoryStore'))).toBe(true)
}, 120_000)
it('cuts the routes into chunks the entry does not load', async () => {
const { script, chunks, entryStaticBytes } = await bundleMobileWebApp()
expect(chunks.length).toBeGreaterThan(1)
// The entry's own bytes plus the chunks it imports statically, which is what the browser
// parses before any route paints. Every route chunk is outside it.
expect(entryStaticBytes).toBeGreaterThan(script.byteLength)
const allBytes =
script.byteLength + chunks.reduce((total, chunk) => total + chunk.bytes.byteLength, 0)
expect(entryStaticBytes).toBeLessThan(allBytes)
}, 120_000)
it('names the chunk each route lands in', async () => {
const { chunks, routeChunks, routeKeys } = await bundleMobileWebApp()
expect(Object.keys(routeChunks).sort()).toEqual([...routeKeys].sort())
const emitted = new Set(chunks.map((chunk) => chunk.name))
for (const [key, name] of Object.entries(routeChunks)) {
expect(emitted, key).toContain(name)
}
// One chunk per route, never the entry: that is what a client-side navigation fetches.
expect(new Set(Object.values(routeChunks)).size).toBe(routeKeys.length)
}, 120_000)
it('counts only static imports into what loads before the first route', () => {
const metafile = {
outputs: {
'dist/entry.js': {
bytes: 10,
imports: [
{ path: 'dist/shared.js', kind: 'import-statement' },
{ path: 'dist/route.js', kind: 'dynamic-import' }
]
},
'dist/shared.js': {
bytes: 20,
imports: [{ path: 'dist/deep.js', kind: 'import-statement' }]
},
'dist/deep.js': { bytes: 30, imports: [] },
'dist/route.js': { bytes: 40, imports: [] }
}
}
expect([...entryStaticClosure(metafile, 'dist/entry.js')]).toEqual([
'dist/entry.js',
'dist/shared.js',
'dist/deep.js'
])
})
it('does not walk a chunk cycle forever', () => {
const metafile = {
outputs: {
'dist/entry.js': { bytes: 1, imports: [{ path: 'dist/a.js', kind: 'import-statement' }] },
'dist/a.js': { bytes: 1, imports: [{ path: 'dist/entry.js', kind: 'import-statement' }] }
}
}
expect(entryStaticClosure(metafile, 'dist/entry.js').size).toBe(2)
})
itBundling(
'refuses to build a route the lazy manifest would strip an export from',
async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
await writeFile(
join(directory, 'index.tsx'),
'export default function Route() { return null }\n'
)
await expect(bundleMobileWebApp({ appDir: scratch })).resolves.toBeTruthy()
await writeFile(
join(directory, 'settings.tsx'),
'const anchor = { anchor: "index" }\nexport { anchor as unstable_settings }\nexport default function Route() { return null }\n'
)
// The build is where this has to fail: the page it would otherwise emit mounts with the
// export silently gone, which is a blank screen on a phone and nothing in any log.
await expect(bundleMobileWebApp({ appDir: scratch })).rejects.toThrow(
/settings\.tsx.*unstable_settings/s
)
})
},
240_000
)
itBundling(
'refuses a route whose star re-export it cannot read',
async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
await writeFile(join(directory, 'boundary.ts'), 'export const value = 1\n')
await writeFile(
join(directory, 'index.tsx'),
'export * from "./boundary"\nexport default function Route() { return null }\n'
)
await expect(bundleMobileWebApp({ appDir: scratch })).rejects.toThrow(
/index\.tsx.*boundary/s
)
})
},
240_000
)
it('bundles every route module', async () => {
const { routeKeys } = await bundleMobileWebApp()
expect(routeKeys).toEqual(await collectMobileWebAppRouteKeys(appDir))
}, 120_000)
it("bundles a route's .web.tsx sibling instead of the native file, changing the bytes", async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
const route = (marker) => `export default function Route() { return '${marker}' }\n`
await writeFile(join(directory, 'index.tsx'), route('native-route-marker'))
const before = await bundleMobileWebApp({ appDir: scratch })
const has = (bundle, marker) =>
allScriptSource(bundle).some((source) => source.includes(marker))
expect(has(before, 'native-route-marker')).toBe(true)
await writeFile(join(directory, 'index.web.tsx'), route('web-route-marker'))
const after = await bundleMobileWebApp({ appDir: scratch })
expect(has(after, 'web-route-marker')).toBe(true)
expect(has(after, 'native-route-marker')).toBe(false)
// Different script bytes means a different asset sha and so a different buildId.
expect(after.script.equals(before.script)).toBe(false)
})
}, 240_000)
/**
* The same route tree, bundled from two directories at different depths. esbuild's own `[hash]`
* is computed over the metafile's input keys, which are paths relative to absWorkingDir, so two
* checkouts of one commit -- at different depths, or one with mobile/node_modules as a symlink
* and one with it as a directory -- name a byte-identical chunk differently. The rename
* cascades through every importer into a different buildId, and every phone re-downloads a
* bundle whose bytes did not change.
*/
async function bundleFromDepth(root, depth) {
const nested = join(root, ...Array.from({ length: depth }, (_, index) => `d${String(index)}`))
const directory = join(nested, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
// Two routes over one import, which is what makes esbuild emit a shared chunk to name.
await writeFile(join(directory, 'shared.ts'), 'export const marker = "shared-marker"\n')
for (const name of ['index.tsx', 'other.tsx']) {
await writeFile(
join(directory, name),
`import { marker } from "./shared"\nexport default function Route() { return marker + "${name}" }\n`
)
}
return { appDir: nested, bundle: await bundleMobileWebApp({ appDir: nested }) }
}
it('names every output by its bytes, so another checkout path builds the same bundle', async () => {
await withScratch(async (shallow) => {
await withScratch(async (deep) => {
const near = await bundleFromDepth(shallow, 1)
const far = await bundleFromDepth(deep, 5)
const names = ({ bundle }) => [...bundle.chunks, ...bundle.images].map((one) => one.name)
expect(names(far)).toEqual(names(near))
expect(far.bundle.script.equals(near.bundle.script)).toBe(true)
// The whole point: the manifest the phone compares is the same document.
const buildIdFrom = async ({ appDir }) =>
withScratch(async (out) => {
const { manifest } = await buildMobileWebAppBundle({
appDir,
outDir: join(out, 'x'),
// A synthetic tree: the real declarations name screens it does not have.
pageRoutes: []
})
return manifest.buildId
})
expect(await buildIdFrom(far)).toBe(await buildIdFrom(near))
})
})
}, 240_000)
it("names an output the same way the manifest's own asset hash does", async () => {
const { script, chunks } = await bundleMobileWebApp()
// The name is embedded in the importer, so it cannot be recomputed later; this is what says
// the name inside the bytes and the manifest's sha256 of those bytes are the same string.
expect(hashedAsset(script, 'js').path).toBe(`assets/${sha256Hex(script)}.js`)
for (const chunk of chunks) {
expect(chunk.name).toBe(`${sha256Hex(chunk.bytes)}.js`)
}
}, 120_000)
it('asks esbuild for the split the budgets assume', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// Each of these is load-bearing for a budget below: esm and splitting are what make a route a
// chunk, and the metafile is the only thing that says which imports are static.
expect(options.format).toBe('esm')
expect(options.splitting).toBe(true)
expect(options.chunkNames).toBe('[hash]')
expect(options.metafile).toBe(true)
})
it('reads a route source the same way the export guard does', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// The guard parses each route on its own, outside this build. Sharing the table is what stops
// a loader the bundle relies on from being missing there and reported as a syntax error.
for (const [extension, loader] of Object.entries(ROUTE_SOURCE_LOADERS)) {
expect(options.loader[extension], extension).toBe(loader)
}
})
it('applies every shim it names', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
for (const shim of MOBILE_WEB_APP_SHIMS) {
expect(shim.appliesTo(options), `${shim.name} is named but not applied`).toBe(true)
}
})
it('fails the named shim, not the whole build, when its option goes missing', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// Each shim reads an option of its own (two read `banner.js`), so stripping every option
// leaves none applying. Without that, the list could name a shim the build stopped applying.
const stripped = {
...options,
alias: {},
loader: {},
define: {},
banner: {},
plugins: []
}
expect(MOBILE_WEB_APP_SHIMS.filter((shim) => shim.appliesTo(stripped))).toEqual([])
})
it('keeps the shims out of the shipped Phase A bootstrap builder', async () => {
const shipped = await readFile(
join(projectDir, 'config', 'scripts', 'build-mobile-web-bundle.mjs'),
'utf8'
)
for (const { name } of MOBILE_WEB_APP_SHIMS) {
expect(shipped, `the Phase A bootstrap builder mentions ${name}`).not.toContain(name)
}
expect(shipped).not.toContain('react-native-web')
expect(shipped).not.toContain('lucide')
})
it('ships no haptic that reaches for the DOM', async () => {
// expo-haptics' web build fakes an iOS haptic by appending a hidden
// `<label><input type="checkbox" switch>` to document.head, clicking it, and removing it —
// once per call. The file explorer calls triggerSelection on every row tap, and C1.9 already
// traced a swallowed long press on the worktree list to that stray click. `haptics.web.ts` is
// what keeps the whole shim out of the bundle, so this reads the bytes rather than the import.
for (const source of allScriptSource(await bundleMobileWebApp())) {
// The shim's own fingerprint, not `navigator.vibrate`: react-native-web's Vibration export
// calls that too, and it touches no DOM until something invokes it.
expect(source).not.toContain('ariaHidden')
expect(source).not.toContain('pointer: coarse')
expect(source).not.toContain('setAttribute("switch"')
}
}, 120_000)
it('embeds no absolute path from this checkout', async () => {
// Every chunk, not only the entry: the route manifest names each route by absolute path, and
// the chunk that import resolves to is where such a path would survive.
for (const source of allScriptSource(await bundleMobileWebApp())) {
expect(source).not.toContain(projectDir)
}
}, 120_000)
it('builds the same buildId twice', async () => {
const first = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'a') })
)
const second = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'b') })
)
expect(first.manifest.buildId).toBe(second.manifest.buildId)
}, 120_000)
it('loads the entry as a module, so its route imports resolve', async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'module-tag')
const { manifest } = await buildMobileWebAppBundle({ outDir })
const html = await readFile(join(outDir, 'index.html'), 'utf8')
// import() in a classic script is a syntax error, so the tag and the format are one fact.
expect(html).toContain('<script type="module" src="/assets/')
const entry = html.match(/src="\/(assets\/[^"]+)"/)?.[1]
expect(manifest.assets.map((asset) => asset.path)).toContain(entry)
})
}, 120_000)
it('carries the root reset, so the mounted tree has a height to be 1 of', async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'root-reset')
await buildMobileWebAppBundle({ outDir })
const html = await readFile(join(outDir, 'index.html'), 'utf8')
expect(html).toContain(MOBILE_WEB_APP_ROOT_RESET)
// Literals rather than substrings taken off the constant, which would read it back against
// itself and follow any rule dropped from it. Every rule, because the chain is only as
// definite as its weakest link: a height on #root alone resolves against a body that has
// none, and percent of auto is auto. Named one by one so a failure says which rule went.
for (const rule of [
'html,body{height:100%}',
'body{overflow:hidden}',
'#root{display:flex;height:100%;flex:1}'
]) {
expect(MOBILE_WEB_APP_ROOT_RESET, rule).toContain(rule)
}
// The id travels with the rules: it is what marks this block as the template's reset rather
// than something the page grew its own copy of.
expect(MOBILE_WEB_APP_ROOT_RESET).toContain('<style id="expo-reset">')
// In the document itself, not a linked asset: the CSP that allows it is the one already
// relaxed for react-native-web's runtime sheet.
expect(html).not.toContain('<link rel="stylesheet"')
})
}, 120_000)
it('writes the manifest shape the packaging contract reads', async () => {
const { manifest } = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'c') })
)
expect(manifest.schemaVersion).toBe(1)
expect(manifest.entrypoint).toBe('index.html')
expect(manifest.assets.map((asset) => asset.path)).toContain('index.html')
expect(manifest.totalBytes).toBe(
manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
)
}, 120_000)
})
describe('the Phase C budget', () => {
it('sits below the contract per-asset ceiling, so growth trips a build not a phone', () => {
expect(MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES).toBeLessThan(MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES)
})
itBundling(
'is not already exceeded by the current bundle',
async () => {
const { manifest, chunkCount, entryStaticBytes, imageCount, routeKeys } = await withScratch(
(scratch) => buildMobileWebAppBundle({ outDir: join(scratch, 'd') })
)
expect(manifest.totalBytes).toBeLessThanOrEqual(MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES)
expect(manifest.assets.length).toBeLessThanOrEqual(
mobileWebAppBundleMaxAssets(routeKeys.length, imageCount)
)
expect(chunkCount).toBeLessThanOrEqual(mobileWebAppBundleMaxChunks(routeKeys.length))
expect(entryStaticBytes).toBeLessThanOrEqual(MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES)
},
120_000
)
it('says which node may be statically imported, and does not promise a route may', async () => {
const source = await readFile(
join(projectDir, 'config', 'scripts', 'verify-mobile-web-app-bundle.mjs'),
'utf8'
)
// The bound reads like a per-route escape hatch and is not one: 5 of the 14 routes break it
// on their own. What keeps it survivable is that expo-router wants a synchronous export off
// layout nodes only, so the note has to name the layout and the export that drives it.
const doc = source.slice(
0,
source.indexOf('export const MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES')
)
const note = doc.slice(doc.lastIndexOf('/**'))
expect(note).toContain('h/_layout.tsx')
expect(note).toContain('unstable_settings')
})
it('budgets what loads first well under what the whole page weighs', () => {
// The point of the split: the entry budget is the one a route must not grow, and it is a
// fraction of the total the bundle is still allowed to weigh.
expect(MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES).toBeLessThan(
MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES
)
})
it('derives the chunk ceiling from the route count, not from a measured number', async () => {
// A chunk is emitted per distinct set of importers, so the count is not a function of the
// route count alone. Re-measured on this head, by copying the route tree and dropping routes
// from the end of the sorted key list -- both siblings of each, because deleting a .web.tsx
// alone leaves the native file for the builder to resolve and measures a different closure.
// The 14-route reading is the real tree and includes the one script the deferred mermaid
// artifact costs.
for (const [routes, measured] of [
[8, 32],
[10, 43],
[12, 61],
[14, 69]
]) {
expect(mobileWebAppBundleMaxChunks(routes), `${String(routes)} routes`).toBeGreaterThan(
measured
)
}
expect(mobileWebAppBundleMaxChunks(14)).toBe(72)
expect(mobileWebAppBundleMaxChunks(15) - mobileWebAppBundleMaxChunks(14)).toBe(4)
// Between four and nine more per route above, so the ceiling is a bound and not a fit -- and
// at 14 routes it is a close one. 69 measured against 72, with the last two routes having cost
// the 8 the ceiling grants for two: the next route that shares less than its neighbours fails
// here, which is what this is for.
expect(mobileWebAppBundleMaxChunks(14) - mobileWebAppBundleMaxChunks(12)).toBe(8)
})
it('refuses an engine chunked along its own lazy boundaries, and passes one artifact', () => {
// The two builds this ceiling has to tell apart, both measured at 14 routes.
//
// The page reaches mermaid through one pre-bundled artifact and the bundle emits 69 scripts
// (68 of them the page's own split, one the deferred engine). Importing the package instead
// emitted 172: mermaid lazily imports each of its own diagram types and esbuild splits along
// those boundaries, all of it inside the generation the phone has already downloaded. The
// route term is the only term precisely so that the second of those fails here -- a ceiling
// raised to admit 172 would have admitted any split at all.
const ROUTES = 14
const WITH_ONE_ARTIFACT = 69
const CHUNKED_ALONG_THE_ENGINE = 172
expect(WITH_ONE_ARTIFACT).toBeLessThanOrEqual(mobileWebAppBundleMaxChunks(ROUTES))
expect(CHUNKED_ALONG_THE_ENGINE).toBeGreaterThan(mobileWebAppBundleMaxChunks(ROUTES))
// And the assets that came with it: 215 against 112, of the 256 the shell will load.
expect(mobileWebAppBundleMaxAssets(ROUTES, 42)).toBeLessThan(CHUNKED_ALONG_THE_ENGINE + 42 + 1)
})
it('derives the asset ceiling so the chunk ceiling is always the one that trips first', () => {
// A bundle's assets are its chunks, its images and the document. Asserting one constant under
// another did not say that: with 42 images, 4 * 18 + 16 chunks plus 42 plus the document is
// 131 assets, over the flat 128 the ceiling used to be, so from 18 routes on the asset count
// failed first and named the wrong thing.
for (const routeCount of [14, 18, 24, 40]) {
for (const imageCount of [0, 42, 120]) {
const chunks = mobileWebAppBundleMaxChunks(routeCount)
expect(mobileWebAppBundleMaxAssets(routeCount, imageCount)).toBe(chunks + imageCount + 1)
// The ordering claim itself: a bundle at the chunk ceiling is exactly at the asset
// ceiling, so no bundle can pass the chunk check and fail the asset one.
expect(chunks + imageCount + 1).toBeLessThanOrEqual(
mobileWebAppBundleMaxAssets(routeCount, imageCount)
)
}
}
})
itBundling(
'keeps the derived ceiling under the map the phone actually holds',
async () => {
const { manifest, routeKeys, imageCount } = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'e') })
)
const ceiling = mobileWebAppBundleMaxAssets(routeKeys.length, imageCount)
expect(manifest.assets.length).toBeLessThanOrEqual(ceiling)
// The native side refuses a manifest past this, so the derived ceiling has to stay inside it.
expect(ceiling).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_MAX_ASSETS)
// And the build is what has to say so: the guard runs on the counts this bundle measured.
const shellCeiling = await readMobileWebBundleMaxAssets()
expect(assertAssetCeilingFitsShell(routeKeys.length, imageCount, shellCeiling)).toBe(ceiling)
},
120_000
)
it('fails the build when the derived ceiling passes what the phone will accept', async () => {
// The shell hands back null for a manifest over its own ceiling, so a derived ceiling above
// that ships a green build no device can open. At the 42 images the tree carries, 4r + 16 +
// 42 + 1 crosses 256 at 50 routes, which Phase C reaches. A deferred engine kept to one
// artifact leaves that where it is; the 103-script version of it moved the crossing to 24.
expect(await readMobileWebBundleMaxAssets()).toBe(MOBILE_WEB_BUNDLE_MAX_ASSETS)
expect(assertAssetCeilingFitsShell(49, 42, MOBILE_WEB_BUNDLE_MAX_ASSETS)).toBe(255)
expect(() => assertAssetCeilingFitsShell(50, 42, MOBILE_WEB_BUNDLE_MAX_ASSETS)).toThrow(
/259 .*256/
)
})
})
describe('the verifier', () => {
itBundling(
'accepts a bundle it has just built',
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
await buildMobileWebAppBundle({ outDir })
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).resolves.toBeDefined()
})
},
240_000
)
itBundling(
"rejects a buildId the manifest's own asset list does not derive",
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
await buildMobileWebAppBundle({ outDir })
const manifestPath = join(outDir, 'manifest.json')
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
manifest.buildId = 'f'.repeat(64)
await writeFile(manifestPath, JSON.stringify(manifest), 'utf8')
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).rejects.toThrow(
'does not match its asset list'
)
})
},
240_000
)
itBundling(
'rejects a self-consistent bundle a fresh build does not reproduce',
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
const { manifest } = await buildMobileWebAppBundle({ outDir })
// What a stale out/ actually looks like: every digest agrees with its bytes and the
// buildId derives from the asset list, but the source has moved on. Only the two fresh
// builds the verifier runs can tell, which is the check this covers.
const assets = await Promise.all(
manifest.assets.map(async (asset) => ({
...asset,
bytes: await readFile(join(outDir, asset.path))
}))
)
const document = assets.find((asset) => asset.path === manifest.entrypoint)
document.bytes = Buffer.concat([document.bytes, Buffer.from('<!-- drift -->\n', 'utf8')])
document.sha256 = sha256Hex(document.bytes)
document.byteLength = document.bytes.byteLength
const [desktopVersion, protocolWindow] = await Promise.all([
readDesktopVersion(),
readProtocolWindow()
])
await writeMobileWebBundleTree({ outDir, written: assets, desktopVersion, protocolWindow })
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).rejects.toThrow('is stale')
})
},
240_000
)
})
describe('the CRLF guard', () => {
it('covers the three trees whose bytes reach the buildId', () => {
expect(MOBILE_WEB_APP_SOURCE_DIRS.map((dir) => dir.slice(projectDir.length))).toEqual([
join('mobile', 'web-entry'),
join('mobile', 'app'),
join('mobile', 'src')
])
})
it('fails on a CRLF source file', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'route.tsx'), 'export default null\r\n', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).rejects.toThrow('CRLF')
})
})
it('exempts the binary assets .gitattributes pins -text', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'icon.ttf'), Buffer.from([0x00, 0x0d, 0x0a]))
await writeFile(join(scratch, 'shot.png'), Buffer.from([0x0d]))
await expect(assertNoCarriageReturnsInSource(scratch)).resolves.toBeUndefined()
})
})
it('exempts the gitignored generated webview engine modules', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'engine.generated.ts'), 'export const X = "a\r\n"', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).resolves.toBeUndefined()
})
})
})
describe('naming an output by its bytes', () => {
it('refuses two outputs that name each other', () => {
const emitted = (text) => new TextEncoder().encode(text)
const metafile = {
outputs: {
'dist/a.js': { imports: [{ path: 'dist/b.js', kind: 'import-statement' }] },
'dist/b.js': { imports: [{ path: 'dist/a.js', kind: 'import-statement' }] }
}
}
// Neither name can be final before the other is, so a cycle has no content hash to reach.
// esbuild's splitting emits a DAG; this is the hard stop for the day it does not.
expect(() =>
renameOutputsByContent(metafile, [
{ path: 'dist/a.js', contents: emitted('import "/assets/b.js"') },
{ path: 'dist/b.js', contents: emitted('import "/assets/a.js"') }
])
).toThrow(/output cycle/)
})
it('refuses a route it cannot find an output for', async () => {
await withScratch(async (scratch) => {
const module = join(scratch, 'index.tsx')
await writeFile(module, 'export default function Route() { return null }\n')
// The metafile is the only thing that knows which chunk holds a route. Without this the
// route reaches the manifest naming a chunk of undefined, which the phone fetches as a 404.
expect(() =>
routeChunkNames({ outputs: {} }, [{ key: './index.tsx', module }], new Map())
).toThrow(/\.\/index\.tsx reached no output/)
})
})
})
config/scripts/build-mobile-web-bundle.mjs
+37 -8
View File
@@ -16,7 +16,15 @@ const CONTENT_TYPE_BY_EXTENSION = {
css: 'text/css; charset=utf-8',
html: 'text/html; charset=utf-8',
js: 'text/javascript; charset=utf-8',
png: 'image/png'
png: 'image/png',
// The Phase C app bundle emits images as same-origin assets rather than data: URLs, so each one
// is content-hashed and served from here. Fonts are absent by design: the policy sets
// font-src 'none'.
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
gif: 'image/gif',
webp: 'image/webp',
svg: 'image/svg+xml'
}
/**
@@ -41,11 +49,11 @@ export function computeMobileWebBundleBuildId(assets) {
return createHash('sha256').update(serializeMobileWebBundleAssets(assets), 'utf8').digest('hex')
}
function sha256Hex(bytes) {
export function sha256Hex(bytes) {
return createHash('sha256').update(bytes).digest('hex')
}
function contentTypeForExtension(extension) {
export function contentTypeForExtension(extension) {
const contentType = CONTENT_TYPE_BY_EXTENSION[extension]
if (!contentType) {
throw new Error(`[build-mobile-web-bundle] no content type registered for .${extension}`)
@@ -65,7 +73,7 @@ function readIntegerConstant(source, name) {
* Parsed rather than imported because protocol-version.ts is TypeScript and this script runs on
* bare node during packaging, before any build output exists.
*/
async function readProtocolWindow() {
export async function readProtocolWindow() {
const source = await readFile(join(projectDir, 'src', 'shared', 'protocol-version.ts'), 'utf8')
return {
runtimeProtocolVersion: readIntegerConstant(source, 'RUNTIME_PROTOCOL_VERSION'),
@@ -77,7 +85,7 @@ async function readProtocolWindow() {
}
}
async function readDesktopVersion() {
export async function readDesktopVersion() {
const packageJson = JSON.parse(await readFile(join(projectDir, 'package.json'), 'utf8'))
if (typeof packageJson.version !== 'string' || packageJson.version.length === 0) {
throw new Error('[build-mobile-web-bundle] root package.json has no version')
@@ -124,7 +132,7 @@ async function transformEntries(protocolWindow, desktopVersion) {
return { script, stylesheet }
}
function hashedAsset(bytes, extension) {
export function hashedAsset(bytes, extension) {
const sha256 = sha256Hex(bytes)
return {
bytes,
@@ -172,7 +180,27 @@ export async function buildMobileWebBundle({ outDir = defaultOutDir } = {}) {
contentType: contentTypeForExtension('html')
}
const written = [indexAsset, ...hashed]
return writeMobileWebBundleTree({
outDir,
written: [indexAsset, ...hashed],
desktopVersion,
protocolWindow
})
}
/**
* Manifest assembly and the on-disk write, shared by the Phase A bootstrap bundle and the Phase C
* app bundle so both produce the same manifest shape the contract module and verifier read.
*/
export async function writeMobileWebBundleTree({
outDir,
written,
desktopVersion,
protocolWindow,
// Empty for the Phase A bootstrap, which carries no route tree at all: a shell reading it finds
// no screen listed and renders every route natively, which is what it already does.
routes = []
}) {
const assets = written
.map(({ path, sha256, byteLength, contentType }) => ({ path, sha256, byteLength, contentType }))
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
@@ -184,7 +212,8 @@ export async function buildMobileWebBundle({ outDir = defaultOutDir } = {}) {
runtimeProtocolVersion: protocolWindow.runtimeProtocolVersion,
entrypoint: MOBILE_WEB_BUNDLE_ENTRYPOINT,
totalBytes: assets.reduce((total, asset) => total + asset.byteLength, 0),
assets
assets,
routes
}
// Why a full clear: a stale asset left from an earlier build would ship unreferenced inside asar.

Some files were not shown because too many files have changed in this diff Show More