Merge refreshed contrast-cache base and regenerate terminal cell cleanup patches

This commit is contained in:
Neil
2026-09-18 22:02:21 -07:00
2491 changed files with 165223 additions and 16618 deletions
+48
View File
@@ -23,6 +23,9 @@
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
/config/patches/*.patch -text
# Same reason, and pnpm parses these too: a CRLF checkout makes the mobile
# patches unparseable, so Windows packaging dies on ERR_PNPM_INVALID_PATCH.
/mobile/patches/*.patch -text
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
# source patch under xterm-src/ instead. The sibling patches stay diffable.
/config/patches/@xterm__xterm@*.patch -diff
@@ -41,3 +44,48 @@
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
# Mobile web bundle source. Every text byte here is hashed into an asset digest and
# from there into buildId, so a CRLF checkout produces a different bundle id for the
# same commit (91af2897 vs 9d78435e). The PNG is -text because it must not be touched.
/src/mobile-web/index.html text eol=lf
/src/mobile-web/src/*.ts text eol=lf
/src/mobile-web/src/*.css text eol=lf
/src/mobile-web/src/*.png -text
# Mobile web page source. Same buildId hazard as src/mobile-web above: these bytes are
# hashed into the Phase C bundle, so a CRLF Windows checkout would ship a different
# buildId for identical source. web-entry/ does not exist yet; the pin lands ahead of it.
/mobile/src/** text eol=lf
/mobile/app/** text eol=lf
/mobile/web-entry/** text eol=lf
# The blanket pin above would mark a future binary as text; exempt the asset types an
# RN page actually carries, the same way src/mobile-web exempts its PNG.
/mobile/src/**/*.png -text
/mobile/src/**/*.jpg -text
/mobile/src/**/*.jpeg -text
/mobile/src/**/*.gif -text
/mobile/src/**/*.ico -text
/mobile/src/**/*.webp -text
/mobile/src/**/*.ttf -text
/mobile/src/**/*.otf -text
/mobile/src/**/*.woff -text
/mobile/src/**/*.woff2 -text
/mobile/app/**/*.png -text
/mobile/app/**/*.jpg -text
/mobile/app/**/*.jpeg -text
/mobile/app/**/*.gif -text
/mobile/app/**/*.ico -text
/mobile/app/**/*.webp -text
/mobile/app/**/*.ttf -text
/mobile/app/**/*.otf -text
/mobile/app/**/*.woff -text
/mobile/app/**/*.woff2 -text
/mobile/web-entry/**/*.png -text
/mobile/web-entry/**/*.jpg -text
/mobile/web-entry/**/*.jpeg -text
/mobile/web-entry/**/*.gif -text
/mobile/web-entry/**/*.ico -text
/mobile/web-entry/**/*.webp -text
/mobile/web-entry/**/*.ttf -text
/mobile/web-entry/**/*.otf -text
/mobile/web-entry/**/*.woff -text
/mobile/web-entry/**/*.woff2 -text
@@ -0,0 +1,22 @@
name: Install mobile dependencies
description: Frozen pnpm install for the mobile/ project, whose node_modules the mobile web bundle build and the mobile-aware lint passes resolve React Native and Expo from.
runs:
using: composite
steps:
# Why a separate install: mobile is its own pnpm project, so the root install leaves
# mobile/node_modules empty and every mobile import resolves to nothing.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the
# gitignored terminal/mermaid webview engine modules that tracked source imports.
# The drift guard mirrors the root install so a stale mobile lockfile fails by name --
# mobile's lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
shell: bash
working-directory: mobile
run: |
pnpm install --frozen-lockfile
# Job containers can run composite steps from a source mirror without .git.
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
+7
View File
@@ -184,6 +184,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -205,6 +208,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
@@ -16,6 +16,8 @@ on:
expected-rehome-generation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
monitor-run-attempt: { required: true, type: string }
gate-override-reason: { required: false, type: string, default: '' }
gate-override-confirmation: { required: false, type: string, default: '' }
wave-index: { required: true, type: string }
permissions:
@@ -52,7 +54,12 @@ jobs:
WAVE_INDEX: ${{ inputs.wave-index }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
# ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
# 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
DRAIN_PACE_WINDOW_MS: '120000'
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
@@ -72,12 +79,14 @@ jobs:
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
# The caller validated this too; re-check here so a malformed override
# can never reach a mutation through this reusable workflow.
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
test "${DEPLOY_MODE}" != verify
test "${GATE_OVERRIDE_CONFIRMATION}" = \
"SKIP_RELAY_MONITOR_GATE ${TARGET_IMAGE_DIGEST}"
[[ "${GATE_OVERRIDE_REASON}" =~ ^[[:print:]]{12,500}$ ]]
fi
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
echo "so recover each remaining cell with its own fresh monitor" >&2
@@ -109,14 +118,34 @@ jobs:
- uses: hashicorp/setup-terraform@v3
with: { terraform_wrapper: false }
# One approved-cell table, in the wave validator the dispatch gate already uses, so
# a cell's class and its wave's selector delta cannot drift apart between the two.
- name: Resolve this cell's admission class and wave selector delta
run: |
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
--cell-id "${TARGET_CELL_ID}")"
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
fi
{
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
} >> "${GITHUB_ENV}"
- name: Require fresh aggregate monitor evidence reference
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private aggregate monitor evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -125,7 +154,7 @@ jobs:
run-id: ${{ inputs.monitor-run-id }}
- name: Verify monitor evidence provenance
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
--directory "${OUTPUT_DIRECTORY}" \
@@ -138,7 +167,7 @@ jobs:
--wave-index "${WAVE_INDEX}"
- name: Download this wave's single-use safety authority
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -147,7 +176,7 @@ jobs:
run-id: ${{ github.run_id }}
- name: Require safety evidence consumed by this workflow
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
# Mutations are single-dispatch: a fresh dispatch cannot resume a
# partial batch (the canary authority binds the batch-entry selector
@@ -184,9 +213,32 @@ jobs:
# Freshness-only failures are publish lag, not health, on every wave
# including the first; the CLI still caps the retry at the wave's
# evidence-age budget, so this cannot mutate on aged evidence.
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" --retry-freshness
#
# This live recheck runs on every mutating wave, including a
# break-glass one. With the aggregate gate overridden there is no
# sealed state to read, so the expected selector comes from the
# dispatch inputs the rehome inspect below verifies against the live
# director; every threshold the sample is judged against is unchanged.
if test -n "${GATE_OVERRIDE_CONFIRMATION}"; then
jq -n \
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
| map_values(split(",") | map(select(length > 0)))' \
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
pnpm incident:relay-preflight -- \
--no-monitor-state \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
else
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
fi
- name: Require durable rehome disabled and exact selector
env:
@@ -215,6 +267,11 @@ jobs:
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c17|c18)
EXPECTED_HARD_CAP=600
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
@@ -230,10 +287,6 @@ jobs:
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
if test "${EXPECTED_REGION}" = us-central1; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
@@ -255,6 +308,14 @@ jobs:
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
fi
# The startup template emits rehome trust lines only for a declared source cell, so
# require membership exactly when either side of this roll expects those lines.
if test "${EXPECTED_REGION}" = us-central1 && {
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
}; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
@@ -299,20 +360,66 @@ jobs:
}
CURRENT_RUNTIME="$(admin_post current-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
# A rollback that failed between template apply and admission restore
# leaves the cell already on the rollback image; resume from that
# state instead of demanding the pre-rollback predecessor.
# Two different failures leave the cell on the rollback image, and the image
# alone cannot tell them apart. A rollback that failed between its template
# apply and its admission restore restarted the cell, so that cell is not
# draining and resumes. A wave that stopped after its drain and before its
# template apply never restarted anything, so its cell is still draining and
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
if test "${DEPLOY_MODE}" = rollback \
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
ROLLBACK_RESUME=true
if test "${LIVE_DRAINING}" = true; then
ROLLBACK_STAGE=stranded
else
ROLLBACK_STAGE=resume
fi
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
else
ROLLBACK_RESUME=false
ROLLBACK_STAGE=roll
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
fi
if test "${ROLLBACK_STAGE}" = resume; then
ROLLBACK_RESUME=true
else
ROLLBACK_RESUME=false
fi
# A stranded cell's template still carries the image the cell is serving, so that
# is the predecessor its plan is reviewed against. A template already moved on to
# the target is refused here rather than rolled backwards under a stale review.
if test "${ROLLBACK_STAGE}" = stranded; then
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
else
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
fi
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require the cell pristine in the
# class it is declared to serve in.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
else
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
fi
# Draining sheds connections, and a migration-only cell holds none, so the flag
# carries no precondition there. It also outlives a failed wave, because the drain
# that set it is followed by no restart, which is the state a failed canary leaves.
if test "${DEPLOY_MODE}" = rollback \
|| test "${ENTRY_ADMISSION}" = migration-only; then
PRECHECK_DRAINING=either
else
PRECHECK_DRAINING=forbidden
fi
# A resumed rollback already restarted, so its cell has to come back not draining;
# that is what separates it from a wave that stopped before its template apply.
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
PREDECESSOR_DRAINING_OK=true
else
PREDECESSOR_DRAINING_OK=false
fi
RESTORED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
@@ -333,8 +440,19 @@ jobs:
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
# A migration-only cell is already isolated and is handed back isolated, so both
# halves of its wave see exactly the membership it entered with.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
fi
{
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
# The drain wait and the plan review both read the image this cell actually
# serves, which is the rollback image on a stranded cell and not the current one.
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
# The failsafe consumes these; deriving them here keeps them
# defined for a failure in any later step.
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
@@ -353,8 +471,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
(.region == $region or
($region == "us-central1" and $protocol == 0 and .region == null)) and
@@ -370,8 +487,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'[
if .role != "cell" then "role" else empty end,
if .cellId != $cell then "cellId" else empty end,
@@ -407,16 +523,6 @@ jobs:
fi
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require a pristine general cell.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
PRECHECK_DRAINING=either
else
PRECHECK_ADMISSION=general
PRECHECK_DRAINING=forbidden
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -442,18 +548,25 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Isolating a migration-only cell must be a read-only no-op; a change here would
# mean the live class is not the one this wave planned around.
if test "${ENTRY_ADMISSION}" = migration-only; then
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
fi
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
# The wait has to outlast the pacing window as well as the leases it waits on.
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat either --admission migration-only --draining required \
--activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
--timeout-ms 900000
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
--timeout-ms 1020000
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
@@ -466,6 +579,7 @@ jobs:
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
@@ -511,6 +625,7 @@ jobs:
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--rollback-image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
@@ -538,20 +653,37 @@ jobs:
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}"
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
# A stranded cell already runs the reviewed template, so the apply above replaces
# no instance and the drain flag, which only a restart clears, would survive the
# whole wave. Roll the MIG explicitly on exactly the policy a template change uses.
# Every field is passed: gcloud persists these into the MIG's update policy, and it
# defaults the method to substitute on a group with no stateful config, so omitting
# one drifts the policy off the reviewed one and fails every later targeted plan.
if test "${ROLLBACK_STAGE}" = stranded \
&& test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then
gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--replacement-method recreate --max-surge 0 --max-unavailable 1
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
@@ -626,28 +758,36 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
--cell-incarnation "${TARGET_INCARNATION}"
- name: Restore only the verified selected cell to general admission
- name: Restore only the verified selected cell to its entry admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
# Activating a migration-only cell would promote it to a serving cell for good, so
# restore it with the idempotent isolate that reports the authoritative generation.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORE_MODE=isolate
else
RESTORE_MODE=activate
fi
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
echo "${ACTIVATE_RESULT}"
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
<<< "${ACTIVATE_RESULT}")"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
echo "${RESTORE_RESULT}"
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
<<< "${RESTORE_RESULT}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission general --draining forbidden --activity allowed \
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
--draining forbidden --activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
@@ -69,6 +69,17 @@ on:
description: Exact digest-and-cell-bound mutation confirmation
required: false
type: string
gate-override-reason:
description: >-
Break-glass only: why this wave may skip the aggregate 15-minute monitor
dry-run gate. The live per-wave preflight still runs.
required: false
type: string
gate-override-confirmation:
description: >-
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
required: false
type: string
permissions:
actions: read
@@ -111,11 +122,16 @@ jobs:
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
CONFIRMATION: ${{ inputs.confirmation }}
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
run: |
# Fails closed on a partial or mismatched override, before any mutation.
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
@@ -123,6 +139,35 @@ jobs:
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
fi
- name: Record the monitor gate override in the run summary
if: ${{ inputs.gate-override-confirmation != '' }}
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
MODE: ${{ inputs.mode }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
run: |
{
echo '## Aggregate monitor gate overridden (break-glass)'
echo
echo '| field | value |'
echo '| --- | --- |'
echo "| actor | ${ACTOR} |"
echo "| mode | ${MODE} |"
echo "| cells | ${CELL_IDS} |"
echo "| target digest | \`${TARGET_DIGEST}\` |"
echo "| reason | ${GATE_OVERRIDE_REASON} |"
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
echo
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
echo 'durable rehome disabled, the exact selector generation and membership, the'
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
echo 'single-dispatch mutation.'
} >> "${GITHUB_STEP_SUMMARY}"
- name: Download exact prior canary authority
if: ${{ inputs.mode == 'batch-apply' }}
uses: actions/download-artifact@v4
@@ -136,17 +181,23 @@ jobs:
if: ${{ inputs.mode == 'batch-apply' }}
env:
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
run: |
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
--target-digest "${{ inputs.target-image-digest }}" \
--rollback-digest "${{ inputs.rollback-image-digest }}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}"
--cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" \
--rollback-digest "${ROLLBACK_DIGEST}" \
--selector-generation "${SELECTOR_GENERATION}" \
--rehome-generation "${REHOME_GENERATION}"
- name: Reject previously consumed aggregate safety evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
env:
GH_TOKEN: ${{ github.token }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -163,7 +214,7 @@ jobs:
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
- name: Consume aggregate safety evidence for this exact wave
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -188,6 +239,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '0'
secrets: inherit
@@ -209,6 +262,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '1'
secrets: inherit
@@ -230,6 +285,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '2'
secrets: inherit
@@ -251,6 +308,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '3'
secrets: inherit
@@ -266,6 +325,10 @@ jobs:
with: { node-version: 24 }
- name: Seal exact successful canary authority
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
run: |
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
@@ -276,6 +339,9 @@ jobs:
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
--actor "${ACTOR}" \
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
- uses: actions/upload-artifact@v4
+22 -1
View File
@@ -57,7 +57,28 @@ jobs:
uses: actions/cache@v4
with:
path: dist/win-unpacked
key: win-unpacked-${{ hashFiles('src/**', 'config/**', 'package.json', 'pnpm-lock.yaml') }}
# mobile/ is in the key because beforePack requires out/mobile-web, whose bytes come from
# the mobile install and, once Phase C flips the bundle, from the page trees below; a
# mobile-only change must miss this cache, not reuse a stale installer. src/** and
# config/** already cover src/mobile-web and the two bundle builders.
key: >-
win-unpacked-${{ hashFiles(
'src/**',
'config/**',
'package.json',
'pnpm-lock.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-unpacked.outputs.cache-hit != 'true'
- name: Build unpacked app
if: steps.cache-unpacked.outputs.cache-hit != 'true'
+8
View File
@@ -156,6 +156,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
@@ -179,6 +182,11 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.freshness.outputs.should_build == 'true'
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
@@ -203,6 +203,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Caches the Electron binary and electron-builder's tool downloads (nsis,
# winCodeSign). Same key shape as release-cut's Windows leg.
@@ -229,6 +232,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why the packaging check runs before the 20-minute build: it only needs
# node_modules, and a stale config should cost seconds rather than a build.
- name: Verify dev-channel packaging identity
+7
View File
@@ -164,6 +164,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -185,6 +188,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
+25 -5
View File
@@ -9,6 +9,18 @@ on:
- ready_for_review
paths:
- 'mobile/**'
# Mobile launch contracts exercise the real host dispatcher and durable receipt store.
- 'src/main/agent-launch/**'
- 'src/main/runtime/rpc/**'
- 'src/main/runtime/runtime-rpc/**'
- 'src/main/runtime/runtime-rpc.ts'
- 'src/main/runtime/device-registry.ts'
- 'src/main/runtime/orca-runtime.ts'
- 'src/main/runtime/agent-session-*.ts'
- 'src/main/native-chat/agent-session-wire/**'
- 'src/shared/agent-launch-*.ts'
- 'src/shared/agent-session-*.ts'
- 'src/shared/new-workspace/worktree-create-collision.ts'
# Why: the mobile terminal link parsers are conformance-tested against
# these shared fixtures; desktop-side fixture edits must re-run this suite.
- 'src/shared/terminal-file-link-conformance.ts'
@@ -21,11 +33,6 @@ on:
# schema edit anywhere under here changes mobile's types, so a desktop-only
# change can break mobile's typecheck with no other mobile signal.
- 'src/shared/rpc-contract/**'
# Why: the catalog above holds params only. This file is the sole holder of
# the agent.launch RESULT shape, and mobile imports it as a value, not just
# a type. CROSS_VERSION_WIRE_PREFIXES already treats it as wire-critical, so
# without this one gate classes it that way while this one cannot see it.
- 'src/shared/agent-launch-intent.ts'
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
@@ -92,6 +99,19 @@ jobs:
- name: Typecheck
run: pnpm typecheck
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
# drifted. This fails when a test file that checks today stops checking, when a test leaves
# the program, and on @ts-nocheck; the baseline may only shrink.
- name: Typecheck tests (ratchet)
run: pnpm run check:tests-typecheck
# This includes the bridged replay of the whole recording corpus, which used to be a second
# step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point:
# it fails when a divergence class grows, when a divergence lands in no class at all, or when
# one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is
# ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite,
# so folding it in costs a fraction of that in wall time and one step less to skip.
- name: Test
run: pnpm test
+89 -17
View File
@@ -29,6 +29,7 @@ jobs:
should_run: ${{ steps.filter.outputs.should_run }}
native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
mobile_web_app: ${{ steps.filter.outputs.mobile_web_app }}
static_analysis: ${{ steps.filter.outputs.static_analysis }}
typecheck: ${{ steps.filter.outputs.typecheck }}
git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
@@ -142,24 +143,11 @@ jobs:
- name: Enforce type-aware code-quality baseline
run: pnpm run audit:code-quality:type-aware
# Why: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Mobile is a separate pnpm project,
# so the root install above leaves it empty and every mobile type degrades to
# an `error` type — reported as phantom findings against the changed lines.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
# the gitignored terminal/mermaid webview engine modules that tracked source imports,
# and skipping it degrades those very types the step exists to resolve. The drift
# guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
# lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
# Why here: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Without the install every mobile type
# degrades to an `error` type — reported as phantom findings against the changed lines.
- uses: ./.github/actions/install-mobile-dependencies
if: needs.code_paths.outputs.mobile_dependencies == 'true'
working-directory: mobile
run: |
pnpm install --frozen-lockfile
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
- name: Enforce changed-code quality
run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
@@ -663,6 +651,64 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts \
src/main/orcad/external-chromium-browser-process.integration.test.ts
# Why its own job: it needs mobile/node_modules and a real browser, and the sharded `test`
# matrix would pay for both on every shard to run two files. Dark through Phase C: this proves
# `build:mobile-web:app` on every PR that touches the page, and ships nothing -- packaging still
# builds the Phase A bootstrap via build:mobile-web.
mobile_web_app:
name: mobile web app bundle
needs: [code_paths]
if: needs.code_paths.outputs.mobile_web_app == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
# Why no native-runtime: the builder is esbuild and the render check is a browser. Nothing
# in this job loads node-pty.
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# The entry lives in mobile/ so one React resolves; without this every RN import is nothing.
- uses: ./.github/actions/install-mobile-dependencies
# Why the runner's Google Chrome and not a downloaded chromium: same reason as the orcad
# browser job -- Ubuntu 24.04 only ships an AppArmor userns profile for the Chrome .deb.
# Why fail instead of skip: a silently skipped render check is the failure this job exists
# to prevent.
- name: Resolve Chrome for the render check
run: |
set -euo pipefail
chrome="$(command -v google-chrome || command -v google-chrome-stable || true)"
if [ -z "$chrome" ]; then
echo "::error::No Google Chrome on the runner; the render check would silently skip."
exit 1
fi
"$chrome" --version
echo "ORCA_MOBILE_WEB_RENDER_BROWSER=$chrome" >> "$GITHUB_ENV"
- name: Build and verify the app bundle
run: pnpm run build:mobile-web:app
# The bundling tests skip themselves where mobile dependencies are absent, which is how they
# stay green in the sharded `test` job. This is the job that installs them, so here a missing
# install has to fail rather than skip everything the job exists to run.
- name: Builder, override census and render check
env:
ORCA_MOBILE_WEB_APP_DEPS_REQUIRED: '1'
run: |
pnpm exec vitest run --config config/vitest.config.ts \
config/scripts/build-mobile-web-app-bundle.test.mjs \
config/scripts/mobile-web-app-web-overrides.test.mjs \
config/scripts/mobile-web-app-render.test.mjs
cross-version-wire:
name: cross-version wire compatibility
needs: [code_paths]
@@ -697,6 +743,8 @@ jobs:
tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts
tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts
tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts
tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts
tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts
managed_hook_node18:
name: managed hooks on Node 18
@@ -748,6 +796,13 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why --no-file-parallelism: every file here launches a full Electron stack twice, and each
# probe carries its own in-process deadline. Four at once on a 4-vCPU runner starve each other
@@ -782,6 +837,12 @@ jobs:
- name: Project web client from renderer build
run: pnpm run build:web-from-renderer
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
# step instead of calling build:release, and electron-builder's beforePack guard hard-fails
# without out/mobile-web.
- name: Build mobile web bundle
run: pnpm run build:mobile-web
- name: Build native components
run: pnpm run build:native
@@ -855,6 +916,13 @@ jobs:
with:
native-runtime: node
persist-native-cache: 'false'
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Save compiled Node native modules
if: steps.deps.outputs.native-cache-hit != 'true'
@@ -1013,6 +1081,7 @@ jobs:
- shell_contracts
- test
- orcad_browser
- mobile_web_app
- cross-version-wire
- managed_hook_node18
- package
@@ -1049,6 +1118,8 @@ jobs:
TEST_SHOULD_RUN: ${{ needs.code_paths.outputs.test }}
ORCAD_BROWSER: ${{ needs.orcad_browser.result }}
ORCAD_BROWSER_SHOULD_RUN: ${{ needs.code_paths.outputs.orcad_browser }}
MOBILE_WEB_APP: ${{ needs.mobile_web_app.result }}
MOBILE_WEB_APP_SHOULD_RUN: ${{ needs.code_paths.outputs.mobile_web_app }}
CROSS_VERSION_WIRE: ${{ needs.cross-version-wire.result }}
CROSS_VERSION_WIRE_SHOULD_RUN: ${{ needs.code_paths.outputs.cross-version-wire }}
MANAGED_HOOK_NODE18: ${{ needs.managed_hook_node18.result }}
@@ -1091,6 +1162,7 @@ jobs:
check_job shell_contracts "$SHELL_CONTRACTS" "$SHELL_CONTRACTS_SHOULD_RUN"
check_job test "$TEST" "$TEST_SHOULD_RUN"
check_job orcad_browser "$ORCAD_BROWSER" "$ORCAD_BROWSER_SHOULD_RUN"
check_job mobile_web_app "$MOBILE_WEB_APP" "$MOBILE_WEB_APP_SHOULD_RUN"
check_job cross-version-wire "$CROSS_VERSION_WIRE" "$CROSS_VERSION_WIRE_SHOULD_RUN"
check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN"
check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN"
+40 -7
View File
@@ -871,8 +871,17 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Why: this install runs lifecycle scripts, so node-gyp rebuilds
# native/windows-registry and fetches that Node version's headers from
# nodejs.org. One `read ECONNRESET` there failed this blocking gate and the
# whole cut. Retry like the release build's install below.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for platform golden
run: npx electron-vite build --mode e2e
@@ -1088,8 +1097,14 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Same node-gyp header fetch as the blocking golden gate above.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for terminal rendering evidence
run: npx electron-vite build --mode e2e
@@ -1204,22 +1219,33 @@ jobs:
# ref, so cutting from an older/off-main ref whose tree predates a composite
# action would fail the step with "Can't find 'action.yml'". Restore the
# actions directory from the commit this workflow file itself came from.
# Not Windows-only: every platform now consumes install-mobile-dependencies, so
# any of them can be the one whose cut ref predates the action.
- name: Restore composite actions from the workflow ref
if: matrix.platform == 'win' && github.run_attempt == 1
shell: bash
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
PLATFORM: ${{ matrix.platform }}
run: |
set -euo pipefail
action_path=".github/actions/install-signpath-module/action.yml"
if [ -f "$action_path" ]; then
required=(.github/actions/install-mobile-dependencies/action.yml)
if [ "$PLATFORM" = win ] && [ "$GITHUB_RUN_ATTEMPT" = 1 ]; then
required+=(.github/actions/install-signpath-module/action.yml)
fi
missing=()
for action_path in "${required[@]}"; do
[ -f "$action_path" ] || missing+=("$action_path")
done
if [ "${#missing[@]}" -eq 0 ]; then
echo "Composite actions already present at the cut ref."
exit 0
fi
echo "Cut ref predates $action_path; restoring it from $WORKFLOW_SHA."
echo "Cut ref predates ${missing[*]}; restoring from $WORKFLOW_SHA."
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- .github/actions
test -f "$action_path"
for action_path in "${required[@]}"; do
test -f "$action_path"
done
# pnpm must be on PATH before setup-node so setup-node can locate the store for caching.
- name: Setup pnpm
@@ -1232,6 +1258,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: release builds hit the same native-module postinstall path as
# PR CI, so keep the pinned node-gyp override here too instead of
@@ -1272,6 +1301,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: `pnpm build:release` verifies the Linux computer-use provider by
# importing AT-SPI bindings, which are runtime package deps but are not
# present on stock GitHub Ubuntu release runners.
+7
View File
@@ -47,6 +47,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Cache the Electron binary + electron-builder tool downloads (notarytool,
# winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job, incl. mac.
@@ -74,6 +77,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Verify macOS signing environment
run: node config/scripts/verify-macos-release-env.mjs
env:
+18 -1
View File
@@ -55,6 +55,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -67,6 +70,9 @@ jobs:
uses: actions/cache@v4
with:
path: dist/orca-windows-setup.exe
# The mobile page trees are in the key because beforePack builds the mobile web bundle
# into the installer; src/** and config/** already cover src/mobile-web and the two
# bundle builders. A mobile-only change must miss this cache, not reuse a stale exe.
key: >-
crash-survival-installer-${{ hashFiles(
'src/**',
@@ -85,7 +91,12 @@ jobs:
'.npmrc',
'package.json',
'pnpm-lock.yaml',
'pnpm-workspace.yaml'
'pnpm-workspace.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why: production edits miss the installer cache by design, but Electron
@@ -101,6 +112,12 @@ jobs:
restore-keys: |
crash-survival-electron-builder-
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
@@ -75,6 +75,12 @@ jobs:
path: dist/orca-windows-setup.exe
key: branch-installer-${{ hashFiles('src/**', 'config/**', 'native/**', 'resources/win32/**', 'package.json', 'pnpm-lock.yaml') }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
@@ -57,6 +57,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -78,6 +81,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: rehearsal builds are never published, so the official-build
# secrets (telemetry key, diagnostics URL) are intentionally omitted.
- name: Build app
@@ -10,6 +10,7 @@ import {
INCIDENT_MONITOR_THRESHOLDS,
type IncidentSample
} from './incident-monitor.js'
import { relayOpsEnvironment } from './environment-config.js'
import type { AdmissionSelector } from './incident-selector.js'
const directories: string[] = []
@@ -63,6 +64,38 @@ function stateFile(
return path
}
// Every configured production cell, in the lexicographic order
// normalizeSelectorMembership canonicalises to. Derived from the same durable
// Terraform config the override path reads, so a new cell cannot strand these.
const configuredCellIds = relayOpsEnvironment('production').cells.map(
(cell) => cell.cellId
)
const canonicalCellIds = [...configuredCellIds].sort()
const canonicalMembership = {
existingOnly: canonicalCellIds,
migrationOnly: [],
general: []
}
// What the director reports: a normalised selector, never an echo of what the
// caller expected. An order-sensitive comparison only holds if the override path
// canonicalises its own input the same way.
function canonicalSample(generation = 1): IncidentSample {
const next = sample()
next.selector = { generation, membership: canonicalMembership }
return next
}
function membershipFile(
membership: Record<string, string[]> = canonicalMembership
): string {
const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-selector-'))
directories.push(directory)
const path = join(directory, 'selector.json')
writeFileSync(path, JSON.stringify(membership))
return path
}
function sample(): IncidentSample {
const observedAt = new Date(now).toISOString()
const signal = (value: number) => ({ value, observedAt })
@@ -154,9 +187,9 @@ describe('relay incident live preflight', () => {
)).resolves.toBeUndefined()
})
it('rejects monitor evidence beyond the 25-minute lineage bound', async () => {
it('rejects monitor evidence beyond the 35-minute lineage bound', async () => {
const path = stateFile('strict', {
startedAt: new Date(now - 26 * 60_000 - 1).toISOString()
startedAt: new Date(now - 36 * 60_000 - 1).toISOString()
})
await expect(runIncidentLivePreflight(
['--state-file', path],
@@ -183,24 +216,31 @@ describe('relay incident live preflight', () => {
await expect(runIncidentLivePreflight(
['--state-file', oneRollOld, '--wave-index', '1'], deps
)).resolves.toBeUndefined()
// Both edges of one predecessor job timeout: 5min + 75min exactly.
// Wave 0 edges: the 10-minute bound covers same-cap job start-up latency.
await expect(runIncidentLivePreflight(
['--state-file', agedState(80 * 60_000), '--wave-index', '1'], deps
['--state-file', agedState(10 * 60_000), '--wave-index', '0'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(80 * 60_000 + 1), '--wave-index', '1'], deps
['--state-file', agedState(10 * 60_000 + 1), '--wave-index', '0'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// Both edges of one predecessor job timeout: 10min + 75min exactly.
await expect(runIncidentLivePreflight(
['--state-file', agedState(85 * 60_000), '--wave-index', '1'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(85 * 60_000 + 1), '--wave-index', '1'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
await expect(runIncidentLivePreflight(
['--state-file', agedState(155 * 60_000), '--wave-index', '2'], deps
['--state-file', agedState(160 * 60_000), '--wave-index', '2'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(155 * 60_000 + 1), '--wave-index', '2'], deps
['--state-file', agedState(160 * 60_000 + 1), '--wave-index', '2'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
await expect(runIncidentLivePreflight(
['--state-file', agedState(230 * 60_000), '--wave-index', '3'], deps
['--state-file', agedState(235 * 60_000), '--wave-index', '3'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(230 * 60_000 + 1), '--wave-index', '3'], deps
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-3 argument.
await expect(runIncidentLivePreflight(
@@ -260,9 +300,11 @@ describe('relay incident live preflight', () => {
slowCell.sources['active-probe']!.signals[
'cell.production-gce-c1.latency_ms'
]!.value = 2_568
// A cell probe now re-samples before it fails a wave, so the wait is injected;
// this cell stays slow on every sample and still names what stopped it.
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: async () => slowCell }
{ now: () => now, collect: async () => slowCell, wait: async () => {} }
)).rejects.toThrow(
'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
)
@@ -278,6 +320,150 @@ describe('relay incident live preflight', () => {
)
})
// Why: this one sample decides a mutating wave, so an Asia cell's ~30 s
// "no healthy upstream" window could still fail a wave here even after the
// 15-minute gate learned to ride it out.
describe('cell probe tolerance', () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
// Serves `badSamples` unhealthy cell readings, then healthy ones.
const downThen = (badSamples: number) => {
let index = 0
return async () => {
const next = sample()
if (index++ < badSamples) {
next.sources['active-probe']!.signals['cell.production-gce-c1.health']!
.value = 0
next.sources['active-probe']!.signals['cell.production-gce-c1.ready']!
.value = 0
}
return next
}
}
it('re-samples through a probe outage within the tolerance', async () => {
const waits: number[] = []
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: downThen(tolerance),
wait: async (ms) => {
waits.push(ms)
}
}
)).resolves.toBeUndefined()
expect(waits).toHaveLength(tolerance)
})
it('fails the wave once the probe outage outlasts the tolerance', async () => {
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: downThen(tolerance + 1),
wait: async () => {}
}
)).rejects.toThrow('active-probe/threshold_equal cell.production-gce-c1.health')
})
it('does not re-sample a director probe failure', async () => {
let samples = 0
const down = async () => {
samples++
const next = sample()
next.sources['active-probe']!.signals['director.health']!.value = 0
return next
}
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: down, wait: async () => {} }
)).rejects.toThrow('active-probe/threshold_equal director.health')
expect(samples).toBe(1)
})
it('does not re-sample a non-probe threshold failure', async () => {
let samples = 0
const hot = async () => {
samples++
const next = sample()
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
return next
}
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: hot, wait: async () => {} }
)).rejects.toThrow('cloud-monitoring/threshold_max')
expect(samples).toBe(1)
})
})
// Why: on 2026-09-17 a canary wave died because one director admin read
// returned 404 for a 2 s Cloud SQL pool timeout. Collecting the sample is not
// a health verdict, so a thrown collector spends an attempt instead.
describe('collector failures', () => {
it('re-samples after a thrown collector and then passes', async () => {
const waits: number[] = []
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: async () => {
attempts++
if (attempts === 1) throw new Error('Relay admin telemetry returned 404')
return sample()
},
wait: async (ms) => {
waits.push(ms)
}
}
)).resolves.toBeUndefined()
expect(attempts).toBe(2)
expect(waits).toEqual([15_000])
})
it('fails the wave when every attempt throws, naming the collector', async () => {
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: async () => {
attempts++
throw new Error('Relay admin telemetry returned 404')
},
wait: async () => {}
}
)).rejects.toThrow(
'relay live preflight failed: collector: Relay admin telemetry returned 404'
)
expect(attempts).toBe(1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples)
})
it('does not re-sample a collector failure past the evidence-age budget', async () => {
const agedPath = stateFile('strict', {
startedAt: new Date(now - 26 * 60_000).toISOString(),
windowStartedAt: new Date(now - 25 * 60_000).toISOString(),
lastSampleAt: new Date(now - 10 * 60_000 + 7).toISOString(),
completedAt: new Date(now - 10 * 60_000 + 14).toISOString()
})
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', agedPath],
{
now: () => now,
collect: async () => {
attempts++
throw new Error('Relay admin telemetry returned 404')
},
wait: async () => {}
}
)).rejects.toThrow('relay live preflight failed: collector:')
expect(attempts).toBe(1)
})
})
it('enforces the signed migration policy', async () => {
const inactiveTarget = sample()
inactiveTarget.sources['director-admin']!.signals[
@@ -374,7 +560,7 @@ describe('relay incident live preflight', () => {
})
it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
const completedAt = now - 290_000
const completedAt = now - 590_000
const stale = sample()
stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
const collect = vi.fn(async () => stale)
@@ -420,6 +606,253 @@ describe('relay incident live preflight', () => {
expect(wait).toHaveBeenCalledTimes(4)
})
// Why: the same-cap break-glass skips the sealed 15-minute aggregate evidence,
// so this live recheck is the only thing left standing between the dispatch and
// a mutation. It must judge the fleet exactly as it does with evidence, and it
// must never accept a half-specified override.
describe('break-glass without monitor state', () => {
const overrideArgs = (extra: string[] = [], membership = canonicalMembership) => [
'--no-monitor-state',
'--expected-selector-generation', '1',
'--selector-membership-file', membershipFile(membership),
...extra
]
// The director's reading, plus whatever the override path decided to expect.
const liveCollect = (
mutate: (next: IncidentSample) => IncidentSample = (next) => next
) => async (expected: AdmissionSelector) => {
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return mutate(next)
}
it('accepts one complete fresh green sample with no sealed evidence', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{ now: () => now, collect: liveCollect() }
)).resolves.toBeUndefined()
})
// Why: the live selector is normalised and the comparison is an ordered
// stringify, so an operator's unsorted membership must canonicalise here or
// every override wave reads as selector drift on a healthy fleet.
it('canonicalises an unsorted operator membership', async () => {
const shuffled = {
existingOnly: [...canonicalCellIds].reverse(),
migrationOnly: [],
general: []
}
expect(shuffled.existingOnly).not.toEqual(canonicalCellIds)
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(
overrideArgs([], shuffled),
{
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
}
)).resolves.toBeUndefined()
expect(seen[0]!.membership.existingOnly).toEqual(canonicalCellIds)
})
// Why: normalising is also what enforces every configured cell exactly once,
// which a bare schema parse would have dropped.
it('rejects a membership that is not every configured cell exactly once', async () => {
const duplicated = {
existingOnly: [...canonicalCellIds, canonicalCellIds[0] as string],
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], duplicated),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
const missing = {
existingOnly: canonicalCellIds.slice(1),
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], missing),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
const unknown = {
existingOnly: [...canonicalCellIds.slice(1), 'production-gce-c999'],
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], unknown),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
})
it('fails closed on a live threshold breach', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.99
return next
}),
wait: async () => {}
}
)).rejects.toThrow('cloud-monitoring/threshold_max cloud_sql.cpu')
})
it('fails closed on a live selector mismatch', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.selector = { ...next.selector, generation: 7 }
return next
}),
wait: async () => {}
}
)).rejects.toThrow('selector_mismatch')
})
it('expects the wave-adjusted live selector generation', async () => {
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(
overrideArgs(['--wave-index', '2']),
{
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
}
)).resolves.toBeUndefined()
expect(seen[0]!.generation).toBe(5)
})
it('offsets by the wave delta the cell class declares', async () => {
const generationFor = async (args: string[]) => {
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(args, {
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
})).resolves.toBeUndefined()
return seen[0]!.generation
}
// A migration-only cell's wave isolates and restores nothing, so no predecessor moved it.
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0'])
)).toBe(1)
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2'])
)).toBe(5)
})
it('rejects a selector wave delta no cell class produces', async () => {
for (const delta of ['1', '3', '4', '', '-0', '02']) {
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', delta]),
{ now: () => now }
)).rejects.toThrow('usage:')
}
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']),
{ now: () => now }
)).rejects.toThrow('usage:')
})
it('pins the strictest migration policy', async () => {
// An inactive migration target is tolerable only under recover-forward,
// and an override cannot elect that policy, so this must still fail.
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.sources['director-admin']!.signals[
'cell.production-gce-c1.migration_target_inactive'
]!.value = 30
return next
}),
wait: async () => {}
}
)).rejects.toThrow('director-admin/threshold_max')
})
it('rejects a half-specified override', async () => {
const cases: string[][] = [
['--no-monitor-state'],
['--no-monitor-state', '--expected-selector-generation', '1'],
['--no-monitor-state', '--selector-membership-file', membershipFile()],
// Mixing the two sources would let a caller pass sealed evidence it
// never wants read.
[
'--no-monitor-state',
'--expected-selector-generation', '1',
'--selector-membership-file', membershipFile(),
'--state-file', stateFile()
],
// Override arguments without the flag must not be silently ignored.
['--state-file', stateFile(), '--expected-selector-generation', '1'],
['--no-monitor-state', '--no-monitor-state'],
['--expected-selector-generation', '1']
]
for (const args of cases) {
await expect(runIncidentLivePreflight(
args,
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('usage:')
}
})
it('rejects an unknown option and a negative generation', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(['--skip-everything']),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('usage:')
await expect(runIncidentLivePreflight(
[
'--no-monitor-state',
'--expected-selector-generation', '-1',
'--selector-membership-file', membershipFile()
],
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow()
})
it('re-samples a tolerable failure and then passes', async () => {
let samples = 0
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
samples++
if (samples === 1) {
next.sources['cloud-monitoring']!.signals['director.instances']!.value = 2
}
return next
}),
wait: async () => {}
}
)).resolves.toBeUndefined()
expect(samples).toBe(2)
})
})
it('uses the supplied admin token without minting through gcloud', async () => {
const identityToken = vi.fn(async () => 'minted.token.value')
const gcloud = livePreflightGcloud(
@@ -2,13 +2,21 @@ import { readFile } from 'node:fs/promises'
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { z } from 'zod'
import { relayOpsEnvironment } from './environment-config.js'
import { createGcloudClient } from './gcloud-client.js'
import { suppliedIdentityToken } from './incident-monitor-cli.js'
import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
import {
AdmissionSelectorSchema,
normalizeSelectorMembership,
SelectorMembershipSchema,
type AdmissionSelector
} from './incident-selector.js'
import {
evaluateIncidentSample,
FRESHNESS_FAILURE_CODES,
INCIDENT_MONITOR_THRESHOLDS,
preDrainDryRunPassed,
toleratedStreakKey,
type IncidentFailure,
type IncidentSample
} from './incident-monitor.js'
@@ -16,10 +24,15 @@ import { createIncidentSampleCollector } from './incident-monitor-sources.js'
const FRESHNESS_RETRY_ATTEMPTS = 5
const FRESHNESS_RETRY_INTERVAL_MS = 15_000
const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
// 10 min, not 5: the same-cap job reaches this check ~5 min after the monitor
// completes (runner queue ~2 min, gate job ~80 s, checkout ~60 s); on 2026-09-17
// a green gate died at 302 s. The live samples below hold every wave to now.
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX_PATTERN = /^[0-3]$/
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
export function livePreflightGcloud(
gcloud: ReturnType<typeof createGcloudClient>,
@@ -81,51 +94,80 @@ function describeFailure(failure: IncidentFailure): string {
return [`${failure.source}/${failure.code}`, ...detail].join(' ')
}
export async function runIncidentLivePreflight(
argv: string[],
dependencies: {
now?: () => number
wait?: (ms: number) => Promise<void>
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
gcloud?: ReturnType<typeof createGcloudClient>
environment?: NodeJS.ProcessEnv
} = {}
): Promise<void> {
const PREFLIGHT_USAGE =
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>] [--retry-freshness]' +
' | --no-monitor-state --expected-selector-generation <n>' +
' --selector-membership-file <json> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>]'
const VALUE_OPTIONS = new Set([
'--state-file',
'--wave-index',
'--selector-wave-delta',
'--expected-selector-generation',
'--selector-membership-file'
])
const FLAG_OPTIONS = new Set(['--retry-freshness', '--no-monitor-state'])
// Rejects an unknown option and a repeated one, so a typo can never silently
// widen what this check accepts.
export function parsePreflightArgs(argv: string[]): {
options: Map<string, string>
flags: Set<string>
} {
const args = argv[0] === '--' ? argv.slice(1) : argv
const freshnessRetryCount = args.filter((arg) => arg === '--retry-freshness').length
const rest = args.filter((arg) => arg !== '--retry-freshness')
const stateArgs: string[] = []
let waveIndex = '0'
let waveIndexCount = 0
for (let index = 0; index < rest.length; index += 1) {
if (rest[index] === '--wave-index') {
waveIndexCount += 1
waveIndex = rest[index + 1] ?? ''
index += 1
} else {
stateArgs.push(rest[index] as string)
const options = new Map<string, string>()
const flags = new Set<string>()
for (let index = 0; index < args.length; index += 1) {
const arg = args[index] as string
if (FLAG_OPTIONS.has(arg)) {
if (flags.has(arg)) throw new Error(PREFLIGHT_USAGE)
flags.add(arg)
continue
}
const value = args[index + 1]
if (!VALUE_OPTIONS.has(arg) || options.has(arg) || !value) {
throw new Error(PREFLIGHT_USAGE)
}
options.set(arg, value)
index += 1
}
return { options, flags }
}
// What the live recheck measures the fleet against. Either source supplies the
// exact same fields; only where they come from and how they can go stale differs.
type PreflightPlan = {
environment: 'production'
expectedSelector: AdmissionSelector
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
recoverySourceCellId: string | null
capacityCellId: string | null
// The instant the live samples age from. Monitor evidence ages from the moment
// the 15-minute window closed; an override has no evidence to age, so its
// retry budget starts when this process does.
evidenceAnchorMs: number
}
async function monitorEvidencePreflightPlan(
options: Map<string, string>,
nowMs: number,
waveIndex: string
): Promise<PreflightPlan> {
const stateFile = options.get('--state-file')
if (
freshnessRetryCount > 1 ||
waveIndexCount > 1 ||
!WAVE_INDEX_PATTERN.test(waveIndex) ||
stateArgs.length !== 2 ||
stateArgs[0] !== '--state-file' ||
!stateArgs[1]
) {
throw new Error(
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>] [--retry-freshness]'
)
}
!stateFile ||
options.has('--expected-selector-generation') ||
options.has('--selector-membership-file')
) throw new Error(PREFLIGHT_USAGE)
const state = PreflightStateSchema.parse(
JSON.parse(await readFile(resolve(stateArgs[1]), 'utf8'))
JSON.parse(await readFile(resolve(stateFile), 'utf8'))
)
const now = dependencies.now ?? Date.now
const completedAt = Date.parse(state.completedAt)
const windowStartedAt = Date.parse(state.windowStartedAt)
const lastSampleAt = Date.parse(state.lastSampleAt)
const evidenceAgeMs = now() - completedAt
const evidenceAgeMs = nowMs - completedAt
// Later same-cap waves start after sequential predecessor cell rolls, so the
// freshness bound grows by one cell-job timeout per predecessor; the live
// samples collected below still hold every wave to current health.
@@ -144,19 +186,89 @@ export async function runIncidentLivePreflight(
) {
throw new Error('relay live preflight monitor evidence is incomplete or stale')
}
return {
environment: state.environment,
expectedSelector: state.expectedSelector,
migrationPolicy: state.migrationPolicy,
recoverySourceCellId: state.recoverySourceCellId,
capacityCellId: state.capacityCellId,
evidenceAnchorMs: completedAt
}
}
// Break-glass: the caller authorized skipping the aggregate 15-minute monitor
// gate, so the expected selector comes straight from the dispatch inputs instead
// of sealed evidence. Nothing about this weakens the live sample below, and the
// policy is pinned to strict -- the only one the same-cap rollout ever verifies.
async function overridePreflightPlan(
options: Map<string, string>,
nowMs: number
): Promise<PreflightPlan> {
const generation = options.get('--expected-selector-generation')
const membershipFile = options.get('--selector-membership-file')
if (!generation || !membershipFile || options.has('--state-file')) {
throw new Error(PREFLIGHT_USAGE)
}
// Canonicalise exactly as the monitor CLI does when it seals evidence. The live
// selector read from the director is normalised too and the comparison is an
// ordered stringify, so unsorted operator input would read as selector drift on
// a healthy fleet; normalising is also what enforces every configured cell
// exactly once.
const membership = normalizeSelectorMembership(
SelectorMembershipSchema.parse(
JSON.parse(await readFile(resolve(membershipFile), 'utf8'))
),
new Set(relayOpsEnvironment('production').cells.map((cell) => cell.cellId))
)
return {
environment: 'production',
expectedSelector: AdmissionSelectorSchema.parse({
generation: Number(generation),
membership
}),
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
evidenceAnchorMs: nowMs
}
}
export async function runIncidentLivePreflight(
argv: string[],
dependencies: {
now?: () => number
wait?: (ms: number) => Promise<void>
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
gcloud?: ReturnType<typeof createGcloudClient>
environment?: NodeJS.ProcessEnv
} = {}
): Promise<void> {
const parsed = parsePreflightArgs(argv)
const waveIndex = parsed.options.get('--wave-index') ?? '0'
if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE)
const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2'
if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE)
const now = dependencies.now ?? Date.now
const plan = parsed.flags.has('--no-monitor-state')
? await overridePreflightPlan(parsed.options, now())
: await monitorEvidencePreflightPlan(parsed.options, now(), waveIndex)
const maxEvidenceAgeMs =
MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
const gcloud = livePreflightGcloud(
dependencies.gcloud ?? createGcloudClient(),
dependencies.environment
)
// Each predecessor same-cap apply wave reversibly isolates and restores its
// cell, advancing the selector generation by exactly 2 with membership
// unchanged (rollback is single-cell, so it never reaches a later wave), so
// the live selector comparison must expect the wave-adjusted generation.
// cell with membership unchanged (rollback is single-cell, so it never reaches
// a later wave), so the live selector comparison must expect the wave-adjusted
// generation. A general cell advances it by 2; a migration-only cell is already
// isolated and stays that way, so its wave advances it by 0. A wave is never
// mixed, so one delta covers every predecessor.
const collectOptions = {
environment: state.environment,
environment: plan.environment,
expectedSelector: {
...state.expectedSelector,
generation: state.expectedSelector.generation + 2 * Number(waveIndex)
...plan.expectedSelector,
generation: plan.expectedSelector.generation + Number(selectorWaveDelta) * Number(waveIndex)
},
...(dependencies.now ? { now: dependencies.now } : {})
}
@@ -167,32 +279,71 @@ export async function runIncidentLivePreflight(
const wait = dependencies.wait ?? ((ms: number) => new Promise<void>((resolveWait) => {
setTimeout(resolveWait, ms)
}))
const attempts = freshnessRetryCount === 1 ? FRESHNESS_RETRY_ATTEMPTS : 1
const freshnessAttempts = parsed.flags.has('--retry-freshness') ? FRESHNESS_RETRY_ATTEMPTS : 1
// Why: this single sample decides a mutating wave, so an Asia cell's ~30 s
// "no healthy upstream" window, or a one-minute director instance-replacement
// dip, could fail a wave here even after the 15-minute gate learned to ride it
// out. Hold the two to the same tolerance. Unlike the freshness retry this
// needs no flag, because a tolerated breach is never the operator's call to
// waive.
const cellProbeAttempts = 1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const attempts = Math.max(freshnessAttempts, cellProbeAttempts)
let freshnessRetries = freshnessAttempts - 1
let cellProbeRetries = cellProbeAttempts - 1
// Waiting must never carry the mutation past the same evidence-age bound the
// entry check enforces, so the wave budget also caps the retry window.
const budgetExhausted = (): boolean =>
now() + FRESHNESS_RETRY_INTERVAL_MS - plan.evidenceAnchorMs > maxEvidenceAgeMs
for (let attempt = 1; attempt <= attempts; attempt++) {
// A director admin read can fail on its own (its handler maps a Cloud SQL
// pool timeout onto 404), which says nothing about relay health; spend an
// attempt on it rather than failing the wave on one unlucky sample.
let sample: IncidentSample
try {
sample = await collect()
} catch (error) {
const message = error instanceof Error ? error.message : 'sample collection failed'
if (attempt === attempts || budgetExhausted()) {
throw new Error(`relay live preflight failed: collector: ${message}`)
}
console.warn(
`relay live preflight re-sampling after collector failure (${attempt}/${attempts - 1})`
)
await wait(FRESHNESS_RETRY_INTERVAL_MS)
continue
}
const evaluation = evaluateIncidentSample(
await collect(),
sample,
now(),
state.migrationPolicy,
state.recoverySourceCellId,
state.capacityCellId
plan.migrationPolicy,
plan.recoverySourceCellId,
plan.capacityCellId
)
if (evaluation.status === 'green') return
const freshnessOnly = evaluation.failures.every((failure) =>
const freshnessFailures = evaluation.failures.filter((failure) =>
FRESHNESS_FAILURE_CODES.has(failure.code)
)
// Waiting must never carry the mutation past the same evidence-age bound
// the entry check enforces, so the wave budget also caps the retry window.
const budgetExhausted =
now() + FRESHNESS_RETRY_INTERVAL_MS - completedAt > maxEvidenceAgeMs
if (!freshnessOnly || attempt === attempts || budgetExhausted) {
// Tolerated readings only (per-cell probes and the director instance count).
// The director and auth health probes are absent here on purpose and fail
// the wave on their first bad sample.
const cellProbeFailures = evaluation.failures.filter((failure) =>
!FRESHNESS_FAILURE_CODES.has(failure.code) && toleratedStreakKey(failure) !== null
)
const retryable =
freshnessFailures.length + cellProbeFailures.length === evaluation.failures.length &&
(freshnessFailures.length === 0 || freshnessRetries > 0) &&
(cellProbeFailures.length === 0 || cellProbeRetries > 0)
if (!retryable || attempt === attempts || budgetExhausted()) {
throw new Error(
`relay live preflight failed: ${evaluation.failures
.map(describeFailure)
.join(',')}`
)
}
if (freshnessFailures.length > 0) freshnessRetries--
if (cellProbeFailures.length > 0) cellProbeRetries--
console.warn(
`relay live preflight awaiting fresh evidence (${attempt}/${attempts - 1})`
`relay live preflight re-sampling after tolerable failure (${attempt}/${attempts - 1})`
)
await wait(FRESHNESS_RETRY_INTERVAL_MS)
}
@@ -68,6 +68,20 @@ const StateSchema = z.object({
observed: z.number().optional(),
threshold: z.number().optional()
})),
// Pre-2026-09-17 state files predate cell-probe tolerance; a resumed run that
// carries no streak is one that also restarts its window, so it earns nothing.
probeStreaks: z.record(z.string(), z.number().int().nonnegative()).default({}),
toleratedProbeEvents: z.array(z.object({
recordedAt: z.string(),
windowSequence: z.number().int().nonnegative(),
failures: z.array(z.object({
code: z.string(),
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
signal: z.string().optional(),
observed: z.number().optional(),
threshold: z.number().optional()
}))
})).default([]),
completedAt: z.string().nullable()
})
@@ -463,6 +477,7 @@ export async function runIncidentMonitorCli(
await wait(ms)
},
collect: segmentedCollect,
warn: (message) => console.warn(message),
persist: async (nextState) => await persistState(options.stateFile, nextState),
checkpoint: async (checkpoint) => {
await appendCheckpoint(options.summaryFile, checkpoint)
@@ -458,4 +458,112 @@ describe('incident monitor sources', () => {
expect(serialized).not.toContain(identityToken)
expect(serialized).not.toContain(sensitiveIdentity)
})
// Why: the director maps a Cloud SQL pool connect timeout in cell-status onto
// a 404, so without this the whole sample dies on one 2 s database stall.
it('retries a transient director admin failure and then reports the cell', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
const selector = {
generation: 1,
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
}
const waits: number[] = []
let cellStatusCalls = 0
const fetchImpl: typeof fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
if (body.cellId) {
cellStatusCalls++
if (cellStatusCalls === 1) {
return Response.json(
{ error: 'timeout exceeded when trying to connect' },
{ status: 404 }
)
}
return Response.json({
status: {
enabled: true,
connectionCapacity: { hardCap: 600 },
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
}
})
}
return Response.json({
blocked: 0,
blockedExpiredUnregistered: 0,
registeredTargetInactive: 0
})
}
const result = await directorSignals(
'production',
selector,
gcloud,
now,
fetchImpl,
async (ms) => {
waits.push(ms)
}
)
expect(waits).toEqual([5_000])
expect(cellStatusCalls).toBe(productionCells.length + 1)
expect(result.cells).toHaveLength(productionCells.length)
expect(
result.source.signals['cell.production-gce-c1.connection_hard_cap']
).toMatchObject({ value: 600 })
})
// A rejected admin token is a decision, not weather: retrying it only burns
// the sample budget and hides the misconfiguration.
it('fails immediately on an unauthorized director admin response', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
const selector = {
generation: 1,
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
}
const waits: number[] = []
let requestCount = 0
const fetchImpl: typeof fetch = async (_input, init) => {
requestCount++
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
return Response.json({ error: 'invalid_token' }, { status: 401 })
}
await expect(
directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
waits.push(ms)
})
).rejects.toThrow('Relay admin telemetry returned 401')
expect(requestCount).toBe(2)
expect(waits).toEqual([])
})
// A 404 the director means (wrong role) must not be retried either.
it('does not retry a director-only 404', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
let requestCount = 0
const fetchImpl: typeof fetch = async () => {
requestCount++
return Response.json({ error: 'director_only' }, { status: 404 })
}
await expect(
directorSignals(
'production',
{ generation: 1, membership: { existingOnly: [], migrationOnly: [], general: productionCells } },
gcloud,
now,
fetchImpl,
async () => {}
)
).rejects.toThrow('Relay admin telemetry returned 404')
expect(requestCount).toBe(1)
})
})
@@ -414,21 +414,96 @@ function relaySignals(
return { observedAt, signals }
}
const ADMIN_RETRY_ATTEMPTS = 3
const ADMIN_RETRY_INTERVAL_MS = 5_000
const ADMIN_RETRYABLE_STATUSES = new Set([500, 502, 503, 504])
const AdminErrorBodySchema = z.object({ error: z.string() })
// The director's /v1/admin/cell-status maps any thrown operation error onto 404
// with an {error: message} body, so a Cloud SQL pool connect timeout arrives
// here as a 404. These are the pool-acquire and transient SQLSTATE messages the
// relay app's own isRelayDatabaseTransientError treats as re-runnable.
const ADMIN_TRANSIENT_ERROR_MESSAGES = [
'timeout exceeded when trying to connect',
'Connection terminated due to connection timeout',
'Connection terminated unexpectedly',
'database_temporarily_unavailable',
'deadlock detected',
'canceling statement due to',
'too many connections',
'the database system is shutting down',
'the database system is starting up'
]
const defaultWait = async (ms: number): Promise<void> =>
await new Promise((resolveWait) => setTimeout(resolveWait, ms))
// A dropped socket or the 30 s AbortSignal firing; both leave the request with
// no verdict, and every admin read here is a plain query.
function transientFetchFailure(error: unknown): boolean {
if (error instanceof TypeError) return true
const name = (error as { name?: unknown } | null)?.name
return name === 'AbortError' || name === 'TimeoutError'
}
function retryableAdminStatus(status: number, body: string): boolean {
if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
// 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
if (status !== 404) return false
const parsed = AdminErrorBodySchema.safeParse(
((): unknown => {
try {
return JSON.parse(body)
} catch {
return null
}
})()
)
return (
parsed.success &&
ADMIN_TRANSIENT_ERROR_MESSAGES.some((message) => parsed.data.error.includes(message))
)
}
async function adminPost(
fetchImpl: typeof fetch,
origin: string,
token: string,
path: string,
body: unknown
body: unknown,
wait: (ms: number) => Promise<void> = defaultWait
): Promise<unknown> {
const response = await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
})
if (!response.ok) throw new Error(`Relay admin telemetry returned ${response.status}`)
return await response.json()
for (let attempt = 1; ; attempt++) {
const lastAttempt = attempt >= ADMIN_RETRY_ATTEMPTS
let response: Response
try {
response = await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
})
} catch (error) {
if (lastAttempt || !transientFetchFailure(error)) throw error
console.warn(
`relay admin telemetry retrying ${path} after network failure` +
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
)
await wait(ADMIN_RETRY_INTERVAL_MS)
continue
}
if (response.ok) return await response.json()
// Status only: the body carries a director error message, which must not
// reach the log the way an identity would.
const errorBody = await response.text().catch(() => '')
if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) {
throw new Error(`Relay admin telemetry returned ${response.status}`)
}
console.warn(
`relay admin telemetry retrying ${path} after ${response.status}` +
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
)
await wait(ADMIN_RETRY_INTERVAL_MS)
}
}
export async function directorSignals(
@@ -436,7 +511,8 @@ export async function directorSignals(
expectedSelector: AdmissionSelector,
gcloud: GcloudClient,
nowMs: number,
fetchImpl: typeof fetch
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void> = defaultWait
): Promise<{
source: IncidentSource
selector: AdmissionSelector
@@ -452,7 +528,8 @@ export async function directorSignals(
environment.directorOrigin,
token,
'/v1/admin/admission-selector/status',
{ v: 1 }
{ v: 1 },
wait
)
).selector
const selector = {
@@ -467,10 +544,14 @@ export async function directorSignals(
statuses.push({
cell,
status: CellStatusSchema.parse(
await adminPost(fetchImpl, environment.directorOrigin, token, '/v1/admin/cell-status', {
v: 1,
cellId: cell.cellId
})
await adminPost(
fetchImpl,
environment.directorOrigin,
token,
'/v1/admin/cell-status',
{ v: 1, cellId: cell.cellId },
wait
)
).status
})
}
@@ -495,7 +576,8 @@ export async function directorSignals(
sourceCellId: source.cellId,
targetCellId: target.cellId,
completeReady: false
}
},
wait
)
)
})
@@ -571,6 +653,7 @@ export type IncidentSampleCollectorOptions = {
expectedSelector: AdmissionSelector
fetchImpl?: typeof fetch
now?: () => number
wait?: (ms: number) => Promise<void>
}
export function createIncidentSampleCollector(
@@ -610,7 +693,8 @@ export function createIncidentSampleCollector(
options.expectedSelector,
gcloud,
nowMs,
fetchImpl
fetchImpl,
options.wait ?? defaultWait
),
cloudMetricEntries
])
+533 -14
View File
@@ -273,16 +273,19 @@ describe('incident monitor evaluator', () => {
)
})
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
for (const errors of [1, 2, 3]) {
// Why: measured non-503 5xx per rolling five minutes over the 24 h to 2026-09-17
// was p90 3 / p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze
// 29% of 15-minute gates on chronic director 500 bursts.
it('tolerates the measured chronic director error rate without relaxing other gates', () => {
for (const errors of [1, 3, 5, 9, 15]) {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
}
const excess = healthySample()
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(16)
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
expect.objectContaining({ signal: 'director.errors', observed: 16, threshold: 15 })
)
const auth = healthySample()
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
@@ -358,17 +361,20 @@ describe('incident monitor evaluator', () => {
})
})
// Why: measured latest-sum over the 24 h to 2026-09-17 was p95 212 / p99 262 /
// max 282, so the old bar of 250 sat under the observed peak and froze 21.8% of
// 15-minute gates. 320 still fires at 65% of the 490 usable connections.
it('bounds Cloud SQL backends above measured healthy peaks', () => {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(250)
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(282)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(251)
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(321)
expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
code: 'threshold_max',
source: 'cloud-monitoring',
signal: 'cloud_sql.backends',
observed: 251,
threshold: 250
observed: 321,
threshold: 320
})
})
@@ -807,6 +813,150 @@ describe('incident monitor lifecycle', () => {
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why: dry-run 35258662628 read a healthy fleet clean for 13 minutes, then one
// unreadable Cloud Monitoring sample restarted the window and the restart blew
// the lineage cap, so a green fleet produced no verdict.
it('carries a 15-minute window through a single collector failure', async () => {
let now = startedAt
const warnings: string[] = []
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (now === startedAt + 10 * 60_000) {
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: (message) => {
warnings.push(message)
}
})
expect(result.windowSequence).toBe(0)
expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
expect(result.sampleCount).toBe(16)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(result.continuityEvents).toEqual([{
recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
windowSequence: 0,
tolerated: true,
failures: [{ code: 'collector_failed', source: 'cloud-monitoring' }]
}])
expect(warnings).toEqual([
'incident monitor collector failed: cloud monitoring read failed'
])
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('restarts the window after three consecutive collector failures', async () => {
let now = startedAt
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (failures > 0 && now >= startedAt + 10 * 60_000) {
failures--
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: () => {}
})
const restartMinute = 10 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
expect(result.windowSequence).toBe(1)
expect(result.windowStartedAt).toBe(
new Date(startedAt + restartMinute * 60_000).toISOString()
)
expect(result.completedAt).toBe(
new Date(startedAt + (restartMinute + 15) * 60_000).toISOString()
)
expect(result.sampleCount).toBe(16)
expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
...Array<boolean>(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
false
])
expect(result.continuityEvents.at(-1)!.failures).toEqual([
{ code: 'collector_failed', source: 'cloud-monitoring' }
])
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('still reaches a dry-run verdict after a restart on the last sample', async () => {
let now = startedAt
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (failures > 0 && now >= startedAt + 13 * 60_000) {
failures--
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: () => {}
})
expect(result.windowSequence).toBe(1)
expect(result.windowStartedAt).toBe(new Date(startedAt + 16 * 60_000).toISOString())
expect(result.completedAt).toBe(new Date(startedAt + 31 * 60_000).toISOString())
expect(result.sampleCount).toBe(16)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('gives a signal a fresh budget only after it reads fresh again', async () => {
let now = startedAt
const staleMinutes = new Set([3, 5, 6, 9, 10])
@@ -971,7 +1121,7 @@ describe('incident monitor lifecycle', () => {
expect(result.sampleCount).toBe(16)
})
it('fails a dry run after 25 total minutes of continuity resets', async () => {
it('fails a dry run after 35 total minutes of continuity resets', async () => {
let now = startedAt
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
@@ -985,15 +1135,16 @@ describe('incident monitor lifecycle', () => {
durationMinutes: 15,
intervalMs: 60_000
})
let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
// Two restarts: the first on the window's last sample, the second far enough
// into the replacement window that no third window can finish in the lineage.
const staleMinutes = new Set([13, 14, 15, 24, 25, 26])
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
staleSamples--
if (staleMinutes.has((now - startedAt) / 60_000)) {
return healthySample(
now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
)
@@ -1008,8 +1159,8 @@ describe('incident monitor lifecycle', () => {
new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString()
)
expect(result.frozenAt).not.toBeNull()
expect(result.windowSequence).toBe(1)
expect(result.sampleCount).toBe(13)
expect(result.windowSequence).toBe(2)
expect(result.sampleCount).toBe(9)
expect(result.failures).toContainEqual({
code: 'continuity_deadline_exceeded',
source: 'active-probe',
@@ -1114,3 +1265,371 @@ describe('incident monitor lifecycle', () => {
expect(waits[0]).toBe(45_000)
})
})
// Why: the asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
// us-central1 behind a 2 s timeout, so a saturated pool makes the load balancer
// answer "no healthy upstream" for about 30 s. Every one of 39 pre-roll gates froze
// on that, and 7 of the last 14 froze on this signal alone.
describe('incident monitor cell probe tolerance', () => {
const dryRunState = () =>
initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
// Returns the finished state of a 15-minute dry-run whose cell probe reads
// health=0 and ready=0 on the sample indexes in `badSamples`.
const runWithCellProbeGaps = async (badSamples: Set<number>) => {
let now = startedAt
let index = -1
return await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (badSamples.has(index)) {
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
}
it('passes a dry-run through a probe outage no longer than the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithCellProbeGaps(
new Set(Array.from({ length: tolerance }, (_, offset) => 3 + offset))
)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
// The blip is absorbed, not hidden: the sealed state still carries it.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health',
observed: 0,
threshold: 1
})
)
// A recovered probe hands back the full budget rather than a partial one.
expect(result.probeStreaks).toEqual({})
})
it('freezes once a cell probe fails past the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithCellProbeGaps(
new Set(Array.from({ length: tolerance + 1 }, (_, offset) => 3 + offset))
)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
expect(result.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health',
observed: 0,
threshold: 1
})
)
// Only the samples past the tolerance freeze; the first two are still absorbed.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
})
it('does not accumulate a streak across a recovered sample', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
// Repeated single-sample outages, each separated by a healthy sample, never
// reach the tolerance however many times they recur.
const spaced = new Set([2, 4, 6, 8, 10])
expect(spaced.size).toBeGreaterThan(tolerance)
const result = await runWithCellProbeGaps(spaced)
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why: health, ready and latency_ms all describe the same round trip, so the streak
// is keyed by cell. Keyed per signal, this cell holds every individual streak at one
// and never reaches the tolerance, yet it is unhealthy without a break from sample 2.
it('freezes on a cell that alternates between slow and unanswered', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index < 2) return sample
// Two samples slow, then two samples down, repeating: never the same signal
// twice in a row beyond the tolerance, but never healthy either.
if (Math.floor((index - 2) / 2) % 2 === 0) {
sample.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
signal(9_000, now)
} else {
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
})
// Why: the streak lives in the state file, so a resumed run must not hand a cell
// that was already failing a fresh budget.
it('freezes immediately when a resumed state carries a full streak', async () => {
let now = startedAt
const resumed = {
...dryRunState(),
lastSampleAt: new Date(startedAt).toISOString(),
probeStreaks: {
'active-probe/cell.production-gce-c1':
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
}
}
const result = await runIncidentMonitor(resumed, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
const sample = healthySample(now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health'
})
)
})
it('gives the director and auth probes no tolerance', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index === 3) {
sample.sources['active-probe']!.signals['director.health'] = signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ source: 'active-probe', signal: 'director.health' })
)
})
})
// Why: Cloud Run replaces director instances in place, so the count leaves the
// [5, 6] band for about one sample roughly twice a day, and a deploy overlap
// raises it the same way. Neither is an unhealthy fleet, and freezing on it
// blocks the roll that fixes the measured condition.
describe('incident monitor director instance tolerance', () => {
const dryRunState = () =>
initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
// Finished state of a 15-minute dry run whose director instance count reads
// `counts[index]` on the sample indexes that map has, and 5 everywhere else.
const runWithInstanceCounts = async (
counts: Map<number, number>,
state = dryRunState()
) => {
let now = startedAt
let index = -1
return await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
const count = counts.get(index)
if (count !== undefined) {
sample.sources['cloud-monitoring']!.signals['director.instances'] =
signal(count, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
}
it('passes a dry run through an instance dip no longer than the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 2]))
)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
// Absorbed, not hidden: the sealed state still carries the dip.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_min',
source: 'cloud-monitoring',
signal: 'director.instances',
observed: 2,
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin
})
)
expect(result.probeStreaks).toEqual({})
})
it('passes a dry run through a deploy-overlap overshoot within the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 9]))
)
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_max',
signal: 'director.instances',
observed: 9,
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax
})
)
})
it('freezes once the instance count stays out of band past the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance + 1 }, (_, offset) => [3 + offset, 2]))
)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
expect(result.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_min',
source: 'cloud-monitoring',
signal: 'director.instances',
observed: 2
})
)
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
})
it('does not accumulate a streak across a recovered sample', async () => {
const spaced = new Map([2, 4, 6, 8, 10].map((index) => [index, 2] as const))
expect(spaced.size).toBeGreaterThan(
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
)
const result = await runWithInstanceCounts(new Map(spaced))
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why min and max share one streak: a count that alternates above and below
// the band would otherwise hold each streak at one and never freeze, yet the
// director is never at its configured size.
it('freezes on a count that alternates above and below the band', async () => {
const counts = new Map<number, number>()
for (let index = 2; index < 12; index += 1) {
counts.set(index, index % 2 === 0 ? 2 : 9)
}
const result = await runWithInstanceCounts(counts)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
})
// Why: the streak lives in the state file, so a resumed run must not hand a
// director that was already out of band a fresh budget.
it('freezes immediately when a resumed state carries a full streak', async () => {
const result = await runWithInstanceCounts(new Map([[0, 2]]), {
...dryRunState(),
lastSampleAt: new Date(startedAt).toISOString(),
probeStreaks: {
'cloud-monitoring/director.instances':
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
}
})
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ signal: 'director.instances' })
)
})
it('keeps every other cloud-monitoring signal at zero tolerance', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index === 3) {
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.99, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ source: 'cloud-monitoring', signal: 'cloud_sql.cpu' })
)
})
})
+151 -17
View File
@@ -42,12 +42,18 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
} as const satisfies Record<RelayOpsRegion, number>,
cloudSqlCpuUtilization: 0.8,
cloudSqlMemoryUtilization: 0.9,
// Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
// bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a
// pre-drain gate on baseline noise). 250 clears measured healthy peaks while
// firing well before the verified 400-connection ceiling; the retry signals
// below discriminate incident-class contention.
cloudSqlBackends: 250,
// Why: 320, recalibrated 2026-09-17 from 250. The auth instance sums seven
// databases, and its steady load has grown past the 2026-08-26 measurement the
// old bar came from. Measured latest-sum over the 24 h to 2026-09-17, aligned
// per minute exactly as this signal reads it: p50 118 / p90 165 / p95 212 /
// p99 262 / max 282. 250 was under the observed max, so 1.95% of minutes and
// 21.8% of 15-minute pre-drain gates froze on ordinary load. 320 clears every
// measured healthy minute with 13% of headroom above the peak and still fires
// at 65% of the 490 connections the budget work (#21165) treats as usable out
// of max_connections 500, so exhaustion-class growth is caught with 170
// connections still in hand. The retry signals below discriminate incident-class
// contention. Re-tighten when the auth connection model lands (#21165).
cloudSqlBackends: 320,
// Bound the observed recovery load; deadlocks remain zero-tolerance.
cloudSqlLockWaits: 20,
cloudSqlDeadlocks: 0,
@@ -101,18 +107,48 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
directorCpuUtilization: 0.8,
directorMemoryUtilization: 0.8,
directorConcurrency: 64,
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
directorErrors: 3,
// Why: 15, recalibrated 2026-09-17 from 3. This counts non-503 5xx answers from
// the director over the rolling five-minute query window; 503s are excluded
// because they are the documented back-pressure answer a client retries.
// Measured over the 24 h to 2026-09-17 (272 non-503 5xx against 71 941 503s):
// p50 0 / p90 3 / p95 5 / p99 9 / max 52. A bar of 3 sits at the p90, so 9.2%
// of windows and 29.0% of 15-minute pre-drain gates froze on the chronic 500
// bursts that /v1/assign, /v1/regions and /v1/resolve emit alongside the
// recurring Cloud SQL stall. 15 clears the chronic p99 with margin and drops
// the baseline gate-freeze rate to 1.5%, while leaving the exceptional 20-52
// bursts detectable. A director that is actually broken answers 5xx on a large
// share of its traffic: it serves ~50 requests a minute in 503s alone, so a
// real fault lands in the hundreds per window, an order of magnitude clear of
// this bar.
directorErrors: 15,
authErrors: 0,
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
cellConnections: 500,
cellQueuedBytes: 48 * 1024 * 1024,
migrationBlocked: 0
migrationBlocked: 0,
// Why: one sample is one HTTP round trip from one GitHub runner to one cell, so
// a single bad reading is evidence about that round trip, not about the fleet.
// The asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
// us-central1 over a 176 ms round trip behind a 2 s statement timeout, so a
// saturated pool makes the load balancer answer "no healthy upstream" for about
// 30 s. That answer is an HTTP 503, not a transport failure, so provenance
// cannot separate it from a cell that genuinely serves health=0 -- persistence
// can. At the 60 s sample interval a 30 s outage shows up in one sample and at
// worst two, so a cell's probe must fail more than this many consecutive samples
// before it freezes the run. The streak is per cell, not per signal, so a cell
// that alternates between slow and unanswered still accumulates one. This applies
// to per-cell probes and the director instance count; the director and auth health
// probes stay at zero tolerance.
cellProbeToleranceSamples: 2
} as const
export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000
// Why: 35 minutes, raised 2026-09-17 from 25. A 15-minute window plus one
// restart must fit: a continuity reset on the window's last sample restarts at
// minute 16 and finishes at 31. Under 25 a reset past minute 9 cost the whole
// verdict, which is what run 35258662628 hit on a healthy fleet.
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 35 * 60_000
export type IncidentSourceName =
| 'active-probe'
@@ -212,6 +248,17 @@ export type IncidentMonitorState = {
}[]
frozenAt: string | null
failures: IncidentFailure[]
// Consecutive samples each tolerated reading has currently been failing for,
// keyed by cell so its health, ready and latency readings share one streak, and
// by signal for the director instance count.
probeStreaks: Record<string, number>
// Cell-probe breaches absorbed by the tolerance, kept so a green artifact still
// shows what the gate chose not to freeze on.
toleratedProbeEvents: {
recordedAt: string
windowSequence: number
failures: IncidentFailure[]
}[]
completedAt: string | null
}
@@ -625,6 +672,8 @@ export function initialIncidentMonitorState(input: {
continuityEvents: [],
frozenAt: null,
failures: [],
probeStreaks: {},
toleratedProbeEvents: [],
completedAt: null
}
}
@@ -635,6 +684,7 @@ export type IncidentMonitorDependencies = {
collect(): Promise<IncidentSample>
persist(state: IncidentMonitorState): Promise<void>
checkpoint(summary: IncidentCheckpoint): Promise<void>
warn?(message: string): void
}
function checkpointMinutes(durationMinutes: number): number[] {
@@ -656,10 +706,19 @@ const CONTINUITY_FAILURE_CODES = new Set([
...FRESHNESS_FAILURE_CODES
])
// A whole sample we could not read gets the same consecutive-sample budget as an
// unread signal, for the same reason: one failed collector round trip is evidence
// about that round trip, not about the fleet. `monitor_gap` is excluded because it
// means the run itself stopped sampling, so the window genuinely has a hole.
const TOLERABLE_CONTINUITY_FAILURE_CODES = new Set([
'collector_failed',
...FRESHNESS_FAILURE_CODES
])
// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
// past minute 10 costs the entire verdict, so a healthy fleet produced none on
// 2026-09-05. A signal may miss this many consecutive samples before the window
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap in force
// then, a restart past minute 10 cost the entire verdict, so a healthy fleet
// produced none on 2026-09-05. A signal may miss this many consecutive samples before the window
// restarts; the sample is still evaluated against every threshold it can read,
// and a threshold breach still freezes the run outright.
export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
@@ -668,6 +727,44 @@ function freshnessKey(failure: IncidentFailure): string {
return `${failure.source}/${failure.signal ?? '*'}`
}
// The streak key for a per-cell active-probe reading, or null if the failure is not
// one. A cell probe is a single HTTP round trip and so is subject to
// cellProbeToleranceSamples; director and auth probes return null on purpose,
// because they are the single points of failure this gate exists to catch.
//
// Why the key is the cell and not the signal: health, ready and latency_ms all
// describe the same round trip. Keyed per signal, a cell that alternates between
// answering slowly and not answering at all holds every individual streak at one and
// never reaches the tolerance, so a continuously unhealthy cell passes the gate.
export function cellProbeStreakKey(failure: IncidentFailure): string | null {
const signal = failure.signal
if (failure.source !== 'active-probe' || signal === undefined) return null
if (!signal.startsWith('cell.')) return null
const lastDot = signal.lastIndexOf('.')
if (lastDot < 'cell.'.length) return null
return `${failure.source}/${signal.slice(0, lastDot)}`
}
// Cloud Run replaces director instances in place rather than holding the count,
// so the reading leaves [min, max] for about one sample roughly twice a day, and a
// deploy that briefly serves two revisions raises it the same way. Neither is an
// unhealthy fleet, and on 2026-09-17 this was one of the signals freezing the
// pre-drain gate on a condition the roll exists to fix. Min and max share one
// streak on purpose: a count that alternates above and below the band would
// otherwise hold each individual streak at one and never reach the tolerance.
export function directorInstancesStreakKey(failure: IncidentFailure): string | null {
if (failure.source !== 'cloud-monitoring' || failure.signal !== 'director.instances') {
return null
}
return `${failure.source}/${failure.signal}`
}
// The streak key for any reading subject to cellProbeToleranceSamples, or null
// for a reading that freezes the run on its first bad sample.
export function toleratedStreakKey(failure: IncidentFailure): string | null {
return cellProbeStreakKey(failure) ?? directorInstancesStreakKey(failure)
}
// Rebuild the per-signal tolerated streak from the trailing continuity events so a
// resumed monitor cannot hand a signal a fresh budget.
function resumeFreshnessStreaks(
@@ -777,7 +874,12 @@ export async function runIncidentMonitor(
state.recoverySourceCellId,
state.capacityCellId
)
} catch {
} catch (error) {
dependencies.warn?.(
`incident monitor collector failed: ${
error instanceof Error ? error.message : 'unknown error'
}`
)
evaluation = {
status: 'freeze',
evaluatedAt: new Date(dependencies.now()).toISOString(),
@@ -798,7 +900,8 @@ export async function runIncidentMonitor(
const toleratedKeys = new Set(
state.windowStartedAt !== null &&
continuityFailures.length > 0 &&
continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
continuityFailures.every((failure) =>
TOLERABLE_CONTINUITY_FAILURE_CODES.has(failure.code))
? continuityFailures.map(freshnessKey)
: []
)
@@ -828,9 +931,40 @@ export async function runIncidentMonitor(
}
state.sampleCount++
}
if (thresholdFailures.length > 0) {
const probeFailures = new Map<string, IncidentFailure[]>()
for (const failure of thresholdFailures) {
const key = toleratedStreakKey(failure)
if (key === null) continue
probeFailures.set(key, [...(probeFailures.get(key) ?? []), failure])
}
for (const key of Object.keys(state.probeStreaks)) {
if (!probeFailures.has(key)) delete state.probeStreaks[key]
}
const sustainedProbeFailures: IncidentFailure[] = []
const toleratedProbeFailures: IncidentFailure[] = []
for (const [key, entries] of probeFailures) {
const streak = (state.probeStreaks[key] ?? 0) + 1
state.probeStreaks[key] = streak
const target =
streak > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
? sustainedProbeFailures
: toleratedProbeFailures
target.push(...entries)
}
if (toleratedProbeFailures.length > 0) {
state.toleratedProbeEvents.push({
recordedAt: evaluation.evaluatedAt,
windowSequence: state.windowSequence,
failures: toleratedProbeFailures
})
}
const freezingFailures = [
...thresholdFailures.filter((failure) => toleratedStreakKey(failure) === null),
...sustainedProbeFailures
]
if (freezingFailures.length > 0) {
state.frozenAt ??= evaluation.evaluatedAt
state.failures = [...state.failures, ...thresholdFailures]
state.failures = [...state.failures, ...freezingFailures]
}
if (state.windowStartedAt === null) {
if (dependencies.now() >= lineageDeadlineMs) {
@@ -0,0 +1,152 @@
import { describe, expect, it, vi } from 'vitest'
import type { RelayConfig } from './config.js'
vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string) => token === 'deploy-token',
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
createRegionalRehomeTokenVerifier: () => async () => false,
createRuntimeTokenVerifier: () => async () => false
}))
vi.mock('./relay-token-verifier.js', () => ({
createRelayTokenVerifier: () => async () => null,
readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
}))
import { createRelayApp } from './app.js'
// The message the pool raises when its own dial outruns connectionTimeoutMillis.
// This is the shape that failed a rollout wave as an HTTP 404.
const poolTimeout = () => new Error('Connection terminated due to connection timeout')
function adminRequest(body: unknown): RequestInit {
return {
method: 'POST',
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
body: JSON.stringify(body)
}
}
function appWith(assignments: Record<string, unknown>, overrides: Partial<RelayConfig> = {}) {
return createRelayApp(config(overrides), {
store: {} as never,
assignments: assignments as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
}
describe('admin routes under a database that is briefly out of reach', () => {
it('answers cell-status with a retryable 503 instead of a not-found', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ cellDeploymentStatus }, { publicAssignmentRetryAfterSeconds: 7 })
const response = await app.request(
'/v1/admin/cell-status',
adminRequest({ v: 1, cellId: 'production-gce-c7' })
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('7')
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
})
it('keeps the not-found mapping for a real cell-status failure', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw new Error('unknown_cell')
})
const app = appWith({ cellDeploymentStatus })
const response = await app.request(
'/v1/admin/cell-status',
adminRequest({ v: 1, cellId: 'production-gce-c7' })
)
expect(response.status).toBe(404)
expect(await response.json()).toEqual({ error: 'unknown_cell' })
})
it('answers admission-selector/status with a retryable 503 instead of a conflict', async () => {
const inspectCellAdmissionSelector = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ inspectCellAdmissionSelector })
const response = await app.request(
'/v1/admin/admission-selector/status',
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('5')
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
})
it('keeps the conflict mapping for a real admission-selector/status failure', async () => {
const inspectCellAdmissionSelector = vi.fn(async () => {
throw new Error('admission_selector_attempt_not_found')
})
const app = appWith({ inspectCellAdmissionSelector })
const response = await app.request(
'/v1/admin/admission-selector/status',
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'admission_selector_attempt_not_found' })
})
it('leaves the pre-request guards ahead of the mapping alone', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ cellDeploymentStatus })
// An unauthenticated caller must never learn the database is struggling.
const unauthenticated = await app.request('/v1/admin/cell-status', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, cellId: 'production-gce-c7' })
})
expect(unauthenticated.status).toBe(401)
const invalid = await app.request('/v1/admin/cell-status', adminRequest({ v: 1 }))
expect(invalid.status).toBe(400)
expect(cellDeploymentStatus).not.toHaveBeenCalled()
})
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
publicUrl: 'https://relay.example.test',
cellUrl: 'https://relay.example.test',
region: 'us-central1',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'director',
cellId: 'director',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data',
...overrides
}
}
+118 -61
View File
@@ -27,10 +27,11 @@ import {
createRegionalRehomeTokenVerifier,
createRuntimeTokenVerifier
} from './admin-token-verifier.js'
import type {
CellFenceAttemptEvidence,
RelayAssignment,
RelayAssignmentStore
import {
RelayHomeCellUnavailableError,
type CellFenceAttemptEvidence,
type RelayAssignment,
type RelayAssignmentStore
} from './assignment-store.js'
import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
import { CELL_ADMISSION_STATES } from './cell-admission-selector.js'
@@ -58,6 +59,8 @@ const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
const REGION_CATALOG_CACHE_MS = 30_000
// A drain that outlives the roll step it belongs to is an outage, not a pacing win.
const DRAIN_PACE_WINDOW_MAX_MS = 5 * 60 * 1_000
type AdmissionRejectionLogEntry = {
route: 'assign' | 'resolve'
@@ -72,7 +75,7 @@ export function createRelayApp(
operations: {
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
drain: (graceMs: number, options?: { paceWindowMs?: number }) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
@@ -176,6 +179,21 @@ export function createRelayApp(
context.header('Retry-After', String(stickyRetryAfterSeconds))
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
}
// An admin route that collapses every failure into one status cannot tell a
// real conflict from a database that was briefly out of reach, and the rollout
// tooling retries on 503 only. Transient failures get the answer the public
// routes already give; everything else keeps the route's own mapping.
const rejectAdminOperation = (
context: Context,
error: unknown,
status: 404 | 409
): Response => {
if (!isRelayDatabaseTransientError(error)) {
return context.json({ error: operationError(error) }, status)
}
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
// Aggregate counters cannot separate a handful of pathological hosts from a broad
// population, so every admission rejection names its host and reason. Keyed on
// route:lane:reason rather than host, the log stays bounded under load.
@@ -223,7 +241,15 @@ export function createRelayApp(
})
app.get('/v1/regions', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
return context.json({ v: 1, regions: await regionCatalog() })
try {
return context.json({ v: 1, regions: await regionCatalog() })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
// Same contract as the assignment routes: a database that is briefly out
// of reach is a retry, not a director fault.
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'region_catalog_temporarily_unavailable' }, 503)
}
})
app.post('/v1/assign', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
@@ -336,16 +362,17 @@ export function createRelayApp(
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'assign',
lane,
hinted: Boolean(body.data.reconnect),
relayHostId: claims.relayHostId,
reason: operationError(error)
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
if (isRelayAssignmentCapacityError(error)) {
if (isRelayAssignmentUnavailableError(error)) {
if (lane === 'placement') {
operations.recordRegionSelection?.({ targetRegion, fallback: false })
}
@@ -364,11 +391,13 @@ export function createRelayApp(
fallback: lane === 'placement' && assignment.region !== targetRegion
})
// Grant-side counterpart of the rejection log: reconnect grants are rare
// enough to log and make "which cell is this host on" answerable.
if (lane === 'sticky') {
// enough to log and make "which cell is this host on" answerable. The
// placement-lane ones matter most — they are the only record that a host
// whose sticky lane failed verification landed anywhere at all.
if (body.data.reconnect) {
console.warn(
`[orca-relay] assignment granted lane=sticky host=${relayHostLogDigest(claims.relayHostId)}` +
` cell=${assignment.cellId}`
`[orca-relay] assignment granted lane=${lane} hinted=true` +
` host=${relayHostLogDigest(claims.relayHostId)} cell=${assignment.cellId}`
)
}
const lease = await new SignJWT({
@@ -441,16 +470,17 @@ export function createRelayApp(
leaseExpiresAt: assignment.leaseExpiresAt
})
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
route: 'resolve',
lane: 'none',
hinted: false,
relayHostId: body.data.relayHostId,
reason: operationError(error)
reason: operationError(error),
...homeCellRejectionDetail(error)
})
}
if (isRelayAssignmentCapacityError(error)) {
if (isRelayAssignmentUnavailableError(error)) {
return context.json({ error: operationError(error) }, 503)
}
if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context)
@@ -465,12 +495,18 @@ export function createRelayApp(
return context.json({ error: 'invalid_token' }, 401)
}
const body = z
.object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) })
.object({
v: z.literal(1),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
// Spreads the drain sends, and so the re-dials, over this window.
paceWindowMs: z.number().int().nonnegative().max(DRAIN_PACE_WINDOW_MAX_MS).optional()
})
.strict()
.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
operations.drain(body.data.graceMs)
return context.json({ ok: true })
const paceWindowMs = body.data.paceWindowMs ?? 0
operations.drain(body.data.graceMs, { paceWindowMs })
return context.json({ ok: true, paceWindowMs })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
@@ -497,7 +533,7 @@ export function createRelayApp(
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
@@ -548,7 +584,7 @@ export function createRelayApp(
...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {})
})
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/runtime-status', async (context) => {
@@ -602,7 +638,7 @@ export function createRelayApp(
await operations.assignments.recordCellHeartbeat(body.data)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-rehome-status', async (context) => {
@@ -622,7 +658,7 @@ export function createRelayApp(
await operations.assignments.recordCellRegionalRehomeStatus(body.data)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
@@ -661,7 +697,7 @@ export function createRelayApp(
const control = await operations.assignments.applyRegionalRehomeControl(body.data)
return context.json({ v: 1, control })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/regional-rehome-trust-probe', async (context) => {
@@ -705,7 +741,7 @@ export function createRelayApp(
})
return context.json(result)
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate', async (context) => {
@@ -726,7 +762,7 @@ export function createRelayApp(
)
return context.json({ v: 1, migration })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-complete', async (context) => {
@@ -747,7 +783,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-supersede-cell', async (context) => {
@@ -772,7 +808,7 @@ export function createRelayApp(
)
return context.json({ v: 1, superseded })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/rebalance-dormant', async (context) => {
@@ -793,7 +829,7 @@ export function createRelayApp(
)
return context.json({ v: 1, assignment })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply', async (context) => {
@@ -813,7 +849,7 @@ export function createRelayApp(
const result = await operations.assignments.applyCellAdmissionSelector(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => {
@@ -838,7 +874,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/status', async (context) => {
@@ -860,7 +896,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => {
@@ -891,7 +927,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-state', async (context) => {
@@ -916,7 +952,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => {
@@ -939,7 +975,7 @@ export function createRelayApp(
)
return context.json({ v: 1, cellId: body.data.cellId, expiresAt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => {
@@ -962,7 +998,7 @@ export function createRelayApp(
)
return context.json({ v: 1, cellId: body.data.cellId, committed: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attest', async (context) => {
@@ -991,7 +1027,7 @@ export function createRelayApp(
attempt: result.attempt
})
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => {
@@ -1012,7 +1048,7 @@ export function createRelayApp(
const attempt = await operations.assignments.prepareCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-start', async (context) => {
@@ -1037,7 +1073,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-plan', async (context) => {
@@ -1061,7 +1097,7 @@ export function createRelayApp(
)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-operation', async (context) => {
@@ -1087,7 +1123,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-status', async (context) => {
@@ -1107,7 +1143,7 @@ export function createRelayApp(
const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-abort', async (context) => {
@@ -1128,7 +1164,7 @@ export function createRelayApp(
const attempt = await operations.assignments.abortCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-prepare', async (context) => {
@@ -1154,7 +1190,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-send', async (context) => {
@@ -1174,7 +1210,7 @@ export function createRelayApp(
const attempt = await operations.assignments.beginCellDrainSend(body.data)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-receipt', async (context) => {
@@ -1196,7 +1232,7 @@ export function createRelayApp(
)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-recover-forward', async (context) => {
@@ -1216,7 +1252,7 @@ export function createRelayApp(
const result = await operations.assignments.prepareCellDrainRecovery(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-config', async (context) => {
@@ -1243,7 +1279,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate-cell', async (context) => {
@@ -1265,7 +1301,7 @@ export function createRelayApp(
)
return context.json({ v: 1, started })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-capacity', async (context) => {
@@ -1288,7 +1324,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...capacity })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-status', async (context) => {
@@ -1307,12 +1343,17 @@ export function createRelayApp(
if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) {
return context.json({ error: 'insufficient_permission' }, 403)
}
const status = await operations.assignments.cellEvacuationStatus(
body.data.sourceCellId,
body.data.targetCellId,
body.data.completeReady
)
return context.json({ v: 1, ...status })
try {
const status = await operations.assignments.cellEvacuationStatus(
body.data.sourceCellId,
body.data.targetCellId,
body.data.completeReady
)
return context.json({ v: 1, ...status })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
})
app.post('/v1/admin/cell-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
@@ -1329,7 +1370,7 @@ export function createRelayApp(
const status = await operations.assignments.cellDeploymentStatus(body.data.cellId)
return context.json({ v: 1, status })
} catch (error) {
return context.json({ error: operationError(error) }, 404)
return rejectAdminOperation(context, error, 404)
}
})
return app
@@ -1912,25 +1953,41 @@ function logAssignmentRejection(input: {
hinted: boolean
relayHostId: string
reason: string
cause?: string
cell?: string
suppressed?: number
}): void {
console.warn(
`[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` +
` hinted=${input.hinted} reason=${input.reason}` +
` host=${relayHostLogDigest(input.relayHostId)}` +
(input.cause === undefined ? '' : ` cause=${input.cause}`) +
(input.cell === undefined ? '' : ` cell=${input.cell}`) +
(input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`)
)
}
function isRelayAssignmentCapacityError(error: unknown): boolean {
// The home-cell reason is not capacity, but it is the same answer to the client:
// retry, the director cannot place you right now.
function isRelayAssignmentUnavailableError(error: unknown): boolean {
return (
error instanceof Error &&
['relay_capacity_exhausted', 'relay_connection_headroom_exhausted'].includes(
error.message
)
[
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted',
'relay_home_cell_unavailable'
].includes(error.message)
)
}
function homeCellRejectionDetail(
error: unknown
): { cause: string; cell: string } | Record<string, never> {
return error instanceof RelayHomeCellUnavailableError
? { cause: error.unavailableCause, cell: error.cellId }
: {}
}
function isCanonicalRelayOrigin(value: string): boolean {
const url = new URL(value)
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const HEARTBEAT_TTL_MS = 45_000
const START_MS = 100
const IDENTITY = { userId: 'user-1', relayHostId: 'host000000000001' }
// A connection-limited cell is what makes the committed fence mandatory, and
// that is the branch which used to answer "capacity exhausted".
const FENCED_CELL: RelayCellConfig = {
id: 'home',
url: 'https://home.example.com',
capacityRequests: 1_000,
connectionHardCap: 600,
connectionUnobservedBound: 50
}
const databases: RelayDatabase[] = []
afterEach(async () => {
for (const database of databases.splice(0)) await database.close()
})
interface Harness {
store: RelayAssignmentStore
heartbeat: (cell: RelayCellConfig, ready: boolean) => Promise<void>
setNow: (value: number) => void
}
async function setup(cells: RelayCellConfig[] = [FENCED_CELL]): Promise<Harness> {
const database = await openInMemoryRelayDatabase()
databases.push(database)
let now = START_MS
const store = new RelayAssignmentStore(database, () => now, {
requireLiveCells: true,
heartbeatTtlMs: HEARTBEAT_TTL_MS
})
await store.reconcileCells(cells, true)
const heartbeat = async (cell: RelayCellConfig, ready: boolean): Promise<void> => {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: `1111111${cells.indexOf(cell)}-1111-4111-8111-111111111111`,
startedAt: 50,
ready,
observedRequests: 0,
...(cell.connectionHardCap === undefined
? {}
: {
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionHardCap: cell.connectionHardCap,
connectionUnobservedBound: cell.connectionUnobservedBound
})
})
}
for (const cell of cells) await heartbeat(cell, true)
return { store, heartbeat, setNow: (value: number) => (now = value) }
}
async function assignFailure(store: RelayAssignmentStore): Promise<unknown> {
return await store.assign(IDENTITY).then(
() => new Error('assign unexpectedly succeeded'),
(error: unknown) => error
)
}
function homeCellError(error: unknown): RelayHomeCellUnavailableError {
expect(error).toBeInstanceOf(RelayHomeCellUnavailableError)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the assertion above.
return error as RelayHomeCellUnavailableError
}
describe('home cell unavailable', () => {
it('names a readiness failure rather than reporting fleet capacity', async () => {
const { store, heartbeat, setNow } = await setup()
await store.assign(IDENTITY)
setNow(START_MS + 1_000)
await heartbeat(FENCED_CELL, false)
const error = homeCellError(await assignFailure(store))
expect(error.message).toBe('relay_home_cell_unavailable')
expect(error.unavailableCause).toBe('not_ready')
expect(error.cellId).toBe(FENCED_CELL.id)
})
it('names a heartbeat gap as unheard even though the cell last reported ready', async () => {
const { store, setNow } = await setup()
await store.assign(IDENTITY)
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('unheard')
})
it('names a drained cell as draining ahead of its heartbeat gap', async () => {
const { store, setNow } = await setup()
await store.assign(IDENTITY)
await store.configureCell(FENCED_CELL, false)
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
})
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
await store.assign(IDENTITY)
await expect(
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
).rejects.toThrow('relay_capacity_exhausted')
})
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
const home: RelayCellConfig = {
id: 'home',
url: 'https://home.example.com',
capacityRequests: 1_000
}
const spare: RelayCellConfig = {
id: 'spare',
url: 'https://spare.example.com',
capacityRequests: 1_000
}
const { store, heartbeat, setNow } = await setup([home, spare])
expect((await store.assign(IDENTITY)).cellId).toBe(home.id)
setNow(START_MS + 1_000)
await heartbeat(home, false)
expect((await store.assign(IDENTITY)).cellId).toBe(spare.id)
})
})
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignment } from './assignment-store.js'
import { RelayHomeCellUnavailableError, type RelayAssignment } from './assignment-store.js'
import type { RelayConfig } from './config.js'
const fakes = vi.hoisted(() => ({
@@ -52,6 +52,39 @@ describe('assignment rejection logging', () => {
expect(line).not.toContain(host)
})
it('separates an unavailable home cell from capacity and names its cause', async () => {
const host = 'cccccccccccccccc'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => {
throw new RelayHomeCellUnavailableError('cell-asia-1', 'not_ready')
}),
// The sticky lane refuses a host whose home cell is not live, so this
// arrives hinted on the placement lane.
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(503)
expect(await response.json()).toEqual({ error: 'relay_home_cell_unavailable' })
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment rejected')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('reason=relay_home_cell_unavailable')
expect(line).toContain('cause=not_ready')
expect(line).toContain('cell=cell-asia-1')
expect(line).not.toContain('relay_capacity_exhausted')
expect(line).not.toContain(host)
})
it('logs an unhinted placement rejection without the raw host id', async () => {
const host = 'gggggggggggggggg'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
@@ -221,6 +254,31 @@ describe('assignment grant logging', () => {
expect(line).not.toContain(host)
})
it('logs a hinted grant served by the placement lane', async () => {
const host = 'rrrrrrrrrrrrrrrr'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: vi.fn(async () => assignment('cell-new', host)),
resolve: vi.fn(async () => null)
} as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
expect(response.status).toBe(200)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('assignment granted')
)
expect(line).toContain('lane=placement')
expect(line).toContain('hinted=true')
expect(line).toContain('cell=cell-new')
expect(line).not.toContain(host)
})
it('does not log unhinted placement grants', async () => {
const host = 'pppppppppppppppp'
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+263 -133
View File
@@ -1,5 +1,13 @@
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
import { IDLE_REHOME_PAGE_SIZE, selectIdleRegionalRehomes } from './idle-regional-rehome-selection.js'
import {
selectIdleRegionalRehomes,
type IdleRegionalRehomeCandidate,
type IdleRehomeHostCursor
} from './idle-regional-rehome-selection.js'
import {
RegionalRehomePollTelemetry,
type RegionalRehomePollGate
} from './regional-rehome-poll-telemetry.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import {
previewRegionalRehomeEligibility,
@@ -45,6 +53,15 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
CONTROL_RENEWAL_BATCH_SQL,
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
controlRenewalBatchParams,
orderedControlRenewalRows,
readControlRenewalOutcomes,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
@@ -101,21 +118,8 @@ type RelayAssignmentStoreOptions = {
recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
}
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
| 'database_error'
export type { ControlRenewalOutcome, ControlRenewalRequest }
const CONTROL_RENEWAL_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved'
])
export type RelayAssignment = AssignmentIdentity & {
cellId: string
cellUrl: string
@@ -395,6 +399,25 @@ class AssignmentInventoryScopeChanged extends Error {
}
}
// Why a reason of its own: a host whose home cell is fenced-but-unattested is
// refused regardless of fleet headroom, so reporting it as capacity sends
// operators after capacity that was never short. Every cell boot and every
// readiness dip produces these.
export type RelayHomeCellUnavailableCause =
| 'draining'
| 'booting'
| 'unheard'
| 'not_ready'
export class RelayHomeCellUnavailableError extends Error {
constructor(
readonly cellId: string,
readonly unavailableCause: RelayHomeCellUnavailableCause
) {
super('relay_home_cell_unavailable')
}
}
// Debt holds connection headroom for a control that may still arrive shortly
// after its director-side timeout. Nothing legitimately arrives minutes late
// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that
@@ -925,7 +948,10 @@ export class RelayAssignmentStore {
)) &&
!(await this.cellHasCommittedFence(transaction, current.cellId, now))
) {
throw new Error('relay_capacity_exhausted')
throw new RelayHomeCellUnavailableError(
current.cellId,
await this.homeCellUnavailableCause(transaction, current.cellId, now)
)
}
forcedDeadReassignment = true
}
@@ -2677,7 +2703,13 @@ export class RelayAssignmentStore {
)
if (assignment.cellId !== text(row, 'cell_id')) moved++
} catch (error) {
if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error
// One unplaceable host must not end the sweep for the rest.
if (
!(error instanceof RelayHomeCellUnavailableError) &&
!(error instanceof Error && error.message === 'relay_capacity_exhausted')
) {
throw error
}
}
}
return moved
@@ -3333,28 +3365,57 @@ export class RelayAssignmentStore {
return previewRegionCorrection(this.database, this.now())
}
private idleRegionalCandidateOffset = 0
private idleRegionalCandidateCursor: IdleRehomeHostCursor = null
private readonly regionalRehomePollTelemetry = new RegionalRehomePollTelemetry()
async selectIdleRegionalRehomeCandidates(
processSafety?: RegionalRehomeSafetySnapshot
): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
): Promise<IdleRegionalRehomeCandidate[]> {
const now = this.now()
if (!processSafety || this.regionalRehomeCohortPercent === 0) return []
const gated = (gate: RegionalRehomePollGate): IdleRegionalRehomeCandidate[] => {
this.regionalRehomePollTelemetry.record({ now, gate, candidates: 0 })
return []
}
if (!processSafety) return gated('process-safety-unavailable')
if (this.regionalRehomeCohortPercent === 0) return gated('cohort-zero')
const control = (await this.database.query(
"SELECT enabled, not_before FROM relay_region_rehome_control WHERE control_id = 'global'"
`SELECT enabled, not_before, preference_max_age_ms, host_cooldown_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
))[0]
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) return []
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) {
return gated('control-closed')
}
// The dispatch budget is durable and global, but until now only
// `commitIdleRegionalRehome` consulted it -- after the join had already run and
// the worker had already POSTed every candidate to its source cell. An absent
// row means the budget has never been spent, so it opens the gate.
const worker = (await this.database.query(
`SELECT paused_until, next_dispatch_at FROM relay_region_rehome_worker_state
WHERE worker_id = 'global'`
))[0]
if (worker && (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now)) {
return gated('budget-closed')
}
const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now)
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return []
const candidates = await selectIdleRegionalRehomes({
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return gated('fleet-safety')
const startedAt = performance.now()
const selection = await selectIdleRegionalRehomes({
database: this.database, now, heartbeatTtlMs: this.heartbeatTtlMs,
cohortPercent: this.regionalRehomeCohortPercent, offset: this.idleRegionalCandidateOffset,
cohortPercent: this.regionalRehomeCohortPercent,
preferenceMaxAgeMs: Number(control.preference_max_age_ms),
hostCooldownMs: Number(control.host_cooldown_ms),
cursor: this.idleRegionalCandidateCursor,
connectionHeadroom: await this.connectionHeadroomByCell(this.database),
cellIsClean: regionalRehomeCellSafetyIsClean
})
this.idleRegionalCandidateOffset = candidates.length < IDLE_REHOME_PAGE_SIZE
? 0 : this.idleRegionalCandidateOffset + candidates.length
return candidates
this.idleRegionalCandidateCursor = selection.cursor
this.regionalRehomePollTelemetry.record({
now,
gate: 'open',
candidates: selection.candidates.length,
selectionMs: performance.now() - startedAt
})
return selection.candidates
}
async commitIdleRegionalRehome(
@@ -3478,132 +3539,149 @@ export class RelayAssignmentStore {
})
}
// Kept as the single-row contract for callers and tests: resolves on a
// renewal and throws the outcome (or the driver's own error) otherwise.
async renewControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number }
): Promise<void> {
validateActivityId(input.activityId)
const now = this.now()
const maximumExpiresAt =
now +
ASSIGNMENT_LIMITS.activityLeaseMs +
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
if (
!Number.isSafeInteger(input.expiresAt) ||
input.expiresAt <= now ||
input.expiresAt > maximumExpiresAt
) {
if (!controlRenewalExpiryIsValid(input.expiresAt, now)) {
throw new Error('invalid_activity_expiry')
}
const startedAt = performance.now()
let outcome: ControlRenewalOutcome = 'database_error'
try {
outcome =
this.database.dialect === 'postgres'
? await this.renewPostgresControlActivity(identity, input, now)
: await this.renewTransactionalControlActivity(identity, input, now)
outcome = await this.renewOneControlActivity({ identity, ...input }, now)
if (outcome !== 'renewed') throw new Error(outcome)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) {
outcome = message as ControlRenewalOutcome
}
outcome = controlRenewalOutcomeOfError(error)
throw error
} finally {
this.recordControlRenewal?.(performance.now() - startedAt, outcome)
}
}
private async renewPostgresControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number },
// Renews every due control lease on a cell in one write transaction, returning
// one outcome per input row in input order. Never throws for a multi-row batch:
// a caller routes its own session on its own outcome.
async renewControlActivities(
rows: readonly ControlRenewalRequest[]
): Promise<ControlRenewalOutcome[]> {
const now = this.now()
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
const accepted: Array<ControlRenewalRequest & { index: number }> = []
for (const [index, row] of rows.entries()) {
const rejection = controlRenewalRejection(row, now)
if (rejection) outcomes[index] = rejection
else accepted.push({ ...row, index })
}
const startedAt = performance.now()
try {
if (accepted.length === 0) return outcomes
let results: ControlRenewalOutcome[]
try {
results = await this.executeControlRenewals(accepted, now)
} catch (error) {
if (rows.length === 1) {
outcomes[accepted[0]!.index] = controlRenewalOutcomeOfError(error)
throw error
}
results = accepted.map(() => 'database_error')
}
for (const [position, row] of accepted.entries()) outcomes[row.index] = results[position]!
return outcomes
} finally {
// Every path, so a rethrown lone renewal and an all-invalid batch are
// counted the same as a batch that reached PostgreSQL.
const durationMs = performance.now() - startedAt
for (const outcome of outcomes) this.recordControlRenewal?.(durationMs, outcome)
}
}
private async executeControlRenewals(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
if (rows.length === 1) return [await this.renewOneControlActivity(rows[0]!, now)]
if (this.database.dialect !== 'postgres') {
// Correctness over throughput: the SQLite writer is serialized anyway, and
// this is the dialect the unit suites run on.
return await this.renewControlActivitiesInSeries(rows, now)
}
const ordered = orderedControlRenewalRows(rows.map((row, index) => ({ ...row, index })))
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
try {
const parsed = readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams(ordered, now)
),
ordered.length
)
for (const [position, row] of ordered.entries()) outcomes[row.index] = parsed[position]!
return outcomes
} catch (error) {
// One statement means one contended assignment row can fail the whole
// batch, so a failure degrades to the per-host statements this replaced
// rather than costing every other host on the cell its renewal.
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_batch_failed',
rows: ordered.length,
message: String((error as { message?: unknown }).message)
})
)
await Promise.all(
ordered.map(async (row) => {
try {
outcomes[row.index] = await this.renewOneControlActivity(row, now)
} catch {
outcomes[row.index] = 'database_error'
}
})
)
return outcomes
}
}
private async renewControlActivitiesInSeries(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
const outcomes: ControlRenewalOutcome[] = []
for (const row of rows) {
try {
outcomes.push(await this.renewOneControlActivity(row, now))
} catch {
outcomes.push('database_error')
}
}
return outcomes
}
// Returns the outcome; a driver or pool failure reaches the caller unchanged.
private async renewOneControlActivity(
row: ControlRenewalRequest,
now: number
): Promise<ControlRenewalOutcome> {
const row = (
await this.database.query(
`WITH assignment_state AS MATERIALIZED (
SELECT cell_id, assignment_epoch
FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?
FOR UPDATE
), migration_state AS MATERIALIZED (
SELECT migration.assignment_epoch
FROM relay_assignment_migrations migration
JOIN assignment_state assignment
ON migration.target_cell_id = assignment.cell_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT 1 AS authorized
FROM assignment_state assignment
WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state)
), lease_state AS MATERIALIZED (
SELECT lease.activity_kind, lease.cell_id
FROM relay_assignment_activity_leases lease
CROSS JOIN authorization_state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, ?),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
AND state.activity_kind = 'control' AND state.cell_id = ?
RETURNING 1
), renewed_assignment AS (
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
AND EXISTS (SELECT 1 FROM renewed_lease)
RETURNING 1
)
SELECT CASE
WHEN NOT EXISTS (SELECT 1 FROM assignment_state)
THEN 'assignment_not_found'
WHEN NOT EXISTS (SELECT 1 FROM authorization_state)
THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (SELECT 1 FROM lease_state)
THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state
WHERE activity_kind <> 'control' OR cell_id <> ?
) THEN 'control_activity_moved'
WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome`,
[
identity.userId,
identity.relayHostId,
identity.userId,
identity.relayHostId,
input.cellId,
input.cellId,
identity.userId,
identity.relayHostId,
input.activityId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.activityId,
input.cellId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.cellId
]
)
)[0]
if (!row) throw new Error('missing_control_renewal_outcome')
const outcome = text(row, 'outcome') as ControlRenewalOutcome
if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome')
return outcome
if (this.database.dialect === 'postgres') {
return readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams([row], now)
),
1
)[0]!
}
try {
return await this.renewTransactionalControlActivity(row.identity, row, now)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) throw error
return message as ControlRenewalOutcome
}
}
private async renewTransactionalControlActivity(
@@ -7111,6 +7189,31 @@ export class RelayAssignmentStore {
return rows.length === 1
}
// Reports which of `cellIsLive`'s conditions failed, so the rejection log
// separates an expected drain or boot from a cell whose readiness went out
// from under its hosts.
private async homeCellUnavailableCause(
database: RelayDatabase,
cellId: string,
now: number
): Promise<RelayHomeCellUnavailableCause> {
const row = (
await database.query(
`SELECT cell.enabled, runtime.last_heartbeat_at
FROM relay_cells cell
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
WHERE cell.cell_id = ?`,
[cellId]
)
)[0]
if (!row) return 'booting'
if (integer(row, 'enabled') === 0) return 'draining'
const heartbeatAt = optionalInteger(row, 'last_heartbeat_at')
if (heartbeatAt === undefined) return 'booting'
// Readiness is all that is left: `cellIsLive` already refused this cell.
return heartbeatAt <= now - this.heartbeatTtlMs ? 'unheard' : 'not_ready'
}
private async cellHasActiveFence(cellId: string): Promise<boolean> {
const rows = await this.database.query(
`SELECT fence.cell_id FROM relay_cell_fences fence
@@ -7900,6 +8003,33 @@ function validateActivityId(activityId: string): void {
if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id')
}
function controlRenewalExpiryIsValid(expiresAt: number, now: number): boolean {
const maximumExpiresAt =
now + ASSIGNMENT_LIMITS.activityLeaseMs + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
return Number.isSafeInteger(expiresAt) && expiresAt > now && expiresAt <= maximumExpiresAt
}
// A renewal that threw still owes the metric an outcome: the message carries one
// when the statement decided it, and anything else is the driver failing.
function controlRenewalOutcomeOfError(error: unknown): ControlRenewalOutcome {
const message = String((error as { message?: unknown }).message)
return CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)
? (message as ControlRenewalOutcome)
: 'database_error'
}
function controlRenewalRejection(
row: ControlRenewalRequest,
now: number
): ControlRenewalOutcome | null {
try {
validateActivityId(row.activityId)
} catch {
return 'invalid_activity_id'
}
return controlRenewalExpiryIsValid(row.expiresAt, now) ? null : 'invalid_activity_expiry'
}
function activityKind(row: SqlRow): AssignmentActivityKind {
const value = text(row, 'activity_kind')
if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind')
@@ -0,0 +1,126 @@
import { afterEach, describe, expect, it, vi, type MockInstance } from 'vitest'
import { openRelayDatabaseAtBoot } from './boot-database-open.js'
import type { RelayDatabase } from './database.js'
const input = { dataDir: '/tmp/orca-relay-boot', databaseUrl: 'postgres://relay@localhost/relay' }
// The message the fleet actually saw: pg-pool reports the connect timeout with
// no SQLSTATE, so the classifier has only this text to go on.
const connectTimeout = (): Error => new Error('Connection terminated due to connection timeout')
const database = {} as RelayDatabase
function loggedEvents(warn: MockInstance<typeof console.warn>): string[] {
return warn.mock.calls.map((call) => String(JSON.parse(String(call[0])).event))
}
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
describe('relay boot database open', () => {
it('waits out a cold proxy instead of failing the boot', async () => {
vi.useFakeTimers()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const open = vi
.fn<() => Promise<RelayDatabase>>()
.mockRejectedValueOnce(connectTimeout())
.mockRejectedValueOnce(connectTimeout())
.mockResolvedValue(database)
const opening = openRelayDatabaseAtBoot(input, open)
await vi.runAllTimersAsync()
expect(await opening).toBe(database)
expect(open).toHaveBeenCalledTimes(3)
expect(open).toHaveBeenCalledWith(input)
expect(loggedEvents(warn)).toEqual([
'orca_relay_boot_database_retry',
'orca_relay_boot_database_retry',
'orca_relay_boot_database_recovered'
])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempt: 1,
delayMs: expect.any(Number),
code: 'unknown',
connectionTimeout: true
})
})
it('fails the boot immediately when the database rejects the relay', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const denied = Object.assign(new Error('password authentication failed'), { code: '28P01' })
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(denied)
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(denied)
expect(open).toHaveBeenCalledTimes(1)
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempts: 1,
retryable: false,
code: 'unknown'
})
})
// A retry re-runs the schema apply, which must never re-queue a boot DDL
// behind the writers that beat it; the request path treats these as transient.
it.each(['55P03', '57014', '53300'])(
'refuses to re-queue the schema apply after SQLSTATE %s',
async (code) => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const contention = Object.assign(new Error('lock unavailable'), { code })
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(contention)
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(contention)
expect(open).toHaveBeenCalledTimes(1)
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
attempts: 1,
retryable: false,
code
})
}
)
it('waits out a connection failure the driver does report a SQLSTATE for', async () => {
vi.useFakeTimers()
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const unreachable = Object.assign(new Error('connection refused'), { code: '08006' })
const open = vi
.fn<() => Promise<RelayDatabase>>()
.mockRejectedValueOnce(unreachable)
.mockResolvedValue(database)
const opening = openRelayDatabaseAtBoot(input, open)
await vi.runAllTimersAsync()
expect(await opening).toBe(database)
expect(open).toHaveBeenCalledTimes(2)
})
it('gives up once the retry budget is spent', async () => {
vi.useFakeTimers()
vi.setSystemTime(0)
vi.spyOn(Math, 'random').mockReturnValue(0)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const failure = connectTimeout()
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(failure)
const opening = openRelayDatabaseAtBoot(input, open)
const rejection = expect(opening).rejects.toBe(failure)
await vi.runAllTimersAsync()
await rejection
expect(Date.now()).toBeLessThanOrEqual(45_000)
expect(open.mock.calls.length).toBeGreaterThan(1)
const events = loggedEvents(warn)
expect(events.at(-1)).toBe('orca_relay_boot_database_failed')
expect(events.filter((event) => event === 'orca_relay_boot_database_retry')).toHaveLength(
open.mock.calls.length - 1
)
expect(JSON.parse(String(warn.mock.calls.at(-1)?.[0]))).toMatchObject({
attempts: open.mock.calls.length,
retryable: true,
connectionTimeout: true
})
})
})
@@ -0,0 +1,71 @@
import { openRelayDatabase, type RelayDatabase, type RelayDatabaseOpenInput } from './database.js'
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
import {
isPostgresPoolConnectFailure,
isPostgresPoolConnectTimeout
} from './postgres-pool-pressure.js'
import { postgresErrorCodeCategory } from './postgres-query-failure.js'
// Only a failure to reach Postgres at all. A retry here re-runs the schema
// apply, and applyPostgresSchema refuses to repeat a DDL lock timeout on
// purpose: relation locks are granted in queue order, so a repeat parks every
// writer behind the same statement again. 55P03, 57014 and 53300 therefore stay
// terminal at boot even though the request path calls them transient.
function isBootDatabaseUnreachable(error: unknown): boolean {
const code = postgresErrorCodeCategory(error)
return isPostgresPoolConnectFailure(error) || code === '08001' || code === '08006'
}
// A cell boots beside a cloud-sql-proxy that is itself still dialling, so the
// first pool acquire can outrun the 2s connect timeout that protects the
// request path. The window is longer than a proxy cold start and shorter than
// the restart loop it replaces.
const BOOT_OPEN_RETRY = {
attempts: 20,
windowMs: 45_000,
baseDelayMs: 250,
maxDelayMs: 4_000,
jitterMs: 250,
isRetryable: isBootDatabaseUnreachable
}
function bootDatabaseErrorFields(error: unknown): Record<string, unknown> {
return {
code: postgresErrorCodeCategory(error),
connectionTimeout: isPostgresPoolConnectTimeout(error)
}
}
export async function openRelayDatabaseAtBoot(
input: RelayDatabaseOpenInput,
open: (input: RelayDatabaseOpenInput) => Promise<RelayDatabase> = openRelayDatabase
): Promise<RelayDatabase> {
return await retryTransientDatabaseStartup(
async () => await open(input),
BOOT_OPEN_RETRY,
{
onRetry: ({ attempt, delayMs, error }) =>
console.warn(
JSON.stringify({
event: 'orca_relay_boot_database_retry',
attempt,
delayMs,
...bootDatabaseErrorFields(error)
})
),
onRecovered: ({ attempts }) =>
console.warn(
JSON.stringify({ event: 'orca_relay_boot_database_recovered', attempts })
),
onGaveUp: ({ attempts, error, retryable }) =>
console.warn(
JSON.stringify({
event: 'orca_relay_boot_database_failed',
attempts,
retryable,
...bootDatabaseErrorFields(error)
})
)
}
)
}
+21 -29
View File
@@ -1,14 +1,19 @@
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { isRelayDatabaseTransientError } from './database.js'
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
type CellAdmissionStartupConfig = Pick<RelayConfig, 'role' | 'cells'>
type CellAdmissionStore = Pick<RelayAssignmentStore, 'reconcileCellsAtStartup'>
const STARTUP_RECONCILE_ATTEMPTS = 20
const STARTUP_RECONCILE_RETRY_WINDOW_MS = 45_000
const STARTUP_RECONCILE_RETRY_BASE_MS = 250
const STARTUP_RECONCILE_RETRY_JITTER_MS = 250
const STARTUP_RECONCILE_RETRY = {
attempts: 20,
windowMs: 45_000,
// Flat: the contention this waits out is another director's schema lock, which
// clears on its own schedule rather than easing as the wait grows.
baseDelayMs: 250,
maxDelayMs: 250,
jitterMs: 250
}
export function roleOwnsAssignmentMaintenance(role: RelayConfig['role']): boolean {
// Cell workers share the database but the director is the sole authority
@@ -23,34 +28,21 @@ export async function reconcileCellAdmissionAtStartup(
// Admission is operator/director state. A new worker must not enable itself
// before its distinct candidate has passed production preflight.
if (config.role === 'cell') return
const retryDeadline = Date.now() + STARTUP_RECONCILE_RETRY_WINDOW_MS
for (let attempt = 1; attempt <= STARTUP_RECONCILE_ATTEMPTS; attempt += 1) {
try {
await assignments.reconcileCellsAtStartup(config.cells)
if (attempt > 1) {
await retryTransientDatabaseStartup(
async () => await assignments.reconcileCellsAtStartup(config.cells),
STARTUP_RECONCILE_RETRY,
{
onRecovered: ({ attempts }) =>
console.warn(
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts: attempt })
)
}
return
} catch (error) {
const remainingMs = retryDeadline - Date.now()
if (
attempt === STARTUP_RECONCILE_ATTEMPTS ||
remainingMs <= 0 ||
!isRelayDatabaseTransientError(error)
) {
if (isRelayDatabaseTransientError(error)) {
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts })
),
onGaveUp: ({ attempts, retryable }) => {
if (retryable) {
console.warn(
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts: attempt })
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts })
)
}
throw error
}
const delayMs =
STARTUP_RECONCILE_RETRY_BASE_MS +
Math.floor(Math.random() * (STARTUP_RECONCILE_RETRY_JITTER_MS + 1))
await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs)))
}
}
)
}
@@ -3,6 +3,7 @@ import { ASSIGNMENT_LIMITS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
@@ -133,6 +134,10 @@ describePostgres('expired control lease after a database outage', () => {
internals.heartbeat(session)
}
// A due renewal leaves the heartbeat as a batch enqueue, so a poll has to
// outlast the batch window before it can call the renewal missing.
const renewalPoll = { timeout: CONTROL_RENEWAL_BATCH_INTERVAL_MS + 4_000 }
const leaseRows = async (relayHostId: string) =>
await database.query(
`SELECT activity_id, cell_id FROM relay_assignment_activity_leases
@@ -149,7 +154,8 @@ describePostgres('expired control lease after a database outage', () => {
.poll(
async () =>
socket.close.mock.calls.length > 0 ||
(await leaseRows(relayHostId)).length === expectedRows
(await leaseRows(relayHostId)).length === expectedRows,
renewalPoll
)
.toBe(true)
}
@@ -212,7 +218,7 @@ describePostgres('expired control lease after a database outage', () => {
).rejects.toThrow('control_activity_moved')
heartbeat(registry, session)
await expect.poll(() => socket.close.mock.calls.length).toBe(1)
await expect.poll(() => socket.close.mock.calls.length, renewalPoll).toBe(1)
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
expect(await leaseRows(identity.relayHostId)).toEqual([
@@ -0,0 +1,291 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type SqlRow
} from './database.js'
const now = 1_900_000_000_000
const expiresAt = now + 105_000
function renewal(userId: string, relayHostId: string, expiry = expiresAt) {
return {
identity: { userId, relayHostId },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: expiry
}
}
// A PostgreSQL-dialect database that answers the renewal statement without a
// server, so the statement count and its parameter arrays are observable.
class RenewalStatementProbe implements RelayDatabase {
readonly dialect = 'postgres' as const
readonly statements: Array<{ sql: string; params: unknown[] }> = []
failuresRemaining = 0
constructor(private readonly outcomeFor: (userId: string) => ControlRenewalOutcome) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push({ sql, params })
if (this.failuresRemaining > 0) {
this.failuresRemaining -= 1
throw new Error('canceling statement due to statement timeout')
}
const userIds = params[0] as string[]
return userIds.map((userId, index) => ({
row_index: String(index + 1),
outcome: this.outcomeFor(userId)
}))
}
async queryLocked(): Promise<SqlRow[]> {
throw new Error('unexpected_locked_query')
}
async transaction<T>(): Promise<T> {
// Renewals must never open one: that is the write transaction per host this
// batch exists to remove.
throw new Error('unexpected_transaction')
}
async close(): Promise<void> {}
}
describe('batched control renewals on PostgreSQL', () => {
it('spends one statement on every host that came due', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000003')
])
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
expect(probe.statements).toHaveLength(1)
expect(probe.statements[0]!.sql).toBe(CONTROL_RENEWAL_BATCH_SQL)
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
})
it('locks assignment rows in primary-key order and still answers in input order', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-c', 'host000000000003'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000001'),
renewal('user-a', 'host000000000001')
])
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
// statement's ORDER BY repeats it: no batch can queue against another in a
// different sequence.
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
expect(probe.statements[0]!.params[1]).toEqual([
'host000000000001',
'host000000000002',
'host000000000001',
'host000000000003'
])
expect(outcomes).toEqual([
'renewed',
'renewed',
'control_activity_moved',
'renewed'
])
})
it('keeps a malformed request out of the statement and fails only that row', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002', now + ASSIGNMENT_LIMITS.activityLeaseMs * 10),
{ ...renewal('user-a', 'host000000000003'), activityId: '' },
renewal('user-a', 'host000000000004')
])
expect(outcomes).toEqual([
'renewed',
'invalid_activity_expiry',
'invalid_activity_id',
'renewed'
])
expect(probe.statements[0]!.params[1]).toEqual(['host000000000001', 'host000000000004'])
})
it('degrades to one statement per host when the batch statement fails', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['renewed', 'renewed'])
expect(probe.statements).toHaveLength(3)
expect(probe.statements[1]!.params[1]).toEqual(['host000000000001'])
expect(probe.statements[2]!.params[1]).toEqual(['host000000000002'])
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_batch_failed',
rows: 2
})
} finally {
warn.mockRestore()
}
})
it('reports a host that fails its own fallback statement without touching the rest', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 2
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes.filter((outcome) => outcome === 'renewed')).toHaveLength(1)
expect(outcomes.filter((outcome) => outcome === 'database_error')).toHaveLength(1)
} finally {
warn.mockRestore()
}
})
it('reports a contended assignment row apart from a missing one', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_lock_unavailable' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
// Retryable: SKIP LOCKED passed over the row rather than queueing the whole
// flush behind whoever held it.
expect(outcomes).toEqual(['renewed', 'assignment_lock_unavailable'])
})
it('counts a lone renewal that threw before rethrowing it', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
// still owes an outcome for the attempt.
await expect(
store.renewControlActivities([renewal('user-a', 'host000000000001')])
).rejects.toThrow('statement timeout')
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
})
it('counts a batch in which every row was rejected before the statement', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
const outcomes = await store.renewControlActivities([
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
renewal('user-a', 'host000000000002', now - 1)
])
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
expect(probe.statements).toHaveLength(0)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'invalid_activity_id',
'invalid_activity_expiry'
])
})
it('counts one renewal metric per row against the flush latency', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
)
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'renewed',
'assignment_not_found'
])
})
})
describe('batched control renewals on SQLite', () => {
it('renews every host through the transactional path', async () => {
let clock = now
const database = await openInMemoryRelayDatabase()
try {
const store = new RelayAssignmentStore(database, () => clock)
await store.reconcileCells([
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
])
const hosts = ['host000000000001', 'host000000000002']
const requests = []
for (const relayHostId of hosts) {
const identity = { userId: 'user-a', relayHostId }
const assignment = await store.assign(identity)
await store.activateControl(identity, {
cellId: assignment.cellId,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
requests.push({
identity,
activityId: `control:${assignment.cellId}:1`,
cellId: assignment.cellId,
expiresAt: clock + 105_000
})
}
// A host with no assignment at all must not cost the others their renewal.
requests.push({
identity: { userId: 'user-a', relayHostId: 'host000000000009' },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: clock + 105_000
})
clock += 1_000
const outcomes = await store.renewControlActivities(requests)
expect(outcomes).toEqual(['renewed', 'renewed', 'assignment_not_found'])
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? ORDER BY relay_host_id ASC`,
['user-a']
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
now + 105_000,
now + 105_000
])
} finally {
await database.close()
}
})
})
@@ -0,0 +1,231 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
CONTROL_RENEWAL_BATCH_INTERVAL_MS,
CONTROL_RENEWAL_BATCH_MAX_ROWS,
ControlRenewalBatch,
type ControlRenewalFlush
} from './control-renewal-batch.js'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// Settles into the outcome the caller saw, attached at enqueue so a rejection is
// never momentarily unhandled.
function outcomeOf(renewal: Promise<void>): Promise<string> {
return renewal.then(
() => 'renewed',
(error: unknown) => String((error as { message?: unknown }).message)
)
}
function request(
host: string,
expiresAt = 1_000,
activityId = 'control:cell-a:1'
): ControlRenewalRequest {
return {
identity: { userId: 'user-a', relayHostId: host },
activityId,
cellId: 'cell-a',
expiresAt
}
}
describe('control renewal batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('spends one call on every renewal that came due in the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const settled = [
batch.enqueue(request('host0000000000a1')),
batch.enqueue(request('host0000000000a2')),
batch.enqueue(request('host0000000000a3'))
]
expect(renew).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all(settled)).resolves.toEqual([undefined, undefined, undefined])
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0].map((row) => row.identity.relayHostId)).toEqual([
'host0000000000a1',
'host0000000000a2',
'host0000000000a3'
])
})
it('routes each outcome back to the caller that asked for it', async () => {
const outcomes: ControlRenewalOutcome[] = [
'renewed',
'assignment_not_found',
'control_activity_moved'
]
const batch = new ControlRenewalBatch(async () => outcomes)
const first = outcomeOf(batch.enqueue(request('host0000000000b1')))
const second = outcomeOf(batch.enqueue(request('host0000000000b2')))
const third = outcomeOf(batch.enqueue(request('host0000000000b3')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second, third])).resolves.toEqual([
'renewed',
'assignment_not_found',
'control_activity_moved'
])
})
it('flushes on reaching the row ceiling instead of waiting out the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
for (let row = 0; row < CONTROL_RENEWAL_BATCH_MAX_ROWS - 1; row++) {
void batch.enqueue(request(`host${String(row).padStart(12, '0')}`))
}
expect(renew).not.toHaveBeenCalled()
void batch.enqueue(request('host0000000000zz'))
await vi.advanceTimersByTimeAsync(0)
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0]).toHaveLength(CONTROL_RENEWAL_BATCH_MAX_ROWS)
// The window timer must not fire a second, empty statement.
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledOnce()
})
it('does not hold a new window behind a statement still in PostgreSQL', async () => {
let release!: (outcomes: ControlRenewalOutcome[]) => void
const renew = vi
.fn<(rows: readonly ControlRenewalRequest[]) => Promise<ControlRenewalOutcome[]>>()
.mockImplementationOnce(
async () => await new Promise<ControlRenewalOutcome[]>((resolve) => (release = resolve))
)
.mockResolvedValue(['renewed'])
const batch = new ControlRenewalBatch(renew)
const stalled = batch.enqueue(request('host0000000000c1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
const next = batch.enqueue(request('host0000000000c2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
await expect(next).resolves.toBeUndefined()
release(['renewed'])
await expect(stalled).resolves.toBeUndefined()
})
it('reports the driver failure to every caller in the flush', async () => {
const batch = new ControlRenewalBatch(async () => {
throw new Error('pool timeout')
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const first = outcomeOf(batch.enqueue(request('host0000000000d1')))
const second = outcomeOf(batch.enqueue(request('host0000000000d2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second])).resolves.toEqual([
'pool timeout',
'pool timeout'
])
} finally {
warn.mockRestore()
}
})
it('supersedes a second attempt for one lease and answers both callers', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const earlier = batch.enqueue(request('host0000000000e1', 1_000))
const later = batch.enqueue(request('host0000000000e1', 2_000))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew.mock.calls[0]![0]).toEqual([
expect.objectContaining({ expiresAt: 2_000 })
])
await expect(earlier).resolves.toBeUndefined()
await expect(later).resolves.toBeUndefined()
})
it('holds a second activity for one host back to the next flush', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const first = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:1'))
const second = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:2'))
const other = batch.enqueue(request('host0000000000h2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
// One statement updates a host's assignment row once, so the host appears in
// one flush only; the newer generation leads the next one.
expect(renew.mock.calls[0]![0].map((row) => row.activityId)).toEqual([
'control:cell-a:1',
'control:cell-a:1'
])
await expect(Promise.all([first, other])).resolves.toEqual([undefined, undefined])
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
expect(renew.mock.calls[1]![0].map((row) => row.activityId)).toEqual(['control:cell-a:2'])
await expect(second).resolves.toBeUndefined()
})
it('stays quiet for a fast flush that renewed everything', async () => {
const flushes: ControlRenewalFlush[] = []
const batch = new ControlRenewalBatch(
async () => ['renewed'],
() => ({ cellId: 'cell-a' }),
(flush) => flushes.push(flush)
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void batch.enqueue(request('host0000000000f1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(warn).not.toHaveBeenCalled()
expect(flushes).toEqual([
{ rows: 1, durationMs: expect.any(Number), outcomes: { renewed: 1 } }
])
} finally {
warn.mockRestore()
}
})
it('logs one line with the outcome counts when a flush did not renew everything', async () => {
const batch = new ControlRenewalBatch(
async () => ['renewed', 'control_activity_not_found'],
() => ({ cellId: 'cell-a' })
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void outcomeOf(batch.enqueue(request('host0000000000g1')))
const missing = outcomeOf(batch.enqueue(request('host0000000000g2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(missing).resolves.toBe('control_activity_not_found')
expect(warn).toHaveBeenCalledOnce()
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_flush',
cellId: 'cell-a',
rows: 2,
outcomes: { renewed: 1, control_activity_not_found: 1 }
})
} finally {
warn.mockRestore()
}
})
})
@@ -0,0 +1,148 @@
import { performance } from 'node:perf_hooks'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// One flush per second turns the fleet's control-lease write rate into a
// function of the cell count rather than the host count: a cell's ~10 due
// renewals per second become one write transaction instead of ten. Well inside
// the 105s lease runway, so a host that misses a window is never at risk.
export const CONTROL_RENEWAL_BATCH_INTERVAL_MS = 1_000
// Ceiling on the parameter arrays. Row locks live until the statement commits,
// so this is what bounds how long one flush holds them: measured at 11.5ms for
// 200 rows against a 20,000-row table, and 9.4ms with a host wedged in a
// per-host transaction.
export const CONTROL_RENEWAL_BATCH_MAX_ROWS = 200
// A flush slower than this is the only latency worth a line; the metrics event
// carries the distribution.
const CONTROL_RENEWAL_SLOW_FLUSH_MS = 250
export type ControlRenewalFlush = {
rows: number
durationMs: number
outcomes: Record<string, number>
}
type PendingWaiter = { resolve: () => void; reject: (error: unknown) => void }
type PendingRenewal = { request: ControlRenewalRequest; waiters: PendingWaiter[] }
type QueuedRenewal = { request: ControlRenewalRequest; waiter: PendingWaiter }
// Per host, not per activity: one statement updates a host's assignment row
// once, so two activities for the same host must not share a flush.
function pendingKey(request: ControlRenewalRequest): string {
return [request.identity.userId, request.identity.relayHostId].join('\u0000')
}
// Collects the control-lease renewals a cell owes and spends one statement on
// them. Each caller still gets the single-renewal contract: the promise resolves
// on `renewed` and rejects with the outcome as its message otherwise, so callers
// keep their per-session error routing unchanged.
export class ControlRenewalBatch {
private pending = new Map<string, PendingRenewal>()
// Renewals a host cannot contribute to the flush being built; they open the
// next one.
private deferred: QueuedRenewal[] = []
private timer: ReturnType<typeof setTimeout> | null = null
constructor(
private readonly renew: (
rows: readonly ControlRenewalRequest[]
) => Promise<ControlRenewalOutcome[]>,
private readonly logFields: () => Record<string, unknown> = () => ({}),
private readonly observe?: (flush: ControlRenewalFlush) => void
) {}
enqueue(request: ControlRenewalRequest): Promise<void> {
return new Promise<void>((resolve, reject) => {
this.admit({ request, waiter: { resolve, reject } })
})
}
private admit(queued: QueuedRenewal): void {
const key = pendingKey(queued.request)
const existing = this.pending.get(key)
if (existing && existing.request.activityId !== queued.request.activityId) {
this.deferred.push(queued)
this.scheduleFlush()
return
}
if (existing) {
// A second attempt at the same lease inside one window supersedes the
// first expiry; both callers still hear the outcome they waited for.
existing.request = {
...queued.request,
expiresAt: Math.max(existing.request.expiresAt, queued.request.expiresAt)
}
existing.waiters.push(queued.waiter)
return
}
this.pending.set(key, { request: queued.request, waiters: [queued.waiter] })
if (this.pending.size >= CONTROL_RENEWAL_BATCH_MAX_ROWS) {
void this.flush()
return
}
this.scheduleFlush()
}
private scheduleFlush(): void {
this.timer ??= setTimeout(() => {
this.timer = null
void this.flush()
}, CONTROL_RENEWAL_BATCH_INTERVAL_MS)
this.timer.unref?.()
}
// Flushes run concurrently on purpose: a statement stalled in PostgreSQL must
// not hold back the renewals that came due while it was waiting.
async flush(): Promise<void> {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
const batch = [...this.pending.values()]
this.pending = new Map()
// Re-admitted against the empty map, so a host deferred out of this flush
// leads the next one.
const deferred = this.deferred
this.deferred = []
for (const queued of deferred) this.admit(queued)
if (batch.length === 0) return
const startedAt = performance.now()
let outcomes: ControlRenewalOutcome[]
try {
outcomes = await this.renew(batch.map((entry) => entry.request))
} catch (error) {
for (const entry of batch) for (const waiter of entry.waiters) waiter.reject(error)
this.report(batch.length, performance.now() - startedAt, { flush_failed: batch.length })
return
}
const counts: Record<string, number> = {}
for (const [index, entry] of batch.entries()) {
const outcome = outcomes[index] ?? 'database_error'
counts[outcome] = (counts[outcome] ?? 0) + 1
for (const waiter of entry.waiters) {
if (outcome === 'renewed') waiter.resolve()
else waiter.reject(new Error(outcome))
}
}
this.report(batch.length, performance.now() - startedAt, counts)
}
private report(rows: number, durationMs: number, outcomes: Record<string, number>): void {
this.observe?.({ rows, durationMs, outcomes })
const renewed = outcomes.renewed ?? 0
if (durationMs <= CONTROL_RENEWAL_SLOW_FLUSH_MS && renewed === rows) return
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_flush',
...this.logFields(),
rows,
durationMs: Math.round(durationMs),
outcomes
})
)
}
}
@@ -1,8 +1,11 @@
import { performance } from 'node:perf_hooks'
import { ASSIGNMENT_LIMITS, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import {
openRelayDatabase,
POSTGRES_LOCK_TIMEOUT_MS,
type RelayDatabase,
type RelayLockOptions,
type SqlRow
@@ -22,7 +25,9 @@ const targetCell = {
capacityRequests: 100
}
const userId = 'control-renewal-postgres-user'
const identities = Array.from({ length: 6 }, (_, index) => ({
// Indexes 0-5 belong to the single-renewal cases below, which mutate their
// host's migration and lease state; the batch cases own 6-13.
const identities = Array.from({ length: 14 }, (_, index) => ({
userId,
relayHostId: `controlrenewal${index + 1}`
}))
@@ -72,7 +77,7 @@ class StallFirstRenewalQueryDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (this.stallNext && sql.includes('WITH assignment_state AS MATERIALIZED')) {
if (this.stallNext && sql === CONTROL_RENEWAL_BATCH_SQL) {
this.stallNext = false
this.stalled.resolve()
await this.continue.promise
@@ -103,7 +108,7 @@ class RenewalQueryProbeDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (sql.includes('WITH assignment_state AS MATERIALIZED')) this.renewalQueries++
if (sql === CONTROL_RENEWAL_BATCH_SQL) this.renewalQueries++
return await this.database.query(sql, params)
}
@@ -332,6 +337,164 @@ describePostgres('PostgreSQL control renewal', () => {
).rejects.toThrow('invalid_activity_expiry')
})
it('renews every due host in one autocommitted statement', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
const batch = identities.slice(6, 10)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
batch.map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed', 'renewed'])
expect(probe.renewalQueries).toBe(1)
expect(probe.transactions).toBe(0)
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_id = ? ORDER BY relay_host_id ASC`,
[userId, controlId(sourceCell.id)]
)
expect(
leases
.filter((lease) =>
batch.some((identity) => identity.relayHostId === lease.relay_host_id)
)
.map((lease) => Number(lease.expires_at))
).toEqual([expiresAt, expiresAt, expiresAt, expiresAt])
})
it('reports each host its own verdict inside one batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const live = identities[10]!
const absent = { userId, relayHostId: 'controlrenewalgone' }
const outcomes = await store.renewControlActivities([
{ identity: absent, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{ identity: live, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{
identity: live,
activityId: controlId(targetCell.id),
cellId: targetCell.id,
expiresAt
}
])
expect(outcomes).toEqual([
'assignment_not_found',
'renewed',
'activity_cell_not_authoritative'
])
})
it('passes over a host whose assignment row is held and renews the rest', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const held = identities[11]!
const free = identities[12]!
const locked = signal()
const release = signal()
// Holds the row the way every per-host transactional path does.
const holder = database.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[held.userId, held.relayHostId]
)
locked.resolve()
await release.promise
})
await locked.promise
const startedAt = performance.now()
const outcomes = await store.renewControlActivities(
[held, free].map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
const elapsedMs = performance.now() - startedAt
release.resolve()
await holder
expect(outcomes).toEqual(['assignment_lock_unavailable', 'renewed'])
// It skipped rather than queued: a blocking FOR UPDATE would have spent the
// pool's whole lock_timeout here and failed the free host too.
expect(elapsedMs).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS)
const lease = (
await database.query(
`SELECT expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[free.userId, free.relayHostId, controlId(sourceCell.id)]
)
)[0]
expect(Number(lease!.expires_at)).toBe(expiresAt)
})
it('renews both of one host\u2019s control leases in a single batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
const identity = identities[13]!
// Two live control leases on one host. Written directly because
// activateControl retires the prior generation, and what is under test is the
// statement's row-wise behaviour, not how the second lease came to exist.
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`control:${sourceCell.id}:2`,
sourceCell.id,
now,
now
]
)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
[1, 2].map((generation) => ({
identity,
activityId: `control:${sourceCell.id}:${generation}`,
cellId: sourceCell.id,
expiresAt: expiresAt - generation
}))
)
expect(outcomes).toEqual(['renewed', 'renewed'])
const leases = await database.query(
`SELECT activity_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`,
[identity.userId, identity.relayHostId]
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
expiresAt - 1,
expiresAt - 2
])
// The assignment row is written once, carrying the later of the two.
const row = (
await database.query(
`SELECT lease_expires_at, last_activity_at FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
expect(Number(row!.lease_expires_at)).toBe(expiresAt - 1)
expect(Number(row!.last_activity_at)).toBe(now)
})
it('uses one autocommitted PostgreSQL statement for a steady renewal', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
@@ -0,0 +1,230 @@
import type { AssignmentIdentity } from './assignment-identity-queue.js'
import type { SqlRow } from './database.js'
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
// The host's assignment row was already locked by one of the per-host
// transactional paths. Retryable, and never a reason to close a control: the
// next tick is 15s away and the lease has 105s on it.
| 'assignment_lock_unavailable'
// Decided per row before the statement runs, so one malformed request cannot
// cost the rest of the batch its renewal.
| 'invalid_activity_id'
| 'invalid_activity_expiry'
| 'database_error'
// Outcomes the statement itself can report. `database_error` is raised by the
// driver, and `invalid_activity_expiry` is decided per row before the statement
// is built, so neither can come back as a row.
export const CONTROL_RENEWAL_STATEMENT_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved',
'assignment_lock_unavailable'
])
export type ControlRenewalRequest = {
identity: AssignmentIdentity
activityId: string
cellId: string
expiresAt: number
}
// LOCK ORDER - (user_id, relay_host_id), the primary key of relay_assignments,
// applied here and repeated as the statement's ORDER BY so it holds whether the
// planner walks the primary-key index or sorts under the LockRows node.
//
// The batch never waits for an assignment row: SKIP LOCKED reports a contended
// host separately instead. That is what bounds how long a flush holds its locks
// to its own execution time, because row locks live until the statement commits,
// and it is why one host wedged in a per-host transaction cannot stall the
// renewals of every other host sharing the flush.
//
// With no wait on the assignment pass, the deadlock question reduces to the two
// later passes. Every writer in this store locks a host's assignment row before
// that host's lease rows (`assignmentRow` then `lockAssignmentActivities`), and
// a host whose assignment row is held was skipped, so the batch never reaches
// that host's lease: the lease pass cannot wait either.
// `markMigrationTargetRegistered` is the one writer that locks a migration row
// without the assignment row first. It takes no further locks, so it can delay a
// mid-migration row by up to the pool's lock_timeout but cannot close a cycle.
export function orderedControlRenewalRows<Row extends { identity: AssignmentIdentity }>(
rows: readonly Row[]
): Row[] {
return [...rows].sort(
(left, right) =>
left.identity.userId.localeCompare(right.identity.userId) ||
left.identity.relayHostId.localeCompare(right.identity.relayHostId)
)
}
// One statement renewing every due control lease on this cell, row-wise over the
// unnested parameter arrays. Logic per row is what the single-row predecessor
// did: lock the assignment, admit the caller's cell either as the current cell or
// as the source of an active forward migration, lock that host's control lease,
// push both expiries forward, and report one outcome. The one addition is
// `present_assignment`, an unlocked probe that separates a host with no
// assignment row at all from one whose row SKIP LOCKED passed over - the first
// closes the control, the second retries.
export const CONTROL_RENEWAL_BATCH_SQL = `WITH renewal_input AS MATERIALIZED (
SELECT
renewal.ordinality AS row_index,
renewal.user_id,
renewal.relay_host_id,
renewal.activity_id,
renewal.cell_id,
renewal.expires_at
FROM unnest(?::text[], ?::text[], ?::text[], ?::text[], ?::bigint[])
WITH ORDINALITY AS renewal(
user_id, relay_host_id, activity_id, cell_id, expires_at, ordinality
)
), present_assignment AS MATERIALIZED (
SELECT input.row_index
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
), assignment_state AS MATERIALIZED (
SELECT input.row_index, assignment.cell_id, assignment.assignment_epoch
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
ORDER BY assignment.user_id, assignment.relay_host_id
FOR UPDATE OF assignment SKIP LOCKED
), migration_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
JOIN relay_assignment_migrations migration
ON migration.user_id = input.user_id
AND migration.relay_host_id = input.relay_host_id
AND migration.source_cell_id = input.cell_id
AND migration.target_cell_id = locked.cell_id
AND migration.assignment_epoch = locked.assignment_epoch
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
WHERE locked.cell_id = input.cell_id
OR EXISTS (
SELECT 1 FROM migration_state moving
WHERE moving.row_index = locked.row_index
)
), lease_state AS MATERIALIZED (
SELECT authorized.row_index, lease.activity_kind, lease.cell_id
FROM authorization_state authorized
JOIN renewal_input input ON input.row_index = authorized.row_index
JOIN relay_assignment_activity_leases lease
ON lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, input.expires_at),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
JOIN renewal_input input ON input.row_index = state.row_index
WHERE lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
AND state.activity_kind = 'control' AND state.cell_id = input.cell_id
RETURNING state.row_index
), renewed_assignment AS (
-- Grouped per host: an UPDATE whose FROM offers a target row more than
-- once applies one source row and returns one, so two leases on one
-- host would leave the assignment carrying the wrong expiry. The
-- aggregate hands it exactly one row, carrying the later expiry.
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, renewed.expires_at),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
FROM (
SELECT input.user_id, input.relay_host_id, MAX(input.expires_at) AS expires_at
FROM renewed_lease renewed
JOIN renewal_input input ON input.row_index = renewed.row_index
GROUP BY input.user_id, input.relay_host_id
) renewed
WHERE assignment.user_id = renewed.user_id
AND assignment.relay_host_id = renewed.relay_host_id
RETURNING renewed.user_id
)
SELECT input.row_index, CASE
WHEN NOT EXISTS (
SELECT 1 FROM present_assignment present
WHERE present.row_index = input.row_index
) THEN 'assignment_not_found'
WHEN NOT EXISTS (
SELECT 1 FROM assignment_state locked WHERE locked.row_index = input.row_index
) THEN 'assignment_lock_unavailable'
WHEN NOT EXISTS (
SELECT 1 FROM authorization_state authorized
WHERE authorized.row_index = input.row_index
) THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (
SELECT 1 FROM lease_state state WHERE state.row_index = input.row_index
) THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state state
WHERE state.row_index = input.row_index
AND (state.activity_kind <> 'control' OR state.cell_id <> input.cell_id)
) THEN 'control_activity_moved'
-- Read from renewed_lease, which has one row per input row. The
-- assignment update collapses to one row per host, so it cannot answer
-- for a host that brought two leases to the same batch.
WHEN EXISTS (
SELECT 1 FROM renewed_lease renewed
WHERE renewed.row_index = input.row_index
) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome
FROM renewal_input input
ORDER BY input.row_index`
export function controlRenewalBatchParams(
rows: readonly ControlRenewalRequest[],
now: number
): unknown[] {
return [
rows.map((row) => row.identity.userId),
rows.map((row) => row.identity.relayHostId),
rows.map((row) => row.activityId),
rows.map((row) => row.cellId),
rows.map((row) => row.expiresAt),
now,
now
]
}
// Rows come back ordered by row_index, which is the 1-based position in the
// statement's parameter arrays.
export function readControlRenewalOutcomes(
rows: SqlRow[],
expected: number
): ControlRenewalOutcome[] {
if (rows.length !== expected) throw new Error('missing_control_renewal_outcome')
return rows.map((row, position) => {
if (Number(row.row_index) !== position + 1) {
throw new Error('misordered_control_renewal_outcome')
}
const outcome = row.outcome
if (
typeof outcome !== 'string' ||
!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(outcome as ControlRenewalOutcome)
) {
throw new Error('invalid_control_renewal_outcome')
}
// SAFETY: the membership check above is what narrows this string.
return outcome as ControlRenewalOutcome
})
}
@@ -0,0 +1,283 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
// The outage this guards against: the credential cleanup ran every 30s in all 23 cells and both
// sweeps over relay_invites had no usable index, so each one seq-scanned the whole table inside the
// maintenance transaction. Only a real planner can show the partial indexes take that away, and
// only a real server has ctid.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_credential_sweep_test'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
function scopedUrl(): string {
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
return url.toString()
}
async function onAdmin<T>(operation: (client: pg.Client) => Promise<T>): Promise<T> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
return await operation(client)
} finally {
await client.end()
}
}
describePostgres('credential cleanup against PostgreSQL', () => {
let database: RelayDatabase
let store: RelayCredentialStore
const opened: RelayDatabase[] = []
beforeEach(async () => {
await onAdmin(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
})
database = await openRelayDatabase({ databaseUrl: scopedUrl(), dataDir: '' })
opened.push(database)
store = new RelayCredentialStore(database, () => NOW)
})
afterAll(async () => {
await Promise.all(opened.map((open) => open.close().catch(() => undefined)))
await onAdmin((client) => client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`))
})
async function seedInvites(
count: number,
state: string,
updatedAt: number,
expiresAt = NOW - DAY_MS
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
SELECT ?, ?, 'device-' || n, 'token-' || ? || '-' || n, ?, 0, 3, ?, ?, ?
FROM generate_series(1, ?) AS n`,
[identity.userId, identity.relayHostId, state, state, expiresAt, updatedAt, updatedAt, count]
)
}
// Not through RelayDatabase: it routes anything that is not a SELECT to the row-count path, and
// EXPLAIN on an UPDATE is neither.
async function plan(sql: string, params: unknown[]): Promise<string> {
const client = new pg.Client({ connectionString: scopedUrl() })
await client.connect()
try {
let index = 0
const result = await client.query(
`EXPLAIN ${sql.replace(/\?/g, () => `$${(index += 1)}`)}`,
params
)
return result.rows.map((row) => String(row['QUERY PLAN'])).join('\n')
} finally {
await client.end()
}
}
it('plans both invite sweeps as index scans instead of scanning the whole table', async () => {
// Production's shape: terminal invites outnumber live ones by orders of magnitude, which is
// what makes the partial predicates worth having.
await seedInvites(20_000, 'consumed', NOW)
for (const state of ['available', 'reserved', 'cooldown']) {
await seedInvites(200, state, NOW, NOW + DAY_MS)
}
await database.query(
`UPDATE relay_invites SET reservation_expires_at = ? WHERE state = 'reserved'`,
[NOW + 1]
)
// Only ANALYZE makes the planner's row estimates real; without it a cold table looks tiny and
// a seq scan wins on any index.
await database.query(`ANALYZE relay_invites`)
const expiry = await plan(
`UPDATE relay_invites SET state = 'expired'
WHERE expires_at <= ? AND state IN ('available', 'reserved', 'cooldown')`,
[NOW]
)
const reservation = await plan(
`UPDATE relay_invites SET state = 'cooldown'
WHERE state = 'reserved' AND reservation_expires_at <= ? AND expires_at > ?`,
[NOW, NOW]
)
// Which of the two partial indexes serves the reservation pass is the planner's call: both
// predicates hold only live invites, so either one reads a handful of rows. The invariant is
// that neither pass reads the whole table any more.
for (const sweep of [expiry, reservation]) {
expect(sweep).not.toContain('Seq Scan on relay_invites')
expect(sweep).toMatch(/using relay_invites_sweep_(expiry|reservation)/)
}
expect(expiry).toContain('relay_invites_sweep_expiry')
})
it('plans the basis sweep off the composite index rather than the 1.5 GB heap', async () => {
// The shape that made this the most expensive statement in the sweep: 20,000 settled bases to
// 50 live ones. A partial index on active = 1 looks like the answer to that ratio and is not:
// a basis is inserted active and flipped to 0, so it accumulates the same dead entries, and
// the planner picks the composite index anyway. See the schema comment beside it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'live-' || n, ?, ?, 'device-1', 1, 'invite', ?, 1, ?
FROM generate_series(1, 50) AS n`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(`ANALYZE relay_connection_bases`)
const sweep = await plan(
`UPDATE relay_connection_bases SET active = 0 WHERE active = 1 AND deadline <= ?`,
[NOW]
)
expect(sweep).not.toContain('Seq Scan on relay_connection_bases')
expect(sweep).toContain('using relay_connection_bases_active_deadline')
})
it('plans the drained basis reaper off the composite index, not the heap', async () => {
// Why the composite index stays for now: it is the only one covering active = 0, and the case
// that needs it is the steady state, where every row is inside retention and the reaper must
// learn there is nothing to do. While the backlog drains the planner rightly prefers a bounded
// sequential scan, because it finds its 5,000 rows and stops; measured at 200k rows, the
// drained batch costs 5 buffers with this index and 1,274 without it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 60_000, NOW - 60_000]
)
await database.query(`ANALYZE relay_connection_bases`)
const reaper = await plan(
`DELETE FROM relay_connection_bases WHERE ctid IN (
SELECT ctid FROM relay_connection_bases WHERE active = ? AND deadline <= ? LIMIT 5000
)`,
[0, NOW - DAY_MS]
)
expect(reaper).toContain('relay_connection_bases_active_deadline')
expect(reaper).not.toContain('Seq Scan on relay_connection_bases')
})
it('plans the pending-authorization and rate-window sweeps as index scans', async () => {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'auth-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 1, NOW - 1]
)
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
SELECT 'scope-' || n, 'invite-mint', ?, 1 FROM generate_series(1, 20000) AS n`,
[NOW]
)
await database.query(`ANALYZE relay_direct_authorizations`)
await database.query(`ANALYZE relay_rate_windows`)
const pending = await plan(
`UPDATE relay_direct_authorizations SET consumed_at = ?
WHERE consumed_at IS NULL AND deadline <= ?`,
[NOW, NOW]
)
const windows = await plan(`DELETE FROM relay_rate_windows WHERE window_started_at < ?`, [
NOW - DAY_MS
])
expect(pending).toContain('relay_direct_authorizations_pending_deadline')
expect(pending).not.toContain('Seq Scan on relay_direct_authorizations')
expect(windows).toContain('relay_rate_windows_started')
expect(windows).not.toContain('Seq Scan on relay_rate_windows')
})
it('reaps settled bases and consumed authorizations through ctid', async () => {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES ('live', ?, ?, 'device-1', 1, 'invite', ?, 1, ?)`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'consumed-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES ('pending', ?, ?, 'device-1', 1, ?, NULL)`,
[identity.userId, identity.relayHostId, NOW + 30_000]
)
await store.cleanup()
expect(
await database.query(`SELECT count(*) AS total FROM relay_connection_bases`)
).toEqual([{ total: '3' }])
expect(
await database.query(`SELECT count(*) AS total FROM relay_direct_authorizations`)
).toEqual([{ total: '3' }])
await store.cleanup()
// Only the rows a reader could still accept are left.
expect(
await database.query(`SELECT basis_conn_id FROM relay_connection_bases`)
).toEqual([{ basis_conn_id: 'live' }])
expect(
await database.query(`SELECT direct_auth_id FROM relay_direct_authorizations`)
).toEqual([{ direct_auth_id: 'pending' }])
})
it('reaps terminal invites past retention through ctid, one bounded batch per cycle', async () => {
await seedInvites(5_002, 'consumed', NOW - 30 * DAY_MS)
await seedInvites(3, 'invalidated', NOW - 6 * DAY_MS)
await seedInvites(2, 'available', NOW - 400 * DAY_MS, NOW + DAY_MS)
await store.cleanup()
expect(await database.query(`SELECT count(*) AS total FROM relay_invites`)).toEqual([
{ total: '7' }
])
await store.cleanup()
// The two live invites and the three inside retention survive; the batch remainder is gone.
expect(
await database.query(`SELECT state, count(*) AS total FROM relay_invites GROUP BY state ORDER BY state`)
).toEqual([
{ state: 'available', total: '2' },
{ state: 'invalidated', total: '3' }
])
})
})
@@ -0,0 +1,297 @@
import { describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
async function insertInvite(
database: RelayDatabase,
invite: { token: string; state: string; updatedAt: number; expiresAt?: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`device-${invite.token}`,
invite.token,
invite.state,
0,
3,
invite.expiresAt ?? NOW + DAY_MS,
invite.updatedAt,
invite.updatedAt
]
)
}
async function remainingTokens(database: RelayDatabase): Promise<string[]> {
const rows = await database.query(`SELECT token_hash FROM relay_invites ORDER BY token_hash`)
return rows.map((row) => String(row.token_hash))
}
async function insertBasis(
database: RelayDatabase,
basis: { id: string; active: number; deadline: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
basis.id,
identity.userId,
identity.relayHostId,
'device-1',
1,
'invite',
basis.deadline,
basis.active,
NOW
]
)
}
async function insertDirectAuthorization(
database: RelayDatabase,
auth: { id: string; deadline: number; consumedAt: number | null }
): Promise<void> {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[auth.id, identity.userId, identity.relayHostId, 'device-1', 1, auth.deadline, auth.consumedAt]
)
}
async function remainingIds(database: RelayDatabase, table: string, column: string): Promise<string[]> {
const rows = await database.query(`SELECT ${column} FROM ${table} ORDER BY ${column}`)
return rows.map((row) => String(row[column]))
}
describe('credential cleanup invite reaper', () => {
it('deletes terminal invites past retention and keeps everything else', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 8 * DAY_MS
const recent = NOW - 6 * DAY_MS
for (const state of ['expired', 'consumed', 'invalidated']) {
await insertInvite(database, { token: `stale-${state}`, state, updatedAt: stale })
await insertInvite(database, { token: `recent-${state}`, state, updatedAt: recent })
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual([
'recent-consumed',
'recent-expired',
'recent-invalidated'
])
await database.close()
})
it('never deletes an invite that a reader could still consume, however old', async () => {
// Retention is measured on updated_at, and a long-lived available invite has an old one. The
// state filter is what keeps the reaper from deleting a credential still in use.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const ancient = NOW - 400 * DAY_MS
for (const state of ['available', 'reserved', 'cooldown']) {
await insertInvite(database, {
token: `live-${state}`,
state,
updatedAt: ancient,
expiresAt: NOW + DAY_MS
})
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual(['live-available', 'live-cooldown', 'live-reserved'])
await database.close()
})
it('bounds one cycle to a single batch and drains the rest on later cycles', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 30 * DAY_MS
for (let index = 0; index < 5_002; index += 1) {
await insertInvite(database, {
token: `consumed-${String(index).padStart(5, '0')}`,
state: 'consumed',
updatedAt: stale
})
}
await store.cleanup()
expect(await remainingTokens(database)).toHaveLength(2)
await store.cleanup()
expect(await remainingTokens(database)).toEqual([])
await database.close()
})
it('still expires credentials the sweep owns, and only those past their deadline', async () => {
// The reaper runs after the sweep in the same call, so this pins that adding it did not
// displace any of the five state transitions the sweep is there for.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertInvite(database, {
token: 'lapsed',
state: 'available',
updatedAt: NOW,
expiresAt: NOW - 1
})
await insertInvite(database, {
token: 'current',
state: 'available',
updatedAt: NOW,
expiresAt: NOW + DAY_MS
})
await database.query(
`UPDATE relay_invites SET state = ?, reservation_expires_at = ? WHERE token_hash = ?`,
['reserved', NOW - 1, 'current']
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
'basis-lapsed', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW - 1, 1, NOW,
'basis-live', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW + 1, 1, NOW
]
)
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-lapsed',
owningControlGeneration: 1,
deadline: NOW - 1
})
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-live',
owningControlGeneration: 1,
deadline: NOW + 1
})
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
VALUES (?, ?, ?, ?), (?, ?, ?, ?)`,
['scope', 'invite-mint', NOW - 2 * DAY_MS, 1, 'scope', 'invite-mint', NOW - 1, 1]
)
await store.cleanup()
expect(
await database.query(`SELECT token_hash, state FROM relay_invites ORDER BY token_hash`)
).toEqual([
{ token_hash: 'current', state: 'cooldown' },
{ token_hash: 'lapsed', state: 'expired' }
])
expect(
await database.query(`SELECT basis_conn_id, active FROM relay_connection_bases ORDER BY basis_conn_id`)
).toEqual([
{ basis_conn_id: 'basis-lapsed', active: 0 },
{ basis_conn_id: 'basis-live', active: 1 }
])
expect(
await database.query(
`SELECT direct_auth_id FROM relay_direct_authorizations
WHERE consumed_at IS NULL ORDER BY direct_auth_id`
)
).toEqual([{ direct_auth_id: 'direct-live' }])
expect(await database.query(`SELECT window_started_at FROM relay_rate_windows`)).toEqual([
{ window_started_at: NOW - 1 }
])
await database.close()
})
it('reaps connection bases whose deadline passed over a day ago, and nothing else', async () => {
// Retention is measured on deadline, and both readers of a basis require deadline >= now, so a
// deadline a day in the past is already unusable however the active flag reads. The active = 0
// clause is what keeps the batch an index range, not what makes the row safe to delete.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertBasis(database, { id: 'stale-inactive', active: 0, deadline: NOW - 2 * DAY_MS })
await insertBasis(database, { id: 'recent-inactive', active: 0, deadline: NOW - 60_000 })
// A long-lived splice: still active hours after the 30s deadline it was created with. The
// sweep deactivates it this cycle and the reaper takes it in the same call, which is safe
// precisely because no reader would have accepted it since its deadline passed.
await insertBasis(database, { id: 'stale-active', active: 1, deadline: NOW - 400 * DAY_MS })
await insertBasis(database, { id: 'live-active', active: 1, deadline: NOW + DAY_MS })
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([
'live-active',
'recent-inactive'
])
// The one row a reader can still use is untouched, active flag included.
expect(
await database.query(
`SELECT active FROM relay_connection_bases WHERE basis_conn_id = 'live-active'`
)
).toEqual([{ active: 1 }])
await database.close()
})
it('reaps consumed direct authorizations past retention and never a pending one', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertDirectAuthorization(database, {
id: 'stale-consumed',
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
await insertDirectAuthorization(database, {
id: 'recent-consumed',
deadline: NOW - 60_000,
consumedAt: NOW - 60_000
})
await insertDirectAuthorization(database, {
id: 'pending-ancient',
deadline: NOW + DAY_MS,
consumedAt: null
})
await store.cleanup()
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([
'pending-ancient',
'recent-consumed'
])
await database.close()
})
it('bounds each table to one batch per cycle', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
for (let index = 0; index < 5_001; index += 1) {
const id = String(index).padStart(5, '0')
await insertBasis(database, { id: `basis-${id}`, active: 0, deadline: NOW - 2 * DAY_MS })
await insertDirectAuthorization(database, {
id: `auth-${id}`,
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
}
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toHaveLength(1)
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toHaveLength(1)
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([])
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])
await database.close()
})
})
+49
View File
@@ -12,6 +12,17 @@ const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing
// working for clients whose clocks trail the cell by up to this margin.
const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000
// Terminal invites are read by nothing: every reader re-checks expiry and state at read time, so
// the row only serves the audit trail, which relay_audit_events already keeps. A week is long
// enough to answer a support question about a pairing that failed.
const TERMINAL_INVITE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000
// Why: both readers of a connection basis and of a direct authorization require it still
// active/unconsumed AND inside its deadline, and every deadline is set at most 30s past insert, so
// a settled row can never authorize anything again. A day is margin for forensics, not for reads.
const INACTIVE_AUTHORIZATION_RETENTION_MS = 24 * 60 * 60 * 1000
// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over
// however many cycles it takes.
const REAP_BATCH_ROWS = 5000
export type RelayIdentity = { userId: string; relayHostId: string }
export type CredentialReservation = RelayIdentity & {
@@ -643,6 +654,44 @@ export class RelayCredentialStore {
[now - 24 * 60 * 60 * 1000]
)
})
await this.reapSettledCredentials(now)
}
// Outside the sweep transaction on purpose: each delete is idempotent and independent of the
// state transitions above, so batching them in would only hold their row locks for longer.
private async reapSettledCredentials(now: number): Promise<void> {
await this.reapBatch(
'relay_invites',
'state IN (?, ?, ?) AND updated_at <= ?',
['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]
)
// deadline, not created_at: it is the second column of relay_connection_bases_active_deadline,
// so once the backlog is drained this batch learns there is nothing left to do from the index
// instead of the 1.5 GB heap. Both readers reject a passed deadline, so a day past one is
// unusable whatever the active flag says.
await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [
0,
now - INACTIVE_AUTHORIZATION_RETENTION_MS
])
// consumed_at, not deadline: consumption is what settles this row, and it can happen well
// before the deadline, so measuring from it retains the row for the full window either way.
await this.reapBatch(
'relay_direct_authorizations',
'consumed_at IS NOT NULL AND consumed_at <= ?',
[now - INACTIVE_AUTHORIZATION_RETENTION_MS]
)
}
// ctid/rowid, not the primary key: the physical address lets the delete re-find exactly the batch
// the subquery located instead of re-matching the predicate per row.
private async reapBatch(table: string, predicate: string, params: unknown[]): Promise<void> {
const address = this.database.dialect === 'sqlite' ? 'rowid' : 'ctid'
await this.database.query(
`DELETE FROM ${table} WHERE ${address} IN (
SELECT ${address} FROM ${table} WHERE ${predicate} LIMIT ${REAP_BATCH_ROWS}
)`,
params
)
}
private async installStatusWith(
@@ -134,7 +134,7 @@ describe('PostgreSQL relay deadlines', () => {
statements.every(
(statement) =>
statement === POSTGRES_STATEMENT_STATS_MIGRATION.trim() ||
/^(?:CREATE|ALTER TABLE)\b/i.test(body(statement))
/^(?:CREATE|ALTER TABLE|DROP INDEX)\b/i.test(body(statement))
)
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
@@ -0,0 +1,52 @@
import { isRelayDatabaseTransientError } from './database.js'
export type DatabaseStartupRetryPolicy = {
attempts: number
windowMs: number
baseDelayMs: number
maxDelayMs: number
jitterMs: number
// Which failures this particular startup step may repeat. Not every caller can
// repeat everything the request path calls transient: what the retry re-runs
// decides that, so the call site owns it.
isRetryable?: (error: unknown) => boolean
}
export type DatabaseStartupRetryObserver = {
onRetry?: (event: { attempt: number; delayMs: number; error: unknown }) => void
onRecovered?: (event: { attempts: number }) => void
onGaveUp?: (event: { attempts: number; error: unknown; retryable: boolean }) => void
}
function retryDelayMs(policy: DatabaseStartupRetryPolicy, attempt: number): number {
const backoffMs = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs)
return backoffMs + Math.floor(Math.random() * (policy.jitterMs + 1))
}
// Startup work that a cold dependency - a proxy sidecar that just started, a
// database still accepting the fleet back - can fail once and serve a moment
// later. The wall-clock window, not the attempt count, is the real bound.
export async function retryTransientDatabaseStartup<T>(
operation: () => Promise<T>,
policy: DatabaseStartupRetryPolicy,
observer: DatabaseStartupRetryObserver = {}
): Promise<T> {
const retryDeadline = Date.now() + policy.windowMs
for (let attempt = 1; ; attempt += 1) {
try {
const result = await operation()
if (attempt > 1) observer.onRecovered?.({ attempts: attempt })
return result
} catch (error) {
const remainingMs = retryDeadline - Date.now()
const retryable = (policy.isRetryable ?? isRelayDatabaseTransientError)(error)
if (attempt === policy.attempts || remainingMs <= 0 || !retryable) {
observer.onGaveUp?.({ attempts: attempt, error, retryable })
throw error
}
const delayMs = Math.min(retryDelayMs(policy, attempt), remainingMs)
observer.onRetry?.({ attempt, delayMs, error })
await new Promise((resolve) => setTimeout(resolve, delayMs))
}
}
}
@@ -1,5 +1,20 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import { isRelayDatabaseTransientError } from './database.js'
import { PostgresPoolPressure } from './postgres-pool-pressure.js'
// The only way to mark an error as an acquire failure is to fail a real
// acquire, so the gated cases go through the pressure wrapper the pool uses.
async function failedAcquire(message: string): Promise<unknown> {
const pool = {
totalCount: 0,
idleCount: 0,
waitingCount: 0,
connect: vi.fn(async () => {
throw new Error(message)
})
}
return await new PostgresPoolPressure(pool as never).connect().catch((error: unknown) => error)
}
describe('relay database transient errors', () => {
it.each(['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'])(
@@ -15,4 +30,39 @@ describe('relay database transient errors', () => {
).toBe(true)
expect(isRelayDatabaseTransientError(new TypeError('broken invariant'))).toBe(false)
})
it('classifies a pool connect timeout that node-postgres reports with no code', () => {
// pg-pool raises this only from its own connect path, so no statement ran.
expect(
isRelayDatabaseTransientError(
new Error('Connection terminated due to connection timeout')
)
).toBe(true)
})
it('classifies an early-ended socket only when it ended during the acquire', async () => {
expect(
isRelayDatabaseTransientError(await failedAcquire('Connection terminated unexpectedly'))
).toBe(true)
// The same message mid-statement leaves the commit outcome unknown, so it
// must stay a hard failure rather than invite a retry.
expect(
isRelayDatabaseTransientError(new Error('Connection terminated unexpectedly'))
).toBe(false)
})
it.each([null, undefined, 'a thrown string'])(
'survives %s reaching it instead of an error object',
(thrown) => {
expect(isRelayDatabaseTransientError(thrown)).toBe(false)
}
)
it('keeps a failed acquire that is not transient out of the retry path', async () => {
expect(
isRelayDatabaseTransientError(
await failedAcquire('password authentication failed for user "relay"')
)
).toBe(false)
})
})
+64 -9
View File
@@ -6,6 +6,7 @@ import pg from 'pg'
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
import {
emptyPostgresPoolPressureCounts,
isPostgresPoolConnectFailure,
PostgresPoolPressure,
type PostgresPoolPressureCounts
} from './postgres-pool-pressure.js'
@@ -99,6 +100,18 @@ CREATE TABLE IF NOT EXISTS relay_invites (
CREATE INDEX IF NOT EXISTS relay_invites_device
ON relay_invites(user_id, relay_host_id, relay_device_id);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the credential sweep matches (state, expires_at) every cycle while invites in a terminal
-- state accumulate for the life of the database. Unindexed it seq-scans the whole table inside the
-- maintenance transaction. Partial, so the index holds only the states the sweep can act on.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry
ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown');
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the second sweep pass matches (state, reservation_expires_at) over the same table.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation
ON relay_invites(reservation_expires_at) WHERE state = 'reserved';
CREATE TABLE IF NOT EXISTS relay_devices (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -159,6 +172,11 @@ CREATE TABLE IF NOT EXISTS relay_connection_bases (
-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle
-- and holds the maintenance transaction open long enough to time out
-- assignment lock waits.
-- Why not a partial index on active = 1: a basis is inserted active and flipped to 0, so each
-- deactivation leaves a dead entry in that index too. Measured on production-shaped history it
-- carries the same dead entries as this one, the planner picks this one in every state, and it
-- costs ~65 bytes of WAL per insert. Bloat here is cured by reaping and vacuum, not by a narrower
-- index.
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
ON relay_connection_bases(active, deadline);
@@ -172,6 +190,12 @@ CREATE TABLE IF NOT EXISTS relay_direct_authorizations (
consumed_at BIGINT
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the sweep expires pending authorizations by (consumed_at IS NULL, deadline), and consumed
-- rows are never deleted. Partial, so the index stays the size of the pending set.
CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline
ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL;
CREATE TABLE IF NOT EXISTS relay_confirm_results (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -523,8 +547,9 @@ CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases (
updated_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE INDEX IF NOT EXISTS relay_assignment_activity_expiry
ON relay_assignment_activity_leases(expires_at);
-- expires_at is deliberately unindexed: every control renewal writes it (~471/s), so an index on
-- it makes each renewal a non-HOT update that rewrites index entries. Its only reader is the 30s
-- expiry sweep, which seq-scans 14.8k rows / 7MB in a few milliseconds.
CREATE TABLE IF NOT EXISTS relay_control_connection_reservations (
reservation_id TEXT PRIMARY KEY,
@@ -558,6 +583,12 @@ CREATE TABLE IF NOT EXISTS relay_rate_windows (
PRIMARY KEY (scope_key, window_kind, window_started_at)
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: window_started_at is the PRIMARY KEY's last column, so the sweep's 24h retention delete
-- cannot use it and seq-scans instead.
CREATE INDEX IF NOT EXISTS relay_rate_windows_started
ON relay_rate_windows(window_started_at);
CREATE TABLE IF NOT EXISTS relay_migration_leases (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -644,7 +675,24 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
// Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because
// this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write,
// and all 28 directors reach it at once; a lock timeout here must not restart the instance, which
// would only re-queue the same DDL behind the same writers. Once it wins, the pre-check answers
// absent and no later boot sends it at all.
`-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s
DROP INDEX IF EXISTS relay_assignment_activity_expiry`,
// The drop is what makes HOT legal; this is what makes it possible. A renewal can only reuse the
// row's own page when that page has room for a second version, and at the default fillfactor of
// 100 a freshly filled page has none - measured at 0.5% HOT with the index gone and the default,
// against 100% at 70. Takes SHARE UPDATE EXCLUSIVE, which blocks vacuum and DDL but no reader or
// writer, and only for the catalog write. Applies to pages as they refill, so the table converges
// over its own renewal cycle rather than at boot.
// Deferrable for the same reason, though SHARE UPDATE EXCLUSIVE blocks only vacuum and DDL: it
// buys nothing until the drop lands, so a boot that deferred the drop should defer this too.
`-- schema-deferrable: buys nothing until the drop above lands
ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`
]
// The exact statement list a Postgres boot applies, in order, so the lock-target census can read
@@ -924,13 +972,15 @@ function retryablePostgresTransactionError(error: unknown): boolean {
}
export function isRelayDatabaseTransientError(error: unknown): boolean {
const code = String((error as { code?: unknown }).code)
// Runs inside the query catch, where a thrown null or undefined would turn a
// database failure into a TypeError that buries it.
const code = String((error as { code?: unknown } | null)?.code)
if (['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'].includes(code)) {
return true
}
return String((error as { message?: unknown }).message).includes(
'timeout exceeded when trying to connect'
)
// A pool that cannot hand out a client reports no SQLSTATE at all, so the
// acquire boundary owns that vocabulary.
return isPostgresPoolConnectFailure(error)
}
async function waitForPostgresRetry(random: () => number = Math.random): Promise<void> {
@@ -965,6 +1015,9 @@ class PostgresDatabase implements RelayDatabase {
error,
phase,
sql,
// Passed in rather than re-derived: the log has to say what the routes
// actually did, and one classifier cannot drift from itself.
transient: isRelayDatabaseTransientError(error),
elapsedMs: performance.now() - startedAt,
pool: this.pool
})
@@ -1155,13 +1208,15 @@ async function backfillRelayCellRegions(database: RelayDatabase): Promise<void>
)
}
export async function openRelayDatabase(input: {
export type RelayDatabaseOpenInput = {
databaseUrl?: string
dataDir: string
poolMax?: number
applicationName?: string
statementTimeoutMs?: number
}): Promise<RelayDatabase> {
}
export async function openRelayDatabase(input: RelayDatabaseOpenInput): Promise<RelayDatabase> {
let database: RelayDatabase
if (input.databaseUrl) {
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
@@ -729,3 +729,73 @@ describe('control lease jitter', () => {
vi.advanceTimersByTime(0)
})
})
describe('paced drain and the phones of a host not yet told', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const laterHostId = 'qrstuvwxyz012345'
const laterIdentity = { ...identity, sub: 'user-2', relayHostId: laterHostId }
async function twoHostCell(): Promise<{
h: ReturnType<typeof harness>
told: FakeSocket
untold: FakeSocket
}> {
const h = harness()
const told = await activeHost(h)
const untold = new FakeSocket()
await h.activate(untold as unknown as WebSocket, laterIdentity, null, 1, false, 1, '1.4.197')
// Both hosts now dial in, so the credential mocks have to answer for either.
h.store.resolveResume.mockImplementation(async (hostId: string) => ({
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub
}))
h.store.reserveCredential.mockImplementation(async (hostId: string) => ({
...reservation,
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub,
relayHostId: hostId
}))
return { h, told, untold }
}
async function dial(h: ReturnType<typeof harness>, hostId: string): Promise<FakeSocket> {
const client = new FakeSocket()
await h.registry.acceptClient(client as unknown as WebSocket, hostId, 'credential')
return client
}
it('serves a host whose drain has not been sent and refuses one whose has', async () => {
const { h, told, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
const refused = await dial(h, identity.relayHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(told.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
const served = await dial(h, laterHostId)
expect(served.close).not.toHaveBeenCalled()
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
})
it('refuses that host\'s phones as soon as its own drain is sent', async () => {
const { h, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(40_000)
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
const refused = await dial(h, laterHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
})
it('keeps an unpaced drain refusing every phone at once', async () => {
const { h } = await twoHostCell()
h.registry.drain(0)
for (const hostId of [identity.relayHostId, laterHostId]) {
const refused = await dial(h, hostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
}
})
})
@@ -12,6 +12,12 @@ import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import {
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import { HostSessionRegistry, type HostSession } from './host-session-registry.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
@@ -23,6 +29,12 @@ import {
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
// A due renewal leaves the heartbeat as a batch enqueue, so the store only sees
// the tick once the batch window closes.
async function closeRenewalWindow(): Promise<void> {
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
}
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSING = 2
@@ -109,6 +121,7 @@ function createRegistry(
activate: ActivateSession
acquireActivity: ReturnType<typeof vi.fn>
renewControlActivity: ReturnType<typeof vi.fn>
renewControlActivities: ReturnType<typeof vi.fn>
releaseActivity: ReturnType<typeof vi.fn>
observer: {
recordAuth: ReturnType<typeof vi.fn>
@@ -119,12 +132,38 @@ function createRegistry(
const acquireActivity = vi.fn().mockResolvedValue(undefined)
const renewControlActivity = vi.fn().mockResolvedValue(undefined)
const releaseActivity = vi.fn().mockResolvedValue(true)
// Mirrors the store's own batch semantics over the single-renewal mock: a known
// outcome becomes that row's verdict, and any other failure reaches the caller
// as the driver's error. Keeps every per-call expectation below aimed at the
// renewal a session actually asked for.
const renewControlActivities = vi.fn(
async (rows: readonly ControlRenewalRequest[]): Promise<ControlRenewalOutcome[]> =>
await Promise.all(
rows.map(async (row): Promise<ControlRenewalOutcome> => {
try {
await renewControlActivity(row.identity, {
activityId: row.activityId,
cellId: row.cellId,
expiresAt: row.expiresAt
})
return 'renewed'
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) {
throw error
}
return message as ControlRenewalOutcome
}
})
)
)
const assignments = {
activateControl,
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity
} as unknown as RelayAssignmentStore
const observer = {
@@ -176,6 +215,7 @@ function createRegistry(
activate,
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity,
observer
}
@@ -336,15 +376,15 @@ describe('host session cleanup races', () => {
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
// POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both
// reach drain(), so a second teardown can be scheduled for the same session.
// reach drain(), so a retry re-sends to every session. It must re-arm the pending
// teardown rather than stack a second one: across a paced cell that is 800 orphaned
// timers per retry, each one holding the loop open for the rest of the window.
registry.drain(0)
const scheduled = vi.getTimerCount()
registry.drain(0)
// Pin the premise: if drain ever gains an idempotency guard, the retry schedules no
// second teardown and the assertion below stops defending the write-once snapshot
// while still passing. Compare against the count before the retry rather than an
// absolute, since the session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled + 1)
// Compare against the count before the retry rather than an absolute, since the
// session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled)
vi.advanceTimersByTime(1)
// Asserting registry state, not the log line: FakeSocket closes synchronously, so
@@ -689,7 +729,8 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
expect(renewControlActivity).toHaveBeenCalledWith(
@@ -715,6 +756,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
@@ -737,6 +779,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
@@ -762,6 +805,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 4; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
@@ -791,8 +835,10 @@ describe('host session cleanup races', () => {
try {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
} finally {
warn.mockRestore()
@@ -812,6 +858,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
@@ -831,13 +878,16 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
await vi.advanceTimersByTimeAsync(0)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(3)
registry.drain(0)
@@ -855,6 +905,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
@@ -880,6 +931,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).not.toHaveBeenCalled()
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
@@ -899,6 +951,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -918,6 +971,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -939,6 +993,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 3; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
expect(renewControlActivity).toHaveBeenCalledTimes(2)
@@ -966,6 +1021,7 @@ describe('control renewal cadence across a rebind', () => {
const beat = async (target: FakeSocket): Promise<void> => {
await vi.advanceTimersByTimeAsync(ping)
target.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
// Age the session so its attempt counter is well above zero.
@@ -993,6 +1049,57 @@ describe('control renewal cadence across a rebind', () => {
})
})
describe('control renewals shared by one batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('renews two due hosts in one call and leaves a stale one alone', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValueOnce('control:production-gce-c3:1')
.mockResolvedValueOnce('control:production-gce-c3:1')
const { registry, activate, renewControlActivities } = createRegistry(activateControl)
const other = { ...identity, sub: 'user-2', relayHostId: 'ponmlkjihgfedcba' }
const staleSocket = new FakeSocket()
const liveSocket = new FakeSocket()
await activate(staleSocket as unknown as WebSocket, identity, null, 1, false, 1)
await activate(liveSocket as unknown as WebSocket, other, null, 1, false, 1)
const stale = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const live = registry.get({ userId: other.sub, relayHostId: other.relayHostId })!
// Both come due inside the same window, and one socket goes away while the
// statement is still in PostgreSQL.
let release!: () => void
renewControlActivities.mockImplementationOnce(
async (rows: readonly ControlRenewalRequest[]) => {
staleSocket.close()
await new Promise<void>((resolve) => (release = resolve))
return rows.map((): ControlRenewalOutcome => 'renewed')
}
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const staleDueAt = stale.activityRenewalDueAt
await closeRenewalWindow()
release()
await vi.advanceTimersByTimeAsync(0)
expect(renewControlActivities).toHaveBeenCalledOnce()
expect(
renewControlActivities.mock.calls[0]![0].map(
(row: ControlRenewalRequest) => row.identity.relayHostId
)
).toEqual([identity.relayHostId, other.relayHostId])
expect(live.activityRenewalCompletedAttempt).toBe(1)
expect(stale.activityRenewalCompletedAttempt).toBe(0)
expect(stale.activityRenewalDueAt).toBe(staleDueAt)
registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('control lease recovery after the session is gone', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -1019,6 +1126,7 @@ describe('control lease recovery after the session is gone', () => {
new Promise<void>((_resolve, reject) => (failRenewal = reject))
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
const newer = new FakeSocket()
@@ -1696,3 +1804,124 @@ describe('host data attach owner lookup', () => {
expect(h.owner.activeConnIds.size).toBe(0)
})
})
describe('paced drain', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
async function connectHosts(count: number): Promise<{
registry: HostSessionRegistry
sockets: FakeSocket[]
}> {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
const { registry, activate } = createRegistry(activateControl)
const sockets: FakeSocket[] = []
for (let index = 0; index < count; index += 1) {
const socket = new FakeSocket()
sockets.push(socket)
await activate(
socket as unknown as WebSocket,
{ ...identity, sub: `user-${index}` },
null,
1,
false,
1
)
socket.send.mockClear()
}
return { registry, sockets }
}
function drainsSent(sockets: FakeSocket[]): number {
return sockets.filter((socket) =>
socket.send.mock.calls.some(([payload]) => String(payload).includes('"type":"drain"'))
).length
}
it('sends every drain at once when no window is given', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
})
// Windows here stay under the 75s control-silence watchdog, which would otherwise close
// a test socket that never heartbeats before its paced send is due.
it('spreads the sends evenly across the window', async () => {
const { registry, sockets } = await connectHosts(5)
registry.drain(0, { paceWindowMs: 40_000 })
// The first host is sent synchronously; the last lands on the window's closing edge.
expect(drainsSent(sockets)).toBe(1)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(2)
await vi.advanceTimersByTimeAsync(20_000)
expect(drainsSent(sockets)).toBe(4)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(5)
})
it('fences admission for every session before the first paced send lands', async () => {
const { registry, sockets } = await connectHosts(3)
registry.drain(0, { paceWindowMs: 40_000 })
expect(registry.isDraining()).toBe(true)
// A host whose drain has not been sent yet must already be non-authoritative.
const socket = new FakeSocket()
registry.acceptControl(socket as unknown as WebSocket, { ...identity, sub: 'user-late' })
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
expect(drainsSent(sockets)).toBe(1)
})
it('gives each host its own grace after its own send, not after the call', async () => {
const { registry, sockets } = await connectHosts(2)
registry.drain(10_000, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(10_000)
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
// Its own send at 40s plus its own 10s grace, not 10s from the drain call.
await vi.advanceTimersByTimeAsync(39_999)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
await vi.advanceTimersByTimeAsync(10_001)
expect(sockets[1]!.readyState).toBe(sockets[1]!.CLOSED)
})
it('leaves no timer behind once an emergency drain cuts a window short', async () => {
const { registry } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
// Every session is closed, so anything still pending is an orphan of the cut window.
expect(vi.getTimerCount()).toBe(0)
})
it('keeps the first teardown snapshot when a regional drain fires before the fleet one', async () => {
const { registry, sockets } = await connectHosts(1)
const session = registry.get({ userId: 'user-0', relayHostId: identity.relayHostId })!
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
registry.drainHost({
attemptId: 'attempt',
userId: 'user-0',
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 0
})
registry.drain(10)
await vi.advanceTimersByTimeAsync(11)
expect(session.closingCounts).toEqual({ splices: 1, pending: 0 })
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
})
it('lets an emergency drain supersede the sends still queued by a paced one', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
expect(drainsSent(sockets)).toBe(1)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
const sendsAfterEmergency = sockets.map((socket) => socket.send.mock.calls.length)
await vi.advanceTimersByTimeAsync(40_000)
expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency)
})
})
+73 -19
View File
@@ -26,6 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import { ControlRenewalBatch } from './control-renewal-batch.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
@@ -179,6 +180,10 @@ export class HostSessionRegistry {
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly drainTimers = new Set<ReturnType<typeof setTimeout>>()
// Hosts whose drain has been sent. Paced sends land minutes apart, so "this cell is
// draining" is not the same question as "this host has been told to leave".
private readonly drainSentHosts = new Set<string>()
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
@@ -299,6 +304,15 @@ export class HostSessionRegistry {
private readonly cellIncarnation?: string
) {}
// Renewals leave the heartbeat as an enqueue: one statement per cell per
// window replaces one write transaction per host, which is what keeps the
// shared PostgreSQL instance out of buffer-header contention.
private readonly controlRenewals = new ControlRenewalBatch(
async (rows) => await this.assignments.renewControlActivities(rows),
() => this.logIdentity(),
(flush) => this.observer.recordControlRenewalFlush?.(flush)
)
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
private controlLeaseExpiresAt(): number {
const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
@@ -330,7 +344,10 @@ export class HostSessionRegistry {
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
// Not `this.draining`: a paced drain tells hosts minutes apart, and the director keeps
// pointing phones here until their own host has moved. Refusing them for the whole
// window would turn a 2 min drain into a 2 min outage for hosts not yet told.
if (this.drainSentHosts.has(hostId)) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
return
@@ -450,7 +467,7 @@ export class HostSessionRegistry {
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.drainSentHosts.has(hostId) ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
@@ -592,7 +609,7 @@ export class HostSessionRegistry {
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.drainSentHosts.has(identity.relayHostId) ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
@@ -846,17 +863,49 @@ export class HostSessionRegistry {
return { controls, splices, pendingSplices }
}
drain(graceMs: number): void {
drain(graceMs: number, options: { paceWindowMs?: number } = {}): void {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
// A later drain (an emergency one, or shutdown) owns every session again, so nothing
// queued by an earlier paced drain may still fire: it would re-send and, worse, keep
// the event loop alive for the rest of a window the operator just cut short.
for (const timer of this.drainTimers) clearTimeout(timer)
this.drainTimers.clear()
const paceWindowMs = Math.max(0, Math.trunc(options.paceWindowMs ?? 0))
const targets = [...this.sessions.values()].filter((session) => session.state !== 'closed')
// The desktop re-dials the director as soon as it reads `drain`, whatever graceMs says,
// so spreading the send is the only thing that spreads the reconnect load.
const step = paceWindowMs > 0 && targets.length > 1 ? paceWindowMs / (targets.length - 1) : 0
for (const [index, session] of targets.entries()) {
const delay = Math.round(step * index)
if (delay === 0) {
this.sendDrain(session, graceMs)
continue
}
this.scheduleDrainTimer(delay, () => this.sendDrain(session, graceMs))
}
}
// A session is only fenced when it is told, not when the drain starts: until its send
// lands it is an ordinary live host, and its phones have to keep being able to reach it.
private sendDrain(session: HostSession, graceMs: number): void {
if (session.state === 'closed') return
session.authorityRevision += 1
session.state = 'drain-only'
this.drainSentHosts.add(session.relayHostId)
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
this.scheduleDrainTimer(graceMs, () => this.closeDrainedSession(session))
}
// Unref'd so a drain in flight never holds the process open past its own work.
private scheduleDrainTimer(delayMs: number, run: () => void): void {
const timer: ReturnType<typeof setTimeout> = setTimeout(() => {
this.drainTimers.delete(timer)
run()
}, delayMs)
timer.unref?.()
this.drainTimers.add(timer)
}
drainHost(input: {
attemptId: string
userId: string
@@ -1345,15 +1394,13 @@ export class HostSessionRegistry {
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
{
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
}
)
void this.controlRenewals
.enqueue({
identity: { userId: session.identity.sub, relayHostId: session.relayHostId },
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
})
.then(() => {
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
@@ -1419,6 +1466,13 @@ export class HostSessionRegistry {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
return
}
if (error instanceof Error && error.message === 'assignment_lock_unavailable') {
// A per-host transaction held the row, so the batch passed over it
// rather than making every other host in the flush wait. The next
// tick is 15s away against a 105s lease, and the flush line already
// reports the count, so this needs no line of its own.
return
}
console.warn('[orca-relay] control activity renewal failed')
})
// Terminal handler: a throw inside the async catch above (e.g. a
@@ -4,114 +4,296 @@ import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
// How many decision rows one poll is allowed to look at. The poll runs about
// fifty times a minute across the directors, so its cost has to be set by this
// number and not by the size of the fleet or the width of the cohort.
export const IDLE_REHOME_DECISION_WINDOW = 500
// Where the last window ended. A keyset beats OFFSET: `OFFSET n` still has to
// produce and throw away n rows, and n grew by a page on every poll that
// dispatched, so the scan got more expensive the longer the rollout ran.
export type IdleRehomeHostCursor = { userId: string; relayHostId: string } | null
export type IdleRegionalRehomeCandidate = IdleRegionalRehomeRequest & { sourceCellUrl: string }
export type IdleRegionalRehomeSelection = {
candidates: IdleRegionalRehomeCandidate[]
cursor: IdleRehomeHostCursor
}
type SourceCell = {
cellId: string
region: string
cellIncarnation: string
startedAt: number
cellUrl: string
}
type TargetCell = { cellId: string; capacityRequests: number; reservedRequests: number }
type SelectionInput = {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
preferenceMaxAgeMs: number
hostCooldownMs: number
cursor: IdleRehomeHostCursor
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
}
const CELL_INVENTORY_QUERY = `SELECT cell.cell_id, cell.cell_url, cell.enabled,
cell.capacity_requests, cell.reserved_requests, region.region,
admission.admission_state, capability.cell_incarnation AS capability_incarnation,
capability.regional_rehome_protocol
FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
LEFT JOIN relay_cell_capabilities capability ON capability.cell_id = cell.cell_id`
export async function selectIdleRegionalRehomes(
input: SelectionInput
): Promise<IdleRegionalRehomeSelection> {
const cells = await readCellInventory(input)
if (!cells.sources.size || !cells.targetsByRegion.size) return { candidates: [], cursor: null }
const sourceRegions = [...new Set([...cells.sources.values()].map((cell) => cell.region))]
const targetRegions = [...cells.targetsByRegion.keys()]
const decisionFilter = `outcome = 'conclusive' AND policy_version = 1
AND preferred_region IN (${placeholders(targetRegions.length)})
AND incumbent_region IN (${placeholders(sourceRegions.length)})
AND preferred_region <> incumbent_region
AND expires_at > ? AND observed_at >= ? AND cohort_bucket < ?`
const decisionParams = [
...targetRegions,
...sourceRegions,
input.now,
input.now - input.preferenceMaxAgeMs,
input.cohortPercent
]
const after = input.cursor ? [input.cursor.userId, input.cursor.relayHostId] : []
const afterFilter = input.cursor ? 'AND (user_id, relay_host_id) > (?, ?)' : ''
// The window is taken first and on its own so the poll knows where it stopped
// reading, not just where it stopped emitting. Every gate below this point can
// reject a host, and a cursor that only advanced past emitted rows would park
// on a rejected host forever.
const window = await input.database.query(
`SELECT user_id, relay_host_id FROM relay_region_decisions
WHERE ${decisionFilter} ${afterFilter}
ORDER BY user_id, relay_host_id LIMIT ?`,
[...decisionParams, ...after, IDLE_REHOME_DECISION_WINDOW]
)
if (!window.length) return { candidates: [], cursor: null }
const windowEnd = window[window.length - 1]!
const windowWasFull = window.length === IDLE_REHOME_DECISION_WINDOW
const sourceList = [...cells.sources.values()]
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
// The verification names the window's keys rather than repeating its LIMIT:
// the two reads take separate snapshots, and a decision that turned eligible
// between them would otherwise shift the second LIMIT and push the last host
// out of it while the cursor still advanced past it.
`SELECT d.user_id, d.relay_host_id, d.preferred_region, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation
FROM (SELECT user_id, relay_host_id, preferred_region, incumbent_region, assignment_epoch
FROM relay_region_decisions
WHERE ${decisionFilter}
AND (user_id, relay_host_id) IN (${Array.from({ length: window.length }, () => '(?,?)').join(',')})
-- The LIMIT cannot truncate a key set this size; it is here because without
-- it Postgres flattens the subquery, estimates one row out of the join, and
-- drives the whole plan from a sequential scan of the capability table.
ORDER BY user_id, relay_host_id LIMIT ?) d
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND a.assignment_epoch = d.assignment_epoch
JOIN (${inlineRows(SOURCE_CELL_COLUMNS, sourceList.length)}) source
ON source.cell_id = a.cell_id AND source.region = d.incumbent_region
JOIN relay_control_capabilities host ON host.user_id = d.user_id
AND host.relay_host_id = d.relay_host_id AND host.cell_id = a.cell_id
AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = source.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = d.user_id
AND lease.relay_host_id = d.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND lease.expires_at > ? AND lease.updated_at >= source.started_at
WHERE NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = d.user_id AND migration.relay_host_id = d.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
WHERE attempt.user_id = d.user_id AND attempt.relay_host_id = d.relay_host_id
AND attempt.created_at > ?)
ORDER BY d.user_id, d.relay_host_id, host.generation DESC
-- Counted in hosts, because a host with one eligible target has to be able
-- to fill a page on its own. A host with many leaves part of this page
-- unread, and the cursor stops where the page stopped, so it is re-read
-- next poll rather than skipped.
LIMIT ?`,
[
...decisionParams,
...window.flatMap((row) => [row.user_id, row.relay_host_id]),
IDLE_REHOME_DECISION_WINDOW,
...sourceList.flatMap((cell) => [cell.cellId, cell.region, cell.cellIncarnation, cell.startedAt]),
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
input.now - input.hostCooldownMs,
IDLE_REHOME_PAGE_SIZE
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
const units = rows.length ? await sourceRequestUnits(input.database, rows) : new Map<string, number>()
const candidates: IdleRegionalRehomeCandidate[] = []
let stoppedAt: IdleRehomeHostCursor = null
for (const row of rows) {
// Whole hosts only: the lower-priority targets are a host's fallbacks when
// the first one defers, and splitting them across pages loses them.
if (candidates.length >= IDLE_REHOME_PAGE_SIZE) {
return { candidates, cursor: stoppedAt }
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
const source = cells.sources.get(String(row.source_cell_id))!
const sourceUnits = units.get(hostKey(row)) ?? 0
for (const target of cells.targetsByRegion.get(String(row.preferred_region)) ?? []) {
if (target.reservedRequests + 1 + sourceUnits > target.capacityRequests) continue
candidates.push(idleRehomeCandidate(row, source, target.cellId))
}
stoppedAt = { userId: String(row.user_id), relayHostId: String(row.relay_host_id) }
}
// A full verification page may have been cut short of the window's end, so only
// a page that ran the window out may wrap to the head of the keyspace.
if (rows.length === IDLE_REHOME_PAGE_SIZE) return { candidates, cursor: stoppedAt }
return {
candidates,
cursor: windowWasFull
? { userId: String(windowEnd.user_id), relayHostId: String(windowEnd.relay_host_id) }
: null
}
}
// Every cell predicate the candidate join used to re-evaluate per (host, cell)
// pair. There are tens of cells and tens of thousands of hosts, so this is
// resolved once per poll against the four small inventory tables.
async function readCellInventory(
input: SelectionInput
): Promise<{ sources: Map<string, SourceCell>; targetsByRegion: Map<string, TargetCell[]> }> {
const { database, now } = input
const [runtimeRows, safetyRows, inventory] = await Promise.all([
database.query('SELECT * FROM relay_cell_runtime'),
database.query('SELECT * FROM relay_cell_rehome_safety'),
database.query(CELL_INVENTORY_QUERY)
])
const runtimes = new Map(runtimeRows.map((row) => [String(row.cell_id), row]))
const safety = new Map(safetyRows.map((row) => [String(row.cell_id), row]))
const sources = new Map<string, SourceCell>()
const targetsByRegion = new Map<string, TargetCell[]>()
const load = new Map<string, number>()
for (const cell of inventory) {
const cellId = String(cell.cell_id)
const runtime = runtimes.get(cellId)
if (!runtime || !input.cellIsClean(safety.get(cellId), runtime, now)) continue
if (
Number(cell.enabled) !== 1 ||
cell.admission_state !== 'general' ||
cell.region == null ||
Number(runtime.ready) !== 1 ||
Number(runtime.last_heartbeat_at) <= now - input.heartbeatTtlMs ||
cell.capability_incarnation == null ||
String(cell.capability_incarnation) !== String(runtime.cell_incarnation) ||
Number(cell.regional_rehome_protocol) < 3
) {
continue
}
const region = String(cell.region)
sources.set(cellId, {
cellId,
region,
cellIncarnation: String(runtime.cell_incarnation),
startedAt: Number(runtime.started_at),
cellUrl: String(cell.cell_url)
})
if (input.connectionHeadroom.get(cellId) === false) continue
const capacityRequests = Number(cell.capacity_requests)
const reservedRequests = Number(cell.reserved_requests)
const targets = targetsByRegion.get(region) ?? []
targets.push({ cellId, capacityRequests, reservedRequests })
targetsByRegion.set(region, targets)
load.set(cellId, (reservedRequests + Number(runtime.observed_requests)) / capacityRequests)
}
for (const targets of targetsByRegion.values()) {
targets.sort(
(left, right) =>
load.get(left.cellId)! - load.get(right.cellId)! || (left.cellId < right.cellId ? -1 : 1)
)
}
return { sources, targetsByRegion }
}
// One grouped read for the page instead of a correlated aggregate per (host, cell) pair.
async function sourceRequestUnits(
database: RelayDatabase,
rows: SqlRow[]
): Promise<Map<string, number>> {
const seen = new Set<string>()
const params: unknown[] = []
for (const row of rows) {
if (seen.has(hostKey(row))) continue
seen.add(hostKey(row))
params.push(row.user_id, row.relay_host_id, row.source_cell_id)
}
const sums = await database.query(
`SELECT user_id, relay_host_id, COALESCE(SUM(request_units), 0) AS request_units
FROM relay_assignment_activity_leases
WHERE (user_id, relay_host_id, cell_id) IN (${Array.from({ length: seen.size }, () => '(?,?,?)').join(',')})
GROUP BY user_id, relay_host_id`,
params
)
return new Map(sums.map((row) => [hostKey(row), Number(row.request_units)]))
}
const SOURCE_CELL_COLUMNS = [
['cell_id', 'TEXT'],
['region', 'TEXT'],
['cell_incarnation', 'TEXT'],
['started_at', 'BIGINT']
] as const
function placeholders(count: number): string {
return Array.from({ length: count }, () => '?').join(',')
}
// A derived table the planner can hash, in the one syntax both Postgres and the
// SQLite test engine accept (`VALUES ... AS t(col)` and LATERAL are not common to
// both). Only the first branch is cast; both engines take the union's types from it.
function inlineRows(columns: ReadonlyArray<readonly [string, string]>, rows: number): string {
const first = columns.map(([name, type]) => `CAST(? AS ${type}) AS ${name}`)
const rest = Array.from({ length: rows - 1 }, () => `UNION ALL SELECT ${placeholders(columns.length)}`)
return `SELECT ${first.join(', ')} ${rest.join(' ')}`
}
function hostKey(row: SqlRow): string {
return `${String(row.user_id)}${String(row.relay_host_id)}`
}
function idleRehomeCandidate(
row: SqlRow,
source: SourceCell,
targetCellId: string
): IdleRegionalRehomeCandidate {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: source.cellId,
sourceCellIncarnation: source.cellIncarnation,
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: source.cellUrl }
}
@@ -182,6 +182,53 @@ describe('constrained idle regional assignment transaction', () => {
})
})
it.each(['next_dispatch_at', 'paused_until'] as const)(
'skips the candidate join while %s holds the durable dispatch budget closed',
async (column) => {
const { store, database, safety } = await setup()
const query = vi.spyOn(database, 'query')
// One assignment only: setup leaves both fields at 0, and naming the other
// one too would assign this column twice, which Postgres rejects.
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt + 1]
)
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(2)
expect(query.mock.calls[1]![0]).toMatch(/FROM relay_region_rehome_worker_state/s)
}
await database.query(
`UPDATE relay_region_rehome_worker_state SET ${column} = ? WHERE worker_id = 'global'`,
[safety.observedAt]
)
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(2)
}
)
it('polls when the worker state row has never been written', async () => {
const { store, database, safety } = await setup()
await database.query('DELETE FROM relay_region_rehome_worker_state')
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
})
it('leaves the candidate page offset untouched across a closed dispatch budget', async () => {
const { store, database, safety } = await setup()
const first = await store.selectIdleRegionalRehomeCandidates(safety)
await database.query(
`UPDATE relay_region_rehome_worker_state SET next_dispatch_at = ? WHERE worker_id = 'global'`,
[safety.observedAt + 1]
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
await database.query(
`UPDATE relay_region_rehome_worker_state SET next_dispatch_at = 0 WHERE worker_id = 'global'`
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { IDLE_REHOME_DECISION_WINDOW } from './idle-regional-rehome-selection.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
// One source cell and three targets, so a poll has to rank targets per host
// rather than take the single one the two-cell fixture leaves it.
const cells = [
{ id: 'us', url: 'https://us.example.test', region: 'us-central1' as const, capacityRequests: 100 },
{ id: 'asia-busy', url: 'https://asia-busy.example.test', region: 'asia-east2' as const, capacityRequests: 100 },
{ id: 'asia-idle', url: 'https://asia-idle.example.test', region: 'asia-east2' as const, capacityRequests: 100 },
{ id: 'asia-mid', url: 'https://asia-mid.example.test', region: 'asia-east2' as const, capacityRequests: 100 }
]
const incarnations = cells.map((_, index) => `${index + 1}${'1'.repeat(7)}-1111-4111-8111-111111111111`)
const observed = [0, 60, 10, 30]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: observed[index]!
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
return { store, database, safety, now }
}
async function seedHost(
store: RelayAssignmentStore,
identity: { userId: string; relayHostId: string }
): Promise<void> {
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: 'us',
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
// Clone one seeded host's rows under new identities, which is far cheaper than
// driving the full activation path thousands of times.
async function cloneHosts(
database: RelayDatabase,
template: { userId: string; relayHostId: string },
count: number
): Promise<void> {
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const row = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
template.userId,
template.relayHostId
])
)[0]!
const columns = Object.keys(row)
const projection = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
for (let index = 0; index < count; index++) {
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${projection.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`clone-${String(index).padStart(5, '0')}`,
`clonehost${String(index).padStart(7, '0')}`,
template.userId,
template.relayHostId
]
)
}
}
}
describe('idle regional rehome candidate window', () => {
const identity = { userId: 'window-test', relayHostId: 'abcdefghijklmnop' }
it('offers every eligible target for a host, least loaded first', async () => {
const { store, safety } = await setup()
await seedHost(store, identity)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates.map((candidate) => candidate.targetCellId)).toEqual([
'asia-idle',
'asia-mid',
'asia-busy'
])
expect(new Set(candidates.map((candidate) => candidate.sourceCellUrl))).toEqual(
new Set(['https://us.example.test'])
)
// Every candidate is the same move to a different target, so the attempt ids differ.
expect(new Set(candidates.map((candidate) => candidate.attemptId)).size).toBe(3)
})
it('drops only the targets without room for the host plus its source activity', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 4 WHERE user_id = ?',
[identity.userId]
)
// Five units needed: four source units plus the assignment the move reserves.
await database.query("UPDATE relay_cells SET capacity_requests = 4 WHERE cell_id = 'asia-idle'")
const short = await store.selectIdleRegionalRehomeCandidates(safety)
expect(short.map((candidate) => candidate.targetCellId)).toEqual(['asia-mid', 'asia-busy'])
// Exactly enough room is enough; it ranks last because the ratio is per capacity.
await database.query("UPDATE relay_cells SET capacity_requests = 5 WHERE cell_id = 'asia-idle'")
const exact = await store.selectIdleRegionalRehomeCandidates(safety)
expect(exact.map((candidate) => candidate.targetCellId)).toEqual([
'asia-mid',
'asia-busy',
'asia-idle'
])
})
it('reads a bounded window of decisions however many hosts are eligible', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, IDLE_REHOME_DECISION_WINDOW + 200)
const query = vi.spyOn(database, 'query')
await store.selectIdleRegionalRehomeCandidates(safety)
const calls = query.mock.calls.map((call) => call[0])
const window = calls.findIndex((sql) => /FROM relay_region_decisions\s*$/m.test(sql))
expect(window).toBeGreaterThanOrEqual(0)
expect(query.mock.calls[window]![1]!.at(-1)).toBe(IDLE_REHOME_DECISION_WINDOW)
// No statement pages by OFFSET any more: that was the cost that grew with the rollout.
expect(calls.some((sql) => /OFFSET/i.test(sql))).toBe(false)
})
it('keeps the window\'s last host when a decision turns eligible between the two reads', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
// Exactly one full window, whose last key in sort order is the seeded host.
await cloneHosts(database, identity, IDLE_REHOME_DECISION_WINDOW - 1)
await database.query(
"UPDATE relay_assignment_activity_leases SET expires_at = ? WHERE user_id LIKE 'clone-%'",
[now - 1]
)
const template = (
await database.query('SELECT * FROM relay_region_decisions WHERE user_id = ?', [
identity.userId
])
)[0]!
const columns = Object.keys(template)
const query = database.query.bind(database)
let inserted = false
vi.spyOn(database, 'query').mockImplementation(async (sql, params) => {
const rows = await query(sql, params)
// A decision that becomes eligible after the window is read and sorts
// inside it: a second LIMIT would push the window's last host out.
if (!inserted && /^SELECT user_id, relay_host_id FROM relay_region_decisions/.test(sql)) {
inserted = true
await query(
`INSERT INTO relay_region_decisions (${columns.join(', ')})
VALUES (${columns.map(() => '?').join(', ')})`,
columns.map((column) =>
column === 'user_id'
? 'clone-99999'
: column === 'relay_host_id'
? 'latehost99999999'
: template[column]
)
)
}
return rows
})
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates.map((candidate) => candidate.userId)).toEqual([
identity.userId,
identity.userId,
identity.userId
])
})
it('walks the whole population in bounded pages and wraps only at the end', async () => {
const { store, database, safety } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, 120)
const seen = new Set<string>()
let pages = 0
let wrapped = false
// 121 hosts x 3 targets is 363 candidates, so the page cap has to be hit
// several times before the window runs out and the cursor wraps.
for (let poll = 0; poll < 20 && !wrapped; poll++) {
const page = await store.selectIdleRegionalRehomeCandidates(safety)
pages += 1
const before = seen.size
for (const candidate of page) seen.add(`${candidate.userId}/${candidate.targetCellId}`)
if (seen.size === before && page.length > 0) wrapped = true
if (page.length < 3) wrapped = true
}
expect(pages).toBeGreaterThan(1)
expect(seen.size).toBe(121 * 3)
})
it('does not stall on a host the window found but the join rejected', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
await cloneHosts(database, identity, 2)
// The first host in key order loses its control lease, so it can never be a
// candidate; an emitted-rows cursor would sit on it forever.
await database.query('UPDATE relay_assignment_activity_leases SET expires_at = ? WHERE user_id = ?', [
now - 1,
'clone-00000'
])
const first = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first.map((candidate) => candidate.userId)).not.toContain('clone-00000')
expect(new Set(first.map((candidate) => candidate.userId))).toEqual(
new Set(['clone-00001', identity.userId])
)
})
it('excludes a host inside its rehome cooldown and takes it back after', async () => {
const { store, database, safety, now } = await setup()
await seedHost(store, identity)
await database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id, source_cell_incarnation,
target_cell_id, target_cell_incarnation, previous_epoch, assignment_epoch, drain_grace_ms,
send_attempts, created_at, updated_at)
VALUES ('cooled', ?, ?, 'asia-east2', 'us', ?, 'asia-idle', ?, 0, 9, 0, 0, ?, ?)`,
[identity.userId, identity.relayHostId, incarnations[0], incarnations[2], now - 1_000, now]
)
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
await database.query('UPDATE relay_region_rehome_attempts SET created_at = ?', [
now - 604_800_000 - 1
])
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
it('excludes a host outside the cohort', async () => {
const { store, database } = await setup()
await seedHost(store, identity)
const now = 100_000_000 + 86_400_000
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
await database.query('UPDATE relay_region_decisions SET cohort_bucket = 40')
const narrow = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 40 })
expect(await narrow.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
const wide = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 41 })
expect(await wide.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(3)
})
})
+17 -12
View File
@@ -10,10 +10,10 @@ import {
reconcileCellAdmissionAtStartup,
roleOwnsAssignmentMaintenance
} from './cell-admission-startup.js'
import { openRelayDatabaseAtBoot } from './boot-database-open.js'
import {
consumeRelayCellInventoryHold,
consumeRelayDatabasePoolPressure,
openRelayDatabase,
readRelayDatabasePoolPressure
} from './database.js'
import { runAssignmentCleanup } from './assignment-cleanup-steps.js'
@@ -28,7 +28,7 @@ import {
} from './registered-migration-inventory.js'
const config = loadRelayConfig()
const database = await openRelayDatabase({
const database = await openRelayDatabaseAtBoot({
databaseUrl: config.databaseUrl,
dataDir: config.dataDir,
poolMax: config.databasePoolMax,
@@ -46,14 +46,19 @@ const {
ready,
cellIncarnation
} = createRelayServer(config, database)
const cleanupTimer = setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
30_000
)
// Same owner as the assignment sweep: the cleanup only expires credentials that every reader
// already re-checks at read time, so running it in all 23 cells multiplied one table scan by 23
// without changing any answer.
const cleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
jitteredSweepIntervalMs(30_000)
)
: null
const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(() => {
void runAssignmentCleanup(assignments)
@@ -82,7 +87,7 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cleanupTimer.unref()
cleanupTimer?.unref()
assignmentCleanupTimer?.unref()
inventorySnapshotTimer?.unref()
migrationInventoryTimer?.unref()
@@ -128,7 +133,7 @@ server.listen(config.port, () => {
})
const shutdown = (): void => {
clearInterval(cleanupTimer)
if (cleanupTimer) clearInterval(cleanupTimer)
if (assignmentCleanupTimer) clearInterval(assignmentCleanupTimer)
if (inventorySnapshotTimer) clearInterval(inventorySnapshotTimer)
if (migrationInventoryTimer) clearInterval(migrationInventoryTimer)
+53 -2
View File
@@ -9,6 +9,48 @@ export type PostgresPoolPressureCounts = {
databasePoolWaitMsMax: number
}
// A pool that cannot hand out a client throws a bare Error with no SQLSTATE, so
// the message is all node-postgres gives us. Both of these come only from
// pg-pool's connect path, so neither can be a statement that already ran.
const POOL_CONNECT_TIMEOUT_MESSAGES = [
// No pooled client came free within connectionTimeoutMillis.
'timeout exceeded when trying to connect',
// A new client's own handshake outran connectionTimeoutMillis.
'Connection terminated due to connection timeout'
]
// pg raises this whenever a socket ends early, during the handshake and mid
// statement alike, so only the acquire boundary can tell the two apart.
const CONNECTION_TERMINATED_MESSAGE = 'Connection terminated unexpectedly'
// Membership is tracked beside the error rather than on it: an error object may
// be frozen, and a mutated one would leak the marker into logs.
const poolAcquireFailures = new WeakSet<object>()
function errorMessage(error: unknown): string {
return String((error as { message?: unknown } | null)?.message)
}
function isPostgresPoolAcquireFailure(error: unknown): boolean {
return typeof error === 'object' && error !== null && poolAcquireFailures.has(error)
}
// connectionTimeoutMillis firing, either waiting in the queue or dialling.
export function isPostgresPoolConnectTimeout(error: unknown): boolean {
const message = errorMessage(error)
return POOL_CONNECT_TIMEOUT_MESSAGES.some((known) => message.includes(known))
}
// Every way the pool can fail to hand out a usable client. An early-ended
// socket counts only at the acquire boundary: retrying a statement whose commit
// outcome is unknown is not safe.
export function isPostgresPoolConnectFailure(error: unknown): boolean {
if (isPostgresPoolConnectTimeout(error)) return true
return (
errorMessage(error).includes(CONNECTION_TERMINATED_MESSAGE) &&
isPostgresPoolAcquireFailure(error)
)
}
const emptyCounts = (): PostgresPoolPressureCounts => ({
databasePoolTotal: 0,
databasePoolIdle: 0,
@@ -32,14 +74,14 @@ export class PostgresPoolPressure {
async connect(): Promise<pg.PoolClient> {
const waitingBefore = this.pool.waitingCount
const connection = this.pool.connect()
if (this.pool.waitingCount <= waitingBefore) return await connection
if (this.pool.waitingCount <= waitingBefore) return await markedAcquire(connection)
const waiter = Symbol()
const startedAt = this.now()
this.waiters.set(waiter, startedAt)
this.waitersMax = Math.max(this.waitersMax, this.waiters.size)
try {
return await connection
return await markedAcquire(connection)
} finally {
this.waitMsMax = Math.max(this.waitMsMax, this.now() - startedAt)
this.waiters.delete(waiter)
@@ -88,6 +130,15 @@ export class PostgresPoolPressure {
}
}
async function markedAcquire(connection: Promise<pg.PoolClient>): Promise<pg.PoolClient> {
try {
return await connection
} catch (error) {
if (typeof error === 'object' && error !== null) poolAcquireFailures.add(error)
throw error
}
}
export function emptyPostgresPoolPressureCounts(): PostgresPoolPressureCounts {
return emptyCounts()
}
@@ -29,7 +29,8 @@ describePostgres('real PostgreSQL query failure phases', () => {
event: 'orca_relay_postgres_query_failed',
phase: 'execute',
code: '57014',
connectionTimeout: false
connectionTimeout: false,
transient: true
})
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
})
@@ -58,6 +59,7 @@ describePostgres('real PostgreSQL query failure phases', () => {
phase: 'acquire',
code: 'unknown',
connectionTimeout: true,
transient: true,
poolTotal: 1,
poolIdle: 0
})
@@ -55,6 +55,7 @@ describe('PostgreSQL query failure diagnostics', () => {
operation: 'control-renewal',
code: 'unknown',
connectionTimeout: true,
transient: true,
elapsedMs: expect.any(Number),
poolTotal: 10,
poolIdle: 0,
@@ -63,9 +64,15 @@ describe('PostgreSQL query failure diagnostics', () => {
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private')
})
it.each(['57014', '55P03', 'ECONNRESET'])(
// ECONNRESET carries no SQLSTATE the routes retry on, and it arrives after the
// statement went out, so it stays a hard failure. The pair pins that boundary.
it.each([
['57014', true],
['55P03', true],
['ECONNRESET', false]
] as const)(
'identifies execute failure %s and releases its client',
async (code) => {
async (code, transient) => {
const error = Object.assign(new Error('private-token'), { code, detail: sql })
fakes.query.mockRejectedValueOnce(error)
await expect(database.query(sql, ['private-token'])).rejects.toBe(error)
@@ -75,19 +82,60 @@ describe('PostgreSQL query failure diagnostics', () => {
phase: 'execute',
operation: 'control-renewal',
code,
connectionTimeout: false
connectionTimeout: false,
transient
})
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private-token')
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain(sql)
}
)
it('marks a dialling timeout that node-postgres reports with no code', async () => {
// pg-pool raises this when a new client's own handshake outruns the limit.
const error = new Error('Connection terminated due to connection timeout')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
code: 'unknown',
connectionTimeout: true,
transient: true
})
})
it('separates an early-ended socket from a timeout while still calling it transient', async () => {
const error = new Error('Connection terminated unexpectedly')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
code: 'unknown',
connectionTimeout: false,
transient: true
})
})
it('reports an acquire failure that is not transient as a hard failure', async () => {
const error = new Error('password authentication failed')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
connectionTimeout: false,
transient: false
})
})
it('does not emit an arbitrary error code, message, query, or parameter', async () => {
const error = { code: 'private-code', message: 'private-message' }
fakes.query.mockRejectedValueOnce(error)
await expect(database.query('SELECT private_column', ['private-param'])).rejects.toBe(error)
const log = vi.mocked(console.warn).mock.calls[0]![0] as string
expect(JSON.parse(log)).toMatchObject({ operation: 'other', code: 'unknown' })
expect(JSON.parse(log)).toMatchObject({
operation: 'other',
code: 'unknown',
transient: false
})
expect(log).not.toContain('private')
})
+15 -6
View File
@@ -1,3 +1,5 @@
import { isPostgresPoolConnectTimeout } from './postgres-pool-pressure.js'
type QueryFailurePhase = 'acquire' | 'execute'
const ERROR_CODES = new Set([
@@ -19,21 +21,27 @@ const ERROR_CODES = new Set([
'EPIPE'
])
// A recognised SQLSTATE or errno, or 'unknown': whatever else a driver attached
// to `code` is not a bounded log category.
export function postgresErrorCodeCategory(error: unknown): string {
const code =
typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined
return typeof code === 'string' && ERROR_CODES.has(code) ? code : 'unknown'
}
export function reportPostgresQueryFailure(input: {
error: unknown
phase: QueryFailurePhase
sql: string
// The routing verdict, supplied by the caller that owns it.
transient: boolean
elapsedMs: number
pool: { totalCount: number; idleCount: number; waitingCount: number }
}): void {
// Emit only bounded categories: error messages and SQL can contain credentials or identities.
try {
const error = input.error as { code?: unknown; message?: unknown } | null
const code =
typeof error?.code === 'string' && ERROR_CODES.has(error.code) ? error.code : 'unknown'
const connectionTimeout =
typeof error?.message === 'string' &&
error.message.includes('timeout exceeded when trying to connect')
const code = postgresErrorCodeCategory(input.error)
const connectionTimeout = isPostgresPoolConnectTimeout(input.error)
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_query_failed',
@@ -43,6 +51,7 @@ export function reportPostgresQueryFailure(input: {
: 'other',
code,
connectionTimeout,
transient: input.transient,
elapsedMs: Math.max(0, Math.round(input.elapsedMs)),
poolTotal: input.pool.totalCount,
poolIdle: input.pool.idleCount,
@@ -697,6 +697,78 @@ async function postPath(
})
}
describe('cell drain endpoint pacing', () => {
function appWithDrain(): {
app: ReturnType<typeof createRelayApp>
drain: ReturnType<typeof vi.fn>
} {
const drain = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
return { app, drain }
}
it('drains everything at once when the caller asks for no pacing', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', { v: 1, graceMs: 0 })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 0 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 0 })
})
it('passes the requested window through and echoes what it accepted', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 120_000 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 120_000 })
})
it('refuses a window that is negative, fractional, or past the cap', async () => {
for (const paceWindowMs of [-1, 1.5, 300_001]) {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs
})
expect(response.status).toBe(400)
expect(drain).not.toHaveBeenCalled()
}
})
it('accepts the cap itself', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 300_000
})
expect(response.status).toBe(200)
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 300_000 })
})
it('still rejects an unauthenticated pacing request', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'wrong-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(401)
expect(drain).not.toHaveBeenCalled()
})
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest'
import {
REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
RegionalRehomePollTelemetry
} from './regional-rehome-poll-telemetry.js'
describe('regional rehome poll telemetry', () => {
it('names the gate that stopped the poll, not just the empty result', () => {
const lines: string[] = []
const telemetry = new RegionalRehomePollTelemetry((line) => lines.push(line))
let now = 1_000
for (let poll = 0; poll < 3; poll++) {
telemetry.record({ now: (now += 6_000), gate: 'budget-closed', candidates: 0 })
}
telemetry.record({ now: (now += 6_000), gate: 'open', candidates: 0, selectionMs: 12 })
expect(lines).toEqual([])
telemetry.record({
now: now + REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
gate: 'open',
candidates: 7,
selectionMs: 30
})
expect(lines).toHaveLength(1)
expect(JSON.parse(lines[0]!)).toMatchObject({
event: 'orca_relay_regional_rehome_poll_summary',
polls: 5,
'budget-closed': 3,
open: 2,
candidates: 7,
selectionMsMax: 30
})
})
it('starts a fresh window after each summary', () => {
const lines: string[] = []
const telemetry = new RegionalRehomePollTelemetry((line) => lines.push(line))
telemetry.record({ now: 0, gate: 'control-closed', candidates: 0 })
telemetry.record({
now: REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS,
gate: 'control-closed',
candidates: 0
})
telemetry.record({
now: REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS * 2,
gate: 'fleet-safety',
candidates: 0
})
expect(lines).toHaveLength(2)
expect(JSON.parse(lines[1]!)).toMatchObject({
polls: 1,
'control-closed': 0,
'fleet-safety': 1,
candidates: 0,
selectionMsMax: 0,
selectionMsP95: 0
})
})
})
@@ -0,0 +1,69 @@
// A gated poll and a poll that simply found nobody to move both produce zero
// candidates and no attempt row, so an operator watching a stalled rollout
// cannot tell them apart. One aggregated line a minute per director names the
// gate and prices the selection, at a rate a 50-polls-a-minute worker can afford.
export type RegionalRehomePollGate =
| 'open'
| 'cohort-zero'
| 'process-safety-unavailable'
| 'control-closed'
| 'budget-closed'
| 'fleet-safety'
export const REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS = 60_000
const EMPTY_GATES: Record<RegionalRehomePollGate, number> = {
open: 0,
'cohort-zero': 0,
'process-safety-unavailable': 0,
'control-closed': 0,
'budget-closed': 0,
'fleet-safety': 0
}
export class RegionalRehomePollTelemetry {
private windowStartedAt: number | null = null
private gates = { ...EMPTY_GATES }
private candidates = 0
private selectionSamplesMs: number[] = []
constructor(private readonly write: (line: string) => void = (line) => console.warn(line)) {}
record(input: {
now: number
gate: RegionalRehomePollGate
candidates: number
selectionMs?: number
}): void {
if (this.windowStartedAt === null) this.windowStartedAt = input.now
this.gates[input.gate] += 1
this.candidates += input.candidates
if (input.selectionMs !== undefined) this.selectionSamplesMs.push(input.selectionMs)
if (input.now - this.windowStartedAt < REGIONAL_REHOME_POLL_SUMMARY_INTERVAL_MS) return
this.write(
JSON.stringify({
event: 'orca_relay_regional_rehome_poll_summary',
windowMs: input.now - this.windowStartedAt,
polls: Object.values(this.gates).reduce((total, count) => total + count, 0),
...this.gates,
candidates: this.candidates,
selectionMsMax: round(Math.max(0, ...this.selectionSamplesMs)),
selectionMsP95: round(percentile(this.selectionSamplesMs, 0.95))
})
)
this.windowStartedAt = input.now
this.gates = { ...EMPTY_GATES }
this.candidates = 0
this.selectionSamplesMs = []
}
}
function percentile(samples: number[], fraction: number): number {
if (!samples.length) return 0
const sorted = [...samples].sort((a, b) => a - b)
return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))]!
}
function round(value: number): number {
return Math.round(value * 100) / 100
}
@@ -1760,7 +1760,7 @@ function hookAfterCandidateScan(
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
query: async (sql, params) => {
const rows = await delegate.query(sql, params)
if (!fired && sql.includes('SELECT a.user_id, a.relay_host_id')) {
if (!fired && sql.includes('SELECT d.user_id, d.relay_host_id')) {
fired = true
await hook(delegate)
}
@@ -1,6 +1,7 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { ControlRenewalFlush } from './control-renewal-batch.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
import type { RelayReadinessGraceEvent, RelayReadinessObservation } from './relay-readiness.js'
@@ -53,6 +54,7 @@ export interface RelayRuntimeObserver {
recordReconnect(): void
recordSql(durationMs: number, success: boolean): void
recordControlRenewal?(durationMs: number, outcome: ControlRenewalOutcome): void
recordControlRenewalFlush?(flush: ControlRenewalFlush): void
recordControlActivityRecovery?(success: boolean): void
recordAssignmentAdmission?(outcome: AssignmentAdmissionOutcome): void
recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void
@@ -119,6 +121,8 @@ type RelayMetricDeltas = {
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlRenewalFlushLatenciesMs: number[]
controlRenewalFlushRowsMax: number
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
@@ -164,6 +168,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlRenewalFlushLatenciesMs: [],
controlRenewalFlushRowsMax: 0,
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
@@ -274,6 +280,14 @@ export class RelayObservability implements RelayRuntimeObserver {
(this.deltas.controlRenewalsByOutcome[outcome] ?? 0) + 1
}
recordControlRenewalFlush(flush: ControlRenewalFlush): void {
this.deltas.controlRenewalFlushLatenciesMs.push(flush.durationMs)
this.deltas.controlRenewalFlushRowsMax = Math.max(
this.deltas.controlRenewalFlushRowsMax,
flush.rows
)
}
recordControlActivityRecovery(success: boolean): void {
if (success) this.deltas.controlActivityRecoveries++
else this.deltas.controlActivityRecoveryFailures++
@@ -379,6 +393,7 @@ export class RelayObservability implements RelayRuntimeObserver {
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const controlRenewalFlush = latencySummary(deltas.controlRenewalFlushLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -447,9 +462,16 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
// Meaning changed when renewals began batching: for a batched row this is
// the flush's duration, not that row's own statement latency. The
// per-flush fields below are the ones to read for statement cost.
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
controlRenewalFlushesDelta: deltas.controlRenewalFlushLatenciesMs.length,
controlRenewalFlushRowsMax: deltas.controlRenewalFlushRowsMax,
controlRenewalFlushLatencyMsP95: controlRenewalFlush.p95,
controlRenewalFlushLatencyMsMax: controlRenewalFlush.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
@@ -265,6 +265,40 @@ describe('Relay region API', () => {
expect(burst.every(({ status }) => status === 200)).toBe(true)
expect(regionCatalog).toHaveBeenCalledOnce()
})
it('answers a pool that cannot hand out a client with a retryable 503', async () => {
const regionCatalog = vi.fn(async () => {
throw new Error('Connection terminated due to connection timeout')
})
const app = createRelayApp(config({ publicAssignmentRetryAfterSeconds: 7 }), {
store: {} as never,
assignments: { regionCatalog } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/regions')
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('7')
expect(await response.json()).toEqual({ error: 'region_catalog_temporarily_unavailable' })
})
it('still fails loudly when the region catalog breaks for a non-transient reason', async () => {
const regionCatalog = vi.fn(async () => {
throw new TypeError('broken invariant')
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: { regionCatalog } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/regions')
expect(response.status).toBe(500)
})
})
function assignmentRequest(relayHostId: string, extra: Record<string, unknown>): RequestInit {
@@ -1,5 +1,5 @@
import pg from 'pg'
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import {
applyPostgresSchema,
catalogObjectPresence,
@@ -190,6 +190,79 @@ describePostgres('relay boot-time schema against PostgreSQL', () => {
}
})
it('defers the activity-lease migrations and still boots while their table is locked', async () => {
// The migration boot, reproduced: the index is there, the table is locked by someone else, and
// all the drop can do is time out. It has to leave the statement for the next boot rather than
// fail, or 28 directors crash-loop through a stall on a table written ~475/s.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(cold)
// Put the database back in its pre-migration shape, which is what makes the drop lock-taking.
await pool.query(
`CREATE INDEX relay_assignment_activity_expiry
ON ${schema}.relay_assignment_activity_leases(expires_at)`
)
await pool.query(`ALTER TABLE ${schema}.relay_assignment_activity_leases RESET (fillfactor)`)
const warned: string[] = []
const warn = vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const holder = new pg.Client({ connectionString: url })
await holder.connect()
await holder.query('BEGIN')
await holder.query(
`LOCK TABLE ${schema}.relay_assignment_activity_leases IN ACCESS EXCLUSIVE MODE`
)
let summary: Awaited<ReturnType<typeof applyPostgresSchema>>
try {
summary = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
} finally {
await holder.query('ROLLBACK')
await holder.end()
warn.mockRestore()
}
// Both statements deferred, and the boot still applied everything else.
expect(summary.deferred).toBe(2)
expect(summary.ran).toBeGreaterThan(0)
const deferred = warned
.map((line) => JSON.parse(line) as { event?: string; name?: string })
.filter((event) => event.event === 'orca_relay_postgres_schema_object_deferred')
expect(deferred.map((event) => event.name)).toEqual([
'relay_assignment_activity_expiry',
'fillfactor=70'
])
// Nothing was applied, so the next boot has the same work to do, not half of it.
const stillThere = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(stillThere.rowCount).toBe(1)
// And the next boot, with the lock gone, finishes the job.
const retry = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
expect(retry.deferred).toBe(0)
const gone = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(gone.rowCount).toBe(0)
const options = await pool.query(
`SELECT reloptions FROM pg_class WHERE oid = to_regclass($1)`,
[`${schema}.relay_assignment_activity_leases`]
)
expect(options.rows[0]?.reloptions).toEqual(['fillfactor=70'])
await pool.end()
})
it('fails that same boot with 55P03 when the pre-check is not wired in', async () => {
// Keeps the test above from passing vacuously: the lock really does block relay's DDL.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest'
import {
requireSchemaLockTarget,
schemaDeferrable,
schemaLockTarget,
sqlWithoutComments,
takesRelationLock,
@@ -16,9 +17,22 @@ import { relayPostgresSchemaStatements } from './database.js'
// CREATE INDEX CONCURRENTLY first, then add it to SCHEMA and update this list.
const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_device', skipWhen: 'present' },
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_current_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_grace_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_connection_bases', name: 'relay_connection_bases_active_deadline', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_assignment_region_preferences',
@@ -72,12 +86,6 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
name: 'relay_cell_drain_attempt_states_cell',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_assignment_activity_leases',
name: 'relay_assignment_activity_expiry',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_control_connection_reservations',
@@ -90,6 +98,7 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
name: 'relay_control_connection_reservation_assignment',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_rate_windows', name: 'relay_rate_windows_started', skipWhen: 'present' },
{ kind: 'index', table: 'relay_assignment_migrations', name: 'relay_assignment_migrations_active', skipWhen: 'present' },
{
kind: 'index',
@@ -116,7 +125,14 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
},
{ kind: 'column', table: 'relay_region_rehome_control', name: 'host_cooldown_ms', skipWhen: 'present' },
{ kind: 'column', table: 'relay_control_capabilities', name: 'idle_regional_rehome', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' }
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' },
{ kind: 'index-by-name', name: 'relay_assignment_activity_expiry', skipWhen: 'absent' },
{
kind: 'reloption',
table: 'relay_assignment_activity_leases',
name: 'fillfactor=70',
skipWhen: 'present'
}
]
const INDEX_OR_ADD_COLUMN = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|ALTER\s+TABLE\s+[^\s]+\s+ADD\s+COLUMN)/i
@@ -149,8 +165,12 @@ describe('relay boot-time lock targets', () => {
for (const statement of relayPostgresSchemaStatements()) {
const target = schemaLockTarget(statement)
if (!target) continue
expect(target.name).toMatch(/^[a-z_][a-z0-9_]*$/)
expect(target.table).toMatch(/^[a-z_][a-z0-9_]*$/)
// A reloption is the one target whose name is a pair rather than an identifier, because
// pg_class stores reloptions as `name=value` text and the value is half the question.
const shape = target.kind === 'reloption' ? /^[a-z_][a-z0-9_]*=[A-Za-z0-9_.]+$/ : /^[a-z_][a-z0-9_]*$/
expect(target.name).toMatch(shape)
// A DROP INDEX names no table, so there is none to check.
if (target.kind !== 'index-by-name') expect(target.table).toMatch(/^[a-z_][a-z0-9_]*$/)
}
})
@@ -194,7 +214,76 @@ describe('relay boot-time lock targets', () => {
it('leaves every statement classifiable once its leading comments are stripped', () => {
for (const statement of relayPostgresSchemaStatements()) {
expect(sqlWithoutComments(statement)).toMatch(/^(?:CREATE|ALTER|DO)\s/i)
expect(sqlWithoutComments(statement)).toMatch(/^(?:CREATE|ALTER|DROP|DO)\s/i)
}
})
it('pre-checks the activity-expiry drop by name, and skips it once the index is gone', () => {
// A DROP INDEX takes ACCESS EXCLUSIVE on the index's table for as long as the index is there,
// so it is in the census like any other lock-taking statement. Its target resolves by name
// alone, because the statement names no table and needs none.
const drops = relayPostgresSchemaStatements().filter((statement) =>
/^DROP\s/i.test(sqlWithoutComments(statement))
)
expect(drops.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
for (const statement of drops) {
expect(takesRelationLock(statement)).toBe(true)
expect(schemaLockTarget(statement)).toEqual({
kind: 'index-by-name',
name: 'relay_assignment_activity_expiry',
skipWhen: 'absent'
})
}
})
it('marks the out-of-band sweep indexes and the activity-lease migrations deferrable, and nothing else', () => {
// The statements a lock timeout must not turn into a crash loop, and the only ones: every
// other statement still fails the boot loudly, which is what keeps the marker meaningful.
const deferrable = relayPostgresSchemaStatements().filter(schemaDeferrable)
expect(deferrable.map((statement) => sqlWithoutComments(statement).replace(/\s+/g, ' '))).toEqual([
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown')",
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation ON relay_invites(reservation_expires_at) WHERE state = 'reserved'",
'CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL',
'CREATE INDEX IF NOT EXISTS relay_rate_windows_started ON relay_rate_windows(window_started_at)',
'DROP INDEX IF EXISTS relay_assignment_activity_expiry',
'ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)'
])
})
it('derives a target for a partial index, WHERE clause and all', () => {
// The pre-check reads the index name and table from the head of the statement, so a trailing
// WHERE is invisible to it. Asserted because the sweep indexes depend on that: a parser that
// gave a partial index no target would send it unchecked on every boot.
const partial = relayPostgresSchemaStatements().filter((statement) =>
/^CREATE\s+INDEX\b[\s\S]*\bWHERE\b/i.test(sqlWithoutComments(statement))
)
expect(partial.map(schemaLockTarget)).toEqual([
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
}
])
})
it('no longer creates an index on the column every control renewal writes', () => {
// The regression this drop exists to prevent: re-adding it would make ~471 renewals/s non-HOT
// again. A CREATE anywhere in the schema naming that index fails here.
const creates = relayPostgresSchemaStatements().filter((statement) =>
/relay_assignment_activity_expiry/i.test(sqlWithoutComments(statement))
)
expect(creates.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
})
})
+1 -1
View File
@@ -136,7 +136,7 @@ export function createRelayServer(
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drain: (graceMs, options) => sessions.drain(graceMs, options ?? {}),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
@@ -52,4 +52,23 @@ describe('sweep schedule jitter', () => {
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
it('jitters the credential cleanup tick', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const cleanup = /'\[orca-relay\] credential cleanup failed'\s*\),\s*([^\n]*?)\n/.exec(source)
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
// A census, not a list of the timers that happen to be gated today: an ungated sweep runs in
// every cell as well as the director, which multiplies one table scan by the fleet size.
it('gates every periodic sweep in index.ts on the maintenance role', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const timers = source.match(/setInterval\(/g) ?? []
const gated =
source.match(/roleOwnsAssignmentMaintenance\(config\.role\)\s*\?\s*setInterval\(/g) ?? []
expect(timers.length).toBeGreaterThan(0)
expect(gated.length).toBe(timers.length)
})
})
@@ -35,6 +35,9 @@ function cellOrigin(cellId) {
// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
// Matches the cell's own cap on /v1/admin/drain.
const MAX_PACE_WINDOW_MS = 5 * 60 * 1_000
export function parseProductionCapacityCellArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
@@ -64,11 +67,22 @@ export function parseProductionCapacityCellArguments(argv) {
) {
throw new Error('production capacity target origin is not exact')
}
const paceWindowMs = values['pace-window-ms'] === undefined
? 0
: Number(values['pace-window-ms'])
if (
!Number.isSafeInteger(paceWindowMs) ||
paceWindowMs < 0 ||
paceWindowMs > MAX_PACE_WINDOW_MS
) {
throw new Error('--pace-window-ms must be an integer between 0 and 300000')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellOrigin: expectedCellOrigin,
cellId,
mode: values.mode
mode: values.mode,
paceWindowMs
}
}
@@ -82,24 +96,39 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const postAt = async (origin, path, body) =>
await responseJson(
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
),
path
const postRaw = async (origin, path, body) =>
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
)
const postAt = async (origin, path, body) =>
await responseJson(await postRaw(origin, path, body), path)
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
if (config.mode === 'drain') {
const paceWindowMs = config.paceWindowMs ?? 0
if (paceWindowMs > 0) {
const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', {
v: 1,
graceMs: 0,
paceWindowMs
})
if (paced.ok) {
await paced.json().catch(() => ({}))
return { changed: false, drained: true, paceWindowMs }
}
// A cell still on an image without paced drain rejects the unknown field outright.
// An unpaced drain is the behaviour that cell already has, so fall back to it.
if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`)
await paced.json().catch(() => ({}))
}
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
return { changed: false, drained: true }
return { changed: false, drained: true, paceWindowMs: 0 }
}
const before = await inspectAdmissionSelector(post)
const state = selectorCellState(before.selector, config.cellId)
@@ -90,7 +90,8 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: 'https://c7.relay.onorca.dev',
cellId: 'production-gce-c7',
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -112,8 +113,12 @@ describe('production Relay capacity cell admission', () => {
]), /not approved/)
})
it('admits the same-cap Asia cells only under the same-cap allowlist', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => {
for (const cellId of [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
// Migration-only canaries: the US-only capacity rollout never touches them either.
'production-gce-c17', 'production-gce-c18'
]) {
const hostname = cellId.slice('production-gce-'.length)
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -125,10 +130,11 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname}.relay.onorca.dev`,
cellId,
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
}
for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
for (const cellId of ['production-gce-c12', 'production-gce-c30']) {
const hostname = cellId.slice('production-gce-'.length)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -168,13 +174,77 @@ describe('production Relay capacity cell admission', () => {
{ ...config, mode: 'drain' },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0 }
}])
})
it('paces the drain send when the roll asks for a window', async () => {
const fake = canaryFetch()
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0, paceWindowMs: 120_000 }
}])
})
it('drains unpaced when the cell image rejects the pacing field', async () => {
const bodies = []
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async (url, init) => {
assert.equal(new URL(url).pathname, '/v1/admin/drain')
const body = JSON.parse(init.body)
bodies.push(body)
if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400)
return response({ v: 1, draining: true })
}
}
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(bodies, [
{ v: 1, graceMs: 0, paceWindowMs: 120_000 },
{ v: 1, graceMs: 0 }
])
})
it('fails a paced drain that the cell rejects for any other reason', async () => {
await assert.rejects(
prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async () => response({ error: 'invalid_token' }, 401)
}
),
/returned 401/
)
})
it('refuses a pacing window that is not a bounded integer', () => {
const argv = (value) => [
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c26.relay.onorca.dev',
'--cell-id', 'production-gce-c26',
'--mode', 'drain',
'--pace-window-ms', value
]
for (const value of ['-1', '300001', '1.5', 'soon']) {
assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/)
}
assert.equal(parseProductionCapacityCellArguments(argv('300000')).paceWindowMs, 300_000)
})
it('restores only the selected cell to general admission', async () => {
const fake = canaryFetch()
await prepareProductionCapacityCell(
@@ -228,7 +298,7 @@ describe('production Relay capacity cell admission', () => {
}
)
assert.equal(calls, 2)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
})
it('fails when both drain attempts return a transient 503', async () => {
@@ -1,12 +1,20 @@
// A refused home cell is not fleet capacity, so it gets its own bucket rather
// than inflating the capacity count a run is read for.
const ASSIGNMENT_REJECTION_BUCKETS = {
relay_capacity_exhausted: 'assignment_capacity_exhausted',
relay_connection_headroom_exhausted: 'assignment_capacity_exhausted',
relay_home_cell_unavailable: 'assignment_home_cell_unavailable'
}
export function relayLoadFailureReason(error) {
const message = error instanceof Error ? error.message : String(error)
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
if (tokenExchange) return `token_http_${tokenExchange[1]}`
const assignment =
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_capacity_exhausted|relay_connection_headroom_exhausted))?$/.exec(
message
)
if (assignment?.[1] === '503' && assignment[2]) return 'assignment_capacity_exhausted'
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_[a-z_]+))?$/.exec(message)
if (assignment?.[1] === '503' && assignment[2]) {
return ASSIGNMENT_REJECTION_BUCKETS[assignment[2]] ?? `assignment_http_${assignment[1]}`
}
if (assignment) return `assignment_http_${assignment[1]}`
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
if (closed) return `control_close_${closed[1]}`
@@ -11,9 +11,10 @@ const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import
requireFromRelay.resolve('@orca-cloud/relay-contract')
)
const CAPACITY_ASSIGNMENT_ERRORS = [
const REPORTABLE_ASSIGNMENT_ERRORS = [
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted'
'relay_connection_headroom_exhausted',
'relay_home_cell_unavailable'
]
function waitForOpen(socket, timeoutMs = 10_000) {
@@ -744,7 +745,7 @@ export class RelayLoadControlPeer {
'relay assignment timeout',
(status, errorCode) =>
`relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`,
CAPACITY_ASSIGNMENT_ERRORS
REPORTABLE_ASSIGNMENT_ERRORS
)
if (
typeof body.cellUrl !== 'string' ||
@@ -2,14 +2,29 @@ import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without
// displacing anybody. It enters and must leave migration-only, never general.
export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18']
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
...SAME_CAP_MIGRATION_ONLY_CELLS
]
// A general cell's wave isolates and restores it, advancing the selector twice; a
// migration-only cell's isolate and restore are both no-ops, so its wave advances nothing.
export function selectorWaveDelta(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 0 : 2
}
export function entryAdmission(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 'migration-only' : 'general'
}
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
@@ -23,9 +38,33 @@ function cells(value) {
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
// Every later cell offsets from one per-wave selector delta, and the two classes
// have different ones, so a mixed wave has no single offset any cell could use.
if (new Set(parsed.map(selectorWaveDelta)).size > 1) {
throw new Error('same-cap wave cells must be all general or all migration-only')
}
return parsed
}
// Break-glass: the aggregate 15-minute monitor gate is skipped, nothing else is.
// Returns null when no override was requested, and throws on a partial or
// mismatched one so a malformed override can never reach a mutation.
export function gateOverrideAuthorization(input, targetDigest, mutation) {
const reason = input.gateOverrideReason ?? ''
const confirmation = input.gateOverrideConfirmation ?? ''
if (!reason && !confirmation) return null
if (!mutation) throw new Error('verify does not accept a monitor gate override')
if (confirmation !== `SKIP_RELAY_MONITOR_GATE ${targetDigest}`) {
throw new Error('gate override confirmation does not match the exact target digest')
}
// Printable single-line only: this reason is rendered into the run summary and
// sealed into the canary artifact.
if (!/^[\x20-\x7e]{12,500}$/.test(reason)) {
throw new Error('gate override reason must be 12 to 500 printable characters on one line')
}
return { reason, confirmation }
}
export function validateSameCapWave(input) {
if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) {
throw new Error('same-cap wave mode is invalid')
@@ -53,13 +92,14 @@ export function validateSameCapWave(input) {
throw new Error('same-cap confirmation does not match the exact digest and cells')
}
if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation')
const gateOverride = gateOverrideAuthorization(input, targetDigest, mutation)
if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) {
throw new Error('batch mode requires a canary run ID')
}
if (input.mode !== 'batch-apply' && input.canaryRunId) {
throw new Error('only batch mode accepts a canary run ID')
}
return { cells: selected, targetDigest, rollbackDigest }
return { cells: selected, targetDigest, rollbackDigest, gateOverride }
}
export function canaryAuthority(input) {
@@ -81,13 +121,21 @@ export function canaryAuthority(input) {
cellId: wave.cells[0],
targetDigest: wave.targetDigest,
rollbackDigest: wave.rollbackDigest,
selectorGeneration: selectorGeneration + 2,
rehomeGeneration
selectorGeneration: selectorGeneration + selectorWaveDelta(wave.cells[0]),
rehomeGeneration,
// Audit trail, not authority: a batch reusing this canary is authorized by
// its own confirmation, so verification below neither requires nor forbids it.
gateOverride: wave.gateOverride === null ? null : {
...wave.gateOverride,
actor: input.actor ?? ''
}
}
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
// A mixed wave is already rejected, so the batch's first cell names the whole batch's class.
const batchAdmission = entryAdmission(cells(expected.cellIds ?? '')[0])
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
@@ -101,6 +149,14 @@ export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// A migration-only cell carries no hosts and a different cap, so rolling it proves nothing
// about a general batch, and its wave advances a different selector delta.
if (entryAdmission(authority.cellId) !== batchAdmission) {
throw new Error(
`canary authority cell ${authority.cellId} is ${entryAdmission(authority.cellId)}, ` +
`but this batch is ${batchAdmission}`
)
}
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
@@ -132,7 +188,9 @@ export function main(argv = process.argv.slice(2)) {
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
canaryRunId: input['canary-run-id']
canaryRunId: input['canary-run-id'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation']
})
process.stdout.write(`${JSON.stringify(wave.cells)}\n`)
return
@@ -147,14 +205,27 @@ export function main(argv = process.argv.slice(2)) {
commitSha: input['commit-sha'],
runId: input['run-id'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation']
rehomeGeneration: input['rehome-generation'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation'],
actor: input.actor
}))}\n`)
return
}
if (command === 'cell-class') {
const cellId = input['cell-id']
if (!SAME_CAP_CELLS.includes(cellId)) throw new Error('same-cap wave cells are invalid')
process.stdout.write(`${JSON.stringify({
entryAdmission: entryAdmission(cellId),
selectorWaveDelta: selectorWaveDelta(cellId)
})}\n`)
return
}
if (command === 'verify-canary') {
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
commitSha: input['commit-sha'],
runId: input['run-id'],
cellIds: input['cell-ids'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
selectorGeneration: input['selector-generation'],
@@ -1,14 +1,19 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { execFileSync, spawnSync } from 'node:child_process'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { test } from 'node:test'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
canaryAuthority,
entryAdmission,
main,
validateSameCapWave,
verifyCanaryAuthority
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
const targetDigest = `sha256:${'a'.repeat(64)}`
const rollbackDigest = `sha256:${'b'.repeat(64)}`
@@ -52,6 +57,82 @@ test('requires one canary or a bounded reviewed batch', () => {
}), /cells/)
})
test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)
assert.equal(entryAdmission(cellId), 'migration-only', cellId)
assert.deepEqual(validateSameCapWave({
mode: 'canary-apply',
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`
}).cells, [cellId])
}
const cellIds = 'production-gce-c17,production-gce-c18'
assert.deepEqual(validateSameCapWave({
mode: 'batch-apply',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
canaryRunId: '42'
}).cells, ['production-gce-c17', 'production-gce-c18'])
// A mixed wave has no single selector delta for its later cells to offset from.
const mixed = 'production-gce-c7,production-gce-c17'
assert.throws(() => validateSameCapWave({
mode: 'batch-apply',
cellIds: mixed,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed}`,
canaryRunId: '42'
}), /all general or all migration-only/)
})
test('seals a migration-only canary at the generation its wave leaves behind', () => {
const seal = (cellId) => canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
// Isolate and restore are both no-ops on a migration-only cell, so nothing advances.
assert.equal(seal('production-gce-c17').selectorGeneration, 11)
assert.equal(seal('production-gce-c7').selectorGeneration, 13)
// That canary still authorizes a later batch of its own class; it is evidence about the image.
assert.equal(verifyCanaryAuthority(seal('production-gce-c17'), {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c17,production-gce-c18',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
rehomeGeneration: '4'
}).cellId, 'production-gce-c17')
})
test('reports each approved cell\'s class and selector delta', () => {
const printed = []
const write = process.stdout.write.bind(process.stdout)
process.stdout.write = (chunk) => printed.push(String(chunk))
try {
main(['cell-class', '--cell-id', 'production-gce-c17'])
main(['cell-class', '--cell-id', 'production-gce-c7'])
} finally {
process.stdout.write = write
}
assert.deepEqual(printed.map((line) => JSON.parse(line)), [
{ entryAdmission: 'migration-only', selectorWaveDelta: 0 },
{ entryAdmission: 'general', selectorWaveDelta: 2 }
])
assert.throws(() => main(['cell-class', '--cell-id', 'production-gce-c12']), /cells are invalid/)
})
test('binds rollback confirmation to the exact digest and ordered cells', () => {
assert.throws(() => validateSameCapWave({
mode: 'rollback',
@@ -94,6 +175,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.equal(verifyCanaryAuthority(authority, {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
@@ -102,6 +184,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.throws(() => verifyCanaryAuthority(authority, {
commitSha: 'd'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
@@ -116,8 +199,8 @@ test('reuses a canary across selector advances only within the same control epoc
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9',
targetDigest, rollbackDigest, selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
@@ -191,6 +274,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
commitSha,
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
@@ -206,3 +290,256 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
await rm(repository.root, { recursive: true, force: true })
}
})
// Why: the break-glass override is the one input that removes a safety check, so
// a partial or mismatched one must fail before the gate job reaches a mutation.
test('accepts only a complete digest-bound monitor gate override', () => {
const reason = 'rolling the measured Cloud SQL stall fix'
const confirmation = `SKIP_RELAY_MONITOR_GATE ${targetDigest}`
const wave = {
mode: 'canary-apply',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
}
assert.deepEqual(
validateSameCapWave({
...wave,
gateOverrideReason: reason,
gateOverrideConfirmation: confirmation
}).gateOverride,
{ reason, confirmation }
)
// An ordinary wave carries no override at all.
assert.equal(validateSameCapWave(wave).gateOverride, null)
assert.equal(
validateSameCapWave({ ...wave, gateOverrideReason: '', gateOverrideConfirmation: '' })
.gateOverride,
null
)
assert.throws(
() => validateSameCapWave({ ...wave, gateOverrideConfirmation: confirmation }),
/gate override reason/
)
assert.throws(
() => validateSameCapWave({ ...wave, gateOverrideReason: reason }),
/gate override confirmation/
)
// Bound to the digest this wave installs, not to any digest.
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: reason,
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${rollbackDigest}`
}),
/gate override confirmation/
)
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: 'too short',
gateOverrideConfirmation: confirmation
}),
/gate override reason/
)
// The reason is rendered into the run summary, so it stays printable and single-line.
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: `${reason}\n| injected | row |`,
gateOverrideConfirmation: confirmation
}),
/gate override reason/
)
assert.throws(
() => validateSameCapWave({
...wave,
mode: 'verify',
confirmation: '',
gateOverrideReason: reason,
gateOverrideConfirmation: confirmation
}),
/verify does not accept a monitor gate override/
)
})
test('a rollback wave may break the glass on its own target digest', () => {
const reason = 'getting off the bad image during an incident'
assert.deepEqual(
validateSameCapWave({
mode: 'rollback',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7`,
gateOverrideReason: reason,
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`
}).gateOverride,
{ reason, confirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}` }
)
})
// Why: the canary authority never carried a monitor run ID, so a batch can reuse
// a canary rolled under an override. Recording it keeps the audit trail in the
// sealed artifact without making it part of what verification demands.
test('seals the override into the canary authority as audit trail only', () => {
const sealed = {
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'f'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
}
const expected = {
commitSha: 'f'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
rehomeGeneration: '4'
}
const overridden = canaryAuthority({
...sealed,
gateOverrideReason: 'rolling the measured Cloud SQL stall fix',
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`,
actor: 'Jinwoo-H'
})
assert.deepEqual(overridden.gateOverride, {
reason: 'rolling the measured Cloud SQL stall fix',
confirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`,
actor: 'Jinwoo-H'
})
assert.equal(canaryAuthority(sealed).gateOverride, null)
// Neither shape changes what a batch verifies.
assert.equal(verifyCanaryAuthority(overridden, expected).cellId, 'production-gce-c7')
assert.equal(
verifyCanaryAuthority(canaryAuthority(sealed), expected).cellId,
'production-gce-c7'
)
})
function sealedCanary(cellId) {
return canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
}
// Why: a migration-only cell holds zero hosts at a different cap and its wave advances no
// selector, so rolling one is no evidence for a general batch, and the reverse is no evidence
// either. Nothing but the sealed cell id says which class a canary actually proved.
test('refuses a canary sealed on a cell of the other admission class', () => {
const expected = {
commitSha: 'c'.repeat(40),
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '99',
rehomeGeneration: '4'
}
const general = 'production-gce-c8,production-gce-c9'
const migrationOnly = SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: general
}),
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: migrationOnly
}),
/canary authority cell production-gce-c7 is general, but this batch is migration-only/
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: general
}).cellId,
'production-gce-c7'
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: migrationOnly
}).cellId,
'production-gce-c17'
)
// A caller that names no batch at all gets no verdict, rather than an unchecked class.
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), expected),
/same-cap wave cells are invalid/
)
})
// The dispatch workflow is the only caller, so the class check only binds anything if that
// step actually hands the batch over; run the step's own shell exactly as written.
function verifyCanaryStepScript() {
const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
const first = ' node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \\\n'
const start = dispatch.indexOf(first)
assert.notEqual(start, -1, 'the dispatch workflow has no verify-canary step')
const last = ' --rehome-generation "${REHOME_GENERATION}"\n'
const end = dispatch.indexOf(last, start)
assert.notEqual(end, -1, 'the verify-canary step does not end at the rehome generation')
return dispatch.slice(start, end + last.length).replace(/^ {10}/gm, '')
}
async function runVerifyCanaryStep(authority, cellIds) {
const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-verify-'))
try {
await mkdir(join(temporary, 'relay-same-cap-canary'), { recursive: true })
await writeFile(
join(temporary, 'relay-same-cap-canary', 'authority.json'),
JSON.stringify(authority)
)
return spawnSync('bash', ['-euo', 'pipefail', '-c', verifyCanaryStepScript()], {
cwd: new URL('../..', import.meta.url),
env: {
...process.env,
RUNNER_TEMP: temporary,
GITHUB_SHA: authority.commitSha,
CANARY_RUN_ID: authority.runId,
CELL_IDS: cellIds,
TARGET_DIGEST: targetDigest,
ROLLBACK_DIGEST: rollbackDigest,
SELECTOR_GENERATION: '99',
REHOME_GENERATION: '4'
},
encoding: 'utf8'
})
} finally {
await rm(temporary, { recursive: true, force: true })
}
}
test('the batch gate hands its own cells to the canary check', async () => {
const accepted = await runVerifyCanaryStep(
sealedCanary('production-gce-c7'),
'production-gce-c8,production-gce-c9'
)
assert.equal(accepted.status, 0, accepted.stderr)
const crossed = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
'production-gce-c8,production-gce-c9'
)
assert.equal(crossed.status, 1, crossed.stdout)
assert.match(
crossed.stderr,
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
const migrationOnly = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
)
assert.equal(migrationOnly.status, 0, migrationOnly.stderr)
})
@@ -67,11 +67,11 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
)
assert.match(
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(
job,
@@ -87,7 +87,7 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_RESTORE\}"/)
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
assert.match(job, /FAILSAFE_GENERATION/)
@@ -98,8 +98,62 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
// Wave 0 must retry freshness-only failures too: one Cloud Monitoring publish
// lag at the sample instant is not health evidence, and single-shot wave 0
// failed a whole batch on a series that was fresh again a minute later.
assert.match(job, /dry-run\.state\.json" \\\n --wave-index "\$\{WAVE_INDEX\}" --retry-freshness/)
assert.match(
job,
/dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
assert.doesNotMatch(job, /RETRY_ARGS/)
// Break-glass: the override skips the aggregate 15-minute monitor evidence and
// nothing else. The live per-wave recheck still runs on the override path, off
// the dispatch inputs the rehome inspect below verifies against the director.
assert.match(
job,
/if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,160}?--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
// The override is re-validated here, not trusted from the caller, and it is
// bound to the digest this wave installs.
assert.match(
job,
/test "\$\{GATE_OVERRIDE_CONFIRMATION\}" = \\\n {14}"SKIP_RELAY_MONITOR_GATE \$\{TARGET_IMAGE_DIGEST\}"/
)
assert.match(job, /\[\[ "\$\{GATE_OVERRIDE_REASON\}" =~ \^\[\[:print:\]\]\{12,500\}\$ \]\]/)
// Exactly the aggregate-evidence steps are skipped, and only them: every step
// that reads or spends the sealed monitor artifact carries the override guard.
const overrideSkipped = [
'Require fresh aggregate monitor evidence reference',
'Download private aggregate monitor evidence',
'Verify monitor evidence provenance',
"Download this wave's single-use safety authority",
'Require safety evidence consumed by this workflow'
]
for (const name of overrideSkipped) {
assert.match(
job,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.equal(
job.match(/inputs\.gate-override-confirmation == ''/g).length,
overrideSkipped.length
)
// The wrapper validates the override before anything runs, passes it to every
// cell, seals it into the canary artifact, and prints it in the run summary.
assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
assert.equal(
wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
4
)
assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
for (const name of [
'Reject previously consumed aggregate safety evidence',
'Consume aggregate safety evidence for this exact wave'
]) {
assert.match(
wrapper,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.match(job, /timeout-minutes: 75/)
// Both age gates step by the cell job timeout above; the constant is
// duplicated across the two languages, so pin each copy to it.
@@ -3,7 +3,12 @@ import { spawnSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { describe, it } from 'node:test'
import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs'
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
entryAdmission,
selectorWaveDelta
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs'
@@ -15,6 +20,7 @@ const production = readFileSync(
)
const REHOME_SOURCE_CELLS = rehomeSourceCells()
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com'
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
@@ -33,16 +39,28 @@ function rehomeSourceCells() {
// The job cross-checks its pinned pool against the committed map; model the same read.
function tfvarsDatabasePoolMax(cellId) {
const start = production.indexOf(`"${cellId}" = {`)
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
const block = production.slice(start, production.indexOf('\n }', start))
return /database_pool_max\s*=\s*(\d+)/.exec(block)?.[1] ?? '10'
return tfvarsCellBlock(cellId).match(/database_pool_max\s*=\s*(\d+)/)?.[1] ?? '10'
}
function startupScript({ cap, image, trusted, pool }) {
function tfvarsHardCap(cellId) {
const cap = /connection_hard_cap\s*=\s*(\d+)/.exec(tfvarsCellBlock(cellId))?.[1]
assert.notEqual(cap, undefined, `${cellId} has no connection_hard_cap`)
return cap
}
function tfvarsCellBlock(cellId) {
const start = production.indexOf(`"${cellId}" = {`)
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
return production.slice(start, production.indexOf('\n }', start))
}
function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) {
return [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacityIdentity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]),
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -59,7 +77,11 @@ function startupScript({ cap, image, trusted, pool }) {
}
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
function rollPlan({ cellId, cap, protocol, pool }) {
function rollPlan({
cellId, cap, protocol, pool,
beforeCapacityIdentity = CAPACITY_IDENTITY,
afterCapacityIdentity = CAPACITY_IDENTITY
}) {
return {
configuration: {
root_module: {
@@ -90,7 +112,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
image: ROLLBACK_IMAGE,
trusted: protocol >= 1,
// The live template predates the reviewed pool raise, as every asia cell's does.
pool: pool === undefined ? undefined : '10'
pool: pool === undefined ? undefined : '10',
capacityIdentity: beforeCapacityIdentity
})
},
after: {
@@ -98,7 +121,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
cap,
image: TARGET_IMAGE,
trusted: protocol >= 1,
pool
pool,
capacityIdentity: afterCapacityIdentity
}),
self_link: null
},
@@ -145,6 +169,95 @@ function cellShape(cellId) {
return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined }
}
// The class block runs before checkout-independent work and decides the whole wave shape.
function resolveCellClass(cellId) {
return spawnSync('bash', [
'-euo',
'pipefail',
'-c',
`${jobBlock(
' CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \\',
' SELECTOR_WAVE_DELTA="$(jq -er \'.selectorWaveDelta\' <<< "${CELL_CLASS}")"'
)}\necho "\${ENTRY_ADMISSION} \${SELECTOR_WAVE_DELTA}"`
], { cwd: new URL('../..', import.meta.url), env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
}
function drainingBlock() {
return `${jobBlock(
' # Rollback is the documented recovery from a failed canary, which',
' PREDECESSOR_DRAINING_OK=false\n fi'
)}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"`
}
// The three fields gcloud would otherwise default, as the MIG resource declares them.
function migUpdatePolicy() {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const policy = terraform.split(' update_policy {')[1]?.split('\n }')[0] ?? ''
const method = /replacement_method\s+= "([A-Z]+)"/.exec(policy)?.[1]
assert.notEqual(method, undefined, 'the MIG declares no replacement method')
// Both fixed bounds come from the topology locals the MIG resource points at.
const surgeLocal = /max_surge_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
const unavailableLocal =
/max_unavailable_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
assert.notEqual(surgeLocal, undefined, 'the MIG pins no surge local')
assert.notEqual(unavailableLocal, undefined, 'the MIG pins no unavailable local')
const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? ''
const local = (name) => {
const value = new RegExp(`${name}\\s+= (\\d+)`).exec(topology)?.[1]
assert.notEqual(value, undefined, `the topology locals pin no ${name}`)
return value
}
return {
replacementMethod: method.toLowerCase(),
maxSurge: local(surgeLocal),
maxUnavailable: local(unavailableLocal)
}
}
// The stage decides the predecessor, the plan's reviewed rollback image, and whether the
// MIG is rolled explicitly, so run the real block rather than restating its rule.
function stageBlock() {
return `${jobBlock(
' # Two different failures leave the cell on the rollback image, and the image',
' PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"\n fi'
)}\necho "\${ROLLBACK_STAGE} \${ROLLBACK_RESUME} \${PREDECESSOR_IMAGE_DIGEST}` +
` \${PREDECESSOR_REHOME_PROTOCOL} \${PLAN_ROLLBACK_IMAGE}"`
}
function generationBlock() {
return `${jobBlock(
' if test "${DEPLOY_MODE}" = verify; then',
' fi'
)}\necho "\${EFFECTIVE_SELECTOR_GENERATION}"`
}
// The job derives both memberships in one block; run that block alone for each class.
function membership(env) {
const script = `${jobBlock(
' RESTORED_MIGRATION_CELLS="$(jq -rn \\',
' fi'
)}\njq -cn --arg a "\${ISOLATED_MIGRATION_CELLS}" --arg b "\${ISOLATED_GENERAL_CELLS}" \\
--arg c "\${RESTORED_MIGRATION_CELLS}" --arg d "\${RESTORED_GENERAL_CELLS}" \\
'{isolatedMigration:$a,isolatedGeneral:$b,restoredMigration:$c,restoredGeneral:$d}'`
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
env: { ...process.env, ...env },
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
return JSON.parse(resolved.stdout)
}
function jobBlock(firstLine, lastLine) {
const start = workflow.indexOf(`${firstLine}\n`)
assert.notEqual(start, -1, `the job has no ${firstLine.trim()}`)
const end = workflow.indexOf(`\n${lastLine}\n`, start)
assert.notEqual(end, -1, `that block has no ${lastLine.trim()}`)
return workflow.slice(start, end + lastLine.length + 1).replace(/^ {10}/gm, '')
}
describe('same-cap roll scripts accept every same-cap cell', () => {
it('parses every wave cell through the same-cap canary allowlist', () => {
for (const cellId of SAME_CAP_CELLS) {
@@ -159,7 +272,8 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
cellId,
mode
mode,
paceWindowMs: 0
})
}
}
@@ -171,12 +285,13 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.match(
resolved.stdout.trim(),
/^(us-central1 1000 pool=|asia-east2 3000 pool=16)$/,
/^(us-central1 1000 pool=|us-central1 600 pool=|asia-east2 3000 pool=16)$/,
cellId
)
assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId)
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
}
assert.equal(resolveCellShape('production-gce-c17').status, 1)
assert.equal(resolveCellShape('production-gce-c12').status, 1)
assert.equal(resolveCellShape('production-gce-c30').status, 1)
})
@@ -188,10 +303,19 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const end = lines.findIndex((line) => !line.endsWith('\\'))
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--approved-cells same-cap/)
assert.match(call, /--mode (isolate|drain|activate)/)
// The restore call picks its mode from the cell's entry admission class.
assert.match(call, /--mode (isolate|drain|activate|"\$\{RESTORE_MODE\}")/)
}
})
it('paces the drain it sends to the selected cell', () => {
const drain = workflow.split('--mode drain')[1] ?? ''
assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
// The transition wait has to outlast the pacing window on top of the leases it waits on.
assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
})
it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
@@ -213,9 +337,15 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
// the rest before it plans, and the next test covers them at protocol 0.
assert.deepEqual(
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
SAME_CAP_MIGRATION_ONLY_CELLS
)
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const { cap, pool } = cellShape(cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -223,6 +353,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: String(protocol),
@@ -261,15 +392,16 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
const plan = rollPlan({ cellId, cap: 600, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
@@ -278,6 +410,304 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
)
})
it('resolves the class and selector delta the wave validator declares', () => {
for (const cellId of SAME_CAP_CELLS) {
const resolved = resolveCellClass(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.equal(
resolved.stdout.trim(),
`${entryAdmission(cellId)} ${selectorWaveDelta(cellId)}`,
cellId
)
}
assert.equal(resolveCellClass('production-gce-c12').status, 1)
})
it('offsets a later wave by this cell class\'s own selector delta', () => {
for (const [waveIndex, delta] of [['0', 2], ['3', 2], ['0', 0], ['3', 0]]) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', generationBlock()], {
env: {
...process.env,
DEPLOY_MODE: 'apply',
EXPECTED_SELECTOR_GENERATION: '40',
WAVE_INDEX: waveIndex,
SELECTOR_WAVE_DELTA: String(delta)
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
assert.equal(resolved.stdout.trim(), String(40 + delta * Number(waveIndex)))
}
})
it('hands a migration-only cell back the exact membership it entered with', () => {
const entry = {
EXPECTED_MIGRATION_ONLY_CELLS: 'production-gce-c17,production-gce-c18',
EXPECTED_GENERAL_CELLS: 'production-gce-c7,production-gce-c8'
}
const isolated = membership({
...entry,
TARGET_CELL_ID: 'production-gce-c17',
ENTRY_ADMISSION: 'migration-only'
})
assert.deepEqual(isolated, {
isolatedMigration: 'production-gce-c17,production-gce-c18',
isolatedGeneral: 'production-gce-c7,production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
})
// A general cell still leaves migration-only and returns to general.
assert.deepEqual(
membership({
...entry,
TARGET_CELL_ID: 'production-gce-c7',
ENTRY_ADMISSION: 'general'
}),
{
isolatedMigration: 'production-gce-c17,production-gce-c18,production-gce-c7',
isolatedGeneral: 'production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
}
)
})
it('never activates a migration-only cell and proves its isolate changed nothing', () => {
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /if test "\$\{ENTRY_ADMISSION\}" = migration-only; then\n\s+RESTORE_MODE=isolate/)
assert.match(restore, /--admission "\$\{ENTRY_ADMISSION\}"/)
// The pre-mutation check must demand the class the cell is declared to serve in.
assert.match(workflow, /PRECHECK_ADMISSION="\$\{ENTRY_ADMISSION\}"/)
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /migration-only; then\n\s+jq -e '\.changed == false'/)
})
it('requires rehome source membership exactly when a roll carries trust lines', () => {
const step = workflow
.split('name: Resolve immutable same-cap cell configuration')[1]
.split('\n - name:')[0]
const guard = step.indexOf('jq -e --arg cell "${TARGET_CELL_ID}" \'index($cell) != null\'')
assert.notEqual(guard, -1)
// The guard reads both protocols, so it has to sit after they are resolved.
assert.ok(step.indexOf('DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"') < guard)
assert.match(
step.slice(0, guard),
/test "\$\{DESIRED_REHOME_PROTOCOL\}" != 0 \|\| test "\$\{CURRENT_REHOME_PROTOCOL\}" != 0\n\s+\}; then\s+$/
)
})
it('rolls a template stale enough to predate the pinned capacity identity', () => {
// Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no
// capacity identity line, so the roll adds one. Run 35290908836 failed closed here.
const cellId = 'production-gce-c17'
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null })
assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 })
// The line may only be gained. A roll may not rewrite it,
assert.throws(
() => validateCapacityPlan(stale, {
...config,
capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com'
}),
/reviewed image and capacity/
)
// nor drop it from a template that already carries one.
assert.throws(
() => validateCapacityPlan(
rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }),
config
),
/reviewed image and capacity/
)
// A same-cap roll cannot run without the identity pinned at all.
assert.throws(
() => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }),
/invalid service account/
)
})
it('pins the capacity identity on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
for (const invocation of invocations) {
const lines = invocation.split('\n')
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
assert.match(
lines.slice(0, end + 1).join(' '),
/--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/
)
}
// Both steps must read it from the same repository variable the job already requires.
assert.equal(
workflow.split(
'CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}'
).length,
4
)
})
it('decides the predecessor draining rule from the real block, for both classes', () => {
// A zero-host cell sheds nothing, and a failed canary's own drain leaves the flag set
// with no restart behind it; run 35292335415 stopped on exactly that residue.
const cases = [
// mode, entry class, resume, expected [precheck admission, precheck draining, jq ok]
['apply', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['apply', 'general', 'false', ['general', 'forbidden', 'false']],
['verify', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['verify', 'general', 'false', ['general', 'forbidden', 'false']],
// Every rollback path keeps exactly the behaviour it had.
['rollback', 'general', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'general', 'true', ['general-or-migration-only', 'either', 'false']],
['rollback', 'migration-only', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'migration-only', 'true', ['general-or-migration-only', 'either', 'false']]
]
for (const [mode, entry, resume, expected] of cases) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', drainingBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
ENTRY_ADMISSION: entry,
ROLLBACK_RESUME: resume
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${entry}/${resume}: ${resolved.stderr}`)
assert.deepEqual(
resolved.stdout.trim().split(' '),
expected,
`${mode}/${entry}/${resume}`
)
}
})
it('reads one draining decision in both predecessor checks', () => {
const step = workflow
.split('name: Verify exact current generation, digest, cap, and rollback point')[1]
.split('\n - name:')[0]
// The jq assertion and its diagnostic must not be able to disagree.
assert.equal(step.split('--argjson drainingOk "${PREDECESSOR_DRAINING_OK}"').length, 3)
assert.doesNotMatch(step, /drainingOk "\$\(test/)
// The fresh VM is still required not to be draining, on every path.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /--admission migration-only --draining forbidden/)
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /--draining forbidden --activity allowed/)
})
it('classifies every rollback stage from the real block', () => {
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
const target = `sha256:${'7'.repeat(64)}`
const rollback = `sha256:${'0'.repeat(64)}`
const stage = (mode, live, draining) => {
// Exactly how the job assigns them: rollback swaps desired and current.
const desired = mode === 'rollback' ? rollback : target
const current = mode === 'rollback' ? target : rollback
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', stageBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
CURRENT_RUNTIME: JSON.stringify({ imageDigest: live, draining }),
DESIRED_IMAGE_DIGEST: desired,
CURRENT_IMAGE_DIGEST: current,
DESIRED_IMAGE: `${repository}@${desired}`,
IMAGE_REPOSITORY: repository,
DESIRED_REHOME_PROTOCOL: '1',
CURRENT_REHOME_PROTOCOL: '0'
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${live}/${draining}: ${resolved.stderr}`)
return resolved.stdout.trim().split(' ')
}
const roll = (current, protocol) =>
['roll', 'false', current, protocol, `${repository}@${current}`]
// Only the last row differs from main: it used to read `resume` and wedge, because the
// resume path refuses a draining cell and never restarts one.
assert.deepEqual(stage('apply', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('apply', rollback, true), roll(rollback, '0'))
assert.deepEqual(stage('apply', target, false), roll(rollback, '0'))
assert.deepEqual(stage('verify', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('rollback', target, false), roll(target, '0'))
assert.deepEqual(stage('rollback', target, true), roll(target, '0'))
assert.deepEqual(
stage('rollback', rollback, false),
['resume', 'true', rollback, '1', `${repository}@${target}`]
)
assert.deepEqual(
stage('rollback', rollback, true),
['stranded', 'false', rollback, '1', `${repository}@${rollback}`]
)
// A runtime that reports no drain flag at all must never read as stranded.
const [missing] = stage('rollback', rollback, null)
assert.equal(missing, 'resume')
})
it('rolls the MIG itself when a stranded plan changes nothing', () => {
const apply = workflow
.split('name: Apply only the selected same-cap template and MIG')[1]
.split('\n - id:')[0]
// The plan is reviewed against the image the cell serves, not an assumed predecessor.
assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/)
assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/)
assert.match(
apply,
/test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0/
)
// gcloud persists all three fields into the MIG's update policy and defaults the
// method to substitute here, so every one has to match what Terraform declares or the
// recovery drifts the policy and the next targeted plan is refused as an unreviewed
// MIG change. Read the declared values rather than restating them.
assert.match(apply, /rolling-action replace "\$\{MIG_NAME\}"/)
const declared = migUpdatePolicy()
assert.deepEqual(declared, {
replacementMethod: 'recreate',
maxSurge: '0',
maxUnavailable: '1'
})
assert.match(
apply,
new RegExp(
`--replacement-method ${declared.replacementMethod}` +
` --max-surge ${declared.maxSurge} --max-unavailable ${declared.maxUnavailable}`
)
)
// Nothing else may reach the group, and the roll has to be waited on.
assert.equal(apply.split('rolling-action').length, 2)
assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3)
})
it('waits on the image a stranded cell actually serves', () => {
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /--expected-image-digests "\$\{PREDECESSOR_IMAGE_DIGEST\}"/)
// A stranded cell has to come back on a new process, which is what clears the drain.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /test "\$\{TARGET_INCARNATION\}" != "\$\{SOURCE_INCARNATION\}"/)
assert.match(after, /if test "\$\{ROLLBACK_RESUME\}" = true; then/)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
@@ -47,7 +47,10 @@ export function parseCapacityPlanArguments(argv) {
return value
}
if (!values.image) throw new Error('missing --image')
if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) {
if (
['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
!values['capacity-service-account']
) {
throw new Error('missing --capacity-service-account')
}
if (
@@ -239,7 +242,10 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
]
]
if (config.mode === 'bootstrap-cell') {
// A same-cap cell whose template predates this line gains it on its next roll, so the
// before/after comparison ignores it; pinning the exact identity here is what reviews it,
// and what stops a roll dropping or rewriting the line it lets through.
if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
expected.push([
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
@@ -454,18 +460,22 @@ function cellPlan(plan, changes, config) {
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
// Only a template stale enough to predate the line may move it, and only by gaining it;
// requireDesiredStartupScript holds the after value to the exact reviewed identity.
const stripCapacityIdentity =
['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
if (
typeof beforeScript !== 'string' ||
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
normalizedStartupScript(
beforeScript,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap,
stripPool
) !== normalizedStartupScript(
script,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap,
stripPool
@@ -501,7 +511,7 @@ export function validateCapacityPlan(plan, config) {
throw new Error('capacity Terraform plans may change only a cell')
}
if (
config.mode === 'bootstrap-cell' &&
['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
!SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
) {
throw new Error('capacity Terraform plan has an invalid service account')
@@ -428,10 +428,14 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false }) => [
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
@@ -468,6 +472,7 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
@@ -653,10 +658,12 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
@@ -693,6 +700,7 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
@@ -737,6 +745,133 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
@@ -747,6 +882,7 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
@@ -788,10 +924,12 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -837,6 +975,7 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
@@ -891,6 +1030,7 @@ test('the database pool argument is accepted by same-cap-cell mode alone', () =>
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
@@ -8,12 +8,26 @@ covered. These events are diagnostic evidence, not a replacement for total SQL
failure counters.
The event contains only an allowlisted error code, a connection-timeout boolean,
the operation category (`control-renewal` or `other`), total elapsed milliseconds,
and pool total/idle/waiting counts at failure. Total elapsed time includes acquisition.
An acquisition timeout can mean either waiting in the queue or establishing a new
connection; use the pool counts and independent server activity to distinguish them.
Unknown error codes stay `unknown`. Query text, parameters, error messages, and
identifiers are never emitted. Successful queries emit no additional event.
a transient boolean, the operation category (`control-renewal` or `other`), total
elapsed milliseconds, and pool total/idle/waiting counts at failure. Total elapsed
time includes acquisition. An acquisition timeout can mean either waiting in the
queue or establishing a new connection; `connectionTimeout` covers both, and the
pool counts separate them: a queue wait has waiters, a dial does not.
Unknown error codes stay `unknown`.
`transient` is the classification the request routes act on, not a second
opinion: true means retryable, false means terminal. It is not a count of HTTP
responses. Every caller of `PostgresDatabase.query` emits this event, including
background sweeps, startup reconciliation, and admin routes that map a failure
to 409, and none of those produces a 503 or a 500. Counting `transient=false`
therefore over-counts user-facing hard failures; narrow by operation, or join
against the route's own rejection logs, before reading it that way. A pool that
cannot hand out a client carries no error code at all, so `code` stays `unknown`
for that whole class and only these two booleans separate it from a genuine
fault such as a rejected password.
Query text, parameters, error messages, and identifiers are never emitted.
Successful queries emit no additional event.
Use structured GCE logs with `jsonPayload.event="orca_relay_postgres_query_failed"`.
Compare counts by phase, operation, and code with the same cell's renewal outcomes
+131 -12
View File
@@ -39,13 +39,69 @@ days. No tokens, request bodies, logs, user IDs, host IDs, or relay device IDs a
Reruns keep one stable incident ID, restore the immediately preceding private
artifact, verify its commit/run/attempt provenance and content hashes, and pass
`--restart`. A missing or mismatched artifact fails closed. A missing, stale,
or collector-failed sample is durably recorded and resets the active continuous
window. The next fresh sample starts a new 15- or 90-minute window under the
same incident lineage.
or collector-failed sample is durably recorded. Up to two consecutive such
samples per source are tolerated and the window keeps running; a third resets
the active continuous window, and the next fresh sample starts a new 15- or
90-minute window under the same incident lineage. A pre-drain dry run must reach
a verdict within 35 minutes of its lineage start.
Exit code `2` means the gate froze or a dry run failed. Missing, stale, malformed, unauthorized, or
unavailable telemetry fails closed.
## Gate override (break-glass)
`Deploy Relay Production Same-Cap` can skip this 15-minute dry-run gate. Supply both
`gate-override-reason` and `gate-override-confirmation`, where the confirmation is exactly
`SKIP_RELAY_MONITOR_GATE <target-image-digest>`. Supplying one without the other, a
confirmation bound to any other digest, or a reason shorter than 12 characters fails the
run before it touches production. `verify` mode rejects the override outright.
### When it is legitimate
The gate proves the fleet is healthy before a wave mutates it. That proof is the wrong
question in exactly two situations.
- **The roll is the fix for the measured condition.** When a chronic fault is the reason
the gate freezes, waiting for a green 15-minute window means waiting for the condition
the wave removes. On 2026-09-17 the gate froze 44 consecutive times on the recurring
Cloud SQL stall the rolling image addresses.
- **An incident where the director is healthy.** Rolling back off a bad image should not
wait 15 minutes for aggregate evidence about a fleet the operator is already watching.
It is not a way to move faster on an ordinary wave. Use it when you can name the signal
the gate is freezing on and say why this wave is the answer to it.
### What it does not skip
Only the aggregate 15-minute dry-run and its sealed evidence are skipped. Every other
control still runs, unchanged:
- The live per-wave preflight, against the same thresholds this document lists. With no
sealed state to read, the expected selector comes from the dispatch inputs instead, and
the migration policy is pinned to `strict`. That membership is canonicalised exactly as
the monitor canonicalises its own, so it must still name every configured cell exactly
once and the order you type it in does not matter. A live threshold breach or selector
mismatch still fails the wave before any mutation.
- Durable regional rehome disabled, and the exact selector generation and membership,
verified against the live director.
- The reviewed Terraform plan, the exact image digest served by Artifact Registry, the
predecessor runtime check, and the new-incarnation check.
- One cell at a time behind the Cloud SQL rollout lease, with the failed-wave failsafe
that leaves a cell isolated.
- Single-dispatch mutation: a re-run still cannot replay a wave.
### The audit trail
Three places record it, and none of them depend on the operator writing anything down:
- The workflow run's inputs, kept by GitHub for the life of the run.
- The gate job's run summary: actor, mode, cells, target digest, reason, and confirmation.
- The sealed canary artifact, under `gateOverride`, for a `canary-apply` wave.
The canary authority a later batch verifies never carried a monitor run ID, so a batch can
reuse a canary that was rolled under an override. The override is recorded in that
artifact as audit trail, not as authority: each wave is authorized by its own confirmation.
## Local use
The active `gcloud` identity must be a service account that can mint an ID token for the exact
@@ -83,8 +139,9 @@ freezes as before.
A production candidate or multi-target mutation must download the exact
dry-run artifact by workflow run ID and attempt. It verifies the artifact
hashes and provenance, requires a green completed 15-minute state no older
than five minutes, then rechecks the live selector and one complete fresh
sample of every safety signal immediately before running the mutation command.
than ten minutes (plus 75 minutes per predecessor same-cap wave), then
rechecks the live selector and one complete fresh sample of every safety
signal immediately before running the mutation command.
The signed state binds `strict` evidence to ordinary mutations and
`recover-forward` evidence to the exact recover-forward source; neither can
authorize the other.
@@ -102,17 +159,17 @@ durably marked consumed before mutation and cannot authorize another run.
| Endpoint latency | over 2,000 ms |
| Cloud SQL CPU | over 80% |
| Cloud SQL memory | over 90% |
| Cloud SQL backends | over 250 (62% of the verified 400-connection ceiling) |
| Cloud SQL backends | over 320 (65% of the 490 usable of `max_connections` 500) |
| Cloud SQL waiting backends | over 20 |
| Cloud SQL deadlocks | over 0 |
| Relay pool waiters | over 800 |
| Relay pool wait | over 2,500 ms |
| PostgreSQL retries in five minutes | over 2,000 |
| Exhausted PostgreSQL retries in five minutes | over 300 |
| Director instances | outside 56 |
| Director instances | outside 56 for more than two consecutive samples |
| Director CPU or memory | over 80% |
| Director concurrency | over 64 |
| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
| Unexpected director 5xx in five minutes (excludes 503) | over 15 |
| Auth 5xx in five minutes | over 0 |
| Connections per cell process | over 500 |
| Queued bytes per cell process | over 48 MiB |
@@ -122,6 +179,23 @@ durably marked consumed before mutation and cannot authorize another run.
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
heartbeats, and matching live admission.
Two readings are the exception to the freeze-on-first-breach rule above.
A cell's endpoint readings are the first. Health, ready and latency are a single HTTP
round trip from one runner, so they must fail more than two consecutive samples before
they freeze the run; the streak is keyed by cell, so one cell's three readings share it.
The director instance count is the second. Cloud Run replaces an instance in place
rather than holding the count, so the reading leaves the band for about one sample
roughly twice a day, and a deploy that briefly serves two revisions raises it the same
way. Neither is an unhealthy fleet. The minimum and maximum share one streak, so a count
that alternates above and below the band still freezes the run.
Absorbed breaches are recorded in the state artifact under `toleratedProbeEvents`. Every
other signal, including the director and auth health probes, freezes on the first bad
sample. The live preflight that runs before each mutating wave re-samples on the same
tolerance.
## Region placement alert policies
Cloud Monitoring alert policies, not monitor freeze bars: these page from
@@ -197,6 +271,49 @@ without its segment is a compile error in relay-contract, not a silent gap.
## Implementation log
- Gave `collector_failed` the same two-consecutive-sample tolerance as an unread
signal and raised the pre-drain lineage cap from 25 to 35 minutes
(2026-09-17). Basis: dry-run 35258662628 sampled a healthy fleet clean for
13 minutes, then a single unreadable Cloud Monitoring sample restarted the
window, and the restarted window ran past the 25-minute cap at 1 500 002 ms,
so a healthy fleet produced no verdict. One failed collector round trip is
evidence about that round trip, not about the fleet, and it cannot freeze the
gate on its own because it carries no threshold breach. `monitor_gap` keeps
zero tolerance: it means the run stopped sampling, so the window has a real
hole. 35 minutes fits a 15-minute window plus one restart: a reset on the
window's last sample restarts at minute 16 and finishes at 31. The job
timeout is already 100 minutes.
- Recalibrated the Cloud SQL backends freeze from 250 to 320, the unexpected
director 5xx freeze from 3 to 15, and gave per-cell endpoint probes a
two-consecutive-sample tolerance (2026-09-17). Basis: the pre-roll dry-run had
frozen 39 times out of 39, every time on a chronic production condition
unrelated to the roll it gates, so it was adding delay rather than safety.
Measured over the 24 h to 2026-09-17 through the Cloud Monitoring API with the
monitor's own aggregation. `cloud_sql.backends` latest-sum per minute: p50 118 /
p90 165 / p95 212 / p99 262 / max 282, so the old bar of 250 sat under the
observed peak and tripped 1.95% of minutes and 21.8% of 15-minute gates; 320
clears every healthy minute with 13% headroom and still fires at 65% of the 490
usable connections, leaving 170 in hand for the runaway that exhaustion actually
is. `director.errors` non-503 5xx per rolling five minutes: p50 0 / p90 3 /
p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze 9.2% of
windows and 29.0% of gates on the chronic `/v1/assign`, `/v1/regions` and
`/v1/resolve` 500 bursts that accompany the recurring Cloud SQL stall; 15 clears
the chronic p99, drops the gate-freeze rate to 1.5%, and deliberately leaves the
exceptional 20-52 bursts detectable. The director serves roughly 50 requests a
minute in 503s alone, so a genuinely broken director lands in the hundreds per
window. For the cell probes, the asia-east2 cells run readiness as `SELECT 1`
against Cloud SQL in us-central1 over a 176 ms round trip behind a 2 s timeout,
so a saturated pool makes the load balancer answer "no healthy upstream" for
about 30 s; 7 of the last 14 freezes were that. It arrives as a real HTTP 503, so
provenance cannot separate it from a cell serving `health=0` and persistence has
to: at the 60 s interval it spans one sample and at worst two. The streak is
keyed by cell rather than by signal, because keyed per signal a cell that
alternates between slow and unanswered holds every streak at one and never
reaches the tolerance. The live preflight before each mutating wave re-samples on
the same tolerance, so a blip cannot fail a wave there either. Thresholds stay
code constants sealed into every checkpoint rather than workflow inputs, so a
green run stays auditable. Re-tighten the backends bar when the auth connection
model lands (#21165).
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
800 waiters / 2,500 ms (2026-08-27). Basis, measured from
`orca_relay_runtime_metrics` (`databasePoolWaitersMax`,
@@ -212,9 +329,11 @@ without its segment is a compile error in relay-contract, not a silent gap.
Basis, measured from `cloudsql.googleapis.com/database/postgresql/num_backends`
latest-sum over 24 healthy hours: mean ~100, 1-minute spikes to 216, with
10 minutes over the old bar of 160 — enough to freeze roughly one in ten
15-minute pre-drain gates on baseline noise. 250 clears measured healthy
peaks and still fires well before the verified 400-connection ceiling;
pool waiters and pool wait latency keep their strict thresholds.
15-minute pre-drain gates on baseline noise. 250 cleared the healthy peaks
measured then and still fired well before the verified 400-connection ceiling;
pool waiters and pool wait latency keep their strict thresholds. Superseded by
the 2026-09-17 entry above, which re-measured a grown baseline against the
490-connection budget.
- Recalibrated the PostgreSQL-retry freeze from 20 to 300 per five minutes
(2026-08-26). Basis, measured from
`jsonPayload.event="orca_relay_postgres_transaction_retry"` in production
@@ -280,4 +399,4 @@ without its segment is a compile error in relay-contract, not a silent gap.
### Director error allowance (2026-09-12)
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 3037% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 3037% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance was set to three non-503 director 5xx here, and was superseded by the 2026-09-17 recalibration to 15 recorded above. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
+85 -1
View File
@@ -374,7 +374,8 @@ remain general at 1,000/60. The workflow lock, single-use evidence marker, exact
targeted Terraform plan, and per-cell heartbeat/admission oracle are unchanged.
`Deploy Relay Production Same-Cap` rolls only the reviewed US 1,000/60 and Asia 3,000/60 serving
sets without changing a cell's connection shape. Use `canary-apply` for exactly one cell. A successful canary
sets and the two migration-only US 600/60 cells, C17 and C18, without changing a cell's connection
shape. Use `canary-apply` for exactly one cell. A successful canary
seals its commit, target and rollback digests, selector generation, and durable rehome generation;
`batch-apply` accepts only that same authority and rolls two to four cells sequentially. Each cell is
isolated, drained to two restart-safe samples, replaced from a targeted saved plan, and restored only
@@ -384,6 +385,89 @@ director; the workflow never receives or mints a director or stamped-cell runtim
keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via
the same workflow's `rollback` mode.
A roll holds the whole startup script identical before and after except the image, so a
template stale enough to predate a pinned line fails closed rather than absorbing the drift.
The one exception is the capacity identity: a cell that predates it gains it on its next roll,
and the plan validator pins the exact reviewed identity instead of comparing that line, so a
roll can never drop or rewrite it. Any other stale line still fails closed and needs a
convergence apply first.
A drained cell is refused before a roll, because draining means something is already
shedding its connections. A migration-only cell has none to shed, so the flag decides nothing
there and is accepted on entry; the replacement VM is still required not to be draining, and
the incarnation check still proves it was replaced. That also unwedges the state a failed
canary leaves behind, where the wave's own drain set the flag and no restart followed.
C17 and C18 hold no hosts and are not general, so rolling one displaces nobody: they are the
zero-displacement canary for a new image. Their wave enters and leaves migration-only, so its
isolate and its restore are both no-ops and the selector generation does not move; a general
cell's wave still advances it by two. One wave may not mix the two classes, because every cell
after the first offsets from a single per-wave delta. Neither cell is a declared regional-rehome
source, so its template carries no rehome trust lines and it may roll only at rehome protocol `0`;
the job refuses a trusted protocol for it before it plans anything.
### Recovering a wave that died after its drain
A cell's drain flag is a one-way latch on the running process. Only a restart clears it, and
the failsafe that isolates a failed cell does not restart anything. So a wave that stopped
any time after its drain step leaves the cell migration-only and draining, and it stays that
way until the cell is rolled.
Read the failed run before dispatching anything. If its log has a
`"event":"relay_production_capacity_canary","mode":"drain"` line for the cell, the cell is
drained. Then read the cell's live runtime image from
`POST https://<hostname>.relay.onorca.dev/v1/admin/runtime-status`.
1. **Do not re-dispatch `apply`.** It requires the cell general and not draining, and a
drained cell is neither. It will fail closed at the predecessor check.
2. **Dispatch `rollback`,** with the same `target-image-digest` and `rollback-image-digest`
the failed wave used, the live selector generation, and the live tri-state membership
with the failed cell listed under migration-only. The confirmation is
`ROLL_BACK_RELAY_SAME_CAP <rollback-digest> <cell-id>`.
3. The job classifies the cell itself and needs no extra input:
- serving the **rollback** image and draining, it is `stranded`. The wave stopped before
or during its template apply. The job re-isolates, re-drains, applies the reviewed
template, and rolls the MIG explicitly if that template was already in place. The cell
comes back on a new instance, so the drain clears, and it is restored to its entry class.
- serving the **target** image, it is `roll`, the ordinary rollback. The template applied
and the instance was replaced.
- serving the **rollback** image and not draining, it is `resume`: a rollback that failed
after its own template apply. Nothing is applied and nothing restarts.
4. Rollback takes exactly one cell per dispatch. Recover the cells one at a time.
5. If the run died inside `wait-until stable`, the MIG is still rolling on its own. Wait for
it to settle and re-read the runtime before dispatching, or the stage will be read off a
state that is about to change.
6. A `stranded` dispatch that fails at plan review means the template already carries the
target image while the old instance is still up. Wait for the MIG to finish replacing it,
then dispatch again; it will classify as `roll`.
A mutating dispatch still needs a fresh aggregate monitor dry-run unless the break-glass
override below is used.
### Gate override (break-glass)
Every mutating same-cap wave normally consumes a fresh 15-minute aggregate monitor dry-run.
`gate-override-reason` plus `gate-override-confirmation`, the latter exactly
`SKIP_RELAY_MONITOR_GATE <target-image-digest>`, skips that aggregate evidence and nothing
else. A partial or mismatched override fails the run before any mutation, and `verify` mode
rejects it outright.
It is legitimate when the roll is the fix for the condition the gate is freezing on, or
during an incident with the director healthy. It is not a way to move faster on an ordinary
wave.
The live per-wave preflight still runs, against the same thresholds, with the expected
selector taken from the dispatch inputs and the migration policy pinned to `strict`. That
membership is canonicalised the same way the monitor canonicalises its own, so it must name
every configured cell exactly once and its order does not matter.
Durable rehome disabled, the exact selector generation and membership, the reviewed
Terraform plan, the predecessor and new-incarnation checks, the rollout lease, the
failed-wave failsafe, and single-dispatch mutation are all unchanged. The actor, reason,
and confirmation are recorded in the gate job's run summary and, for a canary, sealed into
the canary artifact under `gateOverride`; a batch may reuse a canary rolled under an
override, because that authority never carried a monitor run ID. See
[gate override (break-glass)](./relay-incident-monitor.md#gate-override-break-glass).
The first compatible director rollout uses `bootstrap-runtime-identity=true` with
`BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY`. That one-time path requires the exact stamped-cell
predecessor identity, creates both the cold rollback and candidate on the distinct director identity,
@@ -60,6 +60,10 @@ locals {
control_renewal_latency_ms_p50 = { field = "controlRenewalLatencyMsP50", description = "Control renewal latency p50 in the interval." }
control_renewal_latency_ms_p95 = { field = "controlRenewalLatencyMsP95", description = "Control renewal latency p95 in the interval." }
control_renewal_latency_ms_max = { field = "controlRenewalLatencyMsMax", description = "Maximum control renewal latency in the interval." }
control_renewal_flushes = { field = "controlRenewalFlushesDelta", description = "Batched control-renewal statements issued in the interval, one per cell per flush window." }
control_renewal_flush_rows_max = { field = "controlRenewalFlushRowsMax", description = "Largest number of hosts renewed by a single statement in the interval; the row ceiling is what bounds how long one flush holds its row locks." }
control_renewal_flush_ms_p95 = { field = "controlRenewalFlushLatencyMsP95", description = "Batched control-renewal statement duration p95 in the interval. Row locks live until the statement commits, so this is the lock hold." }
control_renewal_flush_ms_max = { field = "controlRenewalFlushLatencyMsMax", description = "Maximum batched control-renewal statement duration in the interval." }
control_renewals = { field = "controlRenewalsDelta", description = "Control renewal attempts in the interval." }
control_renewal_successes = { field = "controlRenewalSuccessesDelta", description = "Successful control renewals in the interval." }
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
@@ -84,7 +84,7 @@ describe('applyPostgresSchema classification', () => {
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(3)
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('treats an already-applied constraint as skipped rather than an error', async () => {
@@ -94,7 +94,110 @@ describe('applyPostgresSchema classification', () => {
throw postgresError('42710')
})
const summary = await applyPostgresSchema(['ALTER TABLE t ADD CONSTRAINT c CHECK (x > 0)'], query)
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('treats an index another director already dropped as skipped rather than an error', async () => {
// Every director boots at once on a deploy and all of them send the same DROP INDEX IF EXISTS.
// Only one can win; the losers must not fail their boot over a drop that already happened.
const query = vi.fn(async () => {
throw postgresError('42704')
})
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(query).toHaveBeenCalledTimes(1)
})
it('still propagates 42704 from a statement that is not a DROP IF EXISTS', async () => {
// Keeps the case above narrow: an undefined object anywhere else is a real boot failure.
const query = vi.fn(async () => {
throw postgresError('42704')
})
await expect(
applyPostgresSchema(['ALTER TABLE t ADD COLUMN IF NOT EXISTS c BIGINT'], query)
).rejects.toThrow(/42704/)
})
it('leaves a deferrable statement unapplied on a lock timeout instead of failing the boot', async () => {
// The crash loop this prevents: 28 directors reach the same DROP INDEX at once on a table
// under continuous write, all of them time out, and every one restarts to re-queue the same
// DDL behind the same writers.
const warned: string[] = []
vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const query = vi.fn(async () => {
throw postgresError('55P03')
})
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'],
query
)
expect(summary).toEqual({ ran: 0, skipped: 0, deferred: 1 })
expect(query).toHaveBeenCalledTimes(1)
const event = JSON.parse(warned[warned.length - 1] ?? '{}')
expect(event.event).toBe('orca_relay_postgres_schema_object_deferred')
expect(event.code).toBe('55P03')
expect(event.name).toBe('i')
})
it('runs the statements after a deferral, rather than abandoning the boot at that point', async () => {
// A deferral is not a failure, so nothing behind it may be skipped: the schema still has
// tables to create, and a boot that stopped here would come up against a partial schema.
const sent: string[] = []
const query = vi.fn(async (statement: string) => {
sent.push(statement)
if (statement.includes('DROP INDEX')) throw postgresError('55P03')
return undefined
})
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i', 'CREATE TABLE IF NOT EXISTS t (id TEXT)'],
query
)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 1 })
expect(sent).toHaveLength(2)
})
it('still fails the boot on a lock timeout for a statement that is not marked deferrable', async () => {
// Keeps the marker meaningful. An unmarked statement retains the old contract: fail once and
// loudly, because retrying parks every writer behind the same queue again.
vi.spyOn(console, 'error').mockImplementation(() => undefined)
const query = vi.fn(async () => {
throw postgresError('55P03')
})
await expect(applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)).rejects.toMatchObject({
code: '55P03'
})
})
it('defers only on a lock timeout, not on any other error from a deferrable statement', async () => {
// A deferrable statement is not a statement whose failures stop mattering. A permission error
// is still a boot failure.
const query = vi.fn(async () => {
throw postgresError('42501')
})
await expect(
applyPostgresSchema(['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'], query)
).rejects.toThrow(/42501/)
})
it('asks the catalog for a dropped index by name and skips the DROP once it is gone', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
// One parameter, the index name: the statement names no table, and the query references no $2.
expect(asked).toEqual([[expect.stringContaining("relkind = 'i'"), 'i']])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('sends the DROP while the index is still there, which is the boot that has to win', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([{}])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
expect(query).toHaveBeenCalledTimes(1)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('propagates an unrelated error without retrying', async () => {
@@ -168,7 +271,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
expect(asked).toEqual([
[expect.stringContaining('pg_catalog.pg_index'), 'relay_connection_bases', 'relay_bases_active']
])
expect(summary).toEqual({ ran: 1, skipped: 1 })
expect(summary).toEqual({ ran: 1, skipped: 1, deferred: 0 })
})
it('skips an index the catalog reports as invalid rather than rebuilding it', async () => {
@@ -206,7 +309,8 @@ describe('applyPostgresSchema catalog pre-check', () => {
expect(JSON.parse(logged[logged.length - 1] ?? '{}')).toEqual({
event: 'orca_push_postgres_schema_applied',
ran: 1,
skipped: 1
skipped: 1,
deferred: 0
})
})
@@ -219,7 +323,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
[expect.stringContaining('pg_catalog.pg_attribute'), 'relay_control_capabilities', 'idle']
])
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('never probes the catalog for a statement that takes no relation lock', async () => {
@@ -247,7 +351,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
]
])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', async () => {
@@ -265,7 +369,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
{ catalogQuery }
)
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(logged).toContainEqual({
event: 'orca_relay_postgres_schema_object_absent',
kind: 'constraint',
@@ -281,7 +385,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
const statement = 'ALTER TABLE t DROP CONSTRAINT IF EXISTS region_check'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends an ADD CONSTRAINT the catalog does not name yet', async () => {
@@ -290,14 +394,14 @@ describe('applyPostgresSchema catalog pre-check', () => {
const statement = 'ALTER TABLE t ADD CONSTRAINT region_valid CHECK (r IN (1))'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends every statement when no catalog query is supplied', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const summary = await applyPostgresSchema([COMMENTED_TABLE, COMMENTED_INDEX], query)
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 2, skipped: 0 })
expect(summary).toEqual({ ran: 2, skipped: 0, deferred: 0 })
})
})
@@ -316,7 +420,7 @@ describe('applyPostgresSchema concurrent creates', () => {
})
expect(query).toHaveBeenCalledTimes(1)
expect(asked).toHaveLength(2)
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('still retries when the catalog says the object is not there after all', async () => {
@@ -332,7 +436,7 @@ describe('applyPostgresSchema concurrent creates', () => {
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('retries a CREATE TABLE collision without a catalog re-ask, having no target to ask about', async () => {
@@ -25,7 +25,7 @@ export type SchemaStartupOptions = {
wait?: (delayMs: number) => Promise<void>
}
export type SchemaApplySummary = { ran: number; skipped: number }
export type SchemaApplySummary = { ran: number; skipped: number; deferred: number }
function retryDelayMs(attempt: number, random: () => number): number {
const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
@@ -39,6 +39,17 @@ function wait(delayMs: number): Promise<void> {
const CREATE_TABLE_IF_NOT_EXISTS = /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
const CREATE_INDEX_IF_NOT_EXISTS = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
const ALTER_TABLE_ADD_CONSTRAINT = /^ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
const DROP_INDEX_IF_EXISTS = /^DROP\s+INDEX\s+(?:CONCURRENTLY\s+)?IF\s+EXISTS\b/i
// Marked in the schema text, beside the SQL it applies to, and read from the raw statement because
// classification strips comments. Says: this boot may leave the statement unapplied rather than
// fail. Only sound for a statement that is idempotent AND that nothing this boot goes on to do
// depends on, because the database is then simply as it was and the next boot re-sends it.
const DEFERRABLE = /^\s*--[^\n]*\bschema-deferrable\b/
export function schemaDeferrable(statement: string): boolean {
return DEFERRABLE.test(statement)
}
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
@@ -70,6 +81,14 @@ function constraintAlreadyApplied(error: unknown, sql: string): boolean {
)
}
// `IF EXISTS` resolves the name, then locks; between those two steps another director's drop can
// commit and the loser raises 42704 instead of the notice it would have got a moment later. Every
// director boots at once on a deploy, so without this the losers fail their boot over a drop that
// already happened.
function dropAlreadyApplied(error: unknown, sql: string): boolean {
return DROP_INDEX_IF_EXISTS.test(sql) && (error as { code?: unknown } | null)?.code === '42704'
}
function retryableSchemaError(error: unknown, sql: string): boolean {
const value = (error as { code?: unknown; constraint?: unknown } | null) ?? {}
return RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, sql)
@@ -90,7 +109,7 @@ async function nothingToDo(
JSON.stringify({
event: `${eventPrefix}_object_${target.skipWhen}`,
kind: target.kind,
table: target.table,
table: target.kind === 'index-by-name' ? undefined : target.table,
name: target.name,
indisvalid: presence.indisvalid
})
@@ -108,7 +127,7 @@ export async function applyPostgresSchema(
const random = options.random ?? Math.random
const pause = options.wait ?? wait
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
const summary: SchemaApplySummary = { ran: 0, skipped: 0 }
const summary: SchemaApplySummary = { ran: 0, skipped: 0, deferred: 0 }
for (const statement of statements) {
// Throws when an index or column statement's target cannot be read, rather than sending it
@@ -126,7 +145,7 @@ export async function applyPostgresSchema(
summary.ran += 1
break
} catch (error) {
if (constraintAlreadyApplied(error, sql)) {
if (constraintAlreadyApplied(error, sql) || dropAlreadyApplied(error, sql)) {
summary.skipped += 1
break
}
@@ -135,6 +154,24 @@ export async function applyPostgresSchema(
// this boot lost the queue. Relation locks are granted in queue order, so each retry parks
// every writer behind it again for another timeout. Fail once, loudly.
if (code === LOCK_NOT_AVAILABLE && !options.retryLockTimeout) {
// A deferrable statement yields the queue instead of crash-looping the instance. Every
// director boots at once on a migration, so a table under continuous write can hand the
// whole fleet a lock timeout on the one statement that has to win once; failing the boot
// for it restarts the instance, which re-queues the same DDL behind the same writers.
if (schemaDeferrable(statement)) {
console.warn(
JSON.stringify({
event: `${eventPrefix}_object_deferred`,
code,
kind: target?.kind,
name: target?.name,
statement: sql.split('\n')[0],
detail: 'could not take its lock; left unapplied for the next boot to retry'
})
)
summary.deferred += 1
break
}
console.error(
JSON.stringify({
event: `${eventPrefix}_lock_timeout`,
@@ -26,10 +26,26 @@ WHERE attrelid = to_regclass($1) AND attname = $2 AND attnum > 0 AND NOT attisdr
const CONSTRAINT_PRESENT = `SELECT 1 FROM pg_catalog.pg_constraint
WHERE conrelid = to_regclass($1) AND conname = $2`
// By name through the search_path, with no table condition, because a DROP INDEX has no table to
// condition on and does not need one: a name that resolves to no visible index is nothing to drop.
// `relkind = 'i'` keeps a same-named table or view from answering for an index. Partitioned indexes
// are 'I', which this deliberately does not match - relay has none, and dropping one is not a
// boot-time operation.
const INDEX_BY_NAME_PRESENT = `SELECT 1 FROM pg_catalog.pg_class c
WHERE c.relname = $1 AND c.relkind = 'i' AND pg_catalog.pg_table_is_visible(c.oid)`
// reloptions is a text[] of `name=value` pairs, absent entirely while the option is at its
// default. Comparing the whole pair is what makes a changed value re-run: `@>` on a different
// value answers no, and the statement runs and overwrites it.
const RELOPTION_PRESENT = `SELECT 1 FROM pg_catalog.pg_class
WHERE oid = to_regclass($1) AND reloptions @> ARRAY[$2]`
const PRESENCE_SQL = {
index: INDEX_PRESENT,
column: COLUMN_PRESENT,
constraint: CONSTRAINT_PRESENT
constraint: CONSTRAINT_PRESENT,
reloption: RELOPTION_PRESENT,
'index-by-name': INDEX_BY_NAME_PRESENT
} as const
export type SchemaCatalogPresence = { present: boolean; indisvalid: unknown }
@@ -42,7 +58,10 @@ export async function catalogObjectPresence(
target: SchemaLockTarget
): Promise<SchemaCatalogPresence> {
const sql = PRESENCE_SQL[target.kind]
const rows = await query(sql, [target.table, target.name])
// The name-only lookup binds one parameter; every other shape binds the table first. Passing a
// parameter the SQL never references is a bind error, not a harmless extra.
const params = target.kind === 'index-by-name' ? [target.name] : [target.table, target.name]
const rows = await query(sql, params)
const row = rows[0]
return row ? { present: true, indisvalid: row.indisvalid } : { present: false, indisvalid: undefined }
}
@@ -1,5 +1,6 @@
export {
applyPostgresSchema,
schemaDeferrable,
type SchemaApplySummary,
type SchemaStartupOptions
} from './apply-postgres-schema.js'
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
import { schemaDeferrable } from './apply-postgres-schema.js'
import {
requireSchemaLockTarget,
schemaLockTarget,
@@ -417,3 +418,105 @@ describe('dollar-quoted bodies', () => {
)
})
})
describe('schemaLockTarget storage parameters', () => {
it('reads a storage parameter as a name=value target the catalog can be asked about', () => {
expect(schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')).toEqual({
kind: 'reloption',
table: 't',
name: 'fillfactor=70',
skipWhen: 'present'
})
})
it('folds the option name but keeps the value as written, the way pg_class stores the pair', () => {
expect(schemaLockTarget('ALTER TABLE t SET (FillFactor=70)')?.name).toBe('fillfactor=70')
})
it('makes a changed value a different target, so it re-runs instead of skipping', () => {
// The failure this prevents: matching on the option name alone would read `fillfactor=100` as
// already satisfying `fillfactor = 70` and skip the statement for the life of the database.
const seventy = schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')
const eighty = schemaLockTarget('ALTER TABLE t SET (fillfactor = 80)')
expect(seventy?.name).not.toBe(eighty?.name)
})
it('refuses a multi-option SET rather than skipping on only the first option', () => {
// Same reason a multi-action ALTER TABLE is refused: skipping on one option would silently
// drop the others for good.
expect(() =>
requireSchemaLockTarget('ALTER TABLE t SET (fillfactor = 70, autovacuum_enabled = false)')
).toThrow(/unparsed_schema_lock_target/)
})
it('fails the boot on a SET whose shape it cannot read, rather than sending it unchecked', () => {
// A storage parameter takes a relation lock, so no target means the lock is taken on every
// boot. RESET has no value to compare and is not supported.
expect(() => requireSchemaLockTarget('ALTER TABLE t RESET (fillfactor)')).not.toThrow()
expect(() => requireSchemaLockTarget('ALTER TABLE t SET (fillfactor)')).toThrow(
/unparsed_schema_lock_target/
)
})
it('takes a relation lock, so the census requires it to carry a target', () => {
expect(takesRelationLock('ALTER TABLE t SET (fillfactor = 70)')).toBe(true)
})
})
describe('schemaLockTarget dropped indexes', () => {
it('resolves a dropped index by name, with no table to name', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS i')).toEqual({
kind: 'index-by-name',
name: 'i',
skipWhen: 'absent'
})
})
it('reads CONCURRENTLY as a modifier rather than the index name', () => {
expect(schemaLockTarget('DROP INDEX CONCURRENTLY IF EXISTS i')?.name).toBe('i')
})
it('folds an unquoted name and keeps a quoted one, the way relname stores it', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS MyIndex')?.name).toBe('myindex')
expect(schemaLockTarget('DROP INDEX IF EXISTS "MyIndex"')?.name).toBe('MyIndex')
})
it('takes a relation lock, because the index is there on the boot that has to drop it', () => {
expect(takesRelationLock('DROP INDEX IF EXISTS i')).toBe(true)
})
it('requires IF EXISTS, so a bare DROP fails the boot instead of running unchecked', () => {
// Same contract as DROP CONSTRAINT: a bare DROP on a missing index is an error the server is
// supposed to raise, and a pre-check that skipped it would swallow that.
expect(() => requireSchemaLockTarget('DROP INDEX i')).toThrow(/unparsed_schema_lock_target/)
})
it('refuses a multi-index DROP rather than pre-checking only the first name', () => {
// Skipping on one name would leave the other index in place for the life of the database.
expect(() => requireSchemaLockTarget('DROP INDEX IF EXISTS a, b')).toThrow(
/unparsed_schema_lock_target/
)
})
it('derives the target through a leading deferrable marker', () => {
// The real shape in relay's schema: the marker is a comment, so classification must see past
// it or the statement would reach the server with no pre-check at all.
const statement = '-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'
expect(sqlWithoutComments(statement)).toBe('DROP INDEX IF EXISTS i')
expect(schemaLockTarget(statement)?.name).toBe('i')
})
})
describe('schemaDeferrable', () => {
it('reads the marker only from a leading comment, never from the SQL body', () => {
// A name or a string containing the word must not make a statement deferrable.
expect(schemaDeferrable('-- schema-deferrable: reason\nDROP INDEX IF EXISTS i')).toBe(true)
expect(schemaDeferrable('DROP INDEX IF EXISTS schema_deferrable')).toBe(false)
expect(schemaDeferrable("CREATE TABLE t (c TEXT DEFAULT 'schema-deferrable')")).toBe(false)
})
it('treats an unmarked statement as fatal on a lock timeout, which is the default', () => {
expect(schemaDeferrable('DROP INDEX IF EXISTS i')).toBe(false)
expect(schemaDeferrable('ALTER TABLE t SET (fillfactor = 70)')).toBe(false)
})
})
@@ -2,14 +2,20 @@
// already exists before the statement joins the lock queue. `table` is kept exactly as the
// statement wrote it, schema qualification and quoting included, because it is fed to
// `to_regclass`; `name` is the bare identifier the catalog stores in `relname`/`attname`.
export type SchemaLockTarget = {
kind: 'index' | 'column' | 'constraint'
table: string
name: string
// The catalog answer that means this statement has nothing left to do. Creating statements skip
// on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is done.
skipWhen: 'present' | 'absent'
}
export type SchemaLockTarget =
| {
kind: 'index' | 'column' | 'constraint' | 'reloption'
table: string
name: string
// The catalog answer that means this statement has nothing left to do. Creating statements
// skip on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is
// done.
skipWhen: 'present' | 'absent'
}
// A `DROP INDEX` names no table, and needs none: an index name that resolves to nothing is
// nothing to drop, whatever table it used to belong to. Resolution is by name through the
// search_path, which is how the DROP itself would resolve it.
| { kind: 'index-by-name'; name: string; skipWhen: 'absent' }
// Keywords that sit in an identifier position when the optional clause before them is absent.
// Without this, `CREATE UNIQUE INDEX CONCURRENTLY ON t(c)` reads CONCURRENTLY as the index name and
@@ -119,10 +125,27 @@ const DROP_CONSTRAINT = new RegExp(
'i'
)
// `IF EXISTS` is required for the same reason it is on DROP CONSTRAINT: a bare `DROP INDEX` on a
// missing index is an error the server is supposed to raise. Without a target the statement throws
// at boot instead, which tells the author to write `IF EXISTS`.
const DROP_INDEX = new RegExp(`^DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?IF\\s+EXISTS\\s+${QUALIFIED}\\s*$`, 'i')
// One option per statement, and a literal value: the catalog stores reloptions as `name=value`
// text, so the pre-check compares the written pair against that array verbatim. A list of options
// is refused by `hasTopLevelComma` before it reaches here, the same as a multi-action ALTER TABLE.
const SET_RELOPTION = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`SET\\s+\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*([A-Za-z0-9_.]+)\\s*\\)\\s*$`,
'i'
)
// Every statement shape that takes a relation lock before Postgres evaluates its existence test.
// `CREATE TABLE IF NOT EXISTS` is absent on purpose: it resolves a name against the schema and
// takes no lock on an existing table.
const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|ALTER\s+TABLE)\b/i
// `DROP INDEX` is here because it takes ACCESS EXCLUSIVE on the index's table whenever the index is
// actually there, which is every boot until the first one wins. That it takes no lock once the
// index is gone is what the pre-check turns into the steady state, not a reason to omit it.
const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|ALTER\s+TABLE)\b/i
export function takesRelationLock(statement: string): boolean {
return TAKES_RELATION_LOCK.test(sqlWithoutComments(statement))
@@ -177,7 +200,9 @@ const MUST_PARSE = [
/^CREATE\s+(?:UNIQUE\s+)?INDEX\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+COLUMN\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bDROP\s+CONSTRAINT\b/i
/^ALTER\s+TABLE\b[\s\S]*\bDROP\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bSET\s+\(/i,
/^DROP\s+INDEX\b/i
]
// Derived from the statement itself so a renamed index cannot drift away from its pre-check.
@@ -209,6 +234,22 @@ export function schemaLockTarget(statement: string): SchemaLockTarget | undefine
skipWhen: 'absent'
}
}
const droppedIndex = DROP_INDEX.exec(sql)
if (droppedIndex?.[1]) {
return { kind: 'index-by-name', name: catalogName(droppedIndex[1]), skipWhen: 'absent' }
}
const option = SET_RELOPTION.exec(sql)
if (option?.[1] && option[2] && option[3]) {
// Option names are always folded, but the value is stored as written, so only the name goes
// through catalogName. `fillfactor=70` and `fillfactor=80` are different targets, which is
// what makes a changed value re-run rather than skip.
return {
kind: 'reloption',
table: option[1],
name: `${catalogName(option[2])}=${option[3]}`,
skipWhen: 'present'
}
}
return undefined
}
+11 -1
View File
@@ -14,6 +14,10 @@ const {
} = require('./packaged-runtime-node-modules.cjs')
const { verifyLinuxGlibcFloor } = require('./scripts/verify-linux-glibc-floor.cjs')
const { writeMacBuildCompatibility } = require('./scripts/mac-build-compatibility.cjs')
const {
MOBILE_WEB_BUNDLE_DIR,
assertMobileWebBundleBuilt
} = require('./scripts/verify-packaged-mobile-web-bundle.cjs')
const { verifyPackagedPluginResources } = require('./scripts/verify-packaged-plugin-resources.cjs')
const {
verifyPackagedWindowsNodePty
@@ -177,6 +181,9 @@ module.exports = {
// Why: these repo-only inputs are either bundled into out/ or copied via
// extraResources. Shipping them in app.asar bloats the desktop bundle.
'!src{,/**/*}',
// Redundant under !src above, kept explicit: the built bundle ships from out/mobile-web via the
// out rules exactly as out/web does, and the source tree must never be mistaken for it.
'!src/mobile-web{,/**/*}',
'!config{,/**/*}',
'!docs{,/**/*}',
'!mobile{,/**/*}',
@@ -289,8 +296,11 @@ module.exports = {
verifyStaticAppImagePackage(file, arch)
}
},
beforePack: (context) => {
// electron-builder calls this with the context alone. The second parameter is the bundle root,
// so a test can point the guard at a scratch bundle instead of needing the repo's out/ built.
beforePack: (context, mobileWebBundleDir = MOBILE_WEB_BUNDLE_DIR) => {
assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch)
assertMobileWebBundleBuilt(mobileWebBundleDir)
},
afterPack: async (context) => {
const resourcesDir =
+1
View File
@@ -16,6 +16,7 @@
"src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts",
"src/main/agent-hooks/managed-agent-hook-controls.ts",
"src/main/claude-accounts/keychain.ts",
"src/mobile-web/src/bootstrap.ts",
"src/renderer/src/main.tsx",
"src/renderer/src/popout.tsx",
"src/renderer/src/web/main.tsx",
+9
View File
@@ -56,6 +56,15 @@
"anti-slop/no-module-mocking": "off"
}
},
// mock-descendant-sweep.ts (daemon and relay) is a test-only side-effect shim: its whole body
// is one vi.mock that keeps mock PTY PIDs away from the host process table, and it exists so
// 60 suites do not each inline the same hoisted factory. It is never imported by product code.
{
"files": ["**/mock-descendant-sweep.ts"],
"rules": {
"anti-slop/no-module-mocking": "off"
}
},
// The exemptions below are file-scoped rather than inline `oxlint-disable` comments
// because the root lint scan does not load this plugin, so an inline directive naming
// an anti-slop rule always reads back as an unused directive there.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,322 @@
diff --git a/src/IIPHandler.ts b/src/IIPHandler.ts
index 559b907416eb38318f439d060d7f89311ed34c7e..8541b4b0ea0d6b451aaae49007d69df64c5bd088 100644
--- a/src/IIPHandler.ts
+++ b/src/IIPHandler.ts
@@ -34,6 +34,7 @@ const DEFAULT_HEADER: IHeaderFields = {
export class IIPHandler implements IOscHandler, IResetHandler {
+ private _generation = 0;
private _aborted = false;
private _hp = new HeaderParser();
private _header: IHeaderFields = DEFAULT_HEADER;
@@ -55,6 +56,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
}
public reset(): void {
+ this._generation++;
this._hp.reset();
this._dec.release();
this._qoiDec.release();
@@ -198,8 +200,13 @@ export class IIPHandler implements IOscHandler, IResetHandler {
blob = new Blob([this._dec.data8], { type: metrics.mime });
}
this._dec.release();
+ const generation = this._generation;
return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
.then(bm => {
+ if (generation !== this._generation) {
+ bm.close();
+ return true;
+ }
this._storage.addImage(bm);
return true;
})
diff --git a/src/ImageAddon.ts b/src/ImageAddon.ts
index 8fd39543118cd420e36c1614c1af370b6c7bbfbb..0c44d2a81642113417bf8dc10a4faa76d7cc5864 100644
--- a/src/ImageAddon.ts
+++ b/src/ImageAddon.ts
@@ -113,6 +113,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
}
public dispose(): void {
+ for (const handler of this._handlers.values()) handler.reset();
for (const obj of this._disposables) {
obj.dispose();
}
diff --git a/src/ImageRenderer.ts b/src/ImageRenderer.ts
index 5854efaec1fdf9dfcb886023542998a563b6d2f2..3afaf9bd63ffd7a4cdf32bf0ac24cf33a8aa814f 100644
--- a/src/ImageRenderer.ts
+++ b/src/ImageRenderer.ts
@@ -186,16 +186,17 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
const dx = col * width;
const dy = row * height;
// safari bug: never access image source out of bounds
- const finalWidth = count * width + sx > img.width ? img.width - sx : count * width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
+ const finalWidth = count * sourceWidth + sx > img.width ? img.width - sx : count * sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
// Floor all pixel offsets to get stable tile mapping without any overflows.
// Note: For not pixel perfect aligned cells like in the DOM renderer
@@ -204,7 +205,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
- Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight)
+ Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight)
);
}
@@ -219,19 +220,20 @@ export class ImageRenderer extends Disposable implements IDisposable {
}
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
- const finalWidth = width + sx > img.width ? img.width - sx : width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
-
- const canvas = ImageRenderer.createCanvas(this.document, finalWidth, finalHeight);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
+ const finalWidth = sourceWidth + sx > img.width ? img.width - sx : sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
+
+ const canvas = ImageRenderer.createCanvas(this.document, Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight));
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight),
- 0, 0, Math.floor(finalWidth), Math.floor(finalHeight)
+ 0, 0, canvas.width, canvas.height
);
return canvas;
}
@@ -299,11 +301,16 @@ export class ImageRenderer extends Disposable implements IDisposable {
spec.actualCellSize.height = originalHeight;
return;
}
- const canvas = ImageRenderer.createCanvas(
- this.document,
- Math.ceil(spec.orig!.width * currentWidth / originalWidth),
- Math.ceil(spec.orig!.height * currentHeight / originalHeight)
- );
+ const scaledWidth = Math.ceil(spec.orig!.width * currentWidth / originalWidth);
+ const scaledHeight = Math.ceil(spec.orig!.height * currentHeight / originalHeight);
+ // Upscale visible tiles directly; a full zoomed copy can dwarf the image budget.
+ if (scaledWidth * scaledHeight > spec.orig!.width * spec.orig!.height) {
+ spec.actual = spec.orig;
+ spec.actualCellSize.width = originalWidth;
+ spec.actualCellSize.height = originalHeight;
+ return;
+ }
+ const canvas = ImageRenderer.createCanvas(this.document, scaledWidth, scaledHeight);
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height);
@@ -415,7 +422,11 @@ export class ImageRenderer extends Disposable implements IDisposable {
for (let i = 0; i < width; i += bWidth) {
ctx2.drawImage(blueprint, i, 0);
}
- ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap);
+ const placeholder = this._placeholder;
+ ImageRenderer.createImageBitmap(placeholder).then(bitmap => {
+ if (this._placeholder !== placeholder) bitmap?.close();
+ else this._placeholderBitmap = bitmap;
+ }).catch(() => {});
}
public get document(): Document | undefined {
diff --git a/src/kitty/KittyGraphicsHandler.ts b/src/kitty/KittyGraphicsHandler.ts
index de889dfff75d9ecc8ab47a025e6989ffe75bb202..54ebea9c061e5bb92b187cab7a53bc1fa320c4f8 100644
--- a/src/kitty/KittyGraphicsHandler.ts
+++ b/src/kitty/KittyGraphicsHandler.ts
@@ -7,6 +7,7 @@ import { IDisposable } from '@xterm/xterm';
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { CELL_SIZE_DEFAULT } from '../ImageStorage';
+import { imageType } from '../IIPMetrics';
import { KittyImageStorage } from './KittyImageStorage';
import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
import {
@@ -37,6 +38,7 @@ const DECODER_OK = Constants.DECODER_OK as unknown as DecodeStatus.OK;
// Kitty graphics protocol handler with streaming base64 decoding.
export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDisposable {
private _aborted = false;
+ private _generation = 0;
private _decodeError = false;
private _activeDecoder: Base64Decoder | null = null;
@@ -80,6 +82,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
public reset(): void {
+ this._generation++;
this._cleanupAllPending();
if (this._activeDecoder) {
this._activeDecoder.release();
@@ -200,6 +203,25 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
this._activeDecoder = pending.decoder;
}
if (!this._activeDecoder) {
+ // Budget WASM capacity, including one page of decoder state and rounding.
+ const decoderCapacity = this._maxEncodedBytes + 131072;
+ if (decoderCapacity > this._opts.storageLimit * 1000000) {
+ this._aborted = true;
+ if (this._parsedCommand?.id !== undefined) {
+ this._sendResponse(this._parsedCommand.id, 'ENOMEM:pending image budget exceeded', this._parsedCommand.quiet ?? 0);
+ }
+ return;
+ }
+ const maxPending = Math.max(1, Math.floor(this._opts.storageLimit * 1000000 / decoderCapacity));
+ while (this._pendingTransmissions.size >= maxPending) {
+ const oldest = this._pendingTransmissions.entries().next().value;
+ if (!oldest) break;
+ oldest[1].decoder.release();
+ this._removePendingEntry(oldest[0]);
+ if (oldest[1].cmd.id !== undefined) {
+ this._sendResponse(oldest[1].cmd.id, 'ENOMEM:pending image budget exceeded', oldest[1].cmd.quiet ?? 0);
+ }
+ }
this._activeDecoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
this._activeDecoder.init();
}
@@ -550,9 +572,11 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
+ const generation = this._generation;
let bitmap: ImageBitmap | undefined = await this._createBitmap(image);
try {
+ if (generation !== this._generation) throw new Error('image decode canceled');
const cropX = Math.max(0, cmd.x ?? 0);
const cropY = Math.max(0, cmd.y ?? 0);
const cropW = cmd.sourceWidth || (bitmap.width - cropX);
@@ -660,6 +684,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
}
+ if (generation !== this._generation) throw new Error('image decode canceled');
const zIndex = cmd.zIndex ?? 0;
this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex);
bitmap = undefined; // ownership transferred to storage
@@ -693,6 +718,12 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
if (image.format === KittyFormat.PNG) {
+ const metrics = imageType(bytes);
+ // IHDR dimensions are parsed with signed shifts, so a value >= 0x80000000 comes
+ // back negative and a bare `>` pixel-limit test passes it; require positive.
+ if (metrics.mime !== 'image/png' || !(metrics.width > 0) || !(metrics.height > 0) || metrics.width * metrics.height > this._opts.pixelLimit) {
+ throw new RangeError('PNG exceeds pixel limit or has invalid dimensions');
+ }
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
@@ -775,27 +806,45 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
- } catch {
+ } catch (error) {
+ if (error instanceof RangeError) throw error;
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
- const ds = new DecompressionStream(format);
- const writer = ds.writable.getWriter();
- writer.write(compressed as BufferSource);
- writer.close();
-
+ const limit = Math.min(this._opts.kittySizeLimit, this._opts.pixelLimit * 4, this._opts.storageLimit * 1000000);
+ let offsetIn = 0;
+ // Bound inflation within one native transform before its output is budgeted.
+ const source = new ReadableStream<BufferSource>({
+ pull(controller) {
+ if (offsetIn >= compressed.length) {
+ controller.close();
+ return;
+ }
+ const end = Math.min(offsetIn + 4096, compressed.length);
+ controller.enqueue(new Uint8Array(compressed.subarray(offsetIn, end)));
+ offsetIn = end;
+ }
+ });
+ const reader = source.pipeThrough(new DecompressionStream(format)).getReader();
const chunks: Uint8Array[] = [];
- const reader = ds.readable.getReader();
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- chunks.push(value);
+ let totalLength = 0;
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ totalLength += value.byteLength;
+ if (totalLength > limit) {
+ await reader.cancel().catch(() => {});
+ throw new RangeError('decompressed image exceeds byte limit');
+ }
+ chunks.push(value);
+ }
+ } finally {
+ reader.releaseLock();
}
- const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
diff --git a/src/kitty/KittyImageStorage.ts b/src/kitty/KittyImageStorage.ts
index 1f5c09ec9e2700f8f6dbd8436a1802217dfc99ef..016943a77c7e8e27da5899d54cd48d82761a87c8 100644
--- a/src/kitty/KittyImageStorage.ts
+++ b/src/kitty/KittyImageStorage.ts
@@ -83,6 +83,25 @@ export class KittyImageStorage implements IDisposable {
this._evictUndisplayedImages();
}
+ // Encoded images awaiting placement are outside ImageStorage's pixel budget.
+ // Unplaced payloads are evicted first so a new upload cannot erase a visible
+ // image while abandoned blobs still hold budget; placed ones go only when
+ // that is not enough, because the byte cap is a hard bound. The new image is
+ // always stored, so an oversized one overshoots by at most one payload
+ // (itself bounded by kittySizeLimit) rather than being dropped after an OK ack.
+ const byteLimit = this._storage.getLimit() * 1000000;
+ this._images.delete(imageId);
+ let retainedBytes = 0;
+ for (const image of this._images.values()) retainedBytes += image.data.size;
+ for (const evictPlaced of [false, true]) {
+ for (const [oldestId, image] of this._images) {
+ if (retainedBytes + imageData.data.size <= byteLimit) break;
+ if (this._kittyIdToStorageId.has(oldestId) !== evictPlaced) continue;
+ retainedBytes -= image.data.size;
+ this.deleteById(oldestId);
+ }
+ }
+
this._images.set(imageId, {
...imageData,
id: imageId
@@ -1,8 +1,8 @@
diff --git a/src/common/buffer/BufferLine.ts b/src/common/buffer/BufferLine.ts
index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e970b6a2ec6 100644
index 4984f3a49abe8437475c206db1345f5f5fe3979c..c6e91a357f25b4d671ee48242f8eed28721fceaf 100644
--- a/src/common/buffer/BufferLine.ts
+++ b/src/common/buffer/BufferLine.ts
@@ -116,12 +116,16 @@ export class BufferLine implements IBufferLine {
@@ -116,12 +116,18 @@ export class BufferLine implements IBufferLine {
* @deprecated
*/
public set(index: number, value: CharData): void {
@@ -21,7 +21,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._data[index * Constants.CELL_INDICIES + Cell.CONTENT] = value[CHAR_DATA_CHAR_INDEX].charCodeAt(0) | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
}
}
@@ -226,12 +230,21 @@ export class BufferLine implements IBufferLine {
@@ -226,12 +232,23 @@ export class BufferLine implements IBufferLine {
* Set data at `index` to `cell`.
*/
public setCell(index: number, cell: ICellData): void {
@@ -45,7 +45,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
}
this._data[index * Constants.CELL_INDICIES + Cell.CONTENT] = cell.content;
this._data[index * Constants.CELL_INDICIES + Cell.FG] = cell.fg;
@@ -244,9 +257,17 @@ export class BufferLine implements IBufferLine {
@@ -244,9 +261,19 @@ export class BufferLine implements IBufferLine {
* it gets an optimized access method.
*/
public setCellFromCodepoint(index: number, codePoint: number, width: number, attrs: IAttributeData): void {
@@ -65,7 +65,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
}
const $idx = index * Constants.CELL_INDICIES;
this._data[$idx + Cell.CONTENT] = codePoint | (width << Content.WIDTH_SHIFT);
@@ -261,6 +282,7 @@ export class BufferLine implements IBufferLine {
@@ -261,6 +288,9 @@ export class BufferLine implements IBufferLine {
* by the previous `setDataFromCodePoint` call, we can omit it here.
*/
public addCodepointToCell(index: number, codePoint: number, width: number): void {
@@ -75,7 +75,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
let content = this._data[index * Constants.CELL_INDICIES + Cell.CONTENT];
if (content & Content.IS_COMBINED_MASK) {
@@ -288,6 +310,7 @@ export class BufferLine implements IBufferLine {
@@ -288,6 +318,9 @@ export class BufferLine implements IBufferLine {
}
public insertCells(pos: number, n: number, fillCellData: ICellData): void {
@@ -85,7 +85,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
pos %= this.length;
@@ -309,6 +332,8 @@ export class BufferLine implements IBufferLine {
@@ -309,6 +342,8 @@ export class BufferLine implements IBufferLine {
}
}
@@ -94,7 +94,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
// handle fullwidth at line end: reset last cell if it is first cell of a wide char
if (this.getWidth(this.length - 1) === 2) {
this.setCellFromCodepoint(this.length - 1, 0, 1, fillCellData);
@@ -316,6 +341,7 @@ export class BufferLine implements IBufferLine {
@@ -316,6 +351,9 @@ export class BufferLine implements IBufferLine {
}
public deleteCells(pos: number, n: number, fillCellData: ICellData): void {
@@ -104,7 +104,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
pos %= this.length;
if (n < this.length - pos) {
@@ -331,6 +357,8 @@ export class BufferLine implements IBufferLine {
@@ -331,6 +369,8 @@ export class BufferLine implements IBufferLine {
}
}
@@ -113,7 +113,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
// handle fullwidth at pos:
// - reset pos-1 if wide char
// - reset pos if width==0 (previous second cell of a wide char)
@@ -343,6 +371,7 @@ export class BufferLine implements IBufferLine {
@@ -343,6 +383,9 @@ export class BufferLine implements IBufferLine {
}
public replaceCells(start: number, end: number, fillCellData: ICellData, respectProtect: boolean = false): void {
@@ -123,7 +123,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
// full branching on respectProtect==true, hopefully getting fast JIT for standard case
if (respectProtect) {
@@ -383,6 +412,7 @@ export class BufferLine implements IBufferLine {
@@ -383,6 +426,9 @@ export class BufferLine implements IBufferLine {
* excess memory (true after shrinking > Constants.CLEANUP_THRESHOLD).
*/
public resize(cols: number, fillCellData: ICellData): boolean {
@@ -133,7 +133,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
if (cols === this.length) {
return this._data.length * 4 * Constants.CLEANUP_THRESHOLD < this._data.buffer.byteLength;
@@ -443,6 +473,7 @@ export class BufferLine implements IBufferLine {
@@ -443,6 +489,9 @@ export class BufferLine implements IBufferLine {
/** fill a line with fillCharData */
public fill(fillCellData: ICellData, respectProtect: boolean = false): void {
@@ -143,7 +143,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
// full branching on respectProtect==true, hopefully getting fast JIT for standard case
if (respectProtect) {
@@ -515,6 +546,7 @@ export class BufferLine implements IBufferLine {
@@ -515,6 +564,9 @@ export class BufferLine implements IBufferLine {
}
public copyCellsFrom(src: BufferLine, srcCol: number, destCol: number, length: number, applyInReverse: boolean): void {
@@ -153,7 +153,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
const srcData = src._data;
if (applyInReverse) {
@@ -596,9 +628,17 @@ export class BufferLine implements IBufferLine {
@@ -596,9 +648,17 @@ export class BufferLine implements IBufferLine {
const srcStart = srcCol * Constants.CELL_INDICIES;
if (src._data[srcStart + Cell.CONTENT] & Content.IS_COMBINED_MASK) {
this._combined[destCol] = src._combined[srcCol];
@@ -1273,10 +1273,10 @@ index 8a10076e3963e33b4a7d1e4602333eb3f4772dc9..c6dcf18b762e3c56fe22e9c2d49b8e55
}
}
diff --git a/src/common/buffer/BufferLine.ts b/src/common/buffer/BufferLine.ts
index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e970b6a2ec6 100644
index 4984f3a49abe8437475c206db1345f5f5fe3979c..c6e91a357f25b4d671ee48242f8eed28721fceaf 100644
--- a/src/common/buffer/BufferLine.ts
+++ b/src/common/buffer/BufferLine.ts
@@ -116,12 +116,16 @@ export class BufferLine implements IBufferLine {
@@ -116,12 +116,18 @@ export class BufferLine implements IBufferLine {
* @deprecated
*/
public set(index: number, value: CharData): void {
@@ -1295,7 +1295,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._data[index * Constants.CELL_INDICIES + Cell.CONTENT] = value[CHAR_DATA_CHAR_INDEX].charCodeAt(0) | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
}
}
@@ -226,12 +230,21 @@ export class BufferLine implements IBufferLine {
@@ -226,12 +232,23 @@ export class BufferLine implements IBufferLine {
* Set data at `index` to `cell`.
*/
public setCell(index: number, cell: ICellData): void {
@@ -1319,7 +1319,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
}
this._data[index * Constants.CELL_INDICIES + Cell.CONTENT] = cell.content;
this._data[index * Constants.CELL_INDICIES + Cell.FG] = cell.fg;
@@ -244,9 +257,17 @@ export class BufferLine implements IBufferLine {
@@ -244,9 +261,19 @@ export class BufferLine implements IBufferLine {
* it gets an optimized access method.
*/
public setCellFromCodepoint(index: number, codePoint: number, width: number, attrs: IAttributeData): void {
@@ -1339,7 +1339,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
}
const $idx = index * Constants.CELL_INDICIES;
this._data[$idx + Cell.CONTENT] = codePoint | (width << Content.WIDTH_SHIFT);
@@ -261,6 +282,7 @@ export class BufferLine implements IBufferLine {
@@ -261,6 +288,9 @@ export class BufferLine implements IBufferLine {
* by the previous `setDataFromCodePoint` call, we can omit it here.
*/
public addCodepointToCell(index: number, codePoint: number, width: number): void {
@@ -1349,7 +1349,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
let content = this._data[index * Constants.CELL_INDICIES + Cell.CONTENT];
if (content & Content.IS_COMBINED_MASK) {
@@ -288,6 +310,7 @@ export class BufferLine implements IBufferLine {
@@ -288,6 +318,9 @@ export class BufferLine implements IBufferLine {
}
public insertCells(pos: number, n: number, fillCellData: ICellData): void {
@@ -1359,7 +1359,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
pos %= this.length;
@@ -309,6 +332,8 @@ export class BufferLine implements IBufferLine {
@@ -309,6 +342,8 @@ export class BufferLine implements IBufferLine {
}
}
@@ -1368,7 +1368,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
// handle fullwidth at line end: reset last cell if it is first cell of a wide char
if (this.getWidth(this.length - 1) === 2) {
this.setCellFromCodepoint(this.length - 1, 0, 1, fillCellData);
@@ -316,6 +341,7 @@ export class BufferLine implements IBufferLine {
@@ -316,6 +351,9 @@ export class BufferLine implements IBufferLine {
}
public deleteCells(pos: number, n: number, fillCellData: ICellData): void {
@@ -1378,7 +1378,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
pos %= this.length;
if (n < this.length - pos) {
@@ -331,6 +357,8 @@ export class BufferLine implements IBufferLine {
@@ -331,6 +369,8 @@ export class BufferLine implements IBufferLine {
}
}
@@ -1387,7 +1387,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
// handle fullwidth at pos:
// - reset pos-1 if wide char
// - reset pos if width==0 (previous second cell of a wide char)
@@ -343,6 +371,7 @@ export class BufferLine implements IBufferLine {
@@ -343,6 +383,9 @@ export class BufferLine implements IBufferLine {
}
public replaceCells(start: number, end: number, fillCellData: ICellData, respectProtect: boolean = false): void {
@@ -1397,7 +1397,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
// full branching on respectProtect==true, hopefully getting fast JIT for standard case
if (respectProtect) {
@@ -383,6 +412,7 @@ export class BufferLine implements IBufferLine {
@@ -383,6 +426,9 @@ export class BufferLine implements IBufferLine {
* excess memory (true after shrinking > Constants.CLEANUP_THRESHOLD).
*/
public resize(cols: number, fillCellData: ICellData): boolean {
@@ -1407,7 +1407,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
if (cols === this.length) {
return this._data.length * 4 * Constants.CLEANUP_THRESHOLD < this._data.buffer.byteLength;
@@ -443,6 +473,7 @@ export class BufferLine implements IBufferLine {
@@ -443,6 +489,9 @@ export class BufferLine implements IBufferLine {
/** fill a line with fillCharData */
public fill(fillCellData: ICellData, respectProtect: boolean = false): void {
@@ -1417,7 +1417,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
// full branching on respectProtect==true, hopefully getting fast JIT for standard case
if (respectProtect) {
@@ -515,6 +546,7 @@ export class BufferLine implements IBufferLine {
@@ -515,6 +564,9 @@ export class BufferLine implements IBufferLine {
}
public copyCellsFrom(src: BufferLine, srcCol: number, destCol: number, length: number, applyInReverse: boolean): void {
@@ -1427,7 +1427,7 @@ index 4984f3a49abe8437475c206db1345f5f5fe3979c..43fd5c5b286f23f11d407c9098062e97
this._cacheValid = false;
const srcData = src._data;
if (applyInReverse) {
@@ -596,9 +628,17 @@ export class BufferLine implements IBufferLine {
@@ -596,9 +648,17 @@ export class BufferLine implements IBufferLine {
const srcStart = srcCol * Constants.CELL_INDICIES;
if (src._data[srcStart + Cell.CONTENT] & Content.IS_COMBINED_MASK) {
this._combined[destCol] = src._combined[srcCol];
+25
View File
@@ -129,6 +129,31 @@
"args": ["run", "esbuild-package"]
}
]
},
{
"name": "@xterm/addon-image",
"version": "0.10.0-beta.300",
"packageDir": "addons/addon-image",
"sourcePatch": "config/patches/xterm-src/@xterm__addon-image@0.10.0-beta.300.src.patch",
"patch": "config/patches/@xterm__addon-image@0.10.0-beta.300.patch",
"generatedPaths": ["lib/"],
"build": [
{
"cwd": "../..",
"command": "npm",
"args": ["run", "build"]
},
{
"cwd": ".",
"command": "npm",
"args": ["run", "package"]
},
{
"cwd": "../..",
"command": "npm",
"args": ["run", "esbuild-package"]
}
]
}
],
"forbiddenBuildScripts": {
@@ -0,0 +1,453 @@
import { readFile } from 'node:fs/promises'
import { realpathSync } from 'node:fs'
import { basename, extname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import * as esbuild from 'esbuild'
import {
MOBILE_WEB_BUNDLE_ENTRYPOINT,
hashedAsset,
isDirectInvocation,
readDesktopVersion,
readProtocolWindow,
sha256Hex,
writeMobileWebBundleTree,
contentTypeForExtension
} from './build-mobile-web-bundle.mjs'
import {
ROUTE_SOURCE_LOADERS,
assertRoutesCarryNoSynchronousExports,
collectMobileWebAppRoutes,
renderMobileWebAppRouteManifest,
routePathnameFromKey
} from './mobile-web-app-route-manifest.mjs'
import { MOBILE_WEB_PAGE_ROUTES } from './mobile-web-page-routes.mjs'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const mobileDir = join(projectDir, 'mobile')
const defaultAppDir = join(mobileDir, 'app')
const entryPoint = join(mobileDir, 'web-entry', 'index.tsx')
const defaultOutDir = join(projectDir, 'out', 'mobile-web-app')
/**
* Every shim the app bundle needs, each one a documented Metro/RN-Web gap. `appliesTo` reads the
* esbuild option that implements the shim, so the list cannot claim a shim the build does not
* apply and a dropped option fails the named shim rather than the whole build.
*/
export const MOBILE_WEB_APP_SHIMS = [
{
// react-native has no browser build; react-native-web is the whole point of Route A.
name: 'react-native-web-alias',
appliesTo: (options) => options.alias?.['react-native'] === 'react-native-web'
},
{
// RN ships untranspiled JSX inside .js files (expo-router's own build/ included).
name: 'js-as-jsx',
appliesTo: (options) => options.loader?.['.js'] === 'jsx'
},
{
// RN code assumes a Hermes/Metro `global`; the browser only has `globalThis`.
name: 'global-as-globalthis',
appliesTo: (options) => options.define?.global === 'globalThis'
},
{
// RN and Expo modules read process.env at module scope, before any of our code runs.
name: 'process-banner',
appliesTo: (options) => options.banner?.js?.includes('globalThis.process ??=') === true
},
{
// lucide-react-native@1.14.0's barrel re-exports LucideProvider from a context.mjs that does
// not export it. Metro's loose CJS interop tolerates it; esbuild's strict ESM does not.
// Web-build only: patching the package would change what the shipped native app consumes.
name: 'lucide-barrel-provider',
appliesTo: (options) =>
options.plugins?.some((plugin) => plugin.name === LUCIDE_PLUGIN_NAME) === true
},
{
// AsyncStorage's web build is window.localStorage, which the shell's page does not have:
// Android turns DOM storage off and on iOS the origin host is the session id, so anything
// written there is gone on the next remount. The page module holds the app's own values,
// primed by `init` and written back over the `storage` grant.
name: 'async-storage-over-the-bridge',
appliesTo: (options) =>
options.alias?.['@react-native-async-storage/async-storage'] === PAGE_ASYNC_STORAGE_MODULE
},
{
// esbuild has no require.context, so the route tree is generated and injected.
name: 'route-manifest',
appliesTo: (options) =>
options.plugins?.some((plugin) => plugin.name === ROUTE_MANIFEST_PLUGIN_NAME) === true
}
]
const PAGE_ASYNC_STORAGE_MODULE = join(
mobileDir,
'src',
'mobile-web-shell',
'bridge',
'page-async-storage.ts'
)
const ROUTE_MANIFEST_PLUGIN_NAME = 'orca-route-manifest'
const LUCIDE_PLUGIN_NAME = 'orca-lucide-barrel-provider'
/** The entry output's name, so classifying the outputs never has to guess which one it is. */
const ENTRY_CHUNK_NAME = 'entry'
// mobile/web-entry/route-manifest.ts is a real typed file rather than a virtual specifier, so the
// entry typechecks and Metro can still resolve it; only its body is replaced here.
function routeManifestPlugin(manifestSource) {
return {
name: ROUTE_MANIFEST_PLUGIN_NAME,
setup(build) {
build.onLoad({ filter: /web-entry[\\/]route-manifest\.ts$/ }, () => ({
contents: manifestSource,
loader: 'js',
resolveDir: mobileDir
}))
}
}
}
const lucideBarrelPlugin = {
name: LUCIDE_PLUGIN_NAME,
setup(build) {
build.onLoad({ filter: /lucide-react-native[\\/].*[\\/]context\.mjs$/ }, async (args) => ({
contents: `${await readFile(args.path, 'utf8')}\nexport const LucideProvider = ({ children }) => children;\n`,
loader: 'js'
}))
}
}
/** Split out so a test can read the options MOBILE_WEB_APP_SHIMS claims, without a build. */
export function mobileWebAppBuildOptions(routes) {
return {
// Fixed so no absolute path of this checkout can reach the output.
absWorkingDir: mobileDir,
entryPoints: [entryPoint],
bundle: true,
minify: true,
// Virtual: write is false, so outdir only names the emitted files esbuild hands back.
outdir: 'dist',
write: false,
// esm, because `splitting` requires it and a per-route chunk is the point: with iife and
// static imports esbuild emitted one 8.16 MB script for all 14 routes.
format: 'esm',
splitting: true,
// esbuild's `[hash]` is over the metafile's input keys, which are paths relative to
// absWorkingDir, so this name is not a function of the bytes and differs between two
// checkouts of one commit. It is a placeholder: renameOutputsByContent replaces it below.
chunkNames: '[hash]',
// Pinned rather than defaulted, so the entry is found by name and not by elimination.
entryNames: ENTRY_CHUNK_NAME,
target: ['es2022'],
charset: 'utf8',
legalComments: 'none',
// No sourcemap: it is an emitted file and would carry this checkout's absolute paths into the
// bundle. The metafile carries them too but is never written and never hashed; it is the only
// thing that says which output is the entry, which of its imports are static, and which
// outputs each one names.
sourcemap: false,
metafile: true,
logLevel: 'silent',
jsx: 'automatic',
// One React: resolve everything from mobile/node_modules, which is where the entry lives.
nodePaths: [join(mobileDir, 'node_modules')],
alias: {
'react-native': 'react-native-web',
'@react-native-async-storage/async-storage': PAGE_ASYNC_STORAGE_MODULE
},
plugins: [routeManifestPlugin(renderMobileWebAppRouteManifest(routes)), lucideBarrelPlugin],
resolveExtensions: [
'.web.tsx',
'.web.ts',
'.web.jsx',
'.web.js',
'.tsx',
'.ts',
'.jsx',
'.js',
'.json'
],
// Images are emitted as same-origin assets, not data: URLs: the shell's CSP sets
// img-src 'self', which refuses data:. Content-hashed names keep the buildId reproducible.
// A font would fail the build here rather than silently ship under font-src 'none'.
loader: {
...ROUTE_SOURCE_LOADERS,
'.png': 'file',
'.jpg': 'file',
'.jpeg': 'file',
'.gif': 'file',
'.webp': 'file',
'.svg': 'file'
},
assetNames: '[hash]',
// Absolute, because the document is served at every route depth and a path relative to the
// script would resolve against the route instead.
publicPath: '/assets',
banner: {
js: "globalThis.process ??= { env: { NODE_ENV: 'production', EXPO_OS: 'web' }, platform: 'web', version: '', nextTick: (fn) => setTimeout(fn, 0) };"
},
define: {
global: 'globalThis',
__DEV__: 'false',
'process.env.NODE_ENV': '"production"',
'process.env.EXPO_OS': '"web"',
'process.env.EXPO_ROUTER_IMPORT_MODE': '"sync"'
}
}
}
/**
* What the browser must have before the first route can paint: the entry plus every chunk it
* reaches by static import, transitively. A dynamic import is what the split exists to defer, so
* it is where this stops.
*
* The bound the verifier holds is this number and not the entry file alone, because esbuild puts
* the code shared by entry and routes in a chunk the entry imports statically: budgeting the entry
* file on its own would fall as the shared chunk grew.
*/
export function entryStaticClosure(metafile, entryOutputPath) {
const reached = new Set([entryOutputPath])
const queue = [entryOutputPath]
while (queue.length > 0) {
const current = queue.shift()
for (const imported of metafile.outputs[current]?.imports ?? []) {
if (imported.kind !== 'import-statement' || reached.has(imported.path)) {
continue
}
reached.add(imported.path)
queue.push(imported.path)
}
}
return reached
}
/**
* Every emitted output, renamed to the sha256 of its own final bytes.
*
* esbuild's `[hash]` is computed over the metafile's input keys, and those keys are paths
* relative to absWorkingDir. A tree whose mobile/node_modules is a symlink keys most of its
* inputs as `../../<somewhere>/...`, a tree that holds a real directory keys them as
* `node_modules/...`, and a byte-identical chunk comes out under a different name in each. The
* name is embedded in every importer, so the difference cascades into a different buildId for one
* commit -- and every phone re-downloads a bundle whose bytes never changed.
*
* Renaming here is what removes the path from the output. Leaves first, so an importer is hashed
* only once the names written inside it are final: an image before the chunk that loads it, a
* chunk before the chunk that imports it, the entry last. The result is what `hashedAsset` would
* name each of these anyway, which is how the name inside the bytes and the manifest's own sha256
* stay the same string.
*/
export function renameOutputsByContent(metafile, outputFiles) {
const emitted = new Map(
outputFiles.map((file) => [basename(file.path), Buffer.from(file.contents)])
)
const importsOf = new Map(
Object.entries(metafile.outputs).map(([output, { imports }]) => [
basename(output),
(imports ?? []).map((entry) => basename(entry.path)).filter((name) => emitted.has(name))
])
)
const renamed = new Map()
const open = new Set()
function rename(name) {
const done = renamed.get(name)
if (done) {
return done
}
if (open.has(name)) {
// Two outputs naming each other have no content hash at all, so this is a hard stop rather
// than a fallback. esbuild's splitting emits a DAG; nothing in the tree has produced one.
throw new Error(
`[build-mobile-web-app-bundle] ${name} is in an output cycle and cannot be content-named`
)
}
open.add(name)
let bytes = emitted.get(name)
for (const child of importsOf.get(name) ?? []) {
const { name: childName } = rename(child)
// publicPath already rewrote the specifier to this exact shape, and an esbuild output name
// is a token that appears nowhere else.
bytes = Buffer.from(
bytes.toString('utf8').split(`/assets/${child}`).join(`/assets/${childName}`),
'utf8'
)
}
open.delete(name)
const result = { name: `${sha256Hex(bytes)}${extname(name)}`, bytes }
renamed.set(name, result)
return result
}
for (const name of [...emitted.keys()].sort()) {
rename(name)
}
return renamed
}
/**
* Which emitted chunk each route key's `import()` lands in. esbuild puts a route module in exactly
* one output, so the metafile's own inputs answer it; nothing downstream can, because by then
* every name is a hash of bytes and the route's source path is gone from the bundle.
*/
export function routeChunkNames(metafile, routes, renamed) {
const owner = new Map()
for (const [output, { inputs }] of Object.entries(metafile.outputs)) {
for (const input of Object.keys(inputs ?? {})) {
// Absolute, and through realpath on the lookup side below: esbuild writes its input keys
// relative to absWorkingDir after resolving symlinks, so a route reached through one (every
// scratch tree under /var on macOS) is keyed by a path the caller never spelled.
owner.set(resolve(mobileDir, input), basename(output))
}
}
return Object.fromEntries(
routes.map(({ key, module }) => {
const emittedName = owner.get(realpathSync(module))
if (!emittedName) {
throw new Error(`[build-mobile-web-app-bundle] ${key} reached no output`)
}
return [key, renamed.get(emittedName).name]
})
)
}
const isScriptOutput = (path) => path.endsWith('.js')
// appDir is a seam for the tests, which bundle a scratch route tree; production always uses mobile/app.
export async function bundleMobileWebApp({ appDir = defaultAppDir } = {}) {
const routes = await collectMobileWebAppRoutes(appDir)
await assertRoutesCarryNoSynchronousExports(routes)
const result = await esbuild.build(mobileWebAppBuildOptions(routes))
const entryOutputPath = Object.keys(result.metafile.outputs).find(
(path) => basename(path) === `${ENTRY_CHUNK_NAME}.js`
)
if (!entryOutputPath) {
throw new Error('[build-mobile-web-app-bundle] esbuild emitted no entry script')
}
const renamed = renameOutputsByContent(result.metafile, result.outputFiles)
const entry = renamed.get(basename(entryOutputPath))
const byName = (left, right) => (left.name < right.name ? -1 : 1)
const others = [...renamed.entries()]
.filter(([emittedName]) => emittedName !== basename(entryOutputPath))
.map(([emittedName, output]) => ({ emittedName, ...output }))
// Chunks keep their new name into the served path: the entry imports them by it, and
// publicPath has already made that specifier /assets/<name>.
const chunks = others.filter(({ emittedName }) => isScriptOutput(emittedName)).sort(byName)
const images = others.filter(({ emittedName }) => !isScriptOutput(emittedName)).sort(byName)
const closure = entryStaticClosure(result.metafile, entryOutputPath)
return {
script: entry.bytes,
chunks,
images,
// Counted off the renamed bytes rather than the metafile's own sizes, which are from before
// the names inside each output grew. Only the metafile knows which import is static; see
// entryStaticClosure.
entryStaticBytes: [...closure].reduce(
(total, path) => total + (renamed.get(basename(path))?.bytes.byteLength ?? 0),
0
),
routeKeys: routes.map((route) => route.key),
routeChunks: routeChunkNames(result.metafile, routes, renamed)
}
}
/**
* The declared page routes, checked against the tree that was actually bundled.
*
* A declaration naming a screen this bundle has no module for would reach a phone as a route the
* shell opens the page for and the page then paints as Unmatched. Failing the build is the only
* place that mismatch is visible to whoever wrote the declaration.
*/
export function resolveMobileWebPageRoutes(routeKeys, declared = MOBILE_WEB_PAGE_ROUTES) {
const bundled = new Set(routeKeys.map(routePathnameFromKey).filter((path) => path !== null))
for (const route of declared) {
if (!bundled.has(route.pathname)) {
throw new Error(
`[build-mobile-web-app-bundle] declared page route ${route.pathname} has no module in the bundle`
)
}
}
return declared.map((route) => ({ pathname: route.pathname, grants: [...route.grants] }))
}
/**
* `pageRoutes` rides with `appDir`: the declarations name screens in the real route tree, so a
* caller bundling some other tree has none to check against and says so by passing its own.
*/
export async function buildMobileWebAppBundle({
appDir,
outDir = defaultOutDir,
pageRoutes = MOBILE_WEB_PAGE_ROUTES
} = {}) {
const [
desktopVersion,
protocolWindow,
{ script, chunks, images, entryStaticBytes, routeChunks, routeKeys }
] = await Promise.all([
readDesktopVersion(),
readProtocolWindow(),
bundleMobileWebApp({ appDir })
])
// Every output is already named by its own bytes, and a name is written inside whatever imports
// it, so hashedAsset here reproduces the name rather than choosing one.
const scriptAsset = hashedAsset(script, 'js')
const written = [
scriptAsset,
...[...chunks, ...images].map(({ name, bytes }) => hashedAsset(bytes, extname(name).slice(1)))
]
// Root-absolute, unlike the Phase A bootstrap's bare relative src: this document is served at
// every route depth (/h/<hostId>/tasks), where a relative href resolves against the route and
// 404s. A <base> tag would be the other fix, but the shell's CSP sets base-uri 'none'.
// type="module", because the entry is esm and reaches its routes through import(). Same-origin
// module and chunk both load under the shell's script-src 'self'; the policy is unchanged.
const html =
'<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8" />\n' +
'<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />\n' +
'<title>Orca</title>\n</head>\n<body>\n<div id="root"></div>\n' +
`<script type="module" src="/${scriptAsset.path}"></script>\n</body>\n</html>\n`
const indexBytes = Buffer.from(html, 'utf8')
const indexAsset = {
bytes: indexBytes,
path: MOBILE_WEB_BUNDLE_ENTRYPOINT,
sha256: sha256Hex(indexBytes),
byteLength: indexBytes.byteLength,
contentType: contentTypeForExtension('html')
}
const { manifest } = await writeMobileWebBundleTree({
outDir,
written: [indexAsset, ...written],
desktopVersion,
protocolWindow,
routes: resolveMobileWebPageRoutes(routeKeys, pageRoutes)
})
return {
manifest,
outDir,
routeChunks,
routeKeys,
entryStaticBytes,
// The entry counts: it is a chunk the browser fetches, and the budget is about how many.
chunkCount: chunks.length + 1,
// Everything the routes import that is not a script, which is the rest of the asset budget.
imageCount: images.length
}
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
try {
const { manifest, outDir, routeKeys, entryStaticBytes, chunkCount } =
await buildMobileWebAppBundle()
console.log(
`[build-mobile-web-app-bundle] OK — ${String(routeKeys.length)} route(s), ` +
`${String(chunkCount)} chunk(s), ${String(entryStaticBytes)} bytes before the first route, ` +
`${String(manifest.assets.length)} asset(s), ${String(manifest.totalBytes)} bytes, ` +
`buildId ${manifest.buildId} -> ${outDir}`
)
} catch (error) {
// The route guards fail here by design, and every throw on this path already names its
// source, so a stack only buries which route and which export.
console.error(error.message)
process.exit(1)
}
}
@@ -0,0 +1,665 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, relative } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import {
MOBILE_WEB_APP_SHIMS,
bundleMobileWebApp,
buildMobileWebAppBundle,
entryStaticClosure,
mobileWebAppBuildOptions,
renameOutputsByContent,
resolveMobileWebPageRoutes,
routeChunkNames
} from './build-mobile-web-app-bundle.mjs'
import {
MOBILE_WEB_APP_ROUTE_ROOT,
ROUTE_SOURCE_LOADERS,
collectMobileWebAppRouteKeys,
collectMobileWebAppRoutes,
routePathnameFromKey
} from './mobile-web-app-route-manifest.mjs'
import {
MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES,
MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES,
MOBILE_WEB_APP_SOURCE_DIRS,
assertAssetCeilingFitsShell,
mobileWebAppBundleMaxAssets,
mobileWebAppBundleMaxChunks,
readMobileWebBundleMaxAssets,
verifyMobileWebAppBundle
} from './verify-mobile-web-app-bundle.mjs'
import {
BINARY_SOURCE_EXTENSIONS,
assertNoCarriageReturnsInSource
} from './verify-mobile-web-bundle.mjs'
import {
computeMobileWebBundleBuildId,
hashedAsset,
readDesktopVersion,
readProtocolWindow,
sha256Hex,
writeMobileWebBundleTree
} from './build-mobile-web-bundle.mjs'
import {
MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES,
MOBILE_WEB_BUNDLE_MAX_ASSETS
} from '../../src/shared/mobile-web-bundle/manifest-contract.js'
import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const appDir = join(projectDir, 'mobile', 'app')
// The sharded `test` job does not install mobile dependencies, so anything that runs esbuild over
// the route tree is skipped there and run for real in pr.yml's mobile_web_app job.
const bundles = mobileWebAppDependenciesPresent()
const describeBundling = bundles ? describe : describe.skip
const itBundling = bundles ? it : it.skip
/** Every script the page loads. A route's code is in a chunk now, not in the entry. */
function allScriptSource({ script, chunks }) {
return [script, ...chunks.map((chunk) => chunk.bytes)].map((bytes) => bytes.toString('utf8'))
}
async function withScratch(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-app-test-'))
try {
return await run(scratch)
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
describe('the page routes the manifest declares', () => {
it('turns a route key into the URL pattern expo-router gives it', () => {
expect(routePathnameFromKey('./h/[hostId]/index.tsx')).toBe('/h/[hostId]')
expect(routePathnameFromKey('./h/[hostId]/tasks.tsx')).toBe('/h/[hostId]/tasks')
expect(routePathnameFromKey('./h/[hostId]/session/[worktreeId].tsx')).toBe(
'/h/[hostId]/session/[worktreeId]'
)
})
it('answers null for a layout, which is not a screen anyone navigates to', () => {
expect(routePathnameFromKey('./h/_layout.tsx')).toBeNull()
expect(routePathnameFromKey('./h/[hostId]/_layout.tsx')).toBeNull()
})
it('declares only routes the bundle has a module for', async () => {
const keys = await collectMobileWebAppRouteKeys(appDir)
expect(resolveMobileWebPageRoutes(keys)).toEqual([
{ pathname: '/h/[hostId]', grants: ['navigate', 'storage'] }
])
})
it('fails the build on a declaration the bundle cannot render', () => {
// The mismatch reaches a phone as a route the shell opens the page for and the page then
// paints as Unmatched. This is the only place whoever wrote the declaration can see it.
expect(() =>
resolveMobileWebPageRoutes(
['./h/[hostId]/index.tsx'],
[{ pathname: '/h/[hostId]/gone', grants: [] }]
)
).toThrow('has no module in the bundle')
})
itBundling(
'reaches the built manifest, where the build id does not move for it',
async () => {
await withScratch(async (scratch) => {
const { manifest } = await buildMobileWebAppBundle({ outDir: join(scratch, 'bundle') })
expect(manifest.routes).toEqual([
{ pathname: '/h/[hostId]', grants: ['navigate', 'storage'] }
])
// The routes are derived from the same tree the script is built from, so the assets
// already decide them and the id has no reason to carry them as well.
expect(manifest.buildId).toBe(computeMobileWebBundleBuildId(manifest.assets))
})
},
240_000
)
})
describe('the CRLF pin', () => {
it('exempts the same extensions in .gitattributes as the CRLF scan skips', async () => {
const attributes = await readFile(join(projectDir, '.gitattributes'), 'utf8')
for (const tree of MOBILE_WEB_APP_SOURCE_DIRS) {
const pattern = `/${relative(projectDir, tree).split('\\').join('/')}/**`
for (const extension of BINARY_SOURCE_EXTENSIONS) {
// Without the exemption the blanket `text eol=lf` pin above it rewrites the binary and
// every asset hash with it.
expect(attributes, `${pattern}/*${extension} is not exempt`).toContain(
`${pattern}/*${extension} -text`
)
}
}
})
})
describeBundling('the app bundle', () => {
it('resolves react-native to react-native-web and leaves no require.context', async () => {
const sources = allScriptSource(await bundleMobileWebApp())
for (const source of sources) {
expect(source).not.toContain('require.context')
}
// react-native-web's touch responder is proof the alias resolved rather than the native stub.
expect(sources.some((source) => source.includes('ResponderTouchHistoryStore'))).toBe(true)
}, 120_000)
it('cuts the routes into chunks the entry does not load', async () => {
const { script, chunks, entryStaticBytes } = await bundleMobileWebApp()
expect(chunks.length).toBeGreaterThan(1)
// The entry's own bytes plus the chunks it imports statically, which is what the browser
// parses before any route paints. Every route chunk is outside it.
expect(entryStaticBytes).toBeGreaterThan(script.byteLength)
const allBytes =
script.byteLength + chunks.reduce((total, chunk) => total + chunk.bytes.byteLength, 0)
expect(entryStaticBytes).toBeLessThan(allBytes)
}, 120_000)
it('names the chunk each route lands in', async () => {
const { chunks, routeChunks, routeKeys } = await bundleMobileWebApp()
expect(Object.keys(routeChunks).sort()).toEqual([...routeKeys].sort())
const emitted = new Set(chunks.map((chunk) => chunk.name))
for (const [key, name] of Object.entries(routeChunks)) {
expect(emitted, key).toContain(name)
}
// One chunk per route, never the entry: that is what a client-side navigation fetches.
expect(new Set(Object.values(routeChunks)).size).toBe(routeKeys.length)
}, 120_000)
it('counts only static imports into what loads before the first route', () => {
const metafile = {
outputs: {
'dist/entry.js': {
bytes: 10,
imports: [
{ path: 'dist/shared.js', kind: 'import-statement' },
{ path: 'dist/route.js', kind: 'dynamic-import' }
]
},
'dist/shared.js': {
bytes: 20,
imports: [{ path: 'dist/deep.js', kind: 'import-statement' }]
},
'dist/deep.js': { bytes: 30, imports: [] },
'dist/route.js': { bytes: 40, imports: [] }
}
}
expect([...entryStaticClosure(metafile, 'dist/entry.js')]).toEqual([
'dist/entry.js',
'dist/shared.js',
'dist/deep.js'
])
})
it('does not walk a chunk cycle forever', () => {
const metafile = {
outputs: {
'dist/entry.js': { bytes: 1, imports: [{ path: 'dist/a.js', kind: 'import-statement' }] },
'dist/a.js': { bytes: 1, imports: [{ path: 'dist/entry.js', kind: 'import-statement' }] }
}
}
expect(entryStaticClosure(metafile, 'dist/entry.js').size).toBe(2)
})
itBundling(
'refuses to build a route the lazy manifest would strip an export from',
async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
await writeFile(
join(directory, 'index.tsx'),
'export default function Route() { return null }\n'
)
await expect(bundleMobileWebApp({ appDir: scratch })).resolves.toBeTruthy()
await writeFile(
join(directory, 'settings.tsx'),
'const anchor = { anchor: "index" }\nexport { anchor as unstable_settings }\nexport default function Route() { return null }\n'
)
// The build is where this has to fail: the page it would otherwise emit mounts with the
// export silently gone, which is a blank screen on a phone and nothing in any log.
await expect(bundleMobileWebApp({ appDir: scratch })).rejects.toThrow(
/settings\.tsx.*unstable_settings/s
)
})
},
240_000
)
itBundling(
'refuses a route whose star re-export it cannot read',
async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
await writeFile(join(directory, 'boundary.ts'), 'export const value = 1\n')
await writeFile(
join(directory, 'index.tsx'),
'export * from "./boundary"\nexport default function Route() { return null }\n'
)
await expect(bundleMobileWebApp({ appDir: scratch })).rejects.toThrow(
/index\.tsx.*boundary/s
)
})
},
240_000
)
it('bundles every route module', async () => {
const { routeKeys } = await bundleMobileWebApp()
expect(routeKeys).toEqual(await collectMobileWebAppRouteKeys(appDir))
}, 120_000)
it("bundles a route's .web.tsx sibling instead of the native file, changing the bytes", async () => {
await withScratch(async (scratch) => {
const directory = join(scratch, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
const route = (marker) => `export default function Route() { return '${marker}' }\n`
await writeFile(join(directory, 'index.tsx'), route('native-route-marker'))
const before = await bundleMobileWebApp({ appDir: scratch })
const has = (bundle, marker) =>
allScriptSource(bundle).some((source) => source.includes(marker))
expect(has(before, 'native-route-marker')).toBe(true)
await writeFile(join(directory, 'index.web.tsx'), route('web-route-marker'))
const after = await bundleMobileWebApp({ appDir: scratch })
expect(has(after, 'web-route-marker')).toBe(true)
expect(has(after, 'native-route-marker')).toBe(false)
// Different script bytes means a different asset sha and so a different buildId.
expect(after.script.equals(before.script)).toBe(false)
})
}, 240_000)
/**
* The same route tree, bundled from two directories at different depths. esbuild's own `[hash]`
* is computed over the metafile's input keys, which are paths relative to absWorkingDir, so two
* checkouts of one commit -- at different depths, or one with mobile/node_modules as a symlink
* and one with it as a directory -- name a byte-identical chunk differently. The rename
* cascades through every importer into a different buildId, and every phone re-downloads a
* bundle whose bytes did not change.
*/
async function bundleFromDepth(root, depth) {
const nested = join(root, ...Array.from({ length: depth }, (_, index) => `d${String(index)}`))
const directory = join(nested, MOBILE_WEB_APP_ROUTE_ROOT)
await mkdir(directory, { recursive: true })
// Two routes over one import, which is what makes esbuild emit a shared chunk to name.
await writeFile(join(directory, 'shared.ts'), 'export const marker = "shared-marker"\n')
for (const name of ['index.tsx', 'other.tsx']) {
await writeFile(
join(directory, name),
`import { marker } from "./shared"\nexport default function Route() { return marker + "${name}" }\n`
)
}
return { appDir: nested, bundle: await bundleMobileWebApp({ appDir: nested }) }
}
it('names every output by its bytes, so another checkout path builds the same bundle', async () => {
await withScratch(async (shallow) => {
await withScratch(async (deep) => {
const near = await bundleFromDepth(shallow, 1)
const far = await bundleFromDepth(deep, 5)
const names = ({ bundle }) => [...bundle.chunks, ...bundle.images].map((one) => one.name)
expect(names(far)).toEqual(names(near))
expect(far.bundle.script.equals(near.bundle.script)).toBe(true)
// The whole point: the manifest the phone compares is the same document.
const buildIdFrom = async ({ appDir }) =>
withScratch(async (out) => {
const { manifest } = await buildMobileWebAppBundle({
appDir,
outDir: join(out, 'x'),
// A synthetic tree: the real declarations name screens it does not have.
pageRoutes: []
})
return manifest.buildId
})
expect(await buildIdFrom(far)).toBe(await buildIdFrom(near))
})
})
}, 240_000)
it("names an output the same way the manifest's own asset hash does", async () => {
const { script, chunks } = await bundleMobileWebApp()
// The name is embedded in the importer, so it cannot be recomputed later; this is what says
// the name inside the bytes and the manifest's sha256 of those bytes are the same string.
expect(hashedAsset(script, 'js').path).toBe(`assets/${sha256Hex(script)}.js`)
for (const chunk of chunks) {
expect(chunk.name).toBe(`${sha256Hex(chunk.bytes)}.js`)
}
}, 120_000)
it('asks esbuild for the split the budgets assume', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// Each of these is load-bearing for a budget below: esm and splitting are what make a route a
// chunk, and the metafile is the only thing that says which imports are static.
expect(options.format).toBe('esm')
expect(options.splitting).toBe(true)
expect(options.chunkNames).toBe('[hash]')
expect(options.metafile).toBe(true)
})
it('reads a route source the same way the export guard does', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// The guard parses each route on its own, outside this build. Sharing the table is what stops
// a loader the bundle relies on from being missing there and reported as a syntax error.
for (const [extension, loader] of Object.entries(ROUTE_SOURCE_LOADERS)) {
expect(options.loader[extension], extension).toBe(loader)
}
})
it('applies every shim it names', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
for (const shim of MOBILE_WEB_APP_SHIMS) {
expect(shim.appliesTo(options), `${shim.name} is named but not applied`).toBe(true)
}
})
it('fails the named shim, not the whole build, when its option goes missing', async () => {
const options = mobileWebAppBuildOptions(await collectMobileWebAppRoutes(appDir))
// Each shim reads a different option, so removing one leaves the other five true. Without
// that, the list could name a shim the build stopped applying.
const stripped = {
...options,
alias: {},
loader: {},
define: {},
banner: {},
plugins: []
}
expect(MOBILE_WEB_APP_SHIMS.filter((shim) => shim.appliesTo(stripped))).toEqual([])
})
it('keeps the shims out of the shipped Phase A bootstrap builder', async () => {
const shipped = await readFile(
join(projectDir, 'config', 'scripts', 'build-mobile-web-bundle.mjs'),
'utf8'
)
for (const { name } of MOBILE_WEB_APP_SHIMS) {
expect(shipped, `the Phase A bootstrap builder mentions ${name}`).not.toContain(name)
}
expect(shipped).not.toContain('react-native-web')
expect(shipped).not.toContain('lucide')
})
it('embeds no absolute path from this checkout', async () => {
// Every chunk, not only the entry: the route manifest names each route by absolute path, and
// the chunk that import resolves to is where such a path would survive.
for (const source of allScriptSource(await bundleMobileWebApp())) {
expect(source).not.toContain(projectDir)
}
}, 120_000)
it('builds the same buildId twice', async () => {
const first = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'a') })
)
const second = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'b') })
)
expect(first.manifest.buildId).toBe(second.manifest.buildId)
}, 120_000)
it('loads the entry as a module, so its route imports resolve', async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'module-tag')
const { manifest } = await buildMobileWebAppBundle({ outDir })
const html = await readFile(join(outDir, 'index.html'), 'utf8')
// import() in a classic script is a syntax error, so the tag and the format are one fact.
expect(html).toContain('<script type="module" src="/assets/')
const entry = html.match(/src="\/(assets\/[^"]+)"/)?.[1]
expect(manifest.assets.map((asset) => asset.path)).toContain(entry)
})
}, 120_000)
it('writes the manifest shape the packaging contract reads', async () => {
const { manifest } = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'c') })
)
expect(manifest.schemaVersion).toBe(1)
expect(manifest.entrypoint).toBe('index.html')
expect(manifest.assets.map((asset) => asset.path)).toContain('index.html')
expect(manifest.totalBytes).toBe(
manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
)
}, 120_000)
})
describe('the Phase C budget', () => {
it('sits below the contract per-asset ceiling, so growth trips a build not a phone', () => {
expect(MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES).toBeLessThan(MOBILE_WEB_BUNDLE_MAX_ASSET_BYTES)
})
itBundling(
'is not already exceeded by the current bundle',
async () => {
const { manifest, chunkCount, entryStaticBytes, imageCount, routeKeys } = await withScratch(
(scratch) => buildMobileWebAppBundle({ outDir: join(scratch, 'd') })
)
expect(manifest.totalBytes).toBeLessThanOrEqual(MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES)
expect(manifest.assets.length).toBeLessThanOrEqual(
mobileWebAppBundleMaxAssets(routeKeys.length, imageCount)
)
expect(chunkCount).toBeLessThanOrEqual(mobileWebAppBundleMaxChunks(routeKeys.length))
expect(entryStaticBytes).toBeLessThanOrEqual(MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES)
},
120_000
)
it('says which node may be statically imported, and does not promise a route may', async () => {
const source = await readFile(
join(projectDir, 'config', 'scripts', 'verify-mobile-web-app-bundle.mjs'),
'utf8'
)
// The bound reads like a per-route escape hatch and is not one: 5 of the 14 routes break it
// on their own. What keeps it survivable is that expo-router wants a synchronous export off
// layout nodes only, so the note has to name the layout and the export that drives it.
const doc = source.slice(
0,
source.indexOf('export const MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES')
)
const note = doc.slice(doc.lastIndexOf('/**'))
expect(note).toContain('h/_layout.tsx')
expect(note).toContain('unstable_settings')
})
it('budgets what loads first well under what the whole page weighs', () => {
// The point of the split: the entry budget is the one a route must not grow, and it is a
// fraction of the total the bundle is still allowed to weigh.
expect(MOBILE_WEB_APP_BUNDLE_MAX_ENTRY_BYTES).toBeLessThan(
MOBILE_WEB_APP_BUNDLE_MAX_TOTAL_BYTES
)
})
it('derives the chunk ceiling from the route count, not from a measured number', async () => {
// A chunk is emitted per distinct set of importers, so the count is combinatorial rather than
// one per route. Measured while building this: 8 routes emit 23 chunks, 10 emit 40, 12 emit
// 47, 14 emit 53 -- about 3 more per route at the top. The ceiling allows 4 and starts 16
// above zero, so the next few routes land under it instead of failing on a pinned number.
for (const [routes, measured] of [
[8, 23],
[10, 40],
[12, 47],
[14, 53]
]) {
expect(mobileWebAppBundleMaxChunks(routes), `${String(routes)} routes`).toBeGreaterThan(
measured
)
}
expect(mobileWebAppBundleMaxChunks(14)).toBe(72)
expect(mobileWebAppBundleMaxChunks(15) - mobileWebAppBundleMaxChunks(14)).toBe(4)
})
it('derives the asset ceiling so the chunk ceiling is always the one that trips first', () => {
// A bundle's assets are its chunks, its images and the document. Asserting one constant under
// another did not say that: with 42 images, 4 * 18 + 16 chunks plus 42 plus the document is
// 131 assets, over the flat 128 the ceiling used to be, so from 18 routes on the asset count
// failed first and named the wrong thing.
for (const routeCount of [14, 18, 24, 40]) {
for (const imageCount of [0, 42, 120]) {
const chunks = mobileWebAppBundleMaxChunks(routeCount)
expect(mobileWebAppBundleMaxAssets(routeCount, imageCount)).toBe(chunks + imageCount + 1)
// The ordering claim itself: a bundle at the chunk ceiling is exactly at the asset
// ceiling, so no bundle can pass the chunk check and fail the asset one.
expect(chunks + imageCount + 1).toBeLessThanOrEqual(
mobileWebAppBundleMaxAssets(routeCount, imageCount)
)
}
}
})
itBundling(
'keeps the derived ceiling under the map the phone actually holds',
async () => {
const { manifest, routeKeys, imageCount } = await withScratch((scratch) =>
buildMobileWebAppBundle({ outDir: join(scratch, 'e') })
)
const ceiling = mobileWebAppBundleMaxAssets(routeKeys.length, imageCount)
expect(manifest.assets.length).toBeLessThanOrEqual(ceiling)
// The native side refuses a manifest past this, so the derived ceiling has to stay inside it.
expect(ceiling).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_MAX_ASSETS)
// And the build is what has to say so: the guard runs on the counts this bundle measured.
const shellCeiling = await readMobileWebBundleMaxAssets()
expect(assertAssetCeilingFitsShell(routeKeys.length, imageCount, shellCeiling)).toBe(ceiling)
},
120_000
)
it('fails the build when the derived ceiling passes what the phone will accept', async () => {
// The shell hands back null for a manifest over its own ceiling, so a derived ceiling above
// that ships a green build no device can open. At the 42 images the tree carries, 4r + 16 +
// 42 + 1 crosses 256 at 50 routes, which Phase C reaches.
expect(await readMobileWebBundleMaxAssets()).toBe(MOBILE_WEB_BUNDLE_MAX_ASSETS)
expect(assertAssetCeilingFitsShell(49, 42, MOBILE_WEB_BUNDLE_MAX_ASSETS)).toBe(255)
expect(() => assertAssetCeilingFitsShell(50, 42, MOBILE_WEB_BUNDLE_MAX_ASSETS)).toThrow(
/259 .*256/
)
})
})
describe('the verifier', () => {
itBundling(
'accepts a bundle it has just built',
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
await buildMobileWebAppBundle({ outDir })
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).resolves.toBeDefined()
})
},
240_000
)
itBundling(
"rejects a buildId the manifest's own asset list does not derive",
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
await buildMobileWebAppBundle({ outDir })
const manifestPath = join(outDir, 'manifest.json')
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
manifest.buildId = 'f'.repeat(64)
await writeFile(manifestPath, JSON.stringify(manifest), 'utf8')
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).rejects.toThrow(
'does not match its asset list'
)
})
},
240_000
)
itBundling(
'rejects a self-consistent bundle a fresh build does not reproduce',
async () => {
await withScratch(async (scratch) => {
const outDir = join(scratch, 'mobile-web-app')
const { manifest } = await buildMobileWebAppBundle({ outDir })
// What a stale out/ actually looks like: every digest agrees with its bytes and the
// buildId derives from the asset list, but the source has moved on. Only the two fresh
// builds the verifier runs can tell, which is the check this covers.
const assets = await Promise.all(
manifest.assets.map(async (asset) => ({
...asset,
bytes: await readFile(join(outDir, asset.path))
}))
)
const document = assets.find((asset) => asset.path === manifest.entrypoint)
document.bytes = Buffer.concat([document.bytes, Buffer.from('<!-- drift -->\n', 'utf8')])
document.sha256 = sha256Hex(document.bytes)
document.byteLength = document.bytes.byteLength
const [desktopVersion, protocolWindow] = await Promise.all([
readDesktopVersion(),
readProtocolWindow()
])
await writeMobileWebBundleTree({ outDir, written: assets, desktopVersion, protocolWindow })
await expect(verifyMobileWebAppBundle({ bundleDir: outDir })).rejects.toThrow('is stale')
})
},
240_000
)
})
describe('the CRLF guard', () => {
it('covers the three trees whose bytes reach the buildId', () => {
expect(MOBILE_WEB_APP_SOURCE_DIRS.map((dir) => dir.slice(projectDir.length))).toEqual([
join('mobile', 'web-entry'),
join('mobile', 'app'),
join('mobile', 'src')
])
})
it('fails on a CRLF source file', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'route.tsx'), 'export default null\r\n', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).rejects.toThrow('CRLF')
})
})
it('exempts the binary assets .gitattributes pins -text', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'icon.ttf'), Buffer.from([0x00, 0x0d, 0x0a]))
await writeFile(join(scratch, 'shot.png'), Buffer.from([0x0d]))
await expect(assertNoCarriageReturnsInSource(scratch)).resolves.toBeUndefined()
})
})
it('exempts the gitignored generated webview engine modules', async () => {
await withScratch(async (scratch) => {
await writeFile(join(scratch, 'engine.generated.ts'), 'export const X = "a\r\n"', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).resolves.toBeUndefined()
})
})
})
describe('naming an output by its bytes', () => {
it('refuses two outputs that name each other', () => {
const emitted = (text) => new TextEncoder().encode(text)
const metafile = {
outputs: {
'dist/a.js': { imports: [{ path: 'dist/b.js', kind: 'import-statement' }] },
'dist/b.js': { imports: [{ path: 'dist/a.js', kind: 'import-statement' }] }
}
}
// Neither name can be final before the other is, so a cycle has no content hash to reach.
// esbuild's splitting emits a DAG; this is the hard stop for the day it does not.
expect(() =>
renameOutputsByContent(metafile, [
{ path: 'dist/a.js', contents: emitted('import "/assets/b.js"') },
{ path: 'dist/b.js', contents: emitted('import "/assets/a.js"') }
])
).toThrow(/output cycle/)
})
it('refuses a route it cannot find an output for', async () => {
await withScratch(async (scratch) => {
const module = join(scratch, 'index.tsx')
await writeFile(module, 'export default function Route() { return null }\n')
// The metafile is the only thing that knows which chunk holds a route. Without this the
// route reaches the manifest naming a chunk of undefined, which the phone fetches as a 404.
expect(() =>
routeChunkNames({ outputs: {} }, [{ key: './index.tsx', module }], new Map())
).toThrow(/\.\/index\.tsx reached no output/)
})
})
})
+258
View File
@@ -0,0 +1,258 @@
import { createHash } from 'node:crypto'
import { realpathSync } from 'node:fs'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import * as esbuild from 'esbuild'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const sourceDir = join(projectDir, 'src', 'mobile-web')
const defaultOutDir = join(projectDir, 'out', 'mobile-web')
export const MOBILE_WEB_BUNDLE_SCHEMA_VERSION = 1
export const MOBILE_WEB_BUNDLE_ENTRYPOINT = 'index.html'
const CONTENT_TYPE_BY_EXTENSION = {
css: 'text/css; charset=utf-8',
html: 'text/html; charset=utf-8',
js: 'text/javascript; charset=utf-8',
png: 'image/png',
// The Phase C app bundle emits images as same-origin assets rather than data: URLs, which the
// shell's img-src 'self' refuses. Fonts are absent by design: the policy sets font-src 'none'.
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
gif: 'image/gif',
webp: 'image/webp',
svg: 'image/svg+xml'
}
/**
* Canonical serialization the buildId hashes. Key order is fixed and the list is sorted by path,
* so the id is a pure function of content. Must stay byte-identical to the contract module's
* serializer in src/shared/mobile-web-bundle/.
*/
export function serializeMobileWebBundleAssets(assets) {
return JSON.stringify(
[...assets]
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
.map(({ path, sha256, byteLength, contentType }) => ({
path,
sha256,
byteLength,
contentType
}))
)
}
export function computeMobileWebBundleBuildId(assets) {
return createHash('sha256').update(serializeMobileWebBundleAssets(assets), 'utf8').digest('hex')
}
export function sha256Hex(bytes) {
return createHash('sha256').update(bytes).digest('hex')
}
export function contentTypeForExtension(extension) {
const contentType = CONTENT_TYPE_BY_EXTENSION[extension]
if (!contentType) {
throw new Error(`[build-mobile-web-bundle] no content type registered for .${extension}`)
}
return contentType
}
function readIntegerConstant(source, name) {
const match = new RegExp(`export const ${name} = (\\d+)`).exec(source)
if (!match) {
throw new Error(`[build-mobile-web-bundle] ${name} not found in src/shared/protocol-version.ts`)
}
return Number.parseInt(match[1], 10)
}
/**
* Parsed rather than imported because protocol-version.ts is TypeScript and this script runs on
* bare node during packaging, before any build output exists.
*/
export async function readProtocolWindow() {
const source = await readFile(join(projectDir, 'src', 'shared', 'protocol-version.ts'), 'utf8')
return {
runtimeProtocolVersion: readIntegerConstant(source, 'RUNTIME_PROTOCOL_VERSION'),
// The bundle is a client: the floor it cares about is the oldest host protocol it can talk to.
minCompatibleRuntimeProtocolVersion: readIntegerConstant(
source,
'MIN_COMPATIBLE_RUNTIME_SERVER_VERSION'
)
}
}
export async function readDesktopVersion() {
const packageJson = JSON.parse(await readFile(join(projectDir, 'package.json'), 'utf8'))
if (typeof packageJson.version !== 'string' || packageJson.version.length === 0) {
throw new Error('[build-mobile-web-bundle] root package.json has no version')
}
return packageJson.version
}
async function transformEntries(protocolWindow, desktopVersion) {
const result = await esbuild.build({
absWorkingDir: sourceDir,
entryPoints: [join(sourceDir, 'src', 'bootstrap.ts'), join(sourceDir, 'src', 'bootstrap.css')],
bundle: true,
minify: true,
// Virtual: write is false, so outdir only names the emitted files esbuild hands back.
outdir: 'dist',
write: false,
format: 'iife',
target: ['es2022'],
charset: 'utf8',
legalComments: 'none',
// Why no sourcemap and no metafile: both embed absolute paths, which would break reproducibility.
sourcemap: false,
logLevel: 'silent',
define: {
ORCA_MOBILE_WEB_DESKTOP_VERSION: JSON.stringify(desktopVersion),
ORCA_MOBILE_WEB_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.runtimeProtocolVersion
),
ORCA_MOBILE_WEB_MIN_COMPATIBLE_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.minCompatibleRuntimeProtocolVersion
)
}
})
const byExtension = new Map()
for (const file of result.outputFiles) {
const extension = file.path.endsWith('.css') ? 'css' : 'js'
byExtension.set(extension, Buffer.from(file.contents))
}
const script = byExtension.get('js')
const stylesheet = byExtension.get('css')
if (!script || !stylesheet) {
throw new Error('[build-mobile-web-bundle] esbuild did not emit both a script and a stylesheet')
}
return { script, stylesheet }
}
export function hashedAsset(bytes, extension) {
const sha256 = sha256Hex(bytes)
return {
bytes,
path: `assets/${sha256}.${extension}`,
sha256,
byteLength: bytes.byteLength,
contentType: contentTypeForExtension(extension)
}
}
export async function buildMobileWebBundle({ outDir = defaultOutDir } = {}) {
const [desktopVersion, protocolWindow] = await Promise.all([
readDesktopVersion(),
readProtocolWindow()
])
const { script, stylesheet } = await transformEntries(protocolWindow, desktopVersion)
const mark = await readFile(join(sourceDir, 'src', 'orca-mark.png'))
const hashed = [
hashedAsset(script, 'js'),
hashedAsset(stylesheet, 'css'),
hashedAsset(mark, 'png')
]
const [scriptAsset, stylesheetAsset, markAsset] = hashed
const template = await readFile(join(sourceDir, MOBILE_WEB_BUNDLE_ENTRYPOINT), 'utf8')
const substitutions = {
__ORCA_BOOTSTRAP_JS__: scriptAsset.path,
__ORCA_BOOTSTRAP_CSS__: stylesheetAsset.path,
__ORCA_MARK_PNG__: markAsset.path
}
let html = template
for (const [token, value] of Object.entries(substitutions)) {
if (!html.includes(token)) {
throw new Error(`[build-mobile-web-bundle] ${MOBILE_WEB_BUNDLE_ENTRYPOINT} lacks ${token}`)
}
html = html.replaceAll(token, value)
}
const indexBytes = Buffer.from(html, 'utf8')
const indexAsset = {
bytes: indexBytes,
path: MOBILE_WEB_BUNDLE_ENTRYPOINT,
sha256: sha256Hex(indexBytes),
byteLength: indexBytes.byteLength,
contentType: contentTypeForExtension('html')
}
return writeMobileWebBundleTree({
outDir,
written: [indexAsset, ...hashed],
desktopVersion,
protocolWindow
})
}
/**
* Manifest assembly and the on-disk write, shared by the Phase A bootstrap bundle and the Phase C
* app bundle so both produce the same manifest shape the contract module and verifier read.
*/
export async function writeMobileWebBundleTree({
outDir,
written,
desktopVersion,
protocolWindow,
// Empty for the Phase A bootstrap, which carries no route tree at all: a shell reading it finds
// no screen listed and renders every route natively, which is what it already does.
routes = []
}) {
const assets = written
.map(({ path, sha256, byteLength, contentType }) => ({ path, sha256, byteLength, contentType }))
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
const manifest = {
schemaVersion: MOBILE_WEB_BUNDLE_SCHEMA_VERSION,
buildId: computeMobileWebBundleBuildId(assets),
desktopVersion,
minCompatibleRuntimeProtocolVersion: protocolWindow.minCompatibleRuntimeProtocolVersion,
runtimeProtocolVersion: protocolWindow.runtimeProtocolVersion,
entrypoint: MOBILE_WEB_BUNDLE_ENTRYPOINT,
totalBytes: assets.reduce((total, asset) => total + asset.byteLength, 0),
assets,
routes
}
// Why a full clear: a stale asset left from an earlier build would ship unreferenced inside asar.
await rm(outDir, { recursive: true, force: true })
await mkdir(join(outDir, 'assets'), { recursive: true })
for (const asset of written) {
await writeFile(join(outDir, asset.path), asset.bytes)
}
await writeFile(join(outDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
return { manifest, outDir }
}
/**
* Whether this module was run as the entry script. Two ways to get this wrong, both of which end
* with the builder exiting 0 having written nothing: `file://${path}` never matches on Windows,
* where import.meta.url is `file:///C:/...`; and Node resolves symlinks in import.meta.url but not
* in argv[1], so `node /tmp/...` against a /private/tmp realpath compares two different strings.
* Both seams are injectable so win32 and a missing path can be exercised from a posix runner.
*/
export function isDirectInvocation(
moduleUrl,
scriptPath,
{ toFileUrl = pathToFileURL, realpath = realpathSync } = {}
) {
if (!scriptPath) {
return false
}
let resolved = scriptPath
try {
resolved = realpath(scriptPath)
} catch {
// A path that cannot be resolved cannot be this module; fall through to the literal compare.
}
return moduleUrl === toFileUrl(resolved).href
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
const { manifest, outDir } = await buildMobileWebBundle()
console.log(
`[build-mobile-web-bundle] OK — ${String(manifest.assets.length)} asset(s), ` +
`${String(manifest.totalBytes)} bytes, buildId ${manifest.buildId} -> ${outDir}`
)
}
@@ -0,0 +1,265 @@
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
buildMobileWebBundle,
computeMobileWebBundleBuildId,
isDirectInvocation,
serializeMobileWebBundleAssets
} from './build-mobile-web-bundle.mjs'
import {
MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS,
MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES,
assertNoCarriageReturnsInSource
} from './verify-mobile-web-bundle.mjs'
async function buildIntoScratch() {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-build-'))
const bundleDir = join(scratch, 'mobile-web')
const { manifest } = await buildMobileWebBundle({ outDir: bundleDir })
return { scratch, bundleDir, manifest }
}
describe('buildMobileWebBundle', () => {
it('emits a content-addressed bundle whose only stable name is the entrypoint', async () => {
const { scratch, bundleDir, manifest } = await buildIntoScratch()
try {
const root = await readdir(bundleDir)
expect(root.sort()).toEqual(['assets', 'index.html', 'manifest.json'])
for (const name of await readdir(join(bundleDir, 'assets'))) {
const [digest, extension] = name.split('.')
expect(digest).toMatch(/^[0-9a-f]{64}$/)
const bytes = await readFile(join(bundleDir, 'assets', name))
expect(createHash('sha256').update(bytes).digest('hex')).toBe(digest)
expect(extension).toMatch(/^(js|css|png)$/)
}
const html = await readFile(join(bundleDir, 'index.html'), 'utf8')
for (const asset of manifest.assets) {
if (asset.path !== 'index.html') {
expect(html).toContain(asset.path)
}
}
expect(html).not.toContain('__ORCA_')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('carries every manifest field the Phase A contract names', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(Object.keys(manifest)).toEqual([
'schemaVersion',
'buildId',
'desktopVersion',
'minCompatibleRuntimeProtocolVersion',
'runtimeProtocolVersion',
'entrypoint',
'totalBytes',
'assets',
'routes'
])
expect(manifest.schemaVersion).toBe(1)
expect(manifest.entrypoint).toBe('index.html')
// The bootstrap bundle carries no route tree, so a shell reading this one finds no screen
// listed and renders every route natively.
expect(manifest.routes).toEqual([])
const packageJson = JSON.parse(
await readFile(new URL('../../package.json', import.meta.url), 'utf8')
)
expect(manifest.desktopVersion).toBe(packageJson.version)
const protocolSource = await readFile(
new URL('../../src/shared/protocol-version.ts', import.meta.url),
'utf8'
)
expect(protocolSource).toContain(
`export const RUNTIME_PROTOCOL_VERSION = ${String(manifest.runtimeProtocolVersion)}`
)
expect(protocolSource).toContain(
`export const MIN_COMPATIBLE_RUNTIME_SERVER_VERSION = ${String(manifest.minCompatibleRuntimeProtocolVersion)}`
)
expect(manifest.totalBytes).toBe(
manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('produces the same buildId from two independent builds', async () => {
const first = await buildIntoScratch()
const second = await buildIntoScratch()
try {
expect(second.manifest.buildId).toBe(first.manifest.buildId)
expect(second.manifest).toEqual(first.manifest)
} finally {
await rm(first.scratch, { recursive: true, force: true })
await rm(second.scratch, { recursive: true, force: true })
}
})
it('embeds no absolute path from the machine that built it', async () => {
const { scratch, bundleDir } = await buildIntoScratch()
try {
const names = [
'index.html',
'manifest.json',
...(await readdir(join(bundleDir, 'assets'))).map((name) => join('assets', name))
]
for (const name of names) {
const text = (await readFile(join(bundleDir, name))).toString('latin1')
expect(text).not.toContain(scratch)
expect(text).not.toContain(process.cwd())
}
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('stays inside the Phase A budget', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(manifest.assets.length).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS)
expect(manifest.totalBytes).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
describe('computeMobileWebBundleBuildId', () => {
const assets = [
{ path: 'index.html', sha256: 'a'.repeat(64), byteLength: 3, contentType: 'text/html' },
{ path: 'assets/b.js', sha256: 'b'.repeat(64), byteLength: 5, contentType: 'text/javascript' }
]
it('sorts by path, so input order cannot change the id', () => {
expect(computeMobileWebBundleBuildId(assets.toReversed())).toBe(
computeMobileWebBundleBuildId(assets)
)
})
it('serializes a fixed key order regardless of the input object key order', () => {
const reordered = assets.map(({ contentType, byteLength, sha256, path }) => ({
contentType,
byteLength,
sha256,
path
}))
expect(serializeMobileWebBundleAssets(reordered)).toBe(serializeMobileWebBundleAssets(assets))
})
it('changes when any hashed field changes', () => {
const baseline = computeMobileWebBundleBuildId(assets)
for (const field of ['sha256', 'byteLength', 'contentType', 'path']) {
const mutated = assets.map((asset, index) =>
index === 0 ? { ...asset, [field]: field === 'byteLength' ? 4 : `${asset[field]}x` } : asset
)
expect(computeMobileWebBundleBuildId(mutated)).not.toBe(baseline)
}
})
})
describe('isDirectInvocation', () => {
const thisFile = import.meta.filename
it('matches the path this module was loaded from', () => {
expect(isDirectInvocation(import.meta.url, thisFile)).toBe(true)
})
it('does not match a different script', () => {
expect(isDirectInvocation(import.meta.url, join(thisFile, '..', 'other.mjs'))).toBe(false)
})
it('tolerates an absent argv[1]', () => {
expect(isDirectInvocation(import.meta.url, undefined)).toBe(false)
expect(isDirectInvocation(import.meta.url, '')).toBe(false)
})
// Why an injected converter: a win32 path cannot be exercised through node:url's pathToFileURL
// on a posix runner, and CI is ubuntu.
const toWin32FileUrl = (windowsPath) => new URL(`file:///${windowsPath.replaceAll('\\', '/')}`)
it('matches a Windows entry path, which the file:// template form never does', () => {
const scriptPath = 'C:\\orca\\config\\scripts\\build-mobile-web-bundle.mjs'
const moduleUrl = 'file:///C:/orca/config/scripts/build-mobile-web-bundle.mjs'
const keepAsIs = (path) => path
expect(
isDirectInvocation(moduleUrl, scriptPath, {
toFileUrl: toWin32FileUrl,
realpath: keepAsIs
})
).toBe(true)
// The regression this guards: `file://${argv[1]}` yields file://C:\orca\... on Windows,
// so the builder exited 0 having written nothing and packaging failed downstream.
expect(`file://${scriptPath}`).not.toBe(moduleUrl)
})
it('is not written with the file:// template form', async () => {
const source = await readFile(new URL('./build-mobile-web-bundle.mjs', import.meta.url), 'utf8')
expect(source).not.toMatch(/file:\/\/\$\{process\.argv\[1\]\}/)
expect(source).toContain('pathToFileURL')
})
})
describe('mobile web source line endings', () => {
it('accepts the committed source tree', async () => {
await expect(assertNoCarriageReturnsInSource()).resolves.toBeUndefined()
})
it('rejects a CRLF source file, because CRLF changes every asset hash and the buildId', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-eol-'))
try {
await writeFile(join(scratch, 'bootstrap.ts'), 'const a = 1\r\nconst b = 2\r\n', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).rejects.toThrow(
/CRLF in mobile web source/
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('pins eol=lf for every committed text source and -text for the binary', () => {
const files = execFileSync('git', ['ls-files', 'src/mobile-web'], { encoding: 'utf8' })
.split('\n')
.filter(Boolean)
expect(files.length).toBeGreaterThanOrEqual(4)
for (const file of files) {
const attributes = execFileSync('git', ['check-attr', 'text', 'eol', '--', file], {
encoding: 'utf8'
})
if (file.endsWith('.png')) {
expect(attributes).toContain('text: unset')
} else {
expect(attributes).toContain('eol: lf')
}
}
})
})
describe('running the builder through a symlink', () => {
// Node resolves symlinks in import.meta.url but not in argv[1]. Before the guard realpath'd the
// entry path, `node /tmp/<link>` compared /tmp against /private/tmp and the builder exited 0
// having written nothing — a green packaging job with no bundle in it.
it('still recognises the entry module', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-link-'))
try {
const builderUrl = new URL('./build-mobile-web-bundle.mjs', import.meta.url).href
const real = join(scratch, 'entry.mjs')
await writeFile(
real,
`import { isDirectInvocation } from ${JSON.stringify(builderUrl)}\n` +
'process.stdout.write(String(isDirectInvocation(import.meta.url, process.argv[1])))\n',
'utf8'
)
const link = join(scratch, 'entry-link.mjs')
await symlink(real, link)
expect(execFileSync(process.execPath, [link], { encoding: 'utf8' })).toBe('true')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
@@ -32,25 +32,37 @@ describe('CI dependency download caches', () => {
describe('release install targets', () => {
const macCpuFlag = '--cpu=current,x64,arm64'
// Both shapes: `run:` steps and steps wrapped in nick-fields/retry (`with.command`).
const installCommand = (step) => step.with?.command ?? step.run
const installSteps = (name) =>
Object.values(workflow(name).jobs)
.flatMap((job) => job.steps ?? [])
.map((step) => step.with?.command ?? step.run)
.filter((command) => typeof command === 'string' && command.includes('pnpm install '))
.filter((step) => installCommand(step)?.includes('pnpm install '))
const installCommands = (name) => installSteps(name).map(installCommand)
it.each(['adhoc-mac-build', 'daily-mac-build', 'hourly-mac-build', 'release-mac-build'])(
'%s installs both mac CPU variants for the x64+arm64 package config',
(name) => {
const installs = installSteps(name)
const installs = installCommands(name)
expect(installs.length).toBeGreaterThan(0)
expect(installs.some((command) => command.includes(macCpuFlag))).toBe(true)
}
)
// A transient `read ECONNRESET` fetching this Node version's headers for
// native/windows-registry's node-gyp rebuild failed a blocking golden gate and the cut.
it('retries every release-cut install so one transient download cannot fail a cut', () => {
const installs = installSteps('release-cut')
expect(installs.length).toBeGreaterThan(0)
for (const step of installs) {
expect(step.uses).toBe('nick-fields/retry@v4')
expect(step.with.max_attempts).toBeGreaterThan(1)
}
})
it.each(['release-cut', 'dev-channel-win-build', 'windows-signing-rehearsal'])(
'%s keeps installs scoped to the runner host',
(name) => {
const installs = installSteps(name)
const installs = installCommands(name)
expect(installs.length).toBeGreaterThan(0)
for (const command of installs) {
expect(command).not.toContain('--os=')

Some files were not shown because too many files have changed in this diff Show More