starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-22 08:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into np-oom-scan-terminal-tail-termination

Browse Source
This commit is contained in:
OrcaWin
2026-09-17 20:10:54 -07:00
committed by GitHub
parent cdbf8609e7 9c92136009
commit 06cc2b69e6
2160 changed files with 380737 additions and 243287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+7
View File
@@ -41,3 +41,10 @@
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
# Mobile web bundle source. Every text byte here is hashed into an asset digest and
# from there into buildId, so a CRLF checkout produces a different bundle id for the
# same commit (91af2897 vs 9d78435e). The PNG is -text because it must not be touched.
/src/mobile-web/index.html text eol=lf
/src/mobile-web/src/*.ts text eol=lf
/src/mobile-web/src/*.css text eol=lf
/src/mobile-web/src/*.png -text
.github/pull_request_template.md
+4 -4
View File
@@ -1,14 +1,14 @@
## ELI5
<!-- Simple high-level explanation -->
<!-- Simple high-level explanation, in plain language. No jargon. -->
## What Changed
<!-- Describe the change clearly and keep scope tight. -->
<!-- Describe the change clearly and keep scope tight. Cover the before and after as the user experiences it, and the mechanism you changed — not just the symptom. -->
## Why
<!-- What problem does this solve, and why is this approach right? -->
<!-- What problem does this solve, and why is this approach better than the alternatives you considered? -->
## Linked Issue
@@ -47,7 +47,7 @@ Ensure no issues in: Security, Cross-platoform support (Linux, Windows, Mac), Re
## Checklist
- [ ] This PR is small and focused
- [ ] I explained what changed and why (including ELI5)
- [ ] I explained what changed and why (ELI5, the user-facing before/after, the mechanism, and why over the alternatives)
- [ ] Before/after screenshots or videos attached for UI changes, or `N/A` with reason
- [ ] Self-reviewed for correctness, security, and performance
- [ ] Cross-platform, SSH/remote, and path/shortcut impact considered (or N/A)
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+209 -50
View File
@@ -16,6 +16,8 @@ on:
expected-rehome-generation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
monitor-run-attempt: { required: true, type: string }
gate-override-reason: { required: false, type: string, default: '' }
gate-override-confirmation: { required: false, type: string, default: '' }
wave-index: { required: true, type: string }
permissions:
@@ -52,7 +54,12 @@ jobs:
WAVE_INDEX: ${{ inputs.wave-index }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
# ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
# 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
DRAIN_PACE_WINDOW_MS: '120000'
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
@@ -72,12 +79,14 @@ jobs:
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
# The caller validated this too; re-check here so a malformed override
# can never reach a mutation through this reusable workflow.
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
test "${DEPLOY_MODE}" != verify
test "${GATE_OVERRIDE_CONFIRMATION}" = \
"SKIP_RELAY_MONITOR_GATE ${TARGET_IMAGE_DIGEST}"
[[ "${GATE_OVERRIDE_REASON}" =~ ^[[:print:]]{12,500}$ ]]
fi
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
echo "so recover each remaining cell with its own fresh monitor" >&2
@@ -109,14 +118,34 @@ jobs:
- uses: hashicorp/setup-terraform@v3
with: { terraform_wrapper: false }
# One approved-cell table, in the wave validator the dispatch gate already uses, so
# a cell's class and its wave's selector delta cannot drift apart between the two.
- name: Resolve this cell's admission class and wave selector delta
run: |
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
--cell-id "${TARGET_CELL_ID}")"
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
fi
{
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
} >> "${GITHUB_ENV}"
- name: Require fresh aggregate monitor evidence reference
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private aggregate monitor evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -125,7 +154,7 @@ jobs:
run-id: ${{ inputs.monitor-run-id }}
- name: Verify monitor evidence provenance
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
--directory "${OUTPUT_DIRECTORY}" \
@@ -138,7 +167,7 @@ jobs:
--wave-index "${WAVE_INDEX}"
- name: Download this wave's single-use safety authority
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -147,7 +176,7 @@ jobs:
run-id: ${{ github.run_id }}
- name: Require safety evidence consumed by this workflow
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
# Mutations are single-dispatch: a fresh dispatch cannot resume a
# partial batch (the canary authority binds the batch-entry selector
@@ -184,9 +213,32 @@ jobs:
# Freshness-only failures are publish lag, not health, on every wave
# including the first; the CLI still caps the retry at the wave's
# evidence-age budget, so this cannot mutate on aged evidence.
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" --retry-freshness
#
# This live recheck runs on every mutating wave, including a
# break-glass one. With the aggregate gate overridden there is no
# sealed state to read, so the expected selector comes from the
# dispatch inputs the rehome inspect below verifies against the live
# director; every threshold the sample is judged against is unchanged.
if test -n "${GATE_OVERRIDE_CONFIRMATION}"; then
jq -n \
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
| map_values(split(",") | map(select(length > 0)))' \
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
pnpm incident:relay-preflight -- \
--no-monitor-state \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
else
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
fi
- name: Require durable rehome disabled and exact selector
env:
@@ -213,10 +265,17 @@ jobs:
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
EXPECTED_HARD_CAP=1000
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c17|c18)
EXPECTED_HARD_CAP=600
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
EXPECTED_DATABASE_POOL_MAX=16
;;
*) exit 1 ;;
esac
@@ -228,14 +287,14 @@ jobs:
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
if test "${EXPECTED_REGION}" = us-central1; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_UNOBSERVED_BOUND}"
# The startup script emits a pool line only off the root default, so an unpinned cell
# must still be on that default or its plan would carry a line nothing reviews.
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_DATABASE_POOL_MAX:-10}"
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
if test "${DEPLOY_MODE}" = rollback; then
@@ -249,6 +308,14 @@ jobs:
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
fi
# The startup template emits rehome trust lines only for a declared source cell, so
# require membership exactly when either side of this roll expects those lines.
if test "${EXPECTED_REGION}" = us-central1 && {
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
}; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
@@ -264,6 +331,7 @@ jobs:
echo "MIG_NAME=${MIG_NAME}"
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
echo "EXPECTED_REGION=${EXPECTED_REGION}"
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
@@ -292,20 +360,66 @@ jobs:
}
CURRENT_RUNTIME="$(admin_post current-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
# A rollback that failed between template apply and admission restore
# leaves the cell already on the rollback image; resume from that
# state instead of demanding the pre-rollback predecessor.
# Two different failures leave the cell on the rollback image, and the image
# alone cannot tell them apart. A rollback that failed between its template
# apply and its admission restore restarted the cell, so that cell is not
# draining and resumes. A wave that stopped after its drain and before its
# template apply never restarted anything, so its cell is still draining and
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
if test "${DEPLOY_MODE}" = rollback \
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
ROLLBACK_RESUME=true
if test "${LIVE_DRAINING}" = true; then
ROLLBACK_STAGE=stranded
else
ROLLBACK_STAGE=resume
fi
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
else
ROLLBACK_RESUME=false
ROLLBACK_STAGE=roll
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
fi
if test "${ROLLBACK_STAGE}" = resume; then
ROLLBACK_RESUME=true
else
ROLLBACK_RESUME=false
fi
# A stranded cell's template still carries the image the cell is serving, so that
# is the predecessor its plan is reviewed against. A template already moved on to
# the target is refused here rather than rolled backwards under a stale review.
if test "${ROLLBACK_STAGE}" = stranded; then
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
else
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
fi
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require the cell pristine in the
# class it is declared to serve in.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
else
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
fi
# Draining sheds connections, and a migration-only cell holds none, so the flag
# carries no precondition there. It also outlives a failed wave, because the drain
# that set it is followed by no restart, which is the state a failed canary leaves.
if test "${DEPLOY_MODE}" = rollback \
|| test "${ENTRY_ADMISSION}" = migration-only; then
PRECHECK_DRAINING=either
else
PRECHECK_DRAINING=forbidden
fi
# A resumed rollback already restarted, so its cell has to come back not draining;
# that is what separates it from a wave that stopped before its template apply.
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
PREDECESSOR_DRAINING_OK=true
else
PREDECESSOR_DRAINING_OK=false
fi
RESTORED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
@@ -326,8 +440,19 @@ jobs:
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
# A migration-only cell is already isolated and is handed back isolated, so both
# halves of its wave see exactly the membership it entered with.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
fi
{
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
# The drain wait and the plan review both read the image this cell actually
# serves, which is the rollback image on a stranded cell and not the current one.
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
# The failsafe consumes these; deriving them here keeps them
# defined for a failure in any later step.
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
@@ -346,8 +471,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
(.region == $region or
($region == "us-central1" and $protocol == 0 and .region == null)) and
@@ -363,8 +487,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'[
if .role != "cell" then "role" else empty end,
if .cellId != $cell then "cellId" else empty end,
@@ -400,16 +523,6 @@ jobs:
fi
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require a pristine general cell.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
PRECHECK_DRAINING=either
else
PRECHECK_ADMISSION=general
PRECHECK_DRAINING=forbidden
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -435,18 +548,25 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Isolating a migration-only cell must be a read-only no-op; a change here would
# mean the live class is not the one this wave planned around.
if test "${ENTRY_ADMISSION}" = migration-only; then
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
fi
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
# The wait has to outlast the pacing window as well as the leases it waits on.
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat either --admission migration-only --draining required \
--activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
--timeout-ms 900000
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
--timeout-ms 1020000
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
@@ -459,8 +579,14 @@ jobs:
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
# Zero resource changes prove the prior run's apply completed and no
# restart will follow, keeping the incarnation check honest. Root
# outputs may lag a targeted apply, so judge resource_changes only.
@@ -499,9 +625,11 @@ jobs:
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--rollback-image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}" \
| jq -e '.changes == 2' >/dev/null
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -514,25 +642,48 @@ jobs:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
# A stranded cell already runs the reviewed template, so the apply above replaces
# no instance and the drain flag, which only a restart clears, would survive the
# whole wave. Roll the MIG explicitly on exactly the policy a template change uses.
# Every field is passed: gcloud persists these into the MIG's update policy, and it
# defaults the method to substitute on a group with no stateful config, so omitting
# one drifts the policy off the reviewed one and fails every later targeted plan.
if test "${ROLLBACK_STAGE}" = stranded \
&& test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then
gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--replacement-method recreate --max-surge 0 --max-unavailable 1
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
@@ -607,28 +758,36 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
--cell-incarnation "${TARGET_INCARNATION}"
- name: Restore only the verified selected cell to general admission
- name: Restore only the verified selected cell to its entry admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
# Activating a migration-only cell would promote it to a serving cell for good, so
# restore it with the idempotent isolate that reports the authoritative generation.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORE_MODE=isolate
else
RESTORE_MODE=activate
fi
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
echo "${ACTIVATE_RESULT}"
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
<<< "${ACTIVATE_RESULT}")"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
echo "${RESTORE_RESULT}"
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
<<< "${RESTORE_RESULT}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission general --draining forbidden --activity allowed \
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
--draining forbidden --activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+73 -7
View File
@@ -69,6 +69,17 @@ on:
description: Exact digest-and-cell-bound mutation confirmation
required: false
type: string
gate-override-reason:
description: >-
Break-glass only: why this wave may skip the aggregate 15-minute monitor
dry-run gate. The live per-wave preflight still runs.
required: false
type: string
gate-override-confirmation:
description: >-
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
required: false
type: string
permissions:
actions: read
@@ -111,11 +122,16 @@ jobs:
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
CONFIRMATION: ${{ inputs.confirmation }}
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
run: |
# Fails closed on a partial or mismatched override, before any mutation.
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
@@ -123,6 +139,35 @@ jobs:
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
fi
- name: Record the monitor gate override in the run summary
if: ${{ inputs.gate-override-confirmation != '' }}
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
MODE: ${{ inputs.mode }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
run: |
{
echo '## Aggregate monitor gate overridden (break-glass)'
echo
echo '| field | value |'
echo '| --- | --- |'
echo "| actor | ${ACTOR} |"
echo "| mode | ${MODE} |"
echo "| cells | ${CELL_IDS} |"
echo "| target digest | \`${TARGET_DIGEST}\` |"
echo "| reason | ${GATE_OVERRIDE_REASON} |"
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
echo
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
echo 'durable rehome disabled, the exact selector generation and membership, the'
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
echo 'single-dispatch mutation.'
} >> "${GITHUB_STEP_SUMMARY}"
- name: Download exact prior canary authority
if: ${{ inputs.mode == 'batch-apply' }}
uses: actions/download-artifact@v4
@@ -136,17 +181,23 @@ jobs:
if: ${{ inputs.mode == 'batch-apply' }}
env:
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
run: |
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
--target-digest "${{ inputs.target-image-digest }}" \
--rollback-digest "${{ inputs.rollback-image-digest }}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}"
--cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" \
--rollback-digest "${ROLLBACK_DIGEST}" \
--selector-generation "${SELECTOR_GENERATION}" \
--rehome-generation "${REHOME_GENERATION}"
- name: Reject previously consumed aggregate safety evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
env:
GH_TOKEN: ${{ github.token }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -163,7 +214,7 @@ jobs:
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
- name: Consume aggregate safety evidence for this exact wave
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -188,6 +239,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '0'
secrets: inherit
@@ -209,6 +262,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '1'
secrets: inherit
@@ -230,6 +285,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '2'
secrets: inherit
@@ -251,6 +308,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '3'
secrets: inherit
@@ -266,6 +325,10 @@ jobs:
with: { node-version: 24 }
- name: Seal exact successful canary authority
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
run: |
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
@@ -276,6 +339,9 @@ jobs:
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
--actor "${ACTOR}" \
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
- uses: actions/upload-artifact@v4
.github/workflows/mobile.yml
+95 -1
View File
@@ -9,21 +9,55 @@ on:
- ready_for_review
paths:
- 'mobile/**'
# Mobile launch contracts exercise the real host dispatcher and durable receipt store.
- 'src/main/agent-launch/**'
- 'src/main/runtime/rpc/**'
- 'src/main/runtime/runtime-rpc/**'
- 'src/main/runtime/runtime-rpc.ts'
- 'src/main/runtime/device-registry.ts'
- 'src/main/runtime/orca-runtime.ts'
- 'src/main/runtime/agent-session-*.ts'
- 'src/main/native-chat/agent-session-wire/**'
- 'src/shared/agent-launch-*.ts'
- 'src/shared/agent-session-*.ts'
- 'src/shared/new-workspace/worktree-create-collision.ts'
# Why: the mobile terminal link parsers are conformance-tested against
# these shared fixtures; desktop-side fixture edits must re-run this suite.
- 'src/shared/terminal-file-link-conformance.ts'
# Why: mobile imports the negotiated capability names directly and records
# the whole capability read verbatim in its goldens, so a capability added
# desktop-side rewrites a mobile fixture and must re-run this suite.
- 'src/shared/protocol-version.ts'
# Why: mobile's rpc-params-contract.ts is a type-only re-export of the
# generated params catalog, and mobile/tsconfig.json includes **/*.ts. A
# schema edit anywhere under here changes mobile's types, so a desktop-only
# change can break mobile's typecheck with no other mobile signal.
- 'src/shared/rpc-contract/**'
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
- '.github/actions/install-node-dependencies/**'
- '.github/workflows/mobile-ios-release.yml'
# Why main too: a behaviour-change branch legitimately pins its own last fenced commit, and that
# commit only stops being reachable when the branch squash-merges. The pull_request run cannot
# see that; this one is where the pin guard finds it.
push:
branches:
- main
paths:
- 'mobile/**'
- '.github/workflows/mobile.yml'
concurrency:
group: mobile-${{ github.event.pull_request.number || github.ref }}
# Per commit on main, not per branch. GitHub cancels any PENDING run in a group when a new one
# queues, whatever `cancel-in-progress` says, so one shared main group drops the middle merge of
# three -- and a pin that breaks there is exactly what this workflow now checks for.
group: mobile-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
verify:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
env:
@@ -62,6 +96,13 @@ jobs:
- name: Typecheck
run: pnpm typecheck
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
# drifted. This fails when a test file that checks today stops checking, when a test leaves
# the program, and on @ts-nocheck; the baseline may only shrink.
- name: Typecheck tests (ratchet)
run: pnpm run check:tests-typecheck
- name: Test
run: pnpm test
@@ -87,3 +128,56 @@ jobs:
- name: Check formatting
run: pnpm format:check
recording-pin:
name: RPC recording pin
runs-on: ubuntu-latest
defaults:
run:
working-directory: mobile
steps:
- name: Checkout
uses: actions/checkout@v6
with:
# The ancestry verdict is read straight off history. On a shallow checkout
# `git merge-base --is-ancestor` answers from grafted parents, so the guard refuses to
# answer at all rather than reporting a pass it has no evidence for -- and the pinned tree
# below has to be checkable out.
fetch-depth: 0
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
# Seconds. No `--ref`, so the pin is judged against the same tree it was read out of. On a
# pull request that is the merge preview, which already carries main's repins; judging the
# branch head instead fails every branch cut before the day's repin, and its instruction would
# tell the author to pin their own head -- creating the break this guard exists to catch. A
# branch that pins its own commit passes here and fails on the push after the squash, which is
# where the pin actually leaves the history.
- name: Check the recording pin is reachable
shell: bash
run: pnpm exec tsx scripts/rpc-recording-pin-guard.mts ancestry
# ~2 min locally for the record itself, so it is gated rather than run twice over. A pull
# request that moves none of the corpus, the manifest or the recorder cannot move this
# verdict away from the one the base commit already published, and `verify` replays the
# corpus against the branch tree in the meantime. A push to main has no `verify` job and is
# where a squash lands a spliced corpus, so there it always runs.
- name: Reproduce the corpus from the pinned tree
shell: bash
env:
PIN_GUARD_BASE: ${{ github.event.pull_request.base.sha }}
run: |
if [ -n "$PIN_GUARD_BASE" ]; then
pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce --if-changed-since "$PIN_GUARD_BASE"
else
pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce
fi
.github/workflows/pr.yml
+11
View File
@@ -780,6 +780,12 @@ jobs:
- name: Project web client from renderer build
run: pnpm run build:web-from-renderer
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
# step instead of calling build:release, and electron-builder's beforePack guard hard-fails
# without out/mobile-web.
- name: Build mobile web bundle
run: pnpm run build:mobile-web
- name: Build native components
run: pnpm run build:native
@@ -871,6 +877,11 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
config/scripts/rebuild-native-deps-node-pty.test.mjs
config/scripts/ensure-native-runtime-job-ownership.test.mjs
config/scripts/verify-packaged-node-pty-job-ownership.test.mjs
config/scripts/windows-pe-machine.test.mjs
config/scripts/script-module-dependencies.test.mjs
src/main/windows-registry-addon.test.ts
config/scripts/windows-process-tree-gyp-path.test.mjs
config/scripts/windows-process-tree-gyp-rebuild.test.mjs
.gitignore
+1
View File
@@ -122,6 +122,7 @@ docs/**
!docs/reference/windows-cmd-shim-resolution.md
!docs/reference/windows-daemon-host-relocation.md
!docs/reference/windows-edr-posture.md
!docs/reference/windows-msys-job-breakaway.md
!docs/reference/windows-process-enumeration.md
!docs/reference/wsl-runner-verification.md
!docs/reference/remote-wire-compatibility.md
AGENTS.md
+12
View File
@@ -48,6 +48,17 @@ Avoid type assertions except `as const`. Unavoidable casts need a line-specific
- **Lint**: `oxlint`, or `pnpm run check:code-quality:changed` for changed files (full `pnpm lint` is slow); format with `pnpm format`
- **Design system**: `pnpm run lint:design-system` for the full renderer report (not a gate); the changed-lines gate above is what CI enforces
# Writing Pull Requests
Fill in [`.github/pull_request_template.md`](./.github/pull_request_template.md), written for a reviewer who has never seen this code:
- No jargon — plain language, no internal shorthand.
- The before and after as the user experiences it.
- The mechanism you changed, not just the symptom.
- Why this approach over the alternatives you considered.
Cover all four concisely. Don't pad or walk the diff.
# Considerations
## Worktree Safety
@@ -65,6 +76,7 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import. Recognised npm/pnpm `.cmd` shims are resolved to their real target so the spawn skips `cmd.exe` entirely; see [`docs/reference/windows-cmd-shim-resolution.md`](./docs/reference/windows-cmd-shim-resolution.md) before adding a shim shape or debugging one.
- **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
- **Windows MSYS/Git Bash panes**: their children break away from the per-PTY job unless it is created without `JOB_OBJECT_LIMIT_BREAKAWAY_OK`, and a `conpty.node` built before that fix passes every existing gate. Before changing the per-PTY job or debugging `windows-msys-job.win32.test.ts`, read [`docs/reference/windows-msys-job-breakaway.md`](./docs/reference/windows-msys-job-breakaway.md).
- **Windows daemon-host relocation**: the terminal daemon runs from a copy of the app runtime under `%LOCALAPPDATA%`, which is what survives an auto-update. Before touching that copy, its exe name, or the NSIS uninstall macro, read [`docs/reference/windows-daemon-host-relocation.md`](./docs/reference/windows-daemon-host-relocation.md).
- **Windows EDR signal**: don't add `-ExecutionPolicy Bypass`, `-EncodedCommand`, `cmd.exe /c` with escaped free text, per-operation interpreter spawning, or runtime `Add-Type` compilation without reading [`docs/reference/windows-edr-posture.md`](./docs/reference/windows-edr-posture.md) first — behavioural EDR scores each of those, and being signed does not clear them.
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
cloud/apps/push/package.json
+1 -1
View File
@@ -30,6 +30,6 @@
"@types/pg": "^8.20.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/apps/push/src/push-database.ts
+4 -1
View File
@@ -213,7 +213,10 @@ async function applySchemaOnUntimedPool(
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(pushSchemaStatements(), (statement) => database.query(statement), {
eventPrefix: 'orca_push_postgres_schema'
eventPrefix: 'orca_push_postgres_schema',
// Push has no catalog pre-check, so a lock timeout here says nothing about whether the
// object already exists and the old bounded retry is still the right answer.
retryLockTimeout: true
})
} finally {
await database.close().catch(() => undefined)
cloud/apps/relay-fence-broker/package.json
+1 -1
View File
@@ -22,6 +22,6 @@
"@types/node": "^24.10.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/apps/relay-ops/package.json
+1 -1
View File
@@ -24,6 +24,6 @@
"@types/node": "^24.10.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
+444 -11
View File
@@ -10,6 +10,7 @@ import {
INCIDENT_MONITOR_THRESHOLDS,
type IncidentSample
} from './incident-monitor.js'
import { relayOpsEnvironment } from './environment-config.js'
import type { AdmissionSelector } from './incident-selector.js'
const directories: string[] = []
@@ -63,6 +64,38 @@ function stateFile(
return path
}
// Every configured production cell, in the lexicographic order
// normalizeSelectorMembership canonicalises to. Derived from the same durable
// Terraform config the override path reads, so a new cell cannot strand these.
const configuredCellIds = relayOpsEnvironment('production').cells.map(
(cell) => cell.cellId
)
const canonicalCellIds = [...configuredCellIds].sort()
const canonicalMembership = {
existingOnly: canonicalCellIds,
migrationOnly: [],
general: []
}
// What the director reports: a normalised selector, never an echo of what the
// caller expected. An order-sensitive comparison only holds if the override path
// canonicalises its own input the same way.
function canonicalSample(generation = 1): IncidentSample {
const next = sample()
next.selector = { generation, membership: canonicalMembership }
return next
}
function membershipFile(
membership: Record<string, string[]> = canonicalMembership
): string {
const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-selector-'))
directories.push(directory)
const path = join(directory, 'selector.json')
writeFileSync(path, JSON.stringify(membership))
return path
}
function sample(): IncidentSample {
const observedAt = new Date(now).toISOString()
const signal = (value: number) => ({ value, observedAt })
@@ -154,9 +187,9 @@ describe('relay incident live preflight', () => {
)).resolves.toBeUndefined()
})
it('rejects monitor evidence beyond the 25-minute lineage bound', async () => {
it('rejects monitor evidence beyond the 35-minute lineage bound', async () => {
const path = stateFile('strict', {
startedAt: new Date(now - 26 * 60_000 - 1).toISOString()
startedAt: new Date(now - 36 * 60_000 - 1).toISOString()
})
await expect(runIncidentLivePreflight(
['--state-file', path],
@@ -183,24 +216,31 @@ describe('relay incident live preflight', () => {
await expect(runIncidentLivePreflight(
['--state-file', oneRollOld, '--wave-index', '1'], deps
)).resolves.toBeUndefined()
// Both edges of one predecessor job timeout: 5min + 75min exactly.
// Wave 0 edges: the 10-minute bound covers same-cap job start-up latency.
await expect(runIncidentLivePreflight(
['--state-file', agedState(80 * 60_000), '--wave-index', '1'], deps
['--state-file', agedState(10 * 60_000), '--wave-index', '0'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(80 * 60_000 + 1), '--wave-index', '1'], deps
['--state-file', agedState(10 * 60_000 + 1), '--wave-index', '0'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// Both edges of one predecessor job timeout: 10min + 75min exactly.
await expect(runIncidentLivePreflight(
['--state-file', agedState(85 * 60_000), '--wave-index', '1'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(85 * 60_000 + 1), '--wave-index', '1'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
await expect(runIncidentLivePreflight(
['--state-file', agedState(155 * 60_000), '--wave-index', '2'], deps
['--state-file', agedState(160 * 60_000), '--wave-index', '2'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(155 * 60_000 + 1), '--wave-index', '2'], deps
['--state-file', agedState(160 * 60_000 + 1), '--wave-index', '2'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
await expect(runIncidentLivePreflight(
['--state-file', agedState(230 * 60_000), '--wave-index', '3'], deps
['--state-file', agedState(235 * 60_000), '--wave-index', '3'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(230 * 60_000 + 1), '--wave-index', '3'], deps
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-3 argument.
await expect(runIncidentLivePreflight(
@@ -260,9 +300,11 @@ describe('relay incident live preflight', () => {
slowCell.sources['active-probe']!.signals[
'cell.production-gce-c1.latency_ms'
]!.value = 2_568
// A cell probe now re-samples before it fails a wave, so the wait is injected;
// this cell stays slow on every sample and still names what stopped it.
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: async () => slowCell }
{ now: () => now, collect: async () => slowCell, wait: async () => {} }
)).rejects.toThrow(
'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
)
@@ -278,6 +320,150 @@ describe('relay incident live preflight', () => {
)
})
// Why: this one sample decides a mutating wave, so an Asia cell's ~30 s
// "no healthy upstream" window could still fail a wave here even after the
// 15-minute gate learned to ride it out.
describe('cell probe tolerance', () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
// Serves `badSamples` unhealthy cell readings, then healthy ones.
const downThen = (badSamples: number) => {
let index = 0
return async () => {
const next = sample()
if (index++ < badSamples) {
next.sources['active-probe']!.signals['cell.production-gce-c1.health']!
.value = 0
next.sources['active-probe']!.signals['cell.production-gce-c1.ready']!
.value = 0
}
return next
}
}
it('re-samples through a probe outage within the tolerance', async () => {
const waits: number[] = []
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: downThen(tolerance),
wait: async (ms) => {
waits.push(ms)
}
}
)).resolves.toBeUndefined()
expect(waits).toHaveLength(tolerance)
})
it('fails the wave once the probe outage outlasts the tolerance', async () => {
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: downThen(tolerance + 1),
wait: async () => {}
}
)).rejects.toThrow('active-probe/threshold_equal cell.production-gce-c1.health')
})
it('does not re-sample a director probe failure', async () => {
let samples = 0
const down = async () => {
samples++
const next = sample()
next.sources['active-probe']!.signals['director.health']!.value = 0
return next
}
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: down, wait: async () => {} }
)).rejects.toThrow('active-probe/threshold_equal director.health')
expect(samples).toBe(1)
})
it('does not re-sample a non-probe threshold failure', async () => {
let samples = 0
const hot = async () => {
samples++
const next = sample()
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
return next
}
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: hot, wait: async () => {} }
)).rejects.toThrow('cloud-monitoring/threshold_max')
expect(samples).toBe(1)
})
})
// Why: on 2026-09-17 a canary wave died because one director admin read
// returned 404 for a 2 s Cloud SQL pool timeout. Collecting the sample is not
// a health verdict, so a thrown collector spends an attempt instead.
describe('collector failures', () => {
it('re-samples after a thrown collector and then passes', async () => {
const waits: number[] = []
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: async () => {
attempts++
if (attempts === 1) throw new Error('Relay admin telemetry returned 404')
return sample()
},
wait: async (ms) => {
waits.push(ms)
}
}
)).resolves.toBeUndefined()
expect(attempts).toBe(2)
expect(waits).toEqual([15_000])
})
it('fails the wave when every attempt throws, naming the collector', async () => {
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{
now: () => now,
collect: async () => {
attempts++
throw new Error('Relay admin telemetry returned 404')
},
wait: async () => {}
}
)).rejects.toThrow(
'relay live preflight failed: collector: Relay admin telemetry returned 404'
)
expect(attempts).toBe(1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples)
})
it('does not re-sample a collector failure past the evidence-age budget', async () => {
const agedPath = stateFile('strict', {
startedAt: new Date(now - 26 * 60_000).toISOString(),
windowStartedAt: new Date(now - 25 * 60_000).toISOString(),
lastSampleAt: new Date(now - 10 * 60_000 + 7).toISOString(),
completedAt: new Date(now - 10 * 60_000 + 14).toISOString()
})
let attempts = 0
await expect(runIncidentLivePreflight(
['--state-file', agedPath],
{
now: () => now,
collect: async () => {
attempts++
throw new Error('Relay admin telemetry returned 404')
},
wait: async () => {}
}
)).rejects.toThrow('relay live preflight failed: collector:')
expect(attempts).toBe(1)
})
})
it('enforces the signed migration policy', async () => {
const inactiveTarget = sample()
inactiveTarget.sources['director-admin']!.signals[
@@ -374,7 +560,7 @@ describe('relay incident live preflight', () => {
})
it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
const completedAt = now - 290_000
const completedAt = now - 590_000
const stale = sample()
stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
const collect = vi.fn(async () => stale)
@@ -420,6 +606,253 @@ describe('relay incident live preflight', () => {
expect(wait).toHaveBeenCalledTimes(4)
})
// Why: the same-cap break-glass skips the sealed 15-minute aggregate evidence,
// so this live recheck is the only thing left standing between the dispatch and
// a mutation. It must judge the fleet exactly as it does with evidence, and it
// must never accept a half-specified override.
describe('break-glass without monitor state', () => {
const overrideArgs = (extra: string[] = [], membership = canonicalMembership) => [
'--no-monitor-state',
'--expected-selector-generation', '1',
'--selector-membership-file', membershipFile(membership),
...extra
]
// The director's reading, plus whatever the override path decided to expect.
const liveCollect = (
mutate: (next: IncidentSample) => IncidentSample = (next) => next
) => async (expected: AdmissionSelector) => {
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return mutate(next)
}
it('accepts one complete fresh green sample with no sealed evidence', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{ now: () => now, collect: liveCollect() }
)).resolves.toBeUndefined()
})
// Why: the live selector is normalised and the comparison is an ordered
// stringify, so an operator's unsorted membership must canonicalise here or
// every override wave reads as selector drift on a healthy fleet.
it('canonicalises an unsorted operator membership', async () => {
const shuffled = {
existingOnly: [...canonicalCellIds].reverse(),
migrationOnly: [],
general: []
}
expect(shuffled.existingOnly).not.toEqual(canonicalCellIds)
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(
overrideArgs([], shuffled),
{
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
}
)).resolves.toBeUndefined()
expect(seen[0]!.membership.existingOnly).toEqual(canonicalCellIds)
})
// Why: normalising is also what enforces every configured cell exactly once,
// which a bare schema parse would have dropped.
it('rejects a membership that is not every configured cell exactly once', async () => {
const duplicated = {
existingOnly: [...canonicalCellIds, canonicalCellIds[0] as string],
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], duplicated),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
const missing = {
existingOnly: canonicalCellIds.slice(1),
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], missing),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
const unknown = {
existingOnly: [...canonicalCellIds.slice(1), 'production-gce-c999'],
migrationOnly: [],
general: []
}
await expect(runIncidentLivePreflight(
overrideArgs([], unknown),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('every configured cell exactly once')
})
it('fails closed on a live threshold breach', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.99
return next
}),
wait: async () => {}
}
)).rejects.toThrow('cloud-monitoring/threshold_max cloud_sql.cpu')
})
it('fails closed on a live selector mismatch', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.selector = { ...next.selector, generation: 7 }
return next
}),
wait: async () => {}
}
)).rejects.toThrow('selector_mismatch')
})
it('expects the wave-adjusted live selector generation', async () => {
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(
overrideArgs(['--wave-index', '2']),
{
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
}
)).resolves.toBeUndefined()
expect(seen[0]!.generation).toBe(5)
})
it('offsets by the wave delta the cell class declares', async () => {
const generationFor = async (args: string[]) => {
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(args, {
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
})).resolves.toBeUndefined()
return seen[0]!.generation
}
// A migration-only cell's wave isolates and restores nothing, so no predecessor moved it.
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0'])
)).toBe(1)
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2'])
)).toBe(5)
})
it('rejects a selector wave delta no cell class produces', async () => {
for (const delta of ['1', '3', '4', '', '-0', '02']) {
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', delta]),
{ now: () => now }
)).rejects.toThrow('usage:')
}
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']),
{ now: () => now }
)).rejects.toThrow('usage:')
})
it('pins the strictest migration policy', async () => {
// An inactive migration target is tolerable only under recover-forward,
// and an override cannot elect that policy, so this must still fail.
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
next.sources['director-admin']!.signals[
'cell.production-gce-c1.migration_target_inactive'
]!.value = 30
return next
}),
wait: async () => {}
}
)).rejects.toThrow('director-admin/threshold_max')
})
it('rejects a half-specified override', async () => {
const cases: string[][] = [
['--no-monitor-state'],
['--no-monitor-state', '--expected-selector-generation', '1'],
['--no-monitor-state', '--selector-membership-file', membershipFile()],
// Mixing the two sources would let a caller pass sealed evidence it
// never wants read.
[
'--no-monitor-state',
'--expected-selector-generation', '1',
'--selector-membership-file', membershipFile(),
'--state-file', stateFile()
],
// Override arguments without the flag must not be silently ignored.
['--state-file', stateFile(), '--expected-selector-generation', '1'],
['--no-monitor-state', '--no-monitor-state'],
['--expected-selector-generation', '1']
]
for (const args of cases) {
await expect(runIncidentLivePreflight(
args,
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('usage:')
}
})
it('rejects an unknown option and a negative generation', async () => {
await expect(runIncidentLivePreflight(
overrideArgs(['--skip-everything']),
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow('usage:')
await expect(runIncidentLivePreflight(
[
'--no-monitor-state',
'--expected-selector-generation', '-1',
'--selector-membership-file', membershipFile()
],
{ now: () => now, collect: liveCollect() }
)).rejects.toThrow()
})
it('re-samples a tolerable failure and then passes', async () => {
let samples = 0
await expect(runIncidentLivePreflight(
overrideArgs(),
{
now: () => now,
collect: liveCollect((next) => {
samples++
if (samples === 1) {
next.sources['cloud-monitoring']!.signals['director.instances']!.value = 2
}
return next
}),
wait: async () => {}
}
)).resolves.toBeUndefined()
expect(samples).toBe(2)
})
})
it('uses the supplied admin token without minting through gcloud', async () => {
const identityToken = vi.fn(async () => 'minted.token.value')
const gcloud = livePreflightGcloud(
cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
+207 -56
View File
@@ -2,13 +2,21 @@ import { readFile } from 'node:fs/promises'
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { z } from 'zod'
import { relayOpsEnvironment } from './environment-config.js'
import { createGcloudClient } from './gcloud-client.js'
import { suppliedIdentityToken } from './incident-monitor-cli.js'
import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
import {
AdmissionSelectorSchema,
normalizeSelectorMembership,
SelectorMembershipSchema,
type AdmissionSelector
} from './incident-selector.js'
import {
evaluateIncidentSample,
FRESHNESS_FAILURE_CODES,
INCIDENT_MONITOR_THRESHOLDS,
preDrainDryRunPassed,
toleratedStreakKey,
type IncidentFailure,
type IncidentSample
} from './incident-monitor.js'
@@ -16,10 +24,15 @@ import { createIncidentSampleCollector } from './incident-monitor-sources.js'
const FRESHNESS_RETRY_ATTEMPTS = 5
const FRESHNESS_RETRY_INTERVAL_MS = 15_000
const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
// 10 min, not 5: the same-cap job reaches this check ~5 min after the monitor
// completes (runner queue ~2 min, gate job ~80 s, checkout ~60 s); on 2026-09-17
// a green gate died at 302 s. The live samples below hold every wave to now.
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX_PATTERN = /^[0-3]$/
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
export function livePreflightGcloud(
gcloud: ReturnType<typeof createGcloudClient>,
@@ -81,51 +94,80 @@ function describeFailure(failure: IncidentFailure): string {
return [`${failure.source}/${failure.code}`, ...detail].join(' ')
}
export async function runIncidentLivePreflight(
argv: string[],
dependencies: {
now?: () => number
wait?: (ms: number) => Promise<void>
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
gcloud?: ReturnType<typeof createGcloudClient>
environment?: NodeJS.ProcessEnv
} = {}
): Promise<void> {
const PREFLIGHT_USAGE =
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>] [--retry-freshness]' +
' | --no-monitor-state --expected-selector-generation <n>' +
' --selector-membership-file <json> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>]'
const VALUE_OPTIONS = new Set([
'--state-file',
'--wave-index',
'--selector-wave-delta',
'--expected-selector-generation',
'--selector-membership-file'
])
const FLAG_OPTIONS = new Set(['--retry-freshness', '--no-monitor-state'])
// Rejects an unknown option and a repeated one, so a typo can never silently
// widen what this check accepts.
export function parsePreflightArgs(argv: string[]): {
options: Map<string, string>
flags: Set<string>
} {
const args = argv[0] === '--' ? argv.slice(1) : argv
const freshnessRetryCount = args.filter((arg) => arg === '--retry-freshness').length
const rest = args.filter((arg) => arg !== '--retry-freshness')
const stateArgs: string[] = []
let waveIndex = '0'
let waveIndexCount = 0
for (let index = 0; index < rest.length; index += 1) {
if (rest[index] === '--wave-index') {
waveIndexCount += 1
waveIndex = rest[index + 1] ?? ''
index += 1
} else {
stateArgs.push(rest[index] as string)
const options = new Map<string, string>()
const flags = new Set<string>()
for (let index = 0; index < args.length; index += 1) {
const arg = args[index] as string
if (FLAG_OPTIONS.has(arg)) {
if (flags.has(arg)) throw new Error(PREFLIGHT_USAGE)
flags.add(arg)
continue
}
const value = args[index + 1]
if (!VALUE_OPTIONS.has(arg) || options.has(arg) || !value) {
throw new Error(PREFLIGHT_USAGE)
}
options.set(arg, value)
index += 1
}
return { options, flags }
}
// What the live recheck measures the fleet against. Either source supplies the
// exact same fields; only where they come from and how they can go stale differs.
type PreflightPlan = {
environment: 'production'
expectedSelector: AdmissionSelector
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
recoverySourceCellId: string | null
capacityCellId: string | null
// The instant the live samples age from. Monitor evidence ages from the moment
// the 15-minute window closed; an override has no evidence to age, so its
// retry budget starts when this process does.
evidenceAnchorMs: number
}
async function monitorEvidencePreflightPlan(
options: Map<string, string>,
nowMs: number,
waveIndex: string
): Promise<PreflightPlan> {
const stateFile = options.get('--state-file')
if (
freshnessRetryCount > 1 ||
waveIndexCount > 1 ||
!WAVE_INDEX_PATTERN.test(waveIndex) ||
stateArgs.length !== 2 ||
stateArgs[0] !== '--state-file' ||
!stateArgs[1]
) {
throw new Error(
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>] [--retry-freshness]'
)
}
!stateFile ||
options.has('--expected-selector-generation') ||
options.has('--selector-membership-file')
) throw new Error(PREFLIGHT_USAGE)
const state = PreflightStateSchema.parse(
JSON.parse(await readFile(resolve(stateArgs[1]), 'utf8'))
JSON.parse(await readFile(resolve(stateFile), 'utf8'))
)
const now = dependencies.now ?? Date.now
const completedAt = Date.parse(state.completedAt)
const windowStartedAt = Date.parse(state.windowStartedAt)
const lastSampleAt = Date.parse(state.lastSampleAt)
const evidenceAgeMs = now() - completedAt
const evidenceAgeMs = nowMs - completedAt
// Later same-cap waves start after sequential predecessor cell rolls, so the
// freshness bound grows by one cell-job timeout per predecessor; the live
// samples collected below still hold every wave to current health.
@@ -144,19 +186,89 @@ export async function runIncidentLivePreflight(
) {
throw new Error('relay live preflight monitor evidence is incomplete or stale')
}
return {
environment: state.environment,
expectedSelector: state.expectedSelector,
migrationPolicy: state.migrationPolicy,
recoverySourceCellId: state.recoverySourceCellId,
capacityCellId: state.capacityCellId,
evidenceAnchorMs: completedAt
}
}
// Break-glass: the caller authorized skipping the aggregate 15-minute monitor
// gate, so the expected selector comes straight from the dispatch inputs instead
// of sealed evidence. Nothing about this weakens the live sample below, and the
// policy is pinned to strict -- the only one the same-cap rollout ever verifies.
async function overridePreflightPlan(
options: Map<string, string>,
nowMs: number
): Promise<PreflightPlan> {
const generation = options.get('--expected-selector-generation')
const membershipFile = options.get('--selector-membership-file')
if (!generation || !membershipFile || options.has('--state-file')) {
throw new Error(PREFLIGHT_USAGE)
}
// Canonicalise exactly as the monitor CLI does when it seals evidence. The live
// selector read from the director is normalised too and the comparison is an
// ordered stringify, so unsorted operator input would read as selector drift on
// a healthy fleet; normalising is also what enforces every configured cell
// exactly once.
const membership = normalizeSelectorMembership(
SelectorMembershipSchema.parse(
JSON.parse(await readFile(resolve(membershipFile), 'utf8'))
),
new Set(relayOpsEnvironment('production').cells.map((cell) => cell.cellId))
)
return {
environment: 'production',
expectedSelector: AdmissionSelectorSchema.parse({
generation: Number(generation),
membership
}),
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
evidenceAnchorMs: nowMs
}
}
export async function runIncidentLivePreflight(
argv: string[],
dependencies: {
now?: () => number
wait?: (ms: number) => Promise<void>
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
gcloud?: ReturnType<typeof createGcloudClient>
environment?: NodeJS.ProcessEnv
} = {}
): Promise<void> {
const parsed = parsePreflightArgs(argv)
const waveIndex = parsed.options.get('--wave-index') ?? '0'
if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE)
const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2'
if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE)
const now = dependencies.now ?? Date.now
const plan = parsed.flags.has('--no-monitor-state')
? await overridePreflightPlan(parsed.options, now())
: await monitorEvidencePreflightPlan(parsed.options, now(), waveIndex)
const maxEvidenceAgeMs =
MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
const gcloud = livePreflightGcloud(
dependencies.gcloud ?? createGcloudClient(),
dependencies.environment
)
// Each predecessor same-cap apply wave reversibly isolates and restores its
// cell, advancing the selector generation by exactly 2 with membership
// unchanged (rollback is single-cell, so it never reaches a later wave), so
// the live selector comparison must expect the wave-adjusted generation.
// cell with membership unchanged (rollback is single-cell, so it never reaches
// a later wave), so the live selector comparison must expect the wave-adjusted
// generation. A general cell advances it by 2; a migration-only cell is already
// isolated and stays that way, so its wave advances it by 0. A wave is never
// mixed, so one delta covers every predecessor.
const collectOptions = {
environment: state.environment,
environment: plan.environment,
expectedSelector: {
...state.expectedSelector,
generation: state.expectedSelector.generation + 2 * Number(waveIndex)
...plan.expectedSelector,
generation: plan.expectedSelector.generation + Number(selectorWaveDelta) * Number(waveIndex)
},
...(dependencies.now ? { now: dependencies.now } : {})
}
@@ -167,32 +279,71 @@ export async function runIncidentLivePreflight(
const wait = dependencies.wait ?? ((ms: number) => new Promise<void>((resolveWait) => {
setTimeout(resolveWait, ms)
}))
const attempts = freshnessRetryCount === 1 ? FRESHNESS_RETRY_ATTEMPTS : 1
const freshnessAttempts = parsed.flags.has('--retry-freshness') ? FRESHNESS_RETRY_ATTEMPTS : 1
// Why: this single sample decides a mutating wave, so an Asia cell's ~30 s
// "no healthy upstream" window, or a one-minute director instance-replacement
// dip, could fail a wave here even after the 15-minute gate learned to ride it
// out. Hold the two to the same tolerance. Unlike the freshness retry this
// needs no flag, because a tolerated breach is never the operator's call to
// waive.
const cellProbeAttempts = 1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const attempts = Math.max(freshnessAttempts, cellProbeAttempts)
let freshnessRetries = freshnessAttempts - 1
let cellProbeRetries = cellProbeAttempts - 1
// Waiting must never carry the mutation past the same evidence-age bound the
// entry check enforces, so the wave budget also caps the retry window.
const budgetExhausted = (): boolean =>
now() + FRESHNESS_RETRY_INTERVAL_MS - plan.evidenceAnchorMs > maxEvidenceAgeMs
for (let attempt = 1; attempt <= attempts; attempt++) {
// A director admin read can fail on its own (its handler maps a Cloud SQL
// pool timeout onto 404), which says nothing about relay health; spend an
// attempt on it rather than failing the wave on one unlucky sample.
let sample: IncidentSample
try {
sample = await collect()
} catch (error) {
const message = error instanceof Error ? error.message : 'sample collection failed'
if (attempt === attempts || budgetExhausted()) {
throw new Error(`relay live preflight failed: collector: ${message}`)
}
console.warn(
`relay live preflight re-sampling after collector failure (${attempt}/${attempts - 1})`
)
await wait(FRESHNESS_RETRY_INTERVAL_MS)
continue
}
const evaluation = evaluateIncidentSample(
await collect(),
sample,
now(),
state.migrationPolicy,
state.recoverySourceCellId,
state.capacityCellId
plan.migrationPolicy,
plan.recoverySourceCellId,
plan.capacityCellId
)
if (evaluation.status === 'green') return
const freshnessOnly = evaluation.failures.every((failure) =>
const freshnessFailures = evaluation.failures.filter((failure) =>
FRESHNESS_FAILURE_CODES.has(failure.code)
)
// Waiting must never carry the mutation past the same evidence-age bound
// the entry check enforces, so the wave budget also caps the retry window.
const budgetExhausted =
now() + FRESHNESS_RETRY_INTERVAL_MS - completedAt > maxEvidenceAgeMs
if (!freshnessOnly || attempt === attempts || budgetExhausted) {
// Tolerated readings only (per-cell probes and the director instance count).
// The director and auth health probes are absent here on purpose and fail
// the wave on their first bad sample.
const cellProbeFailures = evaluation.failures.filter((failure) =>
!FRESHNESS_FAILURE_CODES.has(failure.code) && toleratedStreakKey(failure) !== null
)
const retryable =
freshnessFailures.length + cellProbeFailures.length === evaluation.failures.length &&
(freshnessFailures.length === 0 || freshnessRetries > 0) &&
(cellProbeFailures.length === 0 || cellProbeRetries > 0)
if (!retryable || attempt === attempts || budgetExhausted()) {
throw new Error(
`relay live preflight failed: ${evaluation.failures
.map(describeFailure)
.join(',')}`
)
}
if (freshnessFailures.length > 0) freshnessRetries--
if (cellProbeFailures.length > 0) cellProbeRetries--
console.warn(
`relay live preflight awaiting fresh evidence (${attempt}/${attempts - 1})`
`relay live preflight re-sampling after tolerable failure (${attempt}/${attempts - 1})`
)
await wait(FRESHNESS_RETRY_INTERVAL_MS)
}
cloud/apps/relay-ops/src/incident-monitor-cli.ts
+15
View File
@@ -68,6 +68,20 @@ const StateSchema = z.object({
observed: z.number().optional(),
threshold: z.number().optional()
})),
// Pre-2026-09-17 state files predate cell-probe tolerance; a resumed run that
// carries no streak is one that also restarts its window, so it earns nothing.
probeStreaks: z.record(z.string(), z.number().int().nonnegative()).default({}),
toleratedProbeEvents: z.array(z.object({
recordedAt: z.string(),
windowSequence: z.number().int().nonnegative(),
failures: z.array(z.object({
code: z.string(),
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
signal: z.string().optional(),
observed: z.number().optional(),
threshold: z.number().optional()
}))
})).default([]),
completedAt: z.string().nullable()
})
@@ -463,6 +477,7 @@ export async function runIncidentMonitorCli(
await wait(ms)
},
collect: segmentedCollect,
warn: (message) => console.warn(message),
persist: async (nextState) => await persistState(options.stateFile, nextState),
checkpoint: async (checkpoint) => {
await appendCheckpoint(options.summaryFile, checkpoint)
cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
+108
View File
@@ -458,4 +458,112 @@ describe('incident monitor sources', () => {
expect(serialized).not.toContain(identityToken)
expect(serialized).not.toContain(sensitiveIdentity)
})
// Why: the director maps a Cloud SQL pool connect timeout in cell-status onto
// a 404, so without this the whole sample dies on one 2 s database stall.
it('retries a transient director admin failure and then reports the cell', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
const selector = {
generation: 1,
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
}
const waits: number[] = []
let cellStatusCalls = 0
const fetchImpl: typeof fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
if (body.cellId) {
cellStatusCalls++
if (cellStatusCalls === 1) {
return Response.json(
{ error: 'timeout exceeded when trying to connect' },
{ status: 404 }
)
}
return Response.json({
status: {
enabled: true,
connectionCapacity: { hardCap: 600 },
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
}
})
}
return Response.json({
blocked: 0,
blockedExpiredUnregistered: 0,
registeredTargetInactive: 0
})
}
const result = await directorSignals(
'production',
selector,
gcloud,
now,
fetchImpl,
async (ms) => {
waits.push(ms)
}
)
expect(waits).toEqual([5_000])
expect(cellStatusCalls).toBe(productionCells.length + 1)
expect(result.cells).toHaveLength(productionCells.length)
expect(
result.source.signals['cell.production-gce-c1.connection_hard_cap']
).toMatchObject({ value: 600 })
})
// A rejected admin token is a decision, not weather: retrying it only burns
// the sample budget and hides the misconfiguration.
it('fails immediately on an unauthorized director admin response', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
const selector = {
generation: 1,
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
}
const waits: number[] = []
let requestCount = 0
const fetchImpl: typeof fetch = async (_input, init) => {
requestCount++
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
return Response.json({ error: 'invalid_token' }, { status: 401 })
}
await expect(
directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
waits.push(ms)
})
).rejects.toThrow('Relay admin telemetry returned 401')
expect(requestCount).toBe(2)
expect(waits).toEqual([])
})
// A 404 the director means (wrong role) must not be retried either.
it('does not retry a director-only 404', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
let requestCount = 0
const fetchImpl: typeof fetch = async () => {
requestCount++
return Response.json({ error: 'director_only' }, { status: 404 })
}
await expect(
directorSignals(
'production',
{ generation: 1, membership: { existingOnly: [], migrationOnly: [], general: productionCells } },
gcloud,
now,
fetchImpl,
async () => {}
)
).rejects.toThrow('Relay admin telemetry returned 404')
expect(requestCount).toBe(1)
})
})
cloud/apps/relay-ops/src/incident-monitor-sources.ts
+101 -17
View File
@@ -414,21 +414,96 @@ function relaySignals(
return { observedAt, signals }
}
const ADMIN_RETRY_ATTEMPTS = 3
const ADMIN_RETRY_INTERVAL_MS = 5_000
const ADMIN_RETRYABLE_STATUSES = new Set([500, 502, 503, 504])
const AdminErrorBodySchema = z.object({ error: z.string() })
// The director's /v1/admin/cell-status maps any thrown operation error onto 404
// with an {error: message} body, so a Cloud SQL pool connect timeout arrives
// here as a 404. These are the pool-acquire and transient SQLSTATE messages the
// relay app's own isRelayDatabaseTransientError treats as re-runnable.
const ADMIN_TRANSIENT_ERROR_MESSAGES = [
'timeout exceeded when trying to connect',
'Connection terminated due to connection timeout',
'Connection terminated unexpectedly',
'database_temporarily_unavailable',
'deadlock detected',
'canceling statement due to',
'too many connections',
'the database system is shutting down',
'the database system is starting up'
]
const defaultWait = async (ms: number): Promise<void> =>
await new Promise((resolveWait) => setTimeout(resolveWait, ms))
// A dropped socket or the 30 s AbortSignal firing; both leave the request with
// no verdict, and every admin read here is a plain query.
function transientFetchFailure(error: unknown): boolean {
if (error instanceof TypeError) return true
const name = (error as { name?: unknown } | null)?.name
return name === 'AbortError' || name === 'TimeoutError'
}
function retryableAdminStatus(status: number, body: string): boolean {
if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
// 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
if (status !== 404) return false
const parsed = AdminErrorBodySchema.safeParse(
((): unknown => {
try {
return JSON.parse(body)
} catch {
return null
}
})()
)
return (
parsed.success &&
ADMIN_TRANSIENT_ERROR_MESSAGES.some((message) => parsed.data.error.includes(message))
)
}
async function adminPost(
fetchImpl: typeof fetch,
origin: string,
token: string,
path: string,
body: unknown
body: unknown,
wait: (ms: number) => Promise<void> = defaultWait
): Promise<unknown> {
const response = await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
})
if (!response.ok) throw new Error(`Relay admin telemetry returned ${response.status}`)
return await response.json()
for (let attempt = 1; ; attempt++) {
const lastAttempt = attempt >= ADMIN_RETRY_ATTEMPTS
let response: Response
try {
response = await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
})
} catch (error) {
if (lastAttempt || !transientFetchFailure(error)) throw error
console.warn(
`relay admin telemetry retrying ${path} after network failure` +
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
)
await wait(ADMIN_RETRY_INTERVAL_MS)
continue
}
if (response.ok) return await response.json()
// Status only: the body carries a director error message, which must not
// reach the log the way an identity would.
const errorBody = await response.text().catch(() => '')
if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) {
throw new Error(`Relay admin telemetry returned ${response.status}`)
}
console.warn(
`relay admin telemetry retrying ${path} after ${response.status}` +
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
)
await wait(ADMIN_RETRY_INTERVAL_MS)
}
}
export async function directorSignals(
@@ -436,7 +511,8 @@ export async function directorSignals(
expectedSelector: AdmissionSelector,
gcloud: GcloudClient,
nowMs: number,
fetchImpl: typeof fetch
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void> = defaultWait
): Promise<{
source: IncidentSource
selector: AdmissionSelector
@@ -452,7 +528,8 @@ export async function directorSignals(
environment.directorOrigin,
token,
'/v1/admin/admission-selector/status',
{ v: 1 }
{ v: 1 },
wait
)
).selector
const selector = {
@@ -467,10 +544,14 @@ export async function directorSignals(
statuses.push({
cell,
status: CellStatusSchema.parse(
await adminPost(fetchImpl, environment.directorOrigin, token, '/v1/admin/cell-status', {
v: 1,
cellId: cell.cellId
})
await adminPost(
fetchImpl,
environment.directorOrigin,
token,
'/v1/admin/cell-status',
{ v: 1, cellId: cell.cellId },
wait
)
).status
})
}
@@ -495,7 +576,8 @@ export async function directorSignals(
sourceCellId: source.cellId,
targetCellId: target.cellId,
completeReady: false
}
},
wait
)
)
})
@@ -571,6 +653,7 @@ export type IncidentSampleCollectorOptions = {
expectedSelector: AdmissionSelector
fetchImpl?: typeof fetch
now?: () => number
wait?: (ms: number) => Promise<void>
}
export function createIncidentSampleCollector(
@@ -610,7 +693,8 @@ export function createIncidentSampleCollector(
options.expectedSelector,
gcloud,
nowMs,
fetchImpl
fetchImpl,
options.wait ?? defaultWait
),
cloudMetricEntries
])
cloud/apps/relay-ops/src/incident-monitor.test.ts
+533 -14
View File
@@ -273,16 +273,19 @@ describe('incident monitor evaluator', () => {
)
})
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
for (const errors of [1, 2, 3]) {
// Why: measured non-503 5xx per rolling five minutes over the 24 h to 2026-09-17
// was p90 3 / p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze
// 29% of 15-minute gates on chronic director 500 bursts.
it('tolerates the measured chronic director error rate without relaxing other gates', () => {
for (const errors of [1, 3, 5, 9, 15]) {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
}
const excess = healthySample()
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(16)
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
expect.objectContaining({ signal: 'director.errors', observed: 16, threshold: 15 })
)
const auth = healthySample()
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
@@ -358,17 +361,20 @@ describe('incident monitor evaluator', () => {
})
})
// Why: measured latest-sum over the 24 h to 2026-09-17 was p95 212 / p99 262 /
// max 282, so the old bar of 250 sat under the observed peak and froze 21.8% of
// 15-minute gates. 320 still fires at 65% of the 490 usable connections.
it('bounds Cloud SQL backends above measured healthy peaks', () => {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(250)
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(282)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(251)
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(321)
expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
code: 'threshold_max',
source: 'cloud-monitoring',
signal: 'cloud_sql.backends',
observed: 251,
threshold: 250
observed: 321,
threshold: 320
})
})
@@ -807,6 +813,150 @@ describe('incident monitor lifecycle', () => {
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why: dry-run 35258662628 read a healthy fleet clean for 13 minutes, then one
// unreadable Cloud Monitoring sample restarted the window and the restart blew
// the lineage cap, so a green fleet produced no verdict.
it('carries a 15-minute window through a single collector failure', async () => {
let now = startedAt
const warnings: string[] = []
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (now === startedAt + 10 * 60_000) {
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: (message) => {
warnings.push(message)
}
})
expect(result.windowSequence).toBe(0)
expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
expect(result.sampleCount).toBe(16)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(result.continuityEvents).toEqual([{
recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
windowSequence: 0,
tolerated: true,
failures: [{ code: 'collector_failed', source: 'cloud-monitoring' }]
}])
expect(warnings).toEqual([
'incident monitor collector failed: cloud monitoring read failed'
])
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('restarts the window after three consecutive collector failures', async () => {
let now = startedAt
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (failures > 0 && now >= startedAt + 10 * 60_000) {
failures--
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: () => {}
})
const restartMinute = 10 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
expect(result.windowSequence).toBe(1)
expect(result.windowStartedAt).toBe(
new Date(startedAt + restartMinute * 60_000).toISOString()
)
expect(result.completedAt).toBe(
new Date(startedAt + (restartMinute + 15) * 60_000).toISOString()
)
expect(result.sampleCount).toBe(16)
expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
...Array<boolean>(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
false
])
expect(result.continuityEvents.at(-1)!.failures).toEqual([
{ code: 'collector_failed', source: 'cloud-monitoring' }
])
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('still reaches a dry-run verdict after a restart on the last sample', async () => {
let now = startedAt
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (failures > 0 && now >= startedAt + 13 * 60_000) {
failures--
throw new Error('cloud monitoring read failed')
}
return healthySample(now)
},
persist: async () => {},
checkpoint: async () => {},
warn: () => {}
})
expect(result.windowSequence).toBe(1)
expect(result.windowStartedAt).toBe(new Date(startedAt + 16 * 60_000).toISOString())
expect(result.completedAt).toBe(new Date(startedAt + 31 * 60_000).toISOString())
expect(result.sampleCount).toBe(16)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
})
it('gives a signal a fresh budget only after it reads fresh again', async () => {
let now = startedAt
const staleMinutes = new Set([3, 5, 6, 9, 10])
@@ -971,7 +1121,7 @@ describe('incident monitor lifecycle', () => {
expect(result.sampleCount).toBe(16)
})
it('fails a dry run after 25 total minutes of continuity resets', async () => {
it('fails a dry run after 35 total minutes of continuity resets', async () => {
let now = startedAt
const state = initialIncidentMonitorState({
incidentId: 'incident-1',
@@ -985,15 +1135,16 @@ describe('incident monitor lifecycle', () => {
durationMinutes: 15,
intervalMs: 60_000
})
let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
// Two restarts: the first on the window's last sample, the second far enough
// into the replacement window that no third window can finish in the lineage.
const staleMinutes = new Set([13, 14, 15, 24, 25, 26])
const result = await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
staleSamples--
if (staleMinutes.has((now - startedAt) / 60_000)) {
return healthySample(
now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
)
@@ -1008,8 +1159,8 @@ describe('incident monitor lifecycle', () => {
new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString()
)
expect(result.frozenAt).not.toBeNull()
expect(result.windowSequence).toBe(1)
expect(result.sampleCount).toBe(13)
expect(result.windowSequence).toBe(2)
expect(result.sampleCount).toBe(9)
expect(result.failures).toContainEqual({
code: 'continuity_deadline_exceeded',
source: 'active-probe',
@@ -1114,3 +1265,371 @@ describe('incident monitor lifecycle', () => {
expect(waits[0]).toBe(45_000)
})
})
// Why: the asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
// us-central1 behind a 2 s timeout, so a saturated pool makes the load balancer
// answer "no healthy upstream" for about 30 s. Every one of 39 pre-roll gates froze
// on that, and 7 of the last 14 froze on this signal alone.
describe('incident monitor cell probe tolerance', () => {
const dryRunState = () =>
initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
// Returns the finished state of a 15-minute dry-run whose cell probe reads
// health=0 and ready=0 on the sample indexes in `badSamples`.
const runWithCellProbeGaps = async (badSamples: Set<number>) => {
let now = startedAt
let index = -1
return await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (badSamples.has(index)) {
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
}
it('passes a dry-run through a probe outage no longer than the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithCellProbeGaps(
new Set(Array.from({ length: tolerance }, (_, offset) => 3 + offset))
)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
// The blip is absorbed, not hidden: the sealed state still carries it.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health',
observed: 0,
threshold: 1
})
)
// A recovered probe hands back the full budget rather than a partial one.
expect(result.probeStreaks).toEqual({})
})
it('freezes once a cell probe fails past the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithCellProbeGaps(
new Set(Array.from({ length: tolerance + 1 }, (_, offset) => 3 + offset))
)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
expect(result.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health',
observed: 0,
threshold: 1
})
)
// Only the samples past the tolerance freeze; the first two are still absorbed.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
})
it('does not accumulate a streak across a recovered sample', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
// Repeated single-sample outages, each separated by a healthy sample, never
// reach the tolerance however many times they recur.
const spaced = new Set([2, 4, 6, 8, 10])
expect(spaced.size).toBeGreaterThan(tolerance)
const result = await runWithCellProbeGaps(spaced)
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why: health, ready and latency_ms all describe the same round trip, so the streak
// is keyed by cell. Keyed per signal, this cell holds every individual streak at one
// and never reaches the tolerance, yet it is unhealthy without a break from sample 2.
it('freezes on a cell that alternates between slow and unanswered', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index < 2) return sample
// Two samples slow, then two samples down, repeating: never the same signal
// twice in a row beyond the tolerance, but never healthy either.
if (Math.floor((index - 2) / 2) % 2 === 0) {
sample.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
signal(9_000, now)
} else {
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
})
// Why: the streak lives in the state file, so a resumed run must not hand a cell
// that was already failing a fresh budget.
it('freezes immediately when a resumed state carries a full streak', async () => {
let now = startedAt
const resumed = {
...dryRunState(),
lastSampleAt: new Date(startedAt).toISOString(),
probeStreaks: {
'active-probe/cell.production-gce-c1':
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
}
}
const result = await runIncidentMonitor(resumed, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
const sample = healthySample(now)
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
signal(0, now)
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({
source: 'active-probe',
signal: 'cell.production-gce-c1.health'
})
)
})
it('gives the director and auth probes no tolerance', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index === 3) {
sample.sources['active-probe']!.signals['director.health'] = signal(0, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ source: 'active-probe', signal: 'director.health' })
)
})
})
// Why: Cloud Run replaces director instances in place, so the count leaves the
// [5, 6] band for about one sample roughly twice a day, and a deploy overlap
// raises it the same way. Neither is an unhealthy fleet, and freezing on it
// blocks the roll that fixes the measured condition.
describe('incident monitor director instance tolerance', () => {
const dryRunState = () =>
initialIncidentMonitorState({
incidentId: 'incident-1',
environment: 'production',
expectedSelector: selector,
preDrainDryRun: true,
migrationPolicy: 'strict',
recoverySourceCellId: null,
capacityCellId: null,
startedAt: new Date(startedAt).toISOString(),
durationMinutes: 15,
intervalMs: 60_000
})
// Finished state of a 15-minute dry run whose director instance count reads
// `counts[index]` on the sample indexes that map has, and 5 everywhere else.
const runWithInstanceCounts = async (
counts: Map<number, number>,
state = dryRunState()
) => {
let now = startedAt
let index = -1
return await runIncidentMonitor(state, {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
const count = counts.get(index)
if (count !== undefined) {
sample.sources['cloud-monitoring']!.signals['director.instances'] =
signal(count, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
}
it('passes a dry run through an instance dip no longer than the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 2]))
)
expect(result.frozenAt).toBeNull()
expect(result.failures).toEqual([])
expect(preDrainDryRunPassed(result)).toBe(true)
// Absorbed, not hidden: the sealed state still carries the dip.
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_min',
source: 'cloud-monitoring',
signal: 'director.instances',
observed: 2,
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin
})
)
expect(result.probeStreaks).toEqual({})
})
it('passes a dry run through a deploy-overlap overshoot within the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 9]))
)
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_max',
signal: 'director.instances',
observed: 9,
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax
})
)
})
it('freezes once the instance count stays out of band past the tolerance', async () => {
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
const result = await runWithInstanceCounts(
new Map(Array.from({ length: tolerance + 1 }, (_, offset) => [3 + offset, 2]))
)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
expect(result.failures).toContainEqual(
expect.objectContaining({
code: 'threshold_min',
source: 'cloud-monitoring',
signal: 'director.instances',
observed: 2
})
)
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
})
it('does not accumulate a streak across a recovered sample', async () => {
const spaced = new Map([2, 4, 6, 8, 10].map((index) => [index, 2] as const))
expect(spaced.size).toBeGreaterThan(
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
)
const result = await runWithInstanceCounts(new Map(spaced))
expect(result.frozenAt).toBeNull()
expect(preDrainDryRunPassed(result)).toBe(true)
})
// Why min and max share one streak: a count that alternates above and below
// the band would otherwise hold each streak at one and never freeze, yet the
// director is never at its configured size.
it('freezes on a count that alternates above and below the band', async () => {
const counts = new Map<number, number>()
for (let index = 2; index < 12; index += 1) {
counts.set(index, index % 2 === 0 ? 2 : 9)
}
const result = await runWithInstanceCounts(counts)
expect(result.frozenAt).not.toBeNull()
expect(preDrainDryRunPassed(result)).toBe(false)
})
// Why: the streak lives in the state file, so a resumed run must not hand a
// director that was already out of band a fresh budget.
it('freezes immediately when a resumed state carries a full streak', async () => {
const result = await runWithInstanceCounts(new Map([[0, 2]]), {
...dryRunState(),
lastSampleAt: new Date(startedAt).toISOString(),
probeStreaks: {
'cloud-monitoring/director.instances':
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
}
})
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ signal: 'director.instances' })
)
})
it('keeps every other cloud-monitoring signal at zero tolerance', async () => {
let now = startedAt
let index = -1
const result = await runIncidentMonitor(dryRunState(), {
now: () => now,
wait: async (ms) => {
now += ms
},
collect: async () => {
index++
const sample = healthySample(now)
if (index === 3) {
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.99, now)
}
return sample
},
persist: async () => {},
checkpoint: async () => {}
})
expect(result.frozenAt).not.toBeNull()
expect(result.toleratedProbeEvents).toEqual([])
expect(result.failures).toContainEqual(
expect.objectContaining({ source: 'cloud-monitoring', signal: 'cloud_sql.cpu' })
)
})
})
cloud/apps/relay-ops/src/incident-monitor.ts
+151 -17
View File
@@ -42,12 +42,18 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
} as const satisfies Record<RelayOpsRegion, number>,
cloudSqlCpuUtilization: 0.8,
cloudSqlMemoryUtilization: 0.9,
// Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
// bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a
// pre-drain gate on baseline noise). 250 clears measured healthy peaks while
// firing well before the verified 400-connection ceiling; the retry signals
// below discriminate incident-class contention.
cloudSqlBackends: 250,
// Why: 320, recalibrated 2026-09-17 from 250. The auth instance sums seven
// databases, and its steady load has grown past the 2026-08-26 measurement the
// old bar came from. Measured latest-sum over the 24 h to 2026-09-17, aligned
// per minute exactly as this signal reads it: p50 118 / p90 165 / p95 212 /
// p99 262 / max 282. 250 was under the observed max, so 1.95% of minutes and
// 21.8% of 15-minute pre-drain gates froze on ordinary load. 320 clears every
// measured healthy minute with 13% of headroom above the peak and still fires
// at 65% of the 490 connections the budget work (#21165) treats as usable out
// of max_connections 500, so exhaustion-class growth is caught with 170
// connections still in hand. The retry signals below discriminate incident-class
// contention. Re-tighten when the auth connection model lands (#21165).
cloudSqlBackends: 320,
// Bound the observed recovery load; deadlocks remain zero-tolerance.
cloudSqlLockWaits: 20,
cloudSqlDeadlocks: 0,
@@ -101,18 +107,48 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
directorCpuUtilization: 0.8,
directorMemoryUtilization: 0.8,
directorConcurrency: 64,
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
directorErrors: 3,
// Why: 15, recalibrated 2026-09-17 from 3. This counts non-503 5xx answers from
// the director over the rolling five-minute query window; 503s are excluded
// because they are the documented back-pressure answer a client retries.
// Measured over the 24 h to 2026-09-17 (272 non-503 5xx against 71 941 503s):
// p50 0 / p90 3 / p95 5 / p99 9 / max 52. A bar of 3 sits at the p90, so 9.2%
// of windows and 29.0% of 15-minute pre-drain gates froze on the chronic 500
// bursts that /v1/assign, /v1/regions and /v1/resolve emit alongside the
// recurring Cloud SQL stall. 15 clears the chronic p99 with margin and drops
// the baseline gate-freeze rate to 1.5%, while leaving the exceptional 20-52
// bursts detectable. A director that is actually broken answers 5xx on a large
// share of its traffic: it serves ~50 requests a minute in 503s alone, so a
// real fault lands in the hundreds per window, an order of magnitude clear of
// this bar.
directorErrors: 15,
authErrors: 0,
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
cellConnections: 500,
cellQueuedBytes: 48 * 1024 * 1024,
migrationBlocked: 0
migrationBlocked: 0,
// Why: one sample is one HTTP round trip from one GitHub runner to one cell, so
// a single bad reading is evidence about that round trip, not about the fleet.
// The asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
// us-central1 over a 176 ms round trip behind a 2 s statement timeout, so a
// saturated pool makes the load balancer answer "no healthy upstream" for about
// 30 s. That answer is an HTTP 503, not a transport failure, so provenance
// cannot separate it from a cell that genuinely serves health=0 -- persistence
// can. At the 60 s sample interval a 30 s outage shows up in one sample and at
// worst two, so a cell's probe must fail more than this many consecutive samples
// before it freezes the run. The streak is per cell, not per signal, so a cell
// that alternates between slow and unanswered still accumulates one. This applies
// to per-cell probes and the director instance count; the director and auth health
// probes stay at zero tolerance.
cellProbeToleranceSamples: 2
} as const
export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000
// Why: 35 minutes, raised 2026-09-17 from 25. A 15-minute window plus one
// restart must fit: a continuity reset on the window's last sample restarts at
// minute 16 and finishes at 31. Under 25 a reset past minute 9 cost the whole
// verdict, which is what run 35258662628 hit on a healthy fleet.
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 35 * 60_000
export type IncidentSourceName =
| 'active-probe'
@@ -212,6 +248,17 @@ export type IncidentMonitorState = {
}[]
frozenAt: string | null
failures: IncidentFailure[]
// Consecutive samples each tolerated reading has currently been failing for,
// keyed by cell so its health, ready and latency readings share one streak, and
// by signal for the director instance count.
probeStreaks: Record<string, number>
// Cell-probe breaches absorbed by the tolerance, kept so a green artifact still
// shows what the gate chose not to freeze on.
toleratedProbeEvents: {
recordedAt: string
windowSequence: number
failures: IncidentFailure[]
}[]
completedAt: string | null
}
@@ -625,6 +672,8 @@ export function initialIncidentMonitorState(input: {
continuityEvents: [],
frozenAt: null,
failures: [],
probeStreaks: {},
toleratedProbeEvents: [],
completedAt: null
}
}
@@ -635,6 +684,7 @@ export type IncidentMonitorDependencies = {
collect(): Promise<IncidentSample>
persist(state: IncidentMonitorState): Promise<void>
checkpoint(summary: IncidentCheckpoint): Promise<void>
warn?(message: string): void
}
function checkpointMinutes(durationMinutes: number): number[] {
@@ -656,10 +706,19 @@ const CONTINUITY_FAILURE_CODES = new Set([
...FRESHNESS_FAILURE_CODES
])
// A whole sample we could not read gets the same consecutive-sample budget as an
// unread signal, for the same reason: one failed collector round trip is evidence
// about that round trip, not about the fleet. `monitor_gap` is excluded because it
// means the run itself stopped sampling, so the window genuinely has a hole.
const TOLERABLE_CONTINUITY_FAILURE_CODES = new Set([
'collector_failed',
...FRESHNESS_FAILURE_CODES
])
// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
// past minute 10 costs the entire verdict, so a healthy fleet produced none on
// 2026-09-05. A signal may miss this many consecutive samples before the window
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap in force
// then, a restart past minute 10 cost the entire verdict, so a healthy fleet
// produced none on 2026-09-05. A signal may miss this many consecutive samples before the window
// restarts; the sample is still evaluated against every threshold it can read,
// and a threshold breach still freezes the run outright.
export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
@@ -668,6 +727,44 @@ function freshnessKey(failure: IncidentFailure): string {
return `${failure.source}/${failure.signal ?? '*'}`
}
// The streak key for a per-cell active-probe reading, or null if the failure is not
// one. A cell probe is a single HTTP round trip and so is subject to
// cellProbeToleranceSamples; director and auth probes return null on purpose,
// because they are the single points of failure this gate exists to catch.
//
// Why the key is the cell and not the signal: health, ready and latency_ms all
// describe the same round trip. Keyed per signal, a cell that alternates between
// answering slowly and not answering at all holds every individual streak at one and
// never reaches the tolerance, so a continuously unhealthy cell passes the gate.
export function cellProbeStreakKey(failure: IncidentFailure): string | null {
const signal = failure.signal
if (failure.source !== 'active-probe' || signal === undefined) return null
if (!signal.startsWith('cell.')) return null
const lastDot = signal.lastIndexOf('.')
if (lastDot < 'cell.'.length) return null
return `${failure.source}/${signal.slice(0, lastDot)}`
}
// Cloud Run replaces director instances in place rather than holding the count,
// so the reading leaves [min, max] for about one sample roughly twice a day, and a
// deploy that briefly serves two revisions raises it the same way. Neither is an
// unhealthy fleet, and on 2026-09-17 this was one of the signals freezing the
// pre-drain gate on a condition the roll exists to fix. Min and max share one
// streak on purpose: a count that alternates above and below the band would
// otherwise hold each individual streak at one and never reach the tolerance.
export function directorInstancesStreakKey(failure: IncidentFailure): string | null {
if (failure.source !== 'cloud-monitoring' || failure.signal !== 'director.instances') {
return null
}
return `${failure.source}/${failure.signal}`
}
// The streak key for any reading subject to cellProbeToleranceSamples, or null
// for a reading that freezes the run on its first bad sample.
export function toleratedStreakKey(failure: IncidentFailure): string | null {
return cellProbeStreakKey(failure) ?? directorInstancesStreakKey(failure)
}
// Rebuild the per-signal tolerated streak from the trailing continuity events so a
// resumed monitor cannot hand a signal a fresh budget.
function resumeFreshnessStreaks(
@@ -777,7 +874,12 @@ export async function runIncidentMonitor(
state.recoverySourceCellId,
state.capacityCellId
)
} catch {
} catch (error) {
dependencies.warn?.(
`incident monitor collector failed: ${
error instanceof Error ? error.message : 'unknown error'
}`
)
evaluation = {
status: 'freeze',
evaluatedAt: new Date(dependencies.now()).toISOString(),
@@ -798,7 +900,8 @@ export async function runIncidentMonitor(
const toleratedKeys = new Set(
state.windowStartedAt !== null &&
continuityFailures.length > 0 &&
continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
continuityFailures.every((failure) =>
TOLERABLE_CONTINUITY_FAILURE_CODES.has(failure.code))
? continuityFailures.map(freshnessKey)
: []
)
@@ -828,9 +931,40 @@ export async function runIncidentMonitor(
}
state.sampleCount++
}
if (thresholdFailures.length > 0) {
const probeFailures = new Map<string, IncidentFailure[]>()
for (const failure of thresholdFailures) {
const key = toleratedStreakKey(failure)
if (key === null) continue
probeFailures.set(key, [...(probeFailures.get(key) ?? []), failure])
}
for (const key of Object.keys(state.probeStreaks)) {
if (!probeFailures.has(key)) delete state.probeStreaks[key]
}
const sustainedProbeFailures: IncidentFailure[] = []
const toleratedProbeFailures: IncidentFailure[] = []
for (const [key, entries] of probeFailures) {
const streak = (state.probeStreaks[key] ?? 0) + 1
state.probeStreaks[key] = streak
const target =
streak > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
? sustainedProbeFailures
: toleratedProbeFailures
target.push(...entries)
}
if (toleratedProbeFailures.length > 0) {
state.toleratedProbeEvents.push({
recordedAt: evaluation.evaluatedAt,
windowSequence: state.windowSequence,
failures: toleratedProbeFailures
})
}
const freezingFailures = [
...thresholdFailures.filter((failure) => toleratedStreakKey(failure) === null),
...sustainedProbeFailures
]
if (freezingFailures.length > 0) {
state.frozenAt ??= evaluation.evaluatedAt
state.failures = [...state.failures, ...thresholdFailures]
state.failures = [...state.failures, ...freezingFailures]
}
if (state.windowStartedAt === null) {
if (dependencies.now() >= lineageDeadlineMs) {
cloud/apps/relay/package.json
+1 -1
View File
@@ -31,6 +31,6 @@
"@types/ws": "^8.18.1",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/apps/relay/src/admin-route-transient-app.test.ts
+152
View File
@@ -0,0 +1,152 @@
import { describe, expect, it, vi } from 'vitest'
import type { RelayConfig } from './config.js'
vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string) => token === 'deploy-token',
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
createRegionalRehomeTokenVerifier: () => async () => false,
createRuntimeTokenVerifier: () => async () => false
}))
vi.mock('./relay-token-verifier.js', () => ({
createRelayTokenVerifier: () => async () => null,
readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
}))
import { createRelayApp } from './app.js'
// The message the pool raises when its own dial outruns connectionTimeoutMillis.
// This is the shape that failed a rollout wave as an HTTP 404.
const poolTimeout = () => new Error('Connection terminated due to connection timeout')
function adminRequest(body: unknown): RequestInit {
return {
method: 'POST',
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
body: JSON.stringify(body)
}
}
function appWith(assignments: Record<string, unknown>, overrides: Partial<RelayConfig> = {}) {
return createRelayApp(config(overrides), {
store: {} as never,
assignments: assignments as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
}
describe('admin routes under a database that is briefly out of reach', () => {
it('answers cell-status with a retryable 503 instead of a not-found', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ cellDeploymentStatus }, { publicAssignmentRetryAfterSeconds: 7 })
const response = await app.request(
'/v1/admin/cell-status',
adminRequest({ v: 1, cellId: 'production-gce-c7' })
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('7')
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
})
it('keeps the not-found mapping for a real cell-status failure', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw new Error('unknown_cell')
})
const app = appWith({ cellDeploymentStatus })
const response = await app.request(
'/v1/admin/cell-status',
adminRequest({ v: 1, cellId: 'production-gce-c7' })
)
expect(response.status).toBe(404)
expect(await response.json()).toEqual({ error: 'unknown_cell' })
})
it('answers admission-selector/status with a retryable 503 instead of a conflict', async () => {
const inspectCellAdmissionSelector = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ inspectCellAdmissionSelector })
const response = await app.request(
'/v1/admin/admission-selector/status',
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('5')
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
})
it('keeps the conflict mapping for a real admission-selector/status failure', async () => {
const inspectCellAdmissionSelector = vi.fn(async () => {
throw new Error('admission_selector_attempt_not_found')
})
const app = appWith({ inspectCellAdmissionSelector })
const response = await app.request(
'/v1/admin/admission-selector/status',
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'admission_selector_attempt_not_found' })
})
it('leaves the pre-request guards ahead of the mapping alone', async () => {
const cellDeploymentStatus = vi.fn(async () => {
throw poolTimeout()
})
const app = appWith({ cellDeploymentStatus })
// An unauthenticated caller must never learn the database is struggling.
const unauthenticated = await app.request('/v1/admin/cell-status', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, cellId: 'production-gce-c7' })
})
expect(unauthenticated.status).toBe(401)
const invalid = await app.request('/v1/admin/cell-status', adminRequest({ v: 1 }))
expect(invalid.status).toBe(400)
expect(cellDeploymentStatus).not.toHaveBeenCalled()
})
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
publicUrl: 'https://relay.example.test',
cellUrl: 'https://relay.example.test',
region: 'us-central1',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'director',
cellId: 'director',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data',
...overrides
}
}
cloud/apps/relay/src/app.ts
+88 -48
View File
@@ -43,6 +43,7 @@ import {
type AssignmentAdmissionRejection
} from './public-assignment-admission.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayReadinessDependency } from './relay-readiness.js'
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
import {
isRegionalRehomeTrustProbe,
@@ -57,6 +58,8 @@ const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
const REGION_CATALOG_CACHE_MS = 30_000
// A drain that outlives the roll step it belongs to is an outage, not a pacing win.
const DRAIN_PACE_WINDOW_MAX_MS = 5 * 60 * 1_000
type AdmissionRejectionLogEntry = {
route: 'assign' | 'resolve'
@@ -71,7 +74,7 @@ export function createRelayApp(
operations: {
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
drain: (graceMs: number, options?: { paceWindowMs?: number }) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
@@ -96,6 +99,7 @@ export function createRelayApp(
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
readinessDegradation?: () => RelayReadinessDependency[]
recordAssignmentAdmission?: (
outcome: 'sticky' | 'sticky-rejected' | 'placement' | 'placement-rejected'
) => void
@@ -174,6 +178,21 @@ export function createRelayApp(
context.header('Retry-After', String(stickyRetryAfterSeconds))
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
}
// An admin route that collapses every failure into one status cannot tell a
// real conflict from a database that was briefly out of reach, and the rollout
// tooling retries on 503 only. Transient failures get the answer the public
// routes already give; everything else keeps the route's own mapping.
const rejectAdminOperation = (
context: Context,
error: unknown,
status: 404 | 409
): Response => {
if (!isRelayDatabaseTransientError(error)) {
return context.json({ error: operationError(error) }, status)
}
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
// Aggregate counters cannot separate a handful of pathological hosts from a broad
// population, so every admission rejection names its host and reason. Keyed on
// route:lane:reason rather than host, the log stays bounded under load.
@@ -212,14 +231,24 @@ export function createRelayApp(
app.get('/health', (context) =>
context.json({ ok: true, connectionCapacityProtocol: 2 })
)
app.get('/ready', async (context) =>
(await operations.ready())
? context.json({ ok: true })
: context.json({ error: 'dependency_unavailable' }, 503)
)
app.get('/ready', async (context) => {
if (!(await operations.ready())) return context.json({ error: 'dependency_unavailable' }, 503)
const dependency = operations.readinessDegradation?.() ?? []
// Still the 200 the load balancer needs, with the marker that says the answer is remembered.
if (dependency.length === 0) return context.json({ ok: true })
return context.json({ ok: true, degraded: true, dependency })
})
app.get('/v1/regions', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
return context.json({ v: 1, regions: await regionCatalog() })
try {
return context.json({ v: 1, regions: await regionCatalog() })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
// Same contract as the assignment routes: a database that is briefly out
// of reach is a retry, not a director fault.
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
return context.json({ error: 'region_catalog_temporarily_unavailable' }, 503)
}
})
app.post('/v1/assign', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
@@ -461,12 +490,18 @@ export function createRelayApp(
return context.json({ error: 'invalid_token' }, 401)
}
const body = z
.object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) })
.object({
v: z.literal(1),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
// Spreads the drain sends, and so the re-dials, over this window.
paceWindowMs: z.number().int().nonnegative().max(DRAIN_PACE_WINDOW_MAX_MS).optional()
})
.strict()
.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
operations.drain(body.data.graceMs)
return context.json({ ok: true })
const paceWindowMs = body.data.paceWindowMs ?? 0
operations.drain(body.data.graceMs, { paceWindowMs })
return context.json({ ok: true, paceWindowMs })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
@@ -493,7 +528,7 @@ export function createRelayApp(
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
@@ -544,7 +579,7 @@ export function createRelayApp(
...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {})
})
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/runtime-status', async (context) => {
@@ -598,7 +633,7 @@ export function createRelayApp(
await operations.assignments.recordCellHeartbeat(body.data)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-rehome-status', async (context) => {
@@ -618,7 +653,7 @@ export function createRelayApp(
await operations.assignments.recordCellRegionalRehomeStatus(body.data)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
@@ -657,7 +692,7 @@ export function createRelayApp(
const control = await operations.assignments.applyRegionalRehomeControl(body.data)
return context.json({ v: 1, control })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/regional-rehome-trust-probe', async (context) => {
@@ -701,7 +736,7 @@ export function createRelayApp(
})
return context.json(result)
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate', async (context) => {
@@ -722,7 +757,7 @@ export function createRelayApp(
)
return context.json({ v: 1, migration })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-complete', async (context) => {
@@ -743,7 +778,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/migration-supersede-cell', async (context) => {
@@ -768,7 +803,7 @@ export function createRelayApp(
)
return context.json({ v: 1, superseded })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/rebalance-dormant', async (context) => {
@@ -789,7 +824,7 @@ export function createRelayApp(
)
return context.json({ v: 1, assignment })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply', async (context) => {
@@ -809,7 +844,7 @@ export function createRelayApp(
const result = await operations.assignments.applyCellAdmissionSelector(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => {
@@ -834,7 +869,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/status', async (context) => {
@@ -856,7 +891,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => {
@@ -887,7 +922,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-state', async (context) => {
@@ -912,7 +947,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => {
@@ -935,7 +970,7 @@ export function createRelayApp(
)
return context.json({ v: 1, cellId: body.data.cellId, expiresAt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => {
@@ -958,7 +993,7 @@ export function createRelayApp(
)
return context.json({ v: 1, cellId: body.data.cellId, committed: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attest', async (context) => {
@@ -987,7 +1022,7 @@ export function createRelayApp(
attempt: result.attempt
})
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => {
@@ -1008,7 +1043,7 @@ export function createRelayApp(
const attempt = await operations.assignments.prepareCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-start', async (context) => {
@@ -1033,7 +1068,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-plan', async (context) => {
@@ -1057,7 +1092,7 @@ export function createRelayApp(
)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-operation', async (context) => {
@@ -1083,7 +1118,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-status', async (context) => {
@@ -1103,7 +1138,7 @@ export function createRelayApp(
const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-fence-attempt-abort', async (context) => {
@@ -1124,7 +1159,7 @@ export function createRelayApp(
const attempt = await operations.assignments.abortCellFenceAttempt(evidence)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-prepare', async (context) => {
@@ -1150,7 +1185,7 @@ export function createRelayApp(
})
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-send', async (context) => {
@@ -1170,7 +1205,7 @@ export function createRelayApp(
const attempt = await operations.assignments.beginCellDrainSend(body.data)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-receipt', async (context) => {
@@ -1192,7 +1227,7 @@ export function createRelayApp(
)
return context.json({ v: 1, attempt })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/drain-attempt-recover-forward', async (context) => {
@@ -1212,7 +1247,7 @@ export function createRelayApp(
const result = await operations.assignments.prepareCellDrainRecovery(body.data)
return context.json({ v: 1, ...result })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/cell-config', async (context) => {
@@ -1239,7 +1274,7 @@ export function createRelayApp(
)
return context.json({ ok: true })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuate-cell', async (context) => {
@@ -1261,7 +1296,7 @@ export function createRelayApp(
)
return context.json({ v: 1, started })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-capacity', async (context) => {
@@ -1284,7 +1319,7 @@ export function createRelayApp(
)
return context.json({ v: 1, ...capacity })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
return rejectAdminOperation(context, error, 409)
}
})
app.post('/v1/admin/evacuation-status', async (context) => {
@@ -1303,12 +1338,17 @@ export function createRelayApp(
if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) {
return context.json({ error: 'insufficient_permission' }, 403)
}
const status = await operations.assignments.cellEvacuationStatus(
body.data.sourceCellId,
body.data.targetCellId,
body.data.completeReady
)
return context.json({ v: 1, ...status })
try {
const status = await operations.assignments.cellEvacuationStatus(
body.data.sourceCellId,
body.data.targetCellId,
body.data.completeReady
)
return context.json({ v: 1, ...status })
} catch (error) {
if (!isRelayDatabaseTransientError(error)) throw error
return context.json({ error: 'database_temporarily_unavailable' }, 503)
}
})
app.post('/v1/admin/cell-status', async (context) => {
const bearer = readBearer(context.req.header('authorization'))
@@ -1325,7 +1365,7 @@ export function createRelayApp(
const status = await operations.assignments.cellDeploymentStatus(body.data.cellId)
return context.json({ v: 1, status })
} catch (error) {
return context.json({ error: operationError(error) }, 404)
return rejectAdminOperation(context, error, 404)
}
})
return app
cloud/apps/relay/src/assignment-store.ts
+159 -119
View File
@@ -45,6 +45,15 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
CONTROL_RENEWAL_BATCH_SQL,
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
controlRenewalBatchParams,
orderedControlRenewalRows,
readControlRenewalOutcomes,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
@@ -101,21 +110,8 @@ type RelayAssignmentStoreOptions = {
recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
}
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
| 'database_error'
export type { ControlRenewalOutcome, ControlRenewalRequest }
const CONTROL_RENEWAL_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved'
])
export type RelayAssignment = AssignmentIdentity & {
cellId: string
cellUrl: string
@@ -3478,132 +3474,149 @@ export class RelayAssignmentStore {
})
}
// Kept as the single-row contract for callers and tests: resolves on a
// renewal and throws the outcome (or the driver's own error) otherwise.
async renewControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number }
): Promise<void> {
validateActivityId(input.activityId)
const now = this.now()
const maximumExpiresAt =
now +
ASSIGNMENT_LIMITS.activityLeaseMs +
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
if (
!Number.isSafeInteger(input.expiresAt) ||
input.expiresAt <= now ||
input.expiresAt > maximumExpiresAt
) {
if (!controlRenewalExpiryIsValid(input.expiresAt, now)) {
throw new Error('invalid_activity_expiry')
}
const startedAt = performance.now()
let outcome: ControlRenewalOutcome = 'database_error'
try {
outcome =
this.database.dialect === 'postgres'
? await this.renewPostgresControlActivity(identity, input, now)
: await this.renewTransactionalControlActivity(identity, input, now)
outcome = await this.renewOneControlActivity({ identity, ...input }, now)
if (outcome !== 'renewed') throw new Error(outcome)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) {
outcome = message as ControlRenewalOutcome
}
outcome = controlRenewalOutcomeOfError(error)
throw error
} finally {
this.recordControlRenewal?.(performance.now() - startedAt, outcome)
}
}
private async renewPostgresControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number },
// Renews every due control lease on a cell in one write transaction, returning
// one outcome per input row in input order. Never throws for a multi-row batch:
// a caller routes its own session on its own outcome.
async renewControlActivities(
rows: readonly ControlRenewalRequest[]
): Promise<ControlRenewalOutcome[]> {
const now = this.now()
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
const accepted: Array<ControlRenewalRequest & { index: number }> = []
for (const [index, row] of rows.entries()) {
const rejection = controlRenewalRejection(row, now)
if (rejection) outcomes[index] = rejection
else accepted.push({ ...row, index })
}
const startedAt = performance.now()
try {
if (accepted.length === 0) return outcomes
let results: ControlRenewalOutcome[]
try {
results = await this.executeControlRenewals(accepted, now)
} catch (error) {
if (rows.length === 1) {
outcomes[accepted[0]!.index] = controlRenewalOutcomeOfError(error)
throw error
}
results = accepted.map(() => 'database_error')
}
for (const [position, row] of accepted.entries()) outcomes[row.index] = results[position]!
return outcomes
} finally {
// Every path, so a rethrown lone renewal and an all-invalid batch are
// counted the same as a batch that reached PostgreSQL.
const durationMs = performance.now() - startedAt
for (const outcome of outcomes) this.recordControlRenewal?.(durationMs, outcome)
}
}
private async executeControlRenewals(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
if (rows.length === 1) return [await this.renewOneControlActivity(rows[0]!, now)]
if (this.database.dialect !== 'postgres') {
// Correctness over throughput: the SQLite writer is serialized anyway, and
// this is the dialect the unit suites run on.
return await this.renewControlActivitiesInSeries(rows, now)
}
const ordered = orderedControlRenewalRows(rows.map((row, index) => ({ ...row, index })))
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
try {
const parsed = readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams(ordered, now)
),
ordered.length
)
for (const [position, row] of ordered.entries()) outcomes[row.index] = parsed[position]!
return outcomes
} catch (error) {
// One statement means one contended assignment row can fail the whole
// batch, so a failure degrades to the per-host statements this replaced
// rather than costing every other host on the cell its renewal.
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_batch_failed',
rows: ordered.length,
message: String((error as { message?: unknown }).message)
})
)
await Promise.all(
ordered.map(async (row) => {
try {
outcomes[row.index] = await this.renewOneControlActivity(row, now)
} catch {
outcomes[row.index] = 'database_error'
}
})
)
return outcomes
}
}
private async renewControlActivitiesInSeries(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
const outcomes: ControlRenewalOutcome[] = []
for (const row of rows) {
try {
outcomes.push(await this.renewOneControlActivity(row, now))
} catch {
outcomes.push('database_error')
}
}
return outcomes
}
// Returns the outcome; a driver or pool failure reaches the caller unchanged.
private async renewOneControlActivity(
row: ControlRenewalRequest,
now: number
): Promise<ControlRenewalOutcome> {
const row = (
await this.database.query(
`WITH assignment_state AS MATERIALIZED (
SELECT cell_id, assignment_epoch
FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?
FOR UPDATE
), migration_state AS MATERIALIZED (
SELECT migration.assignment_epoch
FROM relay_assignment_migrations migration
JOIN assignment_state assignment
ON migration.target_cell_id = assignment.cell_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT 1 AS authorized
FROM assignment_state assignment
WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state)
), lease_state AS MATERIALIZED (
SELECT lease.activity_kind, lease.cell_id
FROM relay_assignment_activity_leases lease
CROSS JOIN authorization_state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, ?),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
AND state.activity_kind = 'control' AND state.cell_id = ?
RETURNING 1
), renewed_assignment AS (
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
AND EXISTS (SELECT 1 FROM renewed_lease)
RETURNING 1
)
SELECT CASE
WHEN NOT EXISTS (SELECT 1 FROM assignment_state)
THEN 'assignment_not_found'
WHEN NOT EXISTS (SELECT 1 FROM authorization_state)
THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (SELECT 1 FROM lease_state)
THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state
WHERE activity_kind <> 'control' OR cell_id <> ?
) THEN 'control_activity_moved'
WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome`,
[
identity.userId,
identity.relayHostId,
identity.userId,
identity.relayHostId,
input.cellId,
input.cellId,
identity.userId,
identity.relayHostId,
input.activityId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.activityId,
input.cellId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.cellId
]
)
)[0]
if (!row) throw new Error('missing_control_renewal_outcome')
const outcome = text(row, 'outcome') as ControlRenewalOutcome
if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome')
return outcome
if (this.database.dialect === 'postgres') {
return readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams([row], now)
),
1
)[0]!
}
try {
return await this.renewTransactionalControlActivity(row.identity, row, now)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) throw error
return message as ControlRenewalOutcome
}
}
private async renewTransactionalControlActivity(
@@ -7900,6 +7913,33 @@ function validateActivityId(activityId: string): void {
if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id')
}
function controlRenewalExpiryIsValid(expiresAt: number, now: number): boolean {
const maximumExpiresAt =
now + ASSIGNMENT_LIMITS.activityLeaseMs + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
return Number.isSafeInteger(expiresAt) && expiresAt > now && expiresAt <= maximumExpiresAt
}
// A renewal that threw still owes the metric an outcome: the message carries one
// when the statement decided it, and anything else is the driver failing.
function controlRenewalOutcomeOfError(error: unknown): ControlRenewalOutcome {
const message = String((error as { message?: unknown }).message)
return CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)
? (message as ControlRenewalOutcome)
: 'database_error'
}
function controlRenewalRejection(
row: ControlRenewalRequest,
now: number
): ControlRenewalOutcome | null {
try {
validateActivityId(row.activityId)
} catch {
return 'invalid_activity_id'
}
return controlRenewalExpiryIsValid(row.expiresAt, now) ? null : 'invalid_activity_expiry'
}
function activityKind(row: SqlRow): AssignmentActivityKind {
const value = text(row, 'activity_kind')
if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind')
cloud/apps/relay/src/cell-inventory-hold-samples.test.ts
+42
View File
@@ -67,4 +67,46 @@ describe('cell inventory hold samples', () => {
expect(samples.consumeCounts().cellInventoryHolds).toBe(2)
expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts())
})
// Why: this is the case the hold metrics alone cannot see. A NOWAIT grab that
// fails has no duration, so a retry storm used to leave every hold field at
// zero while the lock was saturated.
it('counts failed acquisitions in a window that recorded no holds', () => {
const samples = new CellInventoryHoldSamples()
for (let attempt = 0; attempt < 65; attempt++) samples.recordUnavailable()
const counts = samples.readCounts()
expect(counts.cellInventoryLockUnavailable).toBe(65)
expect(counts.cellInventoryHolds).toBe(0)
expect(counts.cellInventoryHoldMsMax).toBe(0)
})
it('reports failed acquisitions alongside the holds that did succeed', () => {
const samples = samplesOf([12, 34])
samples.recordUnavailable(3)
expect(samples.readCounts()).toMatchObject({
cellInventoryHolds: 2,
cellInventoryHoldMsMax: 34,
cellInventoryLockUnavailable: 3
})
})
it('ignores a failure count that is not a positive number', () => {
const samples = new CellInventoryHoldSamples()
samples.recordUnavailable(0)
samples.recordUnavailable(-2)
samples.recordUnavailable(Number.NaN)
expect(samples.readCounts()).toEqual(emptyCellInventoryHoldCounts())
})
it('resets failed acquisitions on consume', () => {
const samples = new CellInventoryHoldSamples()
samples.recordUnavailable(4)
expect(samples.consumeCounts().cellInventoryLockUnavailable).toBe(4)
expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts())
})
})
cloud/apps/relay/src/cell-inventory-hold-samples.ts
+37 -3
View File
@@ -5,6 +5,14 @@ export type CellInventoryHoldCounts = {
cellInventoryHoldMsMax: number
cellInventoryHoldMsP95: number
cellInventoryHolds: number
// Why: a failed acquisition produces no hold sample, so the hold fields alone
// read healthy while the lock is saturated. Split by wait policy, not by
// caller: fail-fast covers background sweeps that step aside by design AND
// request-path first attempts that retry, so it reads as contention pressure,
// not user-visible failure. An expired bounded wait has already spent its
// budget, so that lane is the one that tracks stalls.
cellInventoryLockUnavailable: number
cellInventoryLockTimeouts: number
}
// Bounded so a flush interval with heavy assignment traffic cannot grow the array
@@ -12,11 +20,19 @@ export type CellInventoryHoldCounts = {
const MAX_SAMPLES = 2_048
export function emptyCellInventoryHoldCounts(): CellInventoryHoldCounts {
return { cellInventoryHoldMsMax: 0, cellInventoryHoldMsP95: 0, cellInventoryHolds: 0 }
return {
cellInventoryHoldMsMax: 0,
cellInventoryHoldMsP95: 0,
cellInventoryHolds: 0,
cellInventoryLockUnavailable: 0,
cellInventoryLockTimeouts: 0
}
}
export class CellInventoryHoldSamples {
private samples: number[] = []
private unavailable = 0
private timeouts = 0
record(holdMs: number): void {
if (!Number.isFinite(holdMs) || holdMs < 0) return
@@ -24,19 +40,37 @@ export class CellInventoryHoldSamples {
this.samples.push(holdMs)
}
// Counted, not sampled: a failed acquisition has no duration to record.
recordUnavailable(count = 1): void {
if (!Number.isFinite(count) || count <= 0) return
this.unavailable += count
}
recordLockTimeout(count = 1): void {
if (!Number.isFinite(count) || count <= 0) return
this.timeouts += count
}
consumeCounts(): CellInventoryHoldCounts {
const counts = this.readCounts()
this.samples = []
this.unavailable = 0
this.timeouts = 0
return counts
}
readCounts(): CellInventoryHoldCounts {
if (this.samples.length === 0) return emptyCellInventoryHoldCounts()
const failures = {
cellInventoryLockUnavailable: this.unavailable,
cellInventoryLockTimeouts: this.timeouts
}
if (this.samples.length === 0) return { ...emptyCellInventoryHoldCounts(), ...failures }
const sorted = [...this.samples].sort((left, right) => left - right)
return {
cellInventoryHoldMsMax: round(sorted[sorted.length - 1]!),
cellInventoryHoldMsP95: round(sorted[Math.ceil(0.95 * sorted.length) - 1] ?? 0),
cellInventoryHolds: sorted.length
cellInventoryHolds: sorted.length,
...failures
}
}
}
cloud/apps/relay/src/cell-inventory-lock-contention.test.ts
+58
View File
@@ -260,6 +260,64 @@ describe('bounded cell-inventory lock wait', () => {
await database.close()
})
// Why: the 55P03 rolls the transaction back, so a drain on the commit path
// alone would report zero for exactly the windows that were contended.
it('reports a NOWAIT deferral that rolled its transaction back', async () => {
const database = await openFakePostgres()
fakes.query.mockImplementation(async (sql: string) => {
if (sql.includes('FOR UPDATE NOWAIT')) {
throw Object.assign(new Error('could not obtain lock'), { code: '55P03' })
}
return { rows: [], rowCount: 0 }
})
await expect(
database.transaction(async (transaction) => {
await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
failIfUnavailable: true,
measureHoldMs: true
})
})
).rejects.toThrow('database_lock_unavailable')
const counts = consumeRelayCellInventoryHold(database)
expect(counts.cellInventoryLockUnavailable).toBe(1)
expect(counts.cellInventoryLockTimeouts).toBe(0)
await database.close()
})
// Why: a bounded request-path wait raises the same 55P03 without NOWAIT. Folding
// it into the deferral counter would hide user-visible stalls among by-design
// sweep skips, which outnumber them by roughly an order of magnitude.
it('counts an expired bounded wait apart from a NOWAIT deferral', async () => {
const database = await openFakePostgres()
fakes.query.mockImplementation(async (sql: string) => {
if (sql.includes('FOR UPDATE') && !sql.includes('NOWAIT')) {
throw Object.assign(new Error('canceling statement due to lock timeout'), {
code: '55P03'
})
}
return { rows: [], rowCount: 0 }
})
await expect(
database.transaction(async (transaction) => {
await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
lockTimeoutMs: 500,
measureHoldMs: true
})
})
).rejects.toThrow()
const counts = consumeRelayCellInventoryHold(database)
// One per attempt, not per request: 55P03 is retryable, so an exhausted
// request contributes POSTGRES_TRANSACTION_ATTEMPTS timeouts. Reading the
// metric as affected-requests would overstate it threefold.
expect(counts.cellInventoryLockTimeouts).toBe(3)
expect(counts.cellInventoryLockUnavailable).toBe(0)
await database.close()
})
it('records no hold for a PostgreSQL transaction that took no measured lock', async () => {
const database = await openFakePostgres()
cloud/apps/relay/src/config.test.ts
+34
View File
@@ -8,6 +8,11 @@ import {
RELAY_PUBLIC_RESOLVE_CONCURRENCY,
RELAY_PUBLIC_RESOLVE_WAIT_MS
} from './config.js'
import {
RELAY_MAX_READINESS_GRACE_MS,
RELAY_READINESS_JWKS_GRACE_MS,
RELAY_READINESS_SQL_GRACE_MS
} from './relay-readiness.js'
function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
return {
@@ -36,6 +41,35 @@ describe('GCE relay capacity configuration', () => {
}
})
it('defaults readiness grace to fifteen minutes for JWKS and three for SQL', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
readinessJwksGraceMs: RELAY_READINESS_JWKS_GRACE_MS,
readinessSqlGraceMs: RELAY_READINESS_SQL_GRACE_MS
})
expect(RELAY_READINESS_SQL_GRACE_MS).toBeLessThan(RELAY_READINESS_JWKS_GRACE_MS)
env.ORCA_RELAY_READINESS_JWKS_GRACE_MS = '0'
env.ORCA_RELAY_READINESS_SQL_GRACE_MS = String(RELAY_MAX_READINESS_GRACE_MS)
expect(loadRelayConfig(env)).toMatchObject({
readinessJwksGraceMs: 0,
readinessSqlGraceMs: RELAY_MAX_READINESS_GRACE_MS
})
for (const invalid of ['-1', String(RELAY_MAX_READINESS_GRACE_MS + 1), '1.5', 'soon']) {
env.ORCA_RELAY_READINESS_JWKS_GRACE_MS = invalid
expect(() => loadRelayConfig(env)).toThrow()
}
})
it('reads an unset readiness grace variable as the default, never as zero', () => {
const env = cellEnvironment(4_000)
env.ORCA_RELAY_READINESS_JWKS_GRACE_MS = ''
env.ORCA_RELAY_READINESS_SQL_GRACE_MS = ''
expect(loadRelayConfig(env)).toMatchObject({
readinessJwksGraceMs: RELAY_READINESS_JWKS_GRACE_MS,
readinessSqlGraceMs: RELAY_READINESS_SQL_GRACE_MS
})
})
it('requires distinct dedicated admin identities and accepts omitted values', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
cloud/apps/relay/src/config.ts
+19
View File
@@ -9,6 +9,11 @@ import {
type RelayCellConnectionHardCap,
type RelayRegion
} from '@orca-cloud/relay-contract'
import {
RELAY_MAX_READINESS_GRACE_MS,
RELAY_READINESS_JWKS_GRACE_MS,
RELAY_READINESS_SQL_GRACE_MS
} from './relay-readiness.js'
export const RELAY_MAX_CELL_CAPACITY_REQUESTS = 100_000
export const RELAY_DATABASE_POOL_MAX = 10
@@ -31,6 +36,14 @@ const OptionalServiceAccountSchema = z.preprocess(
z.string().email().optional()
)
// 0 disables the window and restores the fail-on-first-error readiness answer. An unset variable
// arrives as '' from Cloud Run, which z.coerce would read as 0 rather than as the default.
const readinessGraceSchema = (defaultMs: number) =>
z.preprocess(
(value) => (value === '' ? undefined : value),
z.coerce.number().int().min(0).max(RELAY_MAX_READINESS_GRACE_MS).default(defaultMs)
)
const EnvSchema = z.object({
PORT: z.coerce.number().int().positive().default(8080),
ORCA_RELAY_PUBLIC_URL: z.string().url(),
@@ -81,6 +94,8 @@ const EnvSchema = z.object({
.optional(),
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_RELAY_READINESS_JWKS_GRACE_MS: readinessGraceSchema(RELAY_READINESS_JWKS_GRACE_MS),
ORCA_RELAY_READINESS_SQL_GRACE_MS: readinessGraceSchema(RELAY_READINESS_SQL_GRACE_MS),
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
@@ -187,6 +202,8 @@ export type RelayConfig = {
connectionUnobservedBound?: number
adminJwksUrl: string
databasePoolMax: number
readinessJwksGraceMs?: number
readinessSqlGraceMs?: number
publicAssignmentsEnabled: boolean
regionalPlacementEnabled?: boolean
regionCorrectionCohortPercent?: number
@@ -335,6 +352,8 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
connectionUnobservedBound: ownCell.connectionUnobservedBound,
adminJwksUrl: parsed.ORCA_RELAY_ADMIN_JWKS_URL,
databasePoolMax,
readinessJwksGraceMs: parsed.ORCA_RELAY_READINESS_JWKS_GRACE_MS,
readinessSqlGraceMs: parsed.ORCA_RELAY_READINESS_SQL_GRACE_MS,
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
cloud/apps/relay/src/control-lease-recovery-postgres.test.ts
+8 -2
View File
@@ -3,6 +3,7 @@ import { ASSIGNMENT_LIMITS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
@@ -133,6 +134,10 @@ describePostgres('expired control lease after a database outage', () => {
internals.heartbeat(session)
}
// A due renewal leaves the heartbeat as a batch enqueue, so a poll has to
// outlast the batch window before it can call the renewal missing.
const renewalPoll = { timeout: CONTROL_RENEWAL_BATCH_INTERVAL_MS + 4_000 }
const leaseRows = async (relayHostId: string) =>
await database.query(
`SELECT activity_id, cell_id FROM relay_assignment_activity_leases
@@ -149,7 +154,8 @@ describePostgres('expired control lease after a database outage', () => {
.poll(
async () =>
socket.close.mock.calls.length > 0 ||
(await leaseRows(relayHostId)).length === expectedRows
(await leaseRows(relayHostId)).length === expectedRows,
renewalPoll
)
.toBe(true)
}
@@ -212,7 +218,7 @@ describePostgres('expired control lease after a database outage', () => {
).rejects.toThrow('control_activity_moved')
heartbeat(registry, session)
await expect.poll(() => socket.close.mock.calls.length).toBe(1)
await expect.poll(() => socket.close.mock.calls.length, renewalPoll).toBe(1)
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
expect(await leaseRows(identity.relayHostId)).toEqual([
cloud/apps/relay/src/control-renewal-batch-store.test.ts
+291
View File
@@ -0,0 +1,291 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type SqlRow
} from './database.js'
const now = 1_900_000_000_000
const expiresAt = now + 105_000
function renewal(userId: string, relayHostId: string, expiry = expiresAt) {
return {
identity: { userId, relayHostId },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: expiry
}
}
// A PostgreSQL-dialect database that answers the renewal statement without a
// server, so the statement count and its parameter arrays are observable.
class RenewalStatementProbe implements RelayDatabase {
readonly dialect = 'postgres' as const
readonly statements: Array<{ sql: string; params: unknown[] }> = []
failuresRemaining = 0
constructor(private readonly outcomeFor: (userId: string) => ControlRenewalOutcome) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push({ sql, params })
if (this.failuresRemaining > 0) {
this.failuresRemaining -= 1
throw new Error('canceling statement due to statement timeout')
}
const userIds = params[0] as string[]
return userIds.map((userId, index) => ({
row_index: String(index + 1),
outcome: this.outcomeFor(userId)
}))
}
async queryLocked(): Promise<SqlRow[]> {
throw new Error('unexpected_locked_query')
}
async transaction<T>(): Promise<T> {
// Renewals must never open one: that is the write transaction per host this
// batch exists to remove.
throw new Error('unexpected_transaction')
}
async close(): Promise<void> {}
}
describe('batched control renewals on PostgreSQL', () => {
it('spends one statement on every host that came due', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000003')
])
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
expect(probe.statements).toHaveLength(1)
expect(probe.statements[0]!.sql).toBe(CONTROL_RENEWAL_BATCH_SQL)
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
})
it('locks assignment rows in primary-key order and still answers in input order', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-c', 'host000000000003'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000001'),
renewal('user-a', 'host000000000001')
])
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
// statement's ORDER BY repeats it: no batch can queue against another in a
// different sequence.
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
expect(probe.statements[0]!.params[1]).toEqual([
'host000000000001',
'host000000000002',
'host000000000001',
'host000000000003'
])
expect(outcomes).toEqual([
'renewed',
'renewed',
'control_activity_moved',
'renewed'
])
})
it('keeps a malformed request out of the statement and fails only that row', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002', now + ASSIGNMENT_LIMITS.activityLeaseMs * 10),
{ ...renewal('user-a', 'host000000000003'), activityId: '' },
renewal('user-a', 'host000000000004')
])
expect(outcomes).toEqual([
'renewed',
'invalid_activity_expiry',
'invalid_activity_id',
'renewed'
])
expect(probe.statements[0]!.params[1]).toEqual(['host000000000001', 'host000000000004'])
})
it('degrades to one statement per host when the batch statement fails', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['renewed', 'renewed'])
expect(probe.statements).toHaveLength(3)
expect(probe.statements[1]!.params[1]).toEqual(['host000000000001'])
expect(probe.statements[2]!.params[1]).toEqual(['host000000000002'])
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_batch_failed',
rows: 2
})
} finally {
warn.mockRestore()
}
})
it('reports a host that fails its own fallback statement without touching the rest', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 2
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes.filter((outcome) => outcome === 'renewed')).toHaveLength(1)
expect(outcomes.filter((outcome) => outcome === 'database_error')).toHaveLength(1)
} finally {
warn.mockRestore()
}
})
it('reports a contended assignment row apart from a missing one', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_lock_unavailable' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
// Retryable: SKIP LOCKED passed over the row rather than queueing the whole
// flush behind whoever held it.
expect(outcomes).toEqual(['renewed', 'assignment_lock_unavailable'])
})
it('counts a lone renewal that threw before rethrowing it', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
// still owes an outcome for the attempt.
await expect(
store.renewControlActivities([renewal('user-a', 'host000000000001')])
).rejects.toThrow('statement timeout')
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
})
it('counts a batch in which every row was rejected before the statement', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
const outcomes = await store.renewControlActivities([
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
renewal('user-a', 'host000000000002', now - 1)
])
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
expect(probe.statements).toHaveLength(0)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'invalid_activity_id',
'invalid_activity_expiry'
])
})
it('counts one renewal metric per row against the flush latency', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
)
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'renewed',
'assignment_not_found'
])
})
})
describe('batched control renewals on SQLite', () => {
it('renews every host through the transactional path', async () => {
let clock = now
const database = await openInMemoryRelayDatabase()
try {
const store = new RelayAssignmentStore(database, () => clock)
await store.reconcileCells([
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
])
const hosts = ['host000000000001', 'host000000000002']
const requests = []
for (const relayHostId of hosts) {
const identity = { userId: 'user-a', relayHostId }
const assignment = await store.assign(identity)
await store.activateControl(identity, {
cellId: assignment.cellId,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
requests.push({
identity,
activityId: `control:${assignment.cellId}:1`,
cellId: assignment.cellId,
expiresAt: clock + 105_000
})
}
// A host with no assignment at all must not cost the others their renewal.
requests.push({
identity: { userId: 'user-a', relayHostId: 'host000000000009' },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: clock + 105_000
})
clock += 1_000
const outcomes = await store.renewControlActivities(requests)
expect(outcomes).toEqual(['renewed', 'renewed', 'assignment_not_found'])
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? ORDER BY relay_host_id ASC`,
['user-a']
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
now + 105_000,
now + 105_000
])
} finally {
await database.close()
}
})
})
cloud/apps/relay/src/control-renewal-batch.test.ts
+231
View File
@@ -0,0 +1,231 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
CONTROL_RENEWAL_BATCH_INTERVAL_MS,
CONTROL_RENEWAL_BATCH_MAX_ROWS,
ControlRenewalBatch,
type ControlRenewalFlush
} from './control-renewal-batch.js'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// Settles into the outcome the caller saw, attached at enqueue so a rejection is
// never momentarily unhandled.
function outcomeOf(renewal: Promise<void>): Promise<string> {
return renewal.then(
() => 'renewed',
(error: unknown) => String((error as { message?: unknown }).message)
)
}
function request(
host: string,
expiresAt = 1_000,
activityId = 'control:cell-a:1'
): ControlRenewalRequest {
return {
identity: { userId: 'user-a', relayHostId: host },
activityId,
cellId: 'cell-a',
expiresAt
}
}
describe('control renewal batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('spends one call on every renewal that came due in the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const settled = [
batch.enqueue(request('host0000000000a1')),
batch.enqueue(request('host0000000000a2')),
batch.enqueue(request('host0000000000a3'))
]
expect(renew).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all(settled)).resolves.toEqual([undefined, undefined, undefined])
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0].map((row) => row.identity.relayHostId)).toEqual([
'host0000000000a1',
'host0000000000a2',
'host0000000000a3'
])
})
it('routes each outcome back to the caller that asked for it', async () => {
const outcomes: ControlRenewalOutcome[] = [
'renewed',
'assignment_not_found',
'control_activity_moved'
]
const batch = new ControlRenewalBatch(async () => outcomes)
const first = outcomeOf(batch.enqueue(request('host0000000000b1')))
const second = outcomeOf(batch.enqueue(request('host0000000000b2')))
const third = outcomeOf(batch.enqueue(request('host0000000000b3')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second, third])).resolves.toEqual([
'renewed',
'assignment_not_found',
'control_activity_moved'
])
})
it('flushes on reaching the row ceiling instead of waiting out the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
for (let row = 0; row < CONTROL_RENEWAL_BATCH_MAX_ROWS - 1; row++) {
void batch.enqueue(request(`host${String(row).padStart(12, '0')}`))
}
expect(renew).not.toHaveBeenCalled()
void batch.enqueue(request('host0000000000zz'))
await vi.advanceTimersByTimeAsync(0)
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0]).toHaveLength(CONTROL_RENEWAL_BATCH_MAX_ROWS)
// The window timer must not fire a second, empty statement.
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledOnce()
})
it('does not hold a new window behind a statement still in PostgreSQL', async () => {
let release!: (outcomes: ControlRenewalOutcome[]) => void
const renew = vi
.fn<(rows: readonly ControlRenewalRequest[]) => Promise<ControlRenewalOutcome[]>>()
.mockImplementationOnce(
async () => await new Promise<ControlRenewalOutcome[]>((resolve) => (release = resolve))
)
.mockResolvedValue(['renewed'])
const batch = new ControlRenewalBatch(renew)
const stalled = batch.enqueue(request('host0000000000c1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
const next = batch.enqueue(request('host0000000000c2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
await expect(next).resolves.toBeUndefined()
release(['renewed'])
await expect(stalled).resolves.toBeUndefined()
})
it('reports the driver failure to every caller in the flush', async () => {
const batch = new ControlRenewalBatch(async () => {
throw new Error('pool timeout')
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const first = outcomeOf(batch.enqueue(request('host0000000000d1')))
const second = outcomeOf(batch.enqueue(request('host0000000000d2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second])).resolves.toEqual([
'pool timeout',
'pool timeout'
])
} finally {
warn.mockRestore()
}
})
it('supersedes a second attempt for one lease and answers both callers', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const earlier = batch.enqueue(request('host0000000000e1', 1_000))
const later = batch.enqueue(request('host0000000000e1', 2_000))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew.mock.calls[0]![0]).toEqual([
expect.objectContaining({ expiresAt: 2_000 })
])
await expect(earlier).resolves.toBeUndefined()
await expect(later).resolves.toBeUndefined()
})
it('holds a second activity for one host back to the next flush', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const first = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:1'))
const second = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:2'))
const other = batch.enqueue(request('host0000000000h2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
// One statement updates a host's assignment row once, so the host appears in
// one flush only; the newer generation leads the next one.
expect(renew.mock.calls[0]![0].map((row) => row.activityId)).toEqual([
'control:cell-a:1',
'control:cell-a:1'
])
await expect(Promise.all([first, other])).resolves.toEqual([undefined, undefined])
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
expect(renew.mock.calls[1]![0].map((row) => row.activityId)).toEqual(['control:cell-a:2'])
await expect(second).resolves.toBeUndefined()
})
it('stays quiet for a fast flush that renewed everything', async () => {
const flushes: ControlRenewalFlush[] = []
const batch = new ControlRenewalBatch(
async () => ['renewed'],
() => ({ cellId: 'cell-a' }),
(flush) => flushes.push(flush)
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void batch.enqueue(request('host0000000000f1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(warn).not.toHaveBeenCalled()
expect(flushes).toEqual([
{ rows: 1, durationMs: expect.any(Number), outcomes: { renewed: 1 } }
])
} finally {
warn.mockRestore()
}
})
it('logs one line with the outcome counts when a flush did not renew everything', async () => {
const batch = new ControlRenewalBatch(
async () => ['renewed', 'control_activity_not_found'],
() => ({ cellId: 'cell-a' })
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void outcomeOf(batch.enqueue(request('host0000000000g1')))
const missing = outcomeOf(batch.enqueue(request('host0000000000g2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(missing).resolves.toBe('control_activity_not_found')
expect(warn).toHaveBeenCalledOnce()
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_flush',
cellId: 'cell-a',
rows: 2,
outcomes: { renewed: 1, control_activity_not_found: 1 }
})
} finally {
warn.mockRestore()
}
})
})
cloud/apps/relay/src/control-renewal-batch.ts
+148
View File
@@ -0,0 +1,148 @@
import { performance } from 'node:perf_hooks'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// One flush per second turns the fleet's control-lease write rate into a
// function of the cell count rather than the host count: a cell's ~10 due
// renewals per second become one write transaction instead of ten. Well inside
// the 105s lease runway, so a host that misses a window is never at risk.
export const CONTROL_RENEWAL_BATCH_INTERVAL_MS = 1_000
// Ceiling on the parameter arrays. Row locks live until the statement commits,
// so this is what bounds how long one flush holds them: measured at 11.5ms for
// 200 rows against a 20,000-row table, and 9.4ms with a host wedged in a
// per-host transaction.
export const CONTROL_RENEWAL_BATCH_MAX_ROWS = 200
// A flush slower than this is the only latency worth a line; the metrics event
// carries the distribution.
const CONTROL_RENEWAL_SLOW_FLUSH_MS = 250
export type ControlRenewalFlush = {
rows: number
durationMs: number
outcomes: Record<string, number>
}
type PendingWaiter = { resolve: () => void; reject: (error: unknown) => void }
type PendingRenewal = { request: ControlRenewalRequest; waiters: PendingWaiter[] }
type QueuedRenewal = { request: ControlRenewalRequest; waiter: PendingWaiter }
// Per host, not per activity: one statement updates a host's assignment row
// once, so two activities for the same host must not share a flush.
function pendingKey(request: ControlRenewalRequest): string {
return [request.identity.userId, request.identity.relayHostId].join('\u0000')
}
// Collects the control-lease renewals a cell owes and spends one statement on
// them. Each caller still gets the single-renewal contract: the promise resolves
// on `renewed` and rejects with the outcome as its message otherwise, so callers
// keep their per-session error routing unchanged.
export class ControlRenewalBatch {
private pending = new Map<string, PendingRenewal>()
// Renewals a host cannot contribute to the flush being built; they open the
// next one.
private deferred: QueuedRenewal[] = []
private timer: ReturnType<typeof setTimeout> | null = null
constructor(
private readonly renew: (
rows: readonly ControlRenewalRequest[]
) => Promise<ControlRenewalOutcome[]>,
private readonly logFields: () => Record<string, unknown> = () => ({}),
private readonly observe?: (flush: ControlRenewalFlush) => void
) {}
enqueue(request: ControlRenewalRequest): Promise<void> {
return new Promise<void>((resolve, reject) => {
this.admit({ request, waiter: { resolve, reject } })
})
}
private admit(queued: QueuedRenewal): void {
const key = pendingKey(queued.request)
const existing = this.pending.get(key)
if (existing && existing.request.activityId !== queued.request.activityId) {
this.deferred.push(queued)
this.scheduleFlush()
return
}
if (existing) {
// A second attempt at the same lease inside one window supersedes the
// first expiry; both callers still hear the outcome they waited for.
existing.request = {
...queued.request,
expiresAt: Math.max(existing.request.expiresAt, queued.request.expiresAt)
}
existing.waiters.push(queued.waiter)
return
}
this.pending.set(key, { request: queued.request, waiters: [queued.waiter] })
if (this.pending.size >= CONTROL_RENEWAL_BATCH_MAX_ROWS) {
void this.flush()
return
}
this.scheduleFlush()
}
private scheduleFlush(): void {
this.timer ??= setTimeout(() => {
this.timer = null
void this.flush()
}, CONTROL_RENEWAL_BATCH_INTERVAL_MS)
this.timer.unref?.()
}
// Flushes run concurrently on purpose: a statement stalled in PostgreSQL must
// not hold back the renewals that came due while it was waiting.
async flush(): Promise<void> {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
const batch = [...this.pending.values()]
this.pending = new Map()
// Re-admitted against the empty map, so a host deferred out of this flush
// leads the next one.
const deferred = this.deferred
this.deferred = []
for (const queued of deferred) this.admit(queued)
if (batch.length === 0) return
const startedAt = performance.now()
let outcomes: ControlRenewalOutcome[]
try {
outcomes = await this.renew(batch.map((entry) => entry.request))
} catch (error) {
for (const entry of batch) for (const waiter of entry.waiters) waiter.reject(error)
this.report(batch.length, performance.now() - startedAt, { flush_failed: batch.length })
return
}
const counts: Record<string, number> = {}
for (const [index, entry] of batch.entries()) {
const outcome = outcomes[index] ?? 'database_error'
counts[outcome] = (counts[outcome] ?? 0) + 1
for (const waiter of entry.waiters) {
if (outcome === 'renewed') waiter.resolve()
else waiter.reject(new Error(outcome))
}
}
this.report(batch.length, performance.now() - startedAt, counts)
}
private report(rows: number, durationMs: number, outcomes: Record<string, number>): void {
this.observe?.({ rows, durationMs, outcomes })
const renewed = outcomes.renewed ?? 0
if (durationMs <= CONTROL_RENEWAL_SLOW_FLUSH_MS && renewed === rows) return
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_flush',
...this.logFields(),
rows,
durationMs: Math.round(durationMs),
outcomes
})
)
}
}
cloud/apps/relay/src/control-renewal-postgres.test.ts
+166 -3
View File
@@ -1,8 +1,11 @@
import { performance } from 'node:perf_hooks'
import { ASSIGNMENT_LIMITS, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import {
openRelayDatabase,
POSTGRES_LOCK_TIMEOUT_MS,
type RelayDatabase,
type RelayLockOptions,
type SqlRow
@@ -22,7 +25,9 @@ const targetCell = {
capacityRequests: 100
}
const userId = 'control-renewal-postgres-user'
const identities = Array.from({ length: 6 }, (_, index) => ({
// Indexes 0-5 belong to the single-renewal cases below, which mutate their
// host's migration and lease state; the batch cases own 6-13.
const identities = Array.from({ length: 14 }, (_, index) => ({
userId,
relayHostId: `controlrenewal${index + 1}`
}))
@@ -72,7 +77,7 @@ class StallFirstRenewalQueryDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (this.stallNext && sql.includes('WITH assignment_state AS MATERIALIZED')) {
if (this.stallNext && sql === CONTROL_RENEWAL_BATCH_SQL) {
this.stallNext = false
this.stalled.resolve()
await this.continue.promise
@@ -103,7 +108,7 @@ class RenewalQueryProbeDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (sql.includes('WITH assignment_state AS MATERIALIZED')) this.renewalQueries++
if (sql === CONTROL_RENEWAL_BATCH_SQL) this.renewalQueries++
return await this.database.query(sql, params)
}
@@ -332,6 +337,164 @@ describePostgres('PostgreSQL control renewal', () => {
).rejects.toThrow('invalid_activity_expiry')
})
it('renews every due host in one autocommitted statement', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
const batch = identities.slice(6, 10)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
batch.map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed', 'renewed'])
expect(probe.renewalQueries).toBe(1)
expect(probe.transactions).toBe(0)
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_id = ? ORDER BY relay_host_id ASC`,
[userId, controlId(sourceCell.id)]
)
expect(
leases
.filter((lease) =>
batch.some((identity) => identity.relayHostId === lease.relay_host_id)
)
.map((lease) => Number(lease.expires_at))
).toEqual([expiresAt, expiresAt, expiresAt, expiresAt])
})
it('reports each host its own verdict inside one batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const live = identities[10]!
const absent = { userId, relayHostId: 'controlrenewalgone' }
const outcomes = await store.renewControlActivities([
{ identity: absent, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{ identity: live, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{
identity: live,
activityId: controlId(targetCell.id),
cellId: targetCell.id,
expiresAt
}
])
expect(outcomes).toEqual([
'assignment_not_found',
'renewed',
'activity_cell_not_authoritative'
])
})
it('passes over a host whose assignment row is held and renews the rest', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const held = identities[11]!
const free = identities[12]!
const locked = signal()
const release = signal()
// Holds the row the way every per-host transactional path does.
const holder = database.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[held.userId, held.relayHostId]
)
locked.resolve()
await release.promise
})
await locked.promise
const startedAt = performance.now()
const outcomes = await store.renewControlActivities(
[held, free].map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
const elapsedMs = performance.now() - startedAt
release.resolve()
await holder
expect(outcomes).toEqual(['assignment_lock_unavailable', 'renewed'])
// It skipped rather than queued: a blocking FOR UPDATE would have spent the
// pool's whole lock_timeout here and failed the free host too.
expect(elapsedMs).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS)
const lease = (
await database.query(
`SELECT expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[free.userId, free.relayHostId, controlId(sourceCell.id)]
)
)[0]
expect(Number(lease!.expires_at)).toBe(expiresAt)
})
it('renews both of one host\u2019s control leases in a single batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
const identity = identities[13]!
// Two live control leases on one host. Written directly because
// activateControl retires the prior generation, and what is under test is the
// statement's row-wise behaviour, not how the second lease came to exist.
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`control:${sourceCell.id}:2`,
sourceCell.id,
now,
now
]
)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
[1, 2].map((generation) => ({
identity,
activityId: `control:${sourceCell.id}:${generation}`,
cellId: sourceCell.id,
expiresAt: expiresAt - generation
}))
)
expect(outcomes).toEqual(['renewed', 'renewed'])
const leases = await database.query(
`SELECT activity_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`,
[identity.userId, identity.relayHostId]
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
expiresAt - 1,
expiresAt - 2
])
// The assignment row is written once, carrying the later of the two.
const row = (
await database.query(
`SELECT lease_expires_at, last_activity_at FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
expect(Number(row!.lease_expires_at)).toBe(expiresAt - 1)
expect(Number(row!.last_activity_at)).toBe(now)
})
it('uses one autocommitted PostgreSQL statement for a steady renewal', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
cloud/apps/relay/src/control-renewal-statement.ts
+230
View File
@@ -0,0 +1,230 @@
import type { AssignmentIdentity } from './assignment-identity-queue.js'
import type { SqlRow } from './database.js'
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
// The host's assignment row was already locked by one of the per-host
// transactional paths. Retryable, and never a reason to close a control: the
// next tick is 15s away and the lease has 105s on it.
| 'assignment_lock_unavailable'
// Decided per row before the statement runs, so one malformed request cannot
// cost the rest of the batch its renewal.
| 'invalid_activity_id'
| 'invalid_activity_expiry'
| 'database_error'
// Outcomes the statement itself can report. `database_error` is raised by the
// driver, and `invalid_activity_expiry` is decided per row before the statement
// is built, so neither can come back as a row.
export const CONTROL_RENEWAL_STATEMENT_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved',
'assignment_lock_unavailable'
])
export type ControlRenewalRequest = {
identity: AssignmentIdentity
activityId: string
cellId: string
expiresAt: number
}
// LOCK ORDER - (user_id, relay_host_id), the primary key of relay_assignments,
// applied here and repeated as the statement's ORDER BY so it holds whether the
// planner walks the primary-key index or sorts under the LockRows node.
//
// The batch never waits for an assignment row: SKIP LOCKED reports a contended
// host separately instead. That is what bounds how long a flush holds its locks
// to its own execution time, because row locks live until the statement commits,
// and it is why one host wedged in a per-host transaction cannot stall the
// renewals of every other host sharing the flush.
//
// With no wait on the assignment pass, the deadlock question reduces to the two
// later passes. Every writer in this store locks a host's assignment row before
// that host's lease rows (`assignmentRow` then `lockAssignmentActivities`), and
// a host whose assignment row is held was skipped, so the batch never reaches
// that host's lease: the lease pass cannot wait either.
// `markMigrationTargetRegistered` is the one writer that locks a migration row
// without the assignment row first. It takes no further locks, so it can delay a
// mid-migration row by up to the pool's lock_timeout but cannot close a cycle.
export function orderedControlRenewalRows<Row extends { identity: AssignmentIdentity }>(
rows: readonly Row[]
): Row[] {
return [...rows].sort(
(left, right) =>
left.identity.userId.localeCompare(right.identity.userId) ||
left.identity.relayHostId.localeCompare(right.identity.relayHostId)
)
}
// One statement renewing every due control lease on this cell, row-wise over the
// unnested parameter arrays. Logic per row is what the single-row predecessor
// did: lock the assignment, admit the caller's cell either as the current cell or
// as the source of an active forward migration, lock that host's control lease,
// push both expiries forward, and report one outcome. The one addition is
// `present_assignment`, an unlocked probe that separates a host with no
// assignment row at all from one whose row SKIP LOCKED passed over - the first
// closes the control, the second retries.
export const CONTROL_RENEWAL_BATCH_SQL = `WITH renewal_input AS MATERIALIZED (
SELECT
renewal.ordinality AS row_index,
renewal.user_id,
renewal.relay_host_id,
renewal.activity_id,
renewal.cell_id,
renewal.expires_at
FROM unnest(?::text[], ?::text[], ?::text[], ?::text[], ?::bigint[])
WITH ORDINALITY AS renewal(
user_id, relay_host_id, activity_id, cell_id, expires_at, ordinality
)
), present_assignment AS MATERIALIZED (
SELECT input.row_index
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
), assignment_state AS MATERIALIZED (
SELECT input.row_index, assignment.cell_id, assignment.assignment_epoch
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
ORDER BY assignment.user_id, assignment.relay_host_id
FOR UPDATE OF assignment SKIP LOCKED
), migration_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
JOIN relay_assignment_migrations migration
ON migration.user_id = input.user_id
AND migration.relay_host_id = input.relay_host_id
AND migration.source_cell_id = input.cell_id
AND migration.target_cell_id = locked.cell_id
AND migration.assignment_epoch = locked.assignment_epoch
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
WHERE locked.cell_id = input.cell_id
OR EXISTS (
SELECT 1 FROM migration_state moving
WHERE moving.row_index = locked.row_index
)
), lease_state AS MATERIALIZED (
SELECT authorized.row_index, lease.activity_kind, lease.cell_id
FROM authorization_state authorized
JOIN renewal_input input ON input.row_index = authorized.row_index
JOIN relay_assignment_activity_leases lease
ON lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, input.expires_at),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
JOIN renewal_input input ON input.row_index = state.row_index
WHERE lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
AND state.activity_kind = 'control' AND state.cell_id = input.cell_id
RETURNING state.row_index
), renewed_assignment AS (
-- Grouped per host: an UPDATE whose FROM offers a target row more than
-- once applies one source row and returns one, so two leases on one
-- host would leave the assignment carrying the wrong expiry. The
-- aggregate hands it exactly one row, carrying the later expiry.
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, renewed.expires_at),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
FROM (
SELECT input.user_id, input.relay_host_id, MAX(input.expires_at) AS expires_at
FROM renewed_lease renewed
JOIN renewal_input input ON input.row_index = renewed.row_index
GROUP BY input.user_id, input.relay_host_id
) renewed
WHERE assignment.user_id = renewed.user_id
AND assignment.relay_host_id = renewed.relay_host_id
RETURNING renewed.user_id
)
SELECT input.row_index, CASE
WHEN NOT EXISTS (
SELECT 1 FROM present_assignment present
WHERE present.row_index = input.row_index
) THEN 'assignment_not_found'
WHEN NOT EXISTS (
SELECT 1 FROM assignment_state locked WHERE locked.row_index = input.row_index
) THEN 'assignment_lock_unavailable'
WHEN NOT EXISTS (
SELECT 1 FROM authorization_state authorized
WHERE authorized.row_index = input.row_index
) THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (
SELECT 1 FROM lease_state state WHERE state.row_index = input.row_index
) THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state state
WHERE state.row_index = input.row_index
AND (state.activity_kind <> 'control' OR state.cell_id <> input.cell_id)
) THEN 'control_activity_moved'
-- Read from renewed_lease, which has one row per input row. The
-- assignment update collapses to one row per host, so it cannot answer
-- for a host that brought two leases to the same batch.
WHEN EXISTS (
SELECT 1 FROM renewed_lease renewed
WHERE renewed.row_index = input.row_index
) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome
FROM renewal_input input
ORDER BY input.row_index`
export function controlRenewalBatchParams(
rows: readonly ControlRenewalRequest[],
now: number
): unknown[] {
return [
rows.map((row) => row.identity.userId),
rows.map((row) => row.identity.relayHostId),
rows.map((row) => row.activityId),
rows.map((row) => row.cellId),
rows.map((row) => row.expiresAt),
now,
now
]
}
// Rows come back ordered by row_index, which is the 1-based position in the
// statement's parameter arrays.
export function readControlRenewalOutcomes(
rows: SqlRow[],
expected: number
): ControlRenewalOutcome[] {
if (rows.length !== expected) throw new Error('missing_control_renewal_outcome')
return rows.map((row, position) => {
if (Number(row.row_index) !== position + 1) {
throw new Error('misordered_control_renewal_outcome')
}
const outcome = row.outcome
if (
typeof outcome !== 'string' ||
!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(outcome as ControlRenewalOutcome)
) {
throw new Error('invalid_control_renewal_outcome')
}
// SAFETY: the membership check above is what narrows this string.
return outcome as ControlRenewalOutcome
})
}
cloud/apps/relay/src/credential-cleanup-sweep-postgres.test.ts
+283
View File
@@ -0,0 +1,283 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
// The outage this guards against: the credential cleanup ran every 30s in all 23 cells and both
// sweeps over relay_invites had no usable index, so each one seq-scanned the whole table inside the
// maintenance transaction. Only a real planner can show the partial indexes take that away, and
// only a real server has ctid.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_credential_sweep_test'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
function scopedUrl(): string {
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
return url.toString()
}
async function onAdmin<T>(operation: (client: pg.Client) => Promise<T>): Promise<T> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
return await operation(client)
} finally {
await client.end()
}
}
describePostgres('credential cleanup against PostgreSQL', () => {
let database: RelayDatabase
let store: RelayCredentialStore
const opened: RelayDatabase[] = []
beforeEach(async () => {
await onAdmin(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
})
database = await openRelayDatabase({ databaseUrl: scopedUrl(), dataDir: '' })
opened.push(database)
store = new RelayCredentialStore(database, () => NOW)
})
afterAll(async () => {
await Promise.all(opened.map((open) => open.close().catch(() => undefined)))
await onAdmin((client) => client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`))
})
async function seedInvites(
count: number,
state: string,
updatedAt: number,
expiresAt = NOW - DAY_MS
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
SELECT ?, ?, 'device-' || n, 'token-' || ? || '-' || n, ?, 0, 3, ?, ?, ?
FROM generate_series(1, ?) AS n`,
[identity.userId, identity.relayHostId, state, state, expiresAt, updatedAt, updatedAt, count]
)
}
// Not through RelayDatabase: it routes anything that is not a SELECT to the row-count path, and
// EXPLAIN on an UPDATE is neither.
async function plan(sql: string, params: unknown[]): Promise<string> {
const client = new pg.Client({ connectionString: scopedUrl() })
await client.connect()
try {
let index = 0
const result = await client.query(
`EXPLAIN ${sql.replace(/\?/g, () => `$${(index += 1)}`)}`,
params
)
return result.rows.map((row) => String(row['QUERY PLAN'])).join('\n')
} finally {
await client.end()
}
}
it('plans both invite sweeps as index scans instead of scanning the whole table', async () => {
// Production's shape: terminal invites outnumber live ones by orders of magnitude, which is
// what makes the partial predicates worth having.
await seedInvites(20_000, 'consumed', NOW)
for (const state of ['available', 'reserved', 'cooldown']) {
await seedInvites(200, state, NOW, NOW + DAY_MS)
}
await database.query(
`UPDATE relay_invites SET reservation_expires_at = ? WHERE state = 'reserved'`,
[NOW + 1]
)
// Only ANALYZE makes the planner's row estimates real; without it a cold table looks tiny and
// a seq scan wins on any index.
await database.query(`ANALYZE relay_invites`)
const expiry = await plan(
`UPDATE relay_invites SET state = 'expired'
WHERE expires_at <= ? AND state IN ('available', 'reserved', 'cooldown')`,
[NOW]
)
const reservation = await plan(
`UPDATE relay_invites SET state = 'cooldown'
WHERE state = 'reserved' AND reservation_expires_at <= ? AND expires_at > ?`,
[NOW, NOW]
)
// Which of the two partial indexes serves the reservation pass is the planner's call: both
// predicates hold only live invites, so either one reads a handful of rows. The invariant is
// that neither pass reads the whole table any more.
for (const sweep of [expiry, reservation]) {
expect(sweep).not.toContain('Seq Scan on relay_invites')
expect(sweep).toMatch(/using relay_invites_sweep_(expiry|reservation)/)
}
expect(expiry).toContain('relay_invites_sweep_expiry')
})
it('plans the basis sweep off the composite index rather than the 1.5 GB heap', async () => {
// The shape that made this the most expensive statement in the sweep: 20,000 settled bases to
// 50 live ones. A partial index on active = 1 looks like the answer to that ratio and is not:
// a basis is inserted active and flipped to 0, so it accumulates the same dead entries, and
// the planner picks the composite index anyway. See the schema comment beside it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'live-' || n, ?, ?, 'device-1', 1, 'invite', ?, 1, ?
FROM generate_series(1, 50) AS n`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(`ANALYZE relay_connection_bases`)
const sweep = await plan(
`UPDATE relay_connection_bases SET active = 0 WHERE active = 1 AND deadline <= ?`,
[NOW]
)
expect(sweep).not.toContain('Seq Scan on relay_connection_bases')
expect(sweep).toContain('using relay_connection_bases_active_deadline')
})
it('plans the drained basis reaper off the composite index, not the heap', async () => {
// Why the composite index stays for now: it is the only one covering active = 0, and the case
// that needs it is the steady state, where every row is inside retention and the reaper must
// learn there is nothing to do. While the backlog drains the planner rightly prefers a bounded
// sequential scan, because it finds its 5,000 rows and stops; measured at 200k rows, the
// drained batch costs 5 buffers with this index and 1,274 without it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 60_000, NOW - 60_000]
)
await database.query(`ANALYZE relay_connection_bases`)
const reaper = await plan(
`DELETE FROM relay_connection_bases WHERE ctid IN (
SELECT ctid FROM relay_connection_bases WHERE active = ? AND deadline <= ? LIMIT 5000
)`,
[0, NOW - DAY_MS]
)
expect(reaper).toContain('relay_connection_bases_active_deadline')
expect(reaper).not.toContain('Seq Scan on relay_connection_bases')
})
it('plans the pending-authorization and rate-window sweeps as index scans', async () => {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'auth-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 1, NOW - 1]
)
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
SELECT 'scope-' || n, 'invite-mint', ?, 1 FROM generate_series(1, 20000) AS n`,
[NOW]
)
await database.query(`ANALYZE relay_direct_authorizations`)
await database.query(`ANALYZE relay_rate_windows`)
const pending = await plan(
`UPDATE relay_direct_authorizations SET consumed_at = ?
WHERE consumed_at IS NULL AND deadline <= ?`,
[NOW, NOW]
)
const windows = await plan(`DELETE FROM relay_rate_windows WHERE window_started_at < ?`, [
NOW - DAY_MS
])
expect(pending).toContain('relay_direct_authorizations_pending_deadline')
expect(pending).not.toContain('Seq Scan on relay_direct_authorizations')
expect(windows).toContain('relay_rate_windows_started')
expect(windows).not.toContain('Seq Scan on relay_rate_windows')
})
it('reaps settled bases and consumed authorizations through ctid', async () => {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES ('live', ?, ?, 'device-1', 1, 'invite', ?, 1, ?)`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'consumed-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES ('pending', ?, ?, 'device-1', 1, ?, NULL)`,
[identity.userId, identity.relayHostId, NOW + 30_000]
)
await store.cleanup()
expect(
await database.query(`SELECT count(*) AS total FROM relay_connection_bases`)
).toEqual([{ total: '3' }])
expect(
await database.query(`SELECT count(*) AS total FROM relay_direct_authorizations`)
).toEqual([{ total: '3' }])
await store.cleanup()
// Only the rows a reader could still accept are left.
expect(
await database.query(`SELECT basis_conn_id FROM relay_connection_bases`)
).toEqual([{ basis_conn_id: 'live' }])
expect(
await database.query(`SELECT direct_auth_id FROM relay_direct_authorizations`)
).toEqual([{ direct_auth_id: 'pending' }])
})
it('reaps terminal invites past retention through ctid, one bounded batch per cycle', async () => {
await seedInvites(5_002, 'consumed', NOW - 30 * DAY_MS)
await seedInvites(3, 'invalidated', NOW - 6 * DAY_MS)
await seedInvites(2, 'available', NOW - 400 * DAY_MS, NOW + DAY_MS)
await store.cleanup()
expect(await database.query(`SELECT count(*) AS total FROM relay_invites`)).toEqual([
{ total: '7' }
])
await store.cleanup()
// The two live invites and the three inside retention survive; the batch remainder is gone.
expect(
await database.query(`SELECT state, count(*) AS total FROM relay_invites GROUP BY state ORDER BY state`)
).toEqual([
{ state: 'available', total: '2' },
{ state: 'invalidated', total: '3' }
])
})
})
cloud/apps/relay/src/credential-store-cleanup.test.ts
+297
View File
@@ -0,0 +1,297 @@
import { describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
async function insertInvite(
database: RelayDatabase,
invite: { token: string; state: string; updatedAt: number; expiresAt?: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`device-${invite.token}`,
invite.token,
invite.state,
0,
3,
invite.expiresAt ?? NOW + DAY_MS,
invite.updatedAt,
invite.updatedAt
]
)
}
async function remainingTokens(database: RelayDatabase): Promise<string[]> {
const rows = await database.query(`SELECT token_hash FROM relay_invites ORDER BY token_hash`)
return rows.map((row) => String(row.token_hash))
}
async function insertBasis(
database: RelayDatabase,
basis: { id: string; active: number; deadline: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
basis.id,
identity.userId,
identity.relayHostId,
'device-1',
1,
'invite',
basis.deadline,
basis.active,
NOW
]
)
}
async function insertDirectAuthorization(
database: RelayDatabase,
auth: { id: string; deadline: number; consumedAt: number | null }
): Promise<void> {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[auth.id, identity.userId, identity.relayHostId, 'device-1', 1, auth.deadline, auth.consumedAt]
)
}
async function remainingIds(database: RelayDatabase, table: string, column: string): Promise<string[]> {
const rows = await database.query(`SELECT ${column} FROM ${table} ORDER BY ${column}`)
return rows.map((row) => String(row[column]))
}
describe('credential cleanup invite reaper', () => {
it('deletes terminal invites past retention and keeps everything else', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 8 * DAY_MS
const recent = NOW - 6 * DAY_MS
for (const state of ['expired', 'consumed', 'invalidated']) {
await insertInvite(database, { token: `stale-${state}`, state, updatedAt: stale })
await insertInvite(database, { token: `recent-${state}`, state, updatedAt: recent })
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual([
'recent-consumed',
'recent-expired',
'recent-invalidated'
])
await database.close()
})
it('never deletes an invite that a reader could still consume, however old', async () => {
// Retention is measured on updated_at, and a long-lived available invite has an old one. The
// state filter is what keeps the reaper from deleting a credential still in use.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const ancient = NOW - 400 * DAY_MS
for (const state of ['available', 'reserved', 'cooldown']) {
await insertInvite(database, {
token: `live-${state}`,
state,
updatedAt: ancient,
expiresAt: NOW + DAY_MS
})
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual(['live-available', 'live-cooldown', 'live-reserved'])
await database.close()
})
it('bounds one cycle to a single batch and drains the rest on later cycles', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 30 * DAY_MS
for (let index = 0; index < 5_002; index += 1) {
await insertInvite(database, {
token: `consumed-${String(index).padStart(5, '0')}`,
state: 'consumed',
updatedAt: stale
})
}
await store.cleanup()
expect(await remainingTokens(database)).toHaveLength(2)
await store.cleanup()
expect(await remainingTokens(database)).toEqual([])
await database.close()
})
it('still expires credentials the sweep owns, and only those past their deadline', async () => {
// The reaper runs after the sweep in the same call, so this pins that adding it did not
// displace any of the five state transitions the sweep is there for.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertInvite(database, {
token: 'lapsed',
state: 'available',
updatedAt: NOW,
expiresAt: NOW - 1
})
await insertInvite(database, {
token: 'current',
state: 'available',
updatedAt: NOW,
expiresAt: NOW + DAY_MS
})
await database.query(
`UPDATE relay_invites SET state = ?, reservation_expires_at = ? WHERE token_hash = ?`,
['reserved', NOW - 1, 'current']
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
'basis-lapsed', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW - 1, 1, NOW,
'basis-live', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW + 1, 1, NOW
]
)
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-lapsed',
owningControlGeneration: 1,
deadline: NOW - 1
})
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-live',
owningControlGeneration: 1,
deadline: NOW + 1
})
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
VALUES (?, ?, ?, ?), (?, ?, ?, ?)`,
['scope', 'invite-mint', NOW - 2 * DAY_MS, 1, 'scope', 'invite-mint', NOW - 1, 1]
)
await store.cleanup()
expect(
await database.query(`SELECT token_hash, state FROM relay_invites ORDER BY token_hash`)
).toEqual([
{ token_hash: 'current', state: 'cooldown' },
{ token_hash: 'lapsed', state: 'expired' }
])
expect(
await database.query(`SELECT basis_conn_id, active FROM relay_connection_bases ORDER BY basis_conn_id`)
).toEqual([
{ basis_conn_id: 'basis-lapsed', active: 0 },
{ basis_conn_id: 'basis-live', active: 1 }
])
expect(
await database.query(
`SELECT direct_auth_id FROM relay_direct_authorizations
WHERE consumed_at IS NULL ORDER BY direct_auth_id`
)
).toEqual([{ direct_auth_id: 'direct-live' }])
expect(await database.query(`SELECT window_started_at FROM relay_rate_windows`)).toEqual([
{ window_started_at: NOW - 1 }
])
await database.close()
})
it('reaps connection bases whose deadline passed over a day ago, and nothing else', async () => {
// Retention is measured on deadline, and both readers of a basis require deadline >= now, so a
// deadline a day in the past is already unusable however the active flag reads. The active = 0
// clause is what keeps the batch an index range, not what makes the row safe to delete.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertBasis(database, { id: 'stale-inactive', active: 0, deadline: NOW - 2 * DAY_MS })
await insertBasis(database, { id: 'recent-inactive', active: 0, deadline: NOW - 60_000 })
// A long-lived splice: still active hours after the 30s deadline it was created with. The
// sweep deactivates it this cycle and the reaper takes it in the same call, which is safe
// precisely because no reader would have accepted it since its deadline passed.
await insertBasis(database, { id: 'stale-active', active: 1, deadline: NOW - 400 * DAY_MS })
await insertBasis(database, { id: 'live-active', active: 1, deadline: NOW + DAY_MS })
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([
'live-active',
'recent-inactive'
])
// The one row a reader can still use is untouched, active flag included.
expect(
await database.query(
`SELECT active FROM relay_connection_bases WHERE basis_conn_id = 'live-active'`
)
).toEqual([{ active: 1 }])
await database.close()
})
it('reaps consumed direct authorizations past retention and never a pending one', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertDirectAuthorization(database, {
id: 'stale-consumed',
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
await insertDirectAuthorization(database, {
id: 'recent-consumed',
deadline: NOW - 60_000,
consumedAt: NOW - 60_000
})
await insertDirectAuthorization(database, {
id: 'pending-ancient',
deadline: NOW + DAY_MS,
consumedAt: null
})
await store.cleanup()
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([
'pending-ancient',
'recent-consumed'
])
await database.close()
})
it('bounds each table to one batch per cycle', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
for (let index = 0; index < 5_001; index += 1) {
const id = String(index).padStart(5, '0')
await insertBasis(database, { id: `basis-${id}`, active: 0, deadline: NOW - 2 * DAY_MS })
await insertDirectAuthorization(database, {
id: `auth-${id}`,
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
}
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toHaveLength(1)
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toHaveLength(1)
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([])
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])
await database.close()
})
})
cloud/apps/relay/src/credential-store.ts
+49
View File
@@ -12,6 +12,17 @@ const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing
// working for clients whose clocks trail the cell by up to this margin.
const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000
// Terminal invites are read by nothing: every reader re-checks expiry and state at read time, so
// the row only serves the audit trail, which relay_audit_events already keeps. A week is long
// enough to answer a support question about a pairing that failed.
const TERMINAL_INVITE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000
// Why: both readers of a connection basis and of a direct authorization require it still
// active/unconsumed AND inside its deadline, and every deadline is set at most 30s past insert, so
// a settled row can never authorize anything again. A day is margin for forensics, not for reads.
const INACTIVE_AUTHORIZATION_RETENTION_MS = 24 * 60 * 60 * 1000
// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over
// however many cycles it takes.
const REAP_BATCH_ROWS = 5000
export type RelayIdentity = { userId: string; relayHostId: string }
export type CredentialReservation = RelayIdentity & {
@@ -643,6 +654,44 @@ export class RelayCredentialStore {
[now - 24 * 60 * 60 * 1000]
)
})
await this.reapSettledCredentials(now)
}
// Outside the sweep transaction on purpose: each delete is idempotent and independent of the
// state transitions above, so batching them in would only hold their row locks for longer.
private async reapSettledCredentials(now: number): Promise<void> {
await this.reapBatch(
'relay_invites',
'state IN (?, ?, ?) AND updated_at <= ?',
['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]
)
// deadline, not created_at: it is the second column of relay_connection_bases_active_deadline,
// so once the backlog is drained this batch learns there is nothing left to do from the index
// instead of the 1.5 GB heap. Both readers reject a passed deadline, so a day past one is
// unusable whatever the active flag says.
await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [
0,
now - INACTIVE_AUTHORIZATION_RETENTION_MS
])
// consumed_at, not deadline: consumption is what settles this row, and it can happen well
// before the deadline, so measuring from it retains the row for the full window either way.
await this.reapBatch(
'relay_direct_authorizations',
'consumed_at IS NOT NULL AND consumed_at <= ?',
[now - INACTIVE_AUTHORIZATION_RETENTION_MS]
)
}
// ctid/rowid, not the primary key: the physical address lets the delete re-find exactly the batch
// the subquery located instead of re-matching the predicate per row.
private async reapBatch(table: string, predicate: string, params: unknown[]): Promise<void> {
const address = this.database.dialect === 'sqlite' ? 'rowid' : 'ctid'
await this.database.query(
`DELETE FROM ${table} WHERE ${address} IN (
SELECT ${address} FROM ${table} WHERE ${predicate} LIMIT ${REAP_BATCH_ROWS}
)`,
params
)
}
private async installStatusWith(
cloud/apps/relay/src/database-postgres-timeout.test.ts
+40 -9
View File
@@ -119,16 +119,22 @@ describe('PostgreSQL relay deadlines', () => {
dataDir: './unused'
})
expect(ddl.length).toBeGreaterThan(0)
expect(ddl).toContain(POSTGRES_STATEMENT_STATS_MIGRATION)
// The catalog pre-check reads pg_catalog on the same untimed connection before each
// lock-taking statement, so the schema pool now carries reads as well as DDL.
const probes = ddl.filter((statement) => /^SELECT\b/i.test(statement))
const statements = ddl.filter((statement) => !/^SELECT\b/i.test(statement))
expect(probes.length).toBeGreaterThan(0)
expect(probes.every((statement) => statement.includes('pg_catalog'))).toBe(true)
expect(statements.length).toBeGreaterThan(0)
expect(statements).toContain(POSTGRES_STATEMENT_STATS_MIGRATION.trim())
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(
ddl.every(
statements.every(
(statement) =>
statement === POSTGRES_STATEMENT_STATS_MIGRATION ||
/^(?:CREATE|ALTER TABLE)\b/i.test(body(statement))
statement === POSTGRES_STATEMENT_STATS_MIGRATION.trim() ||
/^(?:CREATE|ALTER TABLE|DROP INDEX)\b/i.test(body(statement))
)
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
@@ -201,11 +207,11 @@ describe('PostgreSQL relay deadlines', () => {
})
describe('PostgreSQL schema startup', () => {
it('retries lock and statement timeouts with bounded backoff', async () => {
it('retries statement timeouts with bounded backoff', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const query = vi
.fn<(statement: string) => Promise<unknown>>()
.mockRejectedValueOnce(Object.assign(new Error('lock timeout'), { code: '55P03' }))
.mockRejectedValueOnce(Object.assign(new Error('statement timeout'), { code: '57014' }))
.mockRejectedValueOnce(Object.assign(new Error('statement timeout'), { code: '57014' }))
.mockResolvedValue(undefined)
const delays: number[] = []
@@ -221,6 +227,31 @@ describe('PostgreSQL schema startup', () => {
expect(delays).toEqual([125, 250])
})
it('fails the boot on a lock timeout instead of re-entering the lock queue', async () => {
// The catalog pre-check already answered that the object is missing, so a lock timeout means
// this boot lost the queue. Relation locks are granted in queue order, so each retry parks
// every writer behind it for another timeout.
const errors: string[] = []
vi.spyOn(console, 'error').mockImplementation((line: string) => {
errors.push(line)
})
const error = Object.assign(new Error('lock timeout'), { code: '55P03' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
const pause = vi.fn(async () => undefined)
await expect(
applyPostgresSchema(['CREATE INDEX IF NOT EXISTS i ON t(c)'], query, { wait: pause })
).rejects.toBe(error)
expect(query).toHaveBeenCalledTimes(1)
expect(pause).not.toHaveBeenCalled()
expect(JSON.parse(errors[0] ?? '{}')).toMatchObject({
event: 'orca_relay_postgres_schema_lock_timeout',
code: '55P03',
statement: 'CREATE INDEX IF NOT EXISTS i ON t(c)'
})
})
it('retries only the PostgreSQL concurrent type-creation collision', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const collision = Object.assign(new Error('duplicate type'), {
@@ -386,7 +417,7 @@ describe('PostgreSQL schema startup', () => {
it('stops retrying at the shared startup deadline', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const error = Object.assign(new Error('lock timeout'), { code: '55P03' })
const error = Object.assign(new Error('statement timeout'), { code: '57014' })
const delays: number[] = []
let now = 0
const query = vi
@@ -413,7 +444,7 @@ describe('PostgreSQL schema startup', () => {
expect(console.warn).toHaveBeenLastCalledWith(
JSON.stringify({
event: 'orca_relay_postgres_schema_retry_exhausted',
code: '55P03',
code: '57014',
attempts: 2
})
)
cloud/apps/relay/src/database-statement-timeout-postgres.test.ts
+18 -23
View File
@@ -42,10 +42,12 @@ describePostgres('PostgreSQL statement deadline', () => {
expect(result).toBe(2)
}, 15_000)
// Why: DDL runs on its own untimed connection. relay_invites carries a
// CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT EXISTS)
// really does queue behind an ACCESS EXCLUSIVE lock on the table.
it('applies the schema behind a held ACCESS EXCLUSIVE lock', async () => {
// Why: relay_invites carries a CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT
// EXISTS) really does queue behind an ACCESS EXCLUSIVE lock on the table. The catalog pre-check
// asks pg_class whether that index is already there, and the read takes no lock on relay_invites,
// so a boot on a migrated database no longer joins the queue at all. It used to, and every writer
// queued behind it in lock order.
it('boots without queueing behind a held ACCESS EXCLUSIVE lock', async () => {
let releaseTable!: () => void
const tableReleased = new Promise<void>((resolve) => {
releaseTable = resolve
@@ -61,7 +63,9 @@ describePostgres('PostgreSQL statement deadline', () => {
})
await tableHeldPromise
const opening = openRelayDatabase({
// Resolving while the lock is still held is the whole proof: a statement that queued would hit
// the schema connection's 1s lock_timeout and fail the boot, which is no longer retried.
const database = await openRelayDatabase({
databaseUrl,
dataDir: '',
applicationName,
@@ -69,27 +73,18 @@ describePostgres('PostgreSQL statement deadline', () => {
// connection must not.
statementTimeoutMs: 200
})
const blockedOnSchemaConnection = async (): Promise<boolean> => {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await databases[0]!.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'
AND application_name = ?`,
[`${applicationName}/schema`]
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
const blocked = await blockedOnSchemaConnection()
databases.push(database)
const waiting = await databases[0]!.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'
AND application_name = ?`,
[`${applicationName}/schema`]
)
expect(Number(waiting[0]!.waiting)).toBe(0)
releaseTable()
await holder
const database = await opening
databases.push(database)
expect(blocked).toBe(true)
// The serving pool still carries the short deadline it was opened with.
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
{ statement_timeout: '200ms' }
cloud/apps/relay/src/database-transient-error.test.ts
+51 -1
View File
@@ -1,5 +1,20 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import { isRelayDatabaseTransientError } from './database.js'
import { PostgresPoolPressure } from './postgres-pool-pressure.js'
// The only way to mark an error as an acquire failure is to fail a real
// acquire, so the gated cases go through the pressure wrapper the pool uses.
async function failedAcquire(message: string): Promise<unknown> {
const pool = {
totalCount: 0,
idleCount: 0,
waitingCount: 0,
connect: vi.fn(async () => {
throw new Error(message)
})
}
return await new PostgresPoolPressure(pool as never).connect().catch((error: unknown) => error)
}
describe('relay database transient errors', () => {
it.each(['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'])(
@@ -15,4 +30,39 @@ describe('relay database transient errors', () => {
).toBe(true)
expect(isRelayDatabaseTransientError(new TypeError('broken invariant'))).toBe(false)
})
it('classifies a pool connect timeout that node-postgres reports with no code', () => {
// pg-pool raises this only from its own connect path, so no statement ran.
expect(
isRelayDatabaseTransientError(
new Error('Connection terminated due to connection timeout')
)
).toBe(true)
})
it('classifies an early-ended socket only when it ended during the acquire', async () => {
expect(
isRelayDatabaseTransientError(await failedAcquire('Connection terminated unexpectedly'))
).toBe(true)
// The same message mid-statement leaves the commit outcome unknown, so it
// must stay a hard failure rather than invite a retry.
expect(
isRelayDatabaseTransientError(new Error('Connection terminated unexpectedly'))
).toBe(false)
})
it.each([null, undefined, 'a thrown string'])(
'survives %s reaching it instead of an error object',
(thrown) => {
expect(isRelayDatabaseTransientError(thrown)).toBe(false)
}
)
it('keeps a failed acquire that is not transient out of the retry path', async () => {
expect(
isRelayDatabaseTransientError(
await failedAcquire('password authentication failed for user "relay"')
)
).toBe(false)
})
})
cloud/apps/relay/src/database.ts
+114 -12
View File
@@ -6,6 +6,7 @@ import pg from 'pg'
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
import {
emptyPostgresPoolPressureCounts,
isPostgresPoolConnectFailure,
PostgresPoolPressure,
type PostgresPoolPressureCounts
} from './postgres-pool-pressure.js'
@@ -68,6 +69,17 @@ export interface RelayDatabase {
close(): Promise<void>
}
// RULE - no new index and no new column on `relay_control_connection_reservations`,
// `relay_confirm_results`, `relay_audit_events`, `relay_connection_bases`, or any other large
// table may be added to SCHEMA or to POSTGRES_SCHEMA_MIGRATIONS. The catalog pre-check skips a
// lock-taking statement only once the object exists, so a brand-new one reports missing on every
// director at once and each runs a non-concurrent build over the whole table. POSTGRES_LOCK_TIMEOUT_MS
// bounds how long that build waits for its lock, not how long it holds it. Build the index out of
// band with CREATE INDEX CONCURRENTLY first, then add it here, where the pre-check skips it forever
// after. relay-schema-lock-targets.test.ts pins the current list, so an addition fails CI.
// Constraint swaps are matched by NAME in pg_constraint, never by body, because the CHECK list is
// generated from REGION_LIST. Changing a constraint's definition under the same name therefore does
// nothing on boot: an operator drops it, and the next boot adds the current definition back.
const SCHEMA = `
CREATE TABLE IF NOT EXISTS relay_invites (
user_id TEXT NOT NULL,
@@ -88,6 +100,18 @@ CREATE TABLE IF NOT EXISTS relay_invites (
CREATE INDEX IF NOT EXISTS relay_invites_device
ON relay_invites(user_id, relay_host_id, relay_device_id);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the credential sweep matches (state, expires_at) every cycle while invites in a terminal
-- state accumulate for the life of the database. Unindexed it seq-scans the whole table inside the
-- maintenance transaction. Partial, so the index holds only the states the sweep can act on.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry
ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown');
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the second sweep pass matches (state, reservation_expires_at) over the same table.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation
ON relay_invites(reservation_expires_at) WHERE state = 'reserved';
CREATE TABLE IF NOT EXISTS relay_devices (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -148,6 +172,11 @@ CREATE TABLE IF NOT EXISTS relay_connection_bases (
-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle
-- and holds the maintenance transaction open long enough to time out
-- assignment lock waits.
-- Why not a partial index on active = 1: a basis is inserted active and flipped to 0, so each
-- deactivation leaves a dead entry in that index too. Measured on production-shaped history it
-- carries the same dead entries as this one, the planner picks this one in every state, and it
-- costs ~65 bytes of WAL per insert. Bloat here is cured by reaping and vacuum, not by a narrower
-- index.
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
ON relay_connection_bases(active, deadline);
@@ -161,6 +190,12 @@ CREATE TABLE IF NOT EXISTS relay_direct_authorizations (
consumed_at BIGINT
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the sweep expires pending authorizations by (consumed_at IS NULL, deadline), and consumed
-- rows are never deleted. Partial, so the index stays the size of the pending set.
CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline
ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL;
CREATE TABLE IF NOT EXISTS relay_confirm_results (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -512,8 +547,9 @@ CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases (
updated_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE INDEX IF NOT EXISTS relay_assignment_activity_expiry
ON relay_assignment_activity_leases(expires_at);
-- expires_at is deliberately unindexed: every control renewal writes it (~471/s), so an index on
-- it makes each renewal a non-HOT update that rewrites index entries. Its only reader is the 30s
-- expiry sweep, which seq-scans 14.8k rows / 7MB in a few milliseconds.
CREATE TABLE IF NOT EXISTS relay_control_connection_reservations (
reservation_id TEXT PRIMARY KEY,
@@ -547,6 +583,12 @@ CREATE TABLE IF NOT EXISTS relay_rate_windows (
PRIMARY KEY (scope_key, window_kind, window_started_at)
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: window_started_at is the PRIMARY KEY's last column, so the sweep's 24h retention delete
-- cannot use it and seq-scans instead.
CREATE INDEX IF NOT EXISTS relay_rate_windows_started
ON relay_rate_windows(window_started_at);
CREATE TABLE IF NOT EXISTS relay_migration_leases (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -633,9 +675,34 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
// Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because
// this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write,
// and all 28 directors reach it at once; a lock timeout here must not restart the instance, which
// would only re-queue the same DDL behind the same writers. Once it wins, the pre-check answers
// absent and no later boot sends it at all.
`-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s
DROP INDEX IF EXISTS relay_assignment_activity_expiry`,
// The drop is what makes HOT legal; this is what makes it possible. A renewal can only reuse the
// row's own page when that page has room for a second version, and at the default fillfactor of
// 100 a freshly filled page has none - measured at 0.5% HOT with the index gone and the default,
// against 100% at 70. Takes SHARE UPDATE EXCLUSIVE, which blocks vacuum and DDL but no reader or
// writer, and only for the catalog write. Applies to pages as they refill, so the table converges
// over its own renewal cycle rather than at boot.
// Deferrable for the same reason, though SHARE UPDATE EXCLUSIVE blocks only vacuum and DDL: it
// buys nothing until the drop lands, so a boot that deferred the drop should defer this too.
`-- schema-deferrable: buys nothing until the drop above lands
ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`
]
// The exact statement list a Postgres boot applies, in order, so the lock-target census can read
// what production runs rather than a copy of it. The SQLite path keeps SCHEMA on its own.
export function relayPostgresSchemaStatements(): string[] {
return [...SCHEMA.split(';'), ...POSTGRES_SCHEMA_MIGRATIONS]
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0)
}
function postgresSql(sql: string): string {
let index = 0
return sql.replace(/\?/g, () => `$${++index}`)
@@ -784,6 +851,8 @@ class SqliteDatabase extends SqliteTransaction {
class PostgresTransaction implements RelayDatabase {
readonly dialect = 'postgres' as const
private heldFromMs: number | undefined
private lockUnavailable = 0
private lockTimeouts = 0
constructor(protected readonly client: pg.PoolClient) {}
@@ -794,6 +863,20 @@ class PostgresTransaction implements RelayDatabase {
return holdMs
}
// Drained by the owning database on both the commit and the rollback path: a
// 55P03 rolls the transaction back, so counting only on success would drop it.
consumeLockUnavailable(): number {
const count = this.lockUnavailable
this.lockUnavailable = 0
return count
}
consumeLockTimeouts(): number {
const count = this.lockTimeouts
this.lockTimeouts = 0
return count
}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
try {
const result = await this.client.query(postgresSql(sql), params)
@@ -829,8 +912,14 @@ class PostgresTransaction implements RelayDatabase {
options.failIfUnavailable &&
String((error as { code?: unknown }).code) === '55P03'
) {
if (options.measureHoldMs) this.lockUnavailable += 1
throw new Error('database_lock_unavailable')
}
// A bounded wait that expires raises the same 55P03 without NOWAIT. This is
// the request path, so it is counted apart from by-design sweep deferrals.
if (bounded && options.measureHoldMs && String((error as { code?: unknown }).code) === '55P03') {
this.lockTimeouts += 1
}
throw error
} finally {
// Restore on the error path too: the transaction may still be retried or
@@ -883,13 +972,15 @@ function retryablePostgresTransactionError(error: unknown): boolean {
}
export function isRelayDatabaseTransientError(error: unknown): boolean {
const code = String((error as { code?: unknown }).code)
// Runs inside the query catch, where a thrown null or undefined would turn a
// database failure into a TypeError that buries it.
const code = String((error as { code?: unknown } | null)?.code)
if (['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'].includes(code)) {
return true
}
return String((error as { message?: unknown }).message).includes(
'timeout exceeded when trying to connect'
)
// A pool that cannot hand out a client reports no SQLSTATE at all, so the
// acquire boundary owns that vocabulary.
return isPostgresPoolConnectFailure(error)
}
async function waitForPostgresRetry(random: () => number = Math.random): Promise<void> {
@@ -924,6 +1015,9 @@ class PostgresDatabase implements RelayDatabase {
error,
phase,
sql,
// Passed in rather than re-derived: the log has to say what the routes
// actually did, and one classifier cannot drift from itself.
transient: isRelayDatabaseTransientError(error),
elapsedMs: performance.now() - startedAt,
pool: this.pool
})
@@ -950,6 +1044,7 @@ class PostgresDatabase implements RelayDatabase {
options.failIfUnavailable &&
String((error as { code?: unknown }).code) === '55P03'
) {
if (options.measureHoldMs) this.holds.recordUnavailable()
throw new Error('database_lock_unavailable')
}
throw error
@@ -968,9 +1063,13 @@ class PostgresDatabase implements RelayDatabase {
const result = await operation(transaction)
await client.query('COMMIT')
this.holds.record(measuredHoldMs(transaction) ?? Number.NaN)
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
return result
} catch (error) {
await client.query('ROLLBACK').catch(() => undefined)
this.holds.recordUnavailable(transaction.consumeLockUnavailable())
this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
if (!retryablePostgresTransactionError(error) || attempt === POSTGRES_TRANSACTION_ATTEMPTS) {
if (retryablePostgresTransactionError(error) && options.reportRetries !== false) {
console.warn(
@@ -1087,11 +1186,14 @@ async function applySchemaOnUntimedPool(
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
[
...SCHEMA.split(';').filter((statement) => statement.trim()),
...POSTGRES_SCHEMA_MIGRATIONS
],
async (statement) => await database.query(statement)
relayPostgresSchemaStatements(),
async (statement) => await database.query(statement),
// Asks the catalog whether each index or column is already there. CREATE INDEX IF NOT EXISTS
// and ALTER TABLE ADD COLUMN IF NOT EXISTS take their relation lock before the server
// evaluates the existence test, so on an already-migrated database the boot still joins the
// lock queue - and relation locks are granted in queue order, so every writer queues behind
// it. The catalog read takes no lock on the table.
{ catalogQuery: async (sql, params) => await database.query(sql, params) }
)
} finally {
await database.close().catch(() => undefined)
cloud/apps/relay/src/host-session-client-accept.test.ts
+70
View File
@@ -729,3 +729,73 @@ describe('control lease jitter', () => {
vi.advanceTimersByTime(0)
})
})
describe('paced drain and the phones of a host not yet told', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const laterHostId = 'qrstuvwxyz012345'
const laterIdentity = { ...identity, sub: 'user-2', relayHostId: laterHostId }
async function twoHostCell(): Promise<{
h: ReturnType<typeof harness>
told: FakeSocket
untold: FakeSocket
}> {
const h = harness()
const told = await activeHost(h)
const untold = new FakeSocket()
await h.activate(untold as unknown as WebSocket, laterIdentity, null, 1, false, 1, '1.4.197')
// Both hosts now dial in, so the credential mocks have to answer for either.
h.store.resolveResume.mockImplementation(async (hostId: string) => ({
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub
}))
h.store.reserveCredential.mockImplementation(async (hostId: string) => ({
...reservation,
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub,
relayHostId: hostId
}))
return { h, told, untold }
}
async function dial(h: ReturnType<typeof harness>, hostId: string): Promise<FakeSocket> {
const client = new FakeSocket()
await h.registry.acceptClient(client as unknown as WebSocket, hostId, 'credential')
return client
}
it('serves a host whose drain has not been sent and refuses one whose has', async () => {
const { h, told, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
const refused = await dial(h, identity.relayHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(told.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
const served = await dial(h, laterHostId)
expect(served.close).not.toHaveBeenCalled()
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
})
it('refuses that host\'s phones as soon as its own drain is sent', async () => {
const { h, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(40_000)
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
const refused = await dial(h, laterHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
})
it('keeps an unpaced drain refusing every phone at once', async () => {
const { h } = await twoHostCell()
h.registry.drain(0)
for (const hostId of [identity.relayHostId, laterHostId]) {
const refused = await dial(h, hostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
}
})
})
cloud/apps/relay/src/host-session-registry.test.ts
+236 -7
View File
@@ -12,6 +12,12 @@ import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import {
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import { HostSessionRegistry, type HostSession } from './host-session-registry.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
@@ -23,6 +29,12 @@ import {
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
// A due renewal leaves the heartbeat as a batch enqueue, so the store only sees
// the tick once the batch window closes.
async function closeRenewalWindow(): Promise<void> {
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
}
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSING = 2
@@ -109,6 +121,7 @@ function createRegistry(
activate: ActivateSession
acquireActivity: ReturnType<typeof vi.fn>
renewControlActivity: ReturnType<typeof vi.fn>
renewControlActivities: ReturnType<typeof vi.fn>
releaseActivity: ReturnType<typeof vi.fn>
observer: {
recordAuth: ReturnType<typeof vi.fn>
@@ -119,12 +132,38 @@ function createRegistry(
const acquireActivity = vi.fn().mockResolvedValue(undefined)
const renewControlActivity = vi.fn().mockResolvedValue(undefined)
const releaseActivity = vi.fn().mockResolvedValue(true)
// Mirrors the store's own batch semantics over the single-renewal mock: a known
// outcome becomes that row's verdict, and any other failure reaches the caller
// as the driver's error. Keeps every per-call expectation below aimed at the
// renewal a session actually asked for.
const renewControlActivities = vi.fn(
async (rows: readonly ControlRenewalRequest[]): Promise<ControlRenewalOutcome[]> =>
await Promise.all(
rows.map(async (row): Promise<ControlRenewalOutcome> => {
try {
await renewControlActivity(row.identity, {
activityId: row.activityId,
cellId: row.cellId,
expiresAt: row.expiresAt
})
return 'renewed'
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) {
throw error
}
return message as ControlRenewalOutcome
}
})
)
)
const assignments = {
activateControl,
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity
} as unknown as RelayAssignmentStore
const observer = {
@@ -176,6 +215,7 @@ function createRegistry(
activate,
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity,
observer
}
@@ -336,15 +376,15 @@ describe('host session cleanup races', () => {
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
// POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both
// reach drain(), so a second teardown can be scheduled for the same session.
// reach drain(), so a retry re-sends to every session. It must re-arm the pending
// teardown rather than stack a second one: across a paced cell that is 800 orphaned
// timers per retry, each one holding the loop open for the rest of the window.
registry.drain(0)
const scheduled = vi.getTimerCount()
registry.drain(0)
// Pin the premise: if drain ever gains an idempotency guard, the retry schedules no
// second teardown and the assertion below stops defending the write-once snapshot
// while still passing. Compare against the count before the retry rather than an
// absolute, since the session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled + 1)
// Compare against the count before the retry rather than an absolute, since the
// session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled)
vi.advanceTimersByTime(1)
// Asserting registry state, not the log line: FakeSocket closes synchronously, so
@@ -689,7 +729,8 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
expect(renewControlActivity).toHaveBeenCalledWith(
@@ -715,6 +756,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
@@ -737,6 +779,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
@@ -762,6 +805,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 4; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
@@ -791,8 +835,10 @@ describe('host session cleanup races', () => {
try {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
} finally {
warn.mockRestore()
@@ -812,6 +858,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
@@ -831,13 +878,16 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
await vi.advanceTimersByTimeAsync(0)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(3)
registry.drain(0)
@@ -855,6 +905,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
@@ -880,6 +931,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).not.toHaveBeenCalled()
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
@@ -899,6 +951,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -918,6 +971,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -939,6 +993,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 3; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
expect(renewControlActivity).toHaveBeenCalledTimes(2)
@@ -966,6 +1021,7 @@ describe('control renewal cadence across a rebind', () => {
const beat = async (target: FakeSocket): Promise<void> => {
await vi.advanceTimersByTimeAsync(ping)
target.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
// Age the session so its attempt counter is well above zero.
@@ -993,6 +1049,57 @@ describe('control renewal cadence across a rebind', () => {
})
})
describe('control renewals shared by one batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('renews two due hosts in one call and leaves a stale one alone', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValueOnce('control:production-gce-c3:1')
.mockResolvedValueOnce('control:production-gce-c3:1')
const { registry, activate, renewControlActivities } = createRegistry(activateControl)
const other = { ...identity, sub: 'user-2', relayHostId: 'ponmlkjihgfedcba' }
const staleSocket = new FakeSocket()
const liveSocket = new FakeSocket()
await activate(staleSocket as unknown as WebSocket, identity, null, 1, false, 1)
await activate(liveSocket as unknown as WebSocket, other, null, 1, false, 1)
const stale = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const live = registry.get({ userId: other.sub, relayHostId: other.relayHostId })!
// Both come due inside the same window, and one socket goes away while the
// statement is still in PostgreSQL.
let release!: () => void
renewControlActivities.mockImplementationOnce(
async (rows: readonly ControlRenewalRequest[]) => {
staleSocket.close()
await new Promise<void>((resolve) => (release = resolve))
return rows.map((): ControlRenewalOutcome => 'renewed')
}
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const staleDueAt = stale.activityRenewalDueAt
await closeRenewalWindow()
release()
await vi.advanceTimersByTimeAsync(0)
expect(renewControlActivities).toHaveBeenCalledOnce()
expect(
renewControlActivities.mock.calls[0]![0].map(
(row: ControlRenewalRequest) => row.identity.relayHostId
)
).toEqual([identity.relayHostId, other.relayHostId])
expect(live.activityRenewalCompletedAttempt).toBe(1)
expect(stale.activityRenewalCompletedAttempt).toBe(0)
expect(stale.activityRenewalDueAt).toBe(staleDueAt)
registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('control lease recovery after the session is gone', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -1019,6 +1126,7 @@ describe('control lease recovery after the session is gone', () => {
new Promise<void>((_resolve, reject) => (failRenewal = reject))
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
const newer = new FakeSocket()
@@ -1696,3 +1804,124 @@ describe('host data attach owner lookup', () => {
expect(h.owner.activeConnIds.size).toBe(0)
})
})
describe('paced drain', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
async function connectHosts(count: number): Promise<{
registry: HostSessionRegistry
sockets: FakeSocket[]
}> {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
const { registry, activate } = createRegistry(activateControl)
const sockets: FakeSocket[] = []
for (let index = 0; index < count; index += 1) {
const socket = new FakeSocket()
sockets.push(socket)
await activate(
socket as unknown as WebSocket,
{ ...identity, sub: `user-${index}` },
null,
1,
false,
1
)
socket.send.mockClear()
}
return { registry, sockets }
}
function drainsSent(sockets: FakeSocket[]): number {
return sockets.filter((socket) =>
socket.send.mock.calls.some(([payload]) => String(payload).includes('"type":"drain"'))
).length
}
it('sends every drain at once when no window is given', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
})
// Windows here stay under the 75s control-silence watchdog, which would otherwise close
// a test socket that never heartbeats before its paced send is due.
it('spreads the sends evenly across the window', async () => {
const { registry, sockets } = await connectHosts(5)
registry.drain(0, { paceWindowMs: 40_000 })
// The first host is sent synchronously; the last lands on the window's closing edge.
expect(drainsSent(sockets)).toBe(1)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(2)
await vi.advanceTimersByTimeAsync(20_000)
expect(drainsSent(sockets)).toBe(4)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(5)
})
it('fences admission for every session before the first paced send lands', async () => {
const { registry, sockets } = await connectHosts(3)
registry.drain(0, { paceWindowMs: 40_000 })
expect(registry.isDraining()).toBe(true)
// A host whose drain has not been sent yet must already be non-authoritative.
const socket = new FakeSocket()
registry.acceptControl(socket as unknown as WebSocket, { ...identity, sub: 'user-late' })
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
expect(drainsSent(sockets)).toBe(1)
})
it('gives each host its own grace after its own send, not after the call', async () => {
const { registry, sockets } = await connectHosts(2)
registry.drain(10_000, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(10_000)
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
// Its own send at 40s plus its own 10s grace, not 10s from the drain call.
await vi.advanceTimersByTimeAsync(39_999)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
await vi.advanceTimersByTimeAsync(10_001)
expect(sockets[1]!.readyState).toBe(sockets[1]!.CLOSED)
})
it('leaves no timer behind once an emergency drain cuts a window short', async () => {
const { registry } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
// Every session is closed, so anything still pending is an orphan of the cut window.
expect(vi.getTimerCount()).toBe(0)
})
it('keeps the first teardown snapshot when a regional drain fires before the fleet one', async () => {
const { registry, sockets } = await connectHosts(1)
const session = registry.get({ userId: 'user-0', relayHostId: identity.relayHostId })!
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
registry.drainHost({
attemptId: 'attempt',
userId: 'user-0',
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 0
})
registry.drain(10)
await vi.advanceTimersByTimeAsync(11)
expect(session.closingCounts).toEqual({ splices: 1, pending: 0 })
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
})
it('lets an emergency drain supersede the sends still queued by a paced one', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
expect(drainsSent(sockets)).toBe(1)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
const sendsAfterEmergency = sockets.map((socket) => socket.send.mock.calls.length)
await vi.advanceTimersByTimeAsync(40_000)
expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency)
})
})
cloud/apps/relay/src/host-session-registry.ts
+73 -19
View File
@@ -26,6 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import { ControlRenewalBatch } from './control-renewal-batch.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
@@ -179,6 +180,10 @@ export class HostSessionRegistry {
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly drainTimers = new Set<ReturnType<typeof setTimeout>>()
// Hosts whose drain has been sent. Paced sends land minutes apart, so "this cell is
// draining" is not the same question as "this host has been told to leave".
private readonly drainSentHosts = new Set<string>()
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
@@ -299,6 +304,15 @@ export class HostSessionRegistry {
private readonly cellIncarnation?: string
) {}
// Renewals leave the heartbeat as an enqueue: one statement per cell per
// window replaces one write transaction per host, which is what keeps the
// shared PostgreSQL instance out of buffer-header contention.
private readonly controlRenewals = new ControlRenewalBatch(
async (rows) => await this.assignments.renewControlActivities(rows),
() => this.logIdentity(),
(flush) => this.observer.recordControlRenewalFlush?.(flush)
)
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
private controlLeaseExpiresAt(): number {
const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
@@ -330,7 +344,10 @@ export class HostSessionRegistry {
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
// Not `this.draining`: a paced drain tells hosts minutes apart, and the director keeps
// pointing phones here until their own host has moved. Refusing them for the whole
// window would turn a 2 min drain into a 2 min outage for hosts not yet told.
if (this.drainSentHosts.has(hostId)) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
return
@@ -450,7 +467,7 @@ export class HostSessionRegistry {
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.drainSentHosts.has(hostId) ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
@@ -592,7 +609,7 @@ export class HostSessionRegistry {
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.drainSentHosts.has(identity.relayHostId) ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
@@ -846,17 +863,49 @@ export class HostSessionRegistry {
return { controls, splices, pendingSplices }
}
drain(graceMs: number): void {
drain(graceMs: number, options: { paceWindowMs?: number } = {}): void {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
// A later drain (an emergency one, or shutdown) owns every session again, so nothing
// queued by an earlier paced drain may still fire: it would re-send and, worse, keep
// the event loop alive for the rest of a window the operator just cut short.
for (const timer of this.drainTimers) clearTimeout(timer)
this.drainTimers.clear()
const paceWindowMs = Math.max(0, Math.trunc(options.paceWindowMs ?? 0))
const targets = [...this.sessions.values()].filter((session) => session.state !== 'closed')
// The desktop re-dials the director as soon as it reads `drain`, whatever graceMs says,
// so spreading the send is the only thing that spreads the reconnect load.
const step = paceWindowMs > 0 && targets.length > 1 ? paceWindowMs / (targets.length - 1) : 0
for (const [index, session] of targets.entries()) {
const delay = Math.round(step * index)
if (delay === 0) {
this.sendDrain(session, graceMs)
continue
}
this.scheduleDrainTimer(delay, () => this.sendDrain(session, graceMs))
}
}
// A session is only fenced when it is told, not when the drain starts: until its send
// lands it is an ordinary live host, and its phones have to keep being able to reach it.
private sendDrain(session: HostSession, graceMs: number): void {
if (session.state === 'closed') return
session.authorityRevision += 1
session.state = 'drain-only'
this.drainSentHosts.add(session.relayHostId)
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
this.scheduleDrainTimer(graceMs, () => this.closeDrainedSession(session))
}
// Unref'd so a drain in flight never holds the process open past its own work.
private scheduleDrainTimer(delayMs: number, run: () => void): void {
const timer: ReturnType<typeof setTimeout> = setTimeout(() => {
this.drainTimers.delete(timer)
run()
}, delayMs)
timer.unref?.()
this.drainTimers.add(timer)
}
drainHost(input: {
attemptId: string
userId: string
@@ -1345,15 +1394,13 @@ export class HostSessionRegistry {
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
{
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
}
)
void this.controlRenewals
.enqueue({
identity: { userId: session.identity.sub, relayHostId: session.relayHostId },
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
})
.then(() => {
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
@@ -1419,6 +1466,13 @@ export class HostSessionRegistry {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
return
}
if (error instanceof Error && error.message === 'assignment_lock_unavailable') {
// A per-host transaction held the row, so the batch passed over it
// rather than making every other host in the flush wait. The next
// tick is 15s away against a 105s lease, and the flush line already
// reports the count, so this needs no line of its own.
return
}
console.warn('[orca-relay] control activity renewal failed')
})
// Terminal handler: a throw inside the async catch above (e.g. a
cloud/apps/relay/src/index.ts
+15 -10
View File
@@ -46,14 +46,19 @@ const {
ready,
cellIncarnation
} = createRelayServer(config, database)
const cleanupTimer = setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
30_000
)
// Same owner as the assignment sweep: the cleanup only expires credentials that every reader
// already re-checks at read time, so running it in all 23 cells multiplied one table scan by 23
// without changing any answer.
const cleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
jitteredSweepIntervalMs(30_000)
)
: null
const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(() => {
void runAssignmentCleanup(assignments)
@@ -82,7 +87,7 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cleanupTimer.unref()
cleanupTimer?.unref()
assignmentCleanupTimer?.unref()
inventorySnapshotTimer?.unref()
migrationInventoryTimer?.unref()
@@ -128,7 +133,7 @@ server.listen(config.port, () => {
})
const shutdown = (): void => {
clearInterval(cleanupTimer)
if (cleanupTimer) clearInterval(cleanupTimer)
if (assignmentCleanupTimer) clearInterval(assignmentCleanupTimer)
if (inventorySnapshotTimer) clearInterval(inventorySnapshotTimer)
if (migrationInventoryTimer) clearInterval(migrationInventoryTimer)
cloud/apps/relay/src/postgres-pool-pressure.ts
+53 -2
View File
@@ -9,6 +9,48 @@ export type PostgresPoolPressureCounts = {
databasePoolWaitMsMax: number
}
// A pool that cannot hand out a client throws a bare Error with no SQLSTATE, so
// the message is all node-postgres gives us. Both of these come only from
// pg-pool's connect path, so neither can be a statement that already ran.
const POOL_CONNECT_TIMEOUT_MESSAGES = [
// No pooled client came free within connectionTimeoutMillis.
'timeout exceeded when trying to connect',
// A new client's own handshake outran connectionTimeoutMillis.
'Connection terminated due to connection timeout'
]
// pg raises this whenever a socket ends early, during the handshake and mid
// statement alike, so only the acquire boundary can tell the two apart.
const CONNECTION_TERMINATED_MESSAGE = 'Connection terminated unexpectedly'
// Membership is tracked beside the error rather than on it: an error object may
// be frozen, and a mutated one would leak the marker into logs.
const poolAcquireFailures = new WeakSet<object>()
function errorMessage(error: unknown): string {
return String((error as { message?: unknown } | null)?.message)
}
function isPostgresPoolAcquireFailure(error: unknown): boolean {
return typeof error === 'object' && error !== null && poolAcquireFailures.has(error)
}
// connectionTimeoutMillis firing, either waiting in the queue or dialling.
export function isPostgresPoolConnectTimeout(error: unknown): boolean {
const message = errorMessage(error)
return POOL_CONNECT_TIMEOUT_MESSAGES.some((known) => message.includes(known))
}
// Every way the pool can fail to hand out a usable client. An early-ended
// socket counts only at the acquire boundary: retrying a statement whose commit
// outcome is unknown is not safe.
export function isPostgresPoolConnectFailure(error: unknown): boolean {
if (isPostgresPoolConnectTimeout(error)) return true
return (
errorMessage(error).includes(CONNECTION_TERMINATED_MESSAGE) &&
isPostgresPoolAcquireFailure(error)
)
}
const emptyCounts = (): PostgresPoolPressureCounts => ({
databasePoolTotal: 0,
databasePoolIdle: 0,
@@ -32,14 +74,14 @@ export class PostgresPoolPressure {
async connect(): Promise<pg.PoolClient> {
const waitingBefore = this.pool.waitingCount
const connection = this.pool.connect()
if (this.pool.waitingCount <= waitingBefore) return await connection
if (this.pool.waitingCount <= waitingBefore) return await markedAcquire(connection)
const waiter = Symbol()
const startedAt = this.now()
this.waiters.set(waiter, startedAt)
this.waitersMax = Math.max(this.waitersMax, this.waiters.size)
try {
return await connection
return await markedAcquire(connection)
} finally {
this.waitMsMax = Math.max(this.waitMsMax, this.now() - startedAt)
this.waiters.delete(waiter)
@@ -88,6 +130,15 @@ export class PostgresPoolPressure {
}
}
async function markedAcquire(connection: Promise<pg.PoolClient>): Promise<pg.PoolClient> {
try {
return await connection
} catch (error) {
if (typeof error === 'object' && error !== null) poolAcquireFailures.add(error)
throw error
}
}
export function emptyPostgresPoolPressureCounts(): PostgresPoolPressureCounts {
return emptyCounts()
}
cloud/apps/relay/src/postgres-query-failure-postgres.test.ts
+3 -1
View File
@@ -29,7 +29,8 @@ describePostgres('real PostgreSQL query failure phases', () => {
event: 'orca_relay_postgres_query_failed',
phase: 'execute',
code: '57014',
connectionTimeout: false
connectionTimeout: false,
transient: true
})
expect(await database.query('SELECT 1 AS ok')).toEqual([{ ok: 1 }])
})
@@ -58,6 +59,7 @@ describePostgres('real PostgreSQL query failure phases', () => {
phase: 'acquire',
code: 'unknown',
connectionTimeout: true,
transient: true,
poolTotal: 1,
poolIdle: 0
})
cloud/apps/relay/src/postgres-query-failure.test.ts
+52 -4
View File
@@ -55,6 +55,7 @@ describe('PostgreSQL query failure diagnostics', () => {
operation: 'control-renewal',
code: 'unknown',
connectionTimeout: true,
transient: true,
elapsedMs: expect.any(Number),
poolTotal: 10,
poolIdle: 0,
@@ -63,9 +64,15 @@ describe('PostgreSQL query failure diagnostics', () => {
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private')
})
it.each(['57014', '55P03', 'ECONNRESET'])(
// ECONNRESET carries no SQLSTATE the routes retry on, and it arrives after the
// statement went out, so it stays a hard failure. The pair pins that boundary.
it.each([
['57014', true],
['55P03', true],
['ECONNRESET', false]
] as const)(
'identifies execute failure %s and releases its client',
async (code) => {
async (code, transient) => {
const error = Object.assign(new Error('private-token'), { code, detail: sql })
fakes.query.mockRejectedValueOnce(error)
await expect(database.query(sql, ['private-token'])).rejects.toBe(error)
@@ -75,19 +82,60 @@ describe('PostgreSQL query failure diagnostics', () => {
phase: 'execute',
operation: 'control-renewal',
code,
connectionTimeout: false
connectionTimeout: false,
transient
})
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain('private-token')
expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain(sql)
}
)
it('marks a dialling timeout that node-postgres reports with no code', async () => {
// pg-pool raises this when a new client's own handshake outruns the limit.
const error = new Error('Connection terminated due to connection timeout')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
code: 'unknown',
connectionTimeout: true,
transient: true
})
})
it('separates an early-ended socket from a timeout while still calling it transient', async () => {
const error = new Error('Connection terminated unexpectedly')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
code: 'unknown',
connectionTimeout: false,
transient: true
})
})
it('reports an acquire failure that is not transient as a hard failure', async () => {
const error = new Error('password authentication failed')
fakes.connectError = error
await expect(database.query(sql)).rejects.toBe(error)
expect(JSON.parse(vi.mocked(console.warn).mock.calls[0]![0] as string)).toMatchObject({
phase: 'acquire',
connectionTimeout: false,
transient: false
})
})
it('does not emit an arbitrary error code, message, query, or parameter', async () => {
const error = { code: 'private-code', message: 'private-message' }
fakes.query.mockRejectedValueOnce(error)
await expect(database.query('SELECT private_column', ['private-param'])).rejects.toBe(error)
const log = vi.mocked(console.warn).mock.calls[0]![0] as string
expect(JSON.parse(log)).toMatchObject({ operation: 'other', code: 'unknown' })
expect(JSON.parse(log)).toMatchObject({
operation: 'other',
code: 'unknown',
transient: false
})
expect(log).not.toContain('private')
})
cloud/apps/relay/src/postgres-query-failure.ts
+6 -3
View File
@@ -1,3 +1,5 @@
import { isPostgresPoolConnectTimeout } from './postgres-pool-pressure.js'
type QueryFailurePhase = 'acquire' | 'execute'
const ERROR_CODES = new Set([
@@ -23,6 +25,8 @@ export function reportPostgresQueryFailure(input: {
error: unknown
phase: QueryFailurePhase
sql: string
// The routing verdict, supplied by the caller that owns it.
transient: boolean
elapsedMs: number
pool: { totalCount: number; idleCount: number; waitingCount: number }
}): void {
@@ -31,9 +35,7 @@ export function reportPostgresQueryFailure(input: {
const error = input.error as { code?: unknown; message?: unknown } | null
const code =
typeof error?.code === 'string' && ERROR_CODES.has(error.code) ? error.code : 'unknown'
const connectionTimeout =
typeof error?.message === 'string' &&
error.message.includes('timeout exceeded when trying to connect')
const connectionTimeout = isPostgresPoolConnectTimeout(error)
console.warn(
JSON.stringify({
event: 'orca_relay_postgres_query_failed',
@@ -43,6 +45,7 @@ export function reportPostgresQueryFailure(input: {
: 'other',
code,
connectionTimeout,
transient: input.transient,
elapsedMs: Math.max(0, Math.round(input.elapsedMs)),
poolTotal: input.pool.totalCount,
poolIdle: input.pool.idleCount,
cloud/apps/relay/src/regional-host-drain-app.test.ts
+72
View File
@@ -697,6 +697,78 @@ async function postPath(
})
}
describe('cell drain endpoint pacing', () => {
function appWithDrain(): {
app: ReturnType<typeof createRelayApp>
drain: ReturnType<typeof vi.fn>
} {
const drain = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
return { app, drain }
}
it('drains everything at once when the caller asks for no pacing', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', { v: 1, graceMs: 0 })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 0 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 0 })
})
it('passes the requested window through and echoes what it accepted', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 120_000 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 120_000 })
})
it('refuses a window that is negative, fractional, or past the cap', async () => {
for (const paceWindowMs of [-1, 1.5, 300_001]) {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs
})
expect(response.status).toBe(400)
expect(drain).not.toHaveBeenCalled()
}
})
it('accepts the cap itself', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 300_000
})
expect(response.status).toBe(200)
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 300_000 })
})
it('still rejects an unauthenticated pacing request', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'wrong-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(401)
expect(drain).not.toHaveBeenCalled()
})
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
cloud/apps/relay/src/regional-rehome-safety.test.ts
+31 -9
View File
@@ -1,7 +1,5 @@
import { describe, expect, it } from 'vitest'
import {
REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT,
REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT,
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
REGIONAL_REHOME_SQL_FAILURES_LIMIT,
regionalRehomeSafetyFailure
@@ -41,14 +39,38 @@ describe('regionalRehomeSafetyFailure', () => {
).toBeNull()
})
it('still fails closed on each pool pressure bound', () => {
for (const overrides of [
{ databasePoolWaitersMax: REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT + 1 },
{ databasePoolWaitMsMax: REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT + 1 }
]) {
expect(regionalRehomeSafetyFailure(safety(overrides), NOW, 19)).toBe(
'database_pool_pressure'
it('passes asia-scale pool pressure, which excludes a cell rather than the fleet', () => {
// Measured 2026-09-16 on the asia-east2 cells: a client pool too narrow for
// a 176ms round trip, with 0.2ms server-side execution. The fleet snapshot
// is a Math.max, so gating on it here stops every region.
expect(
regionalRehomeSafetyFailure(
safety({ databasePoolWaitersMax: 150, databasePoolWaitMsMax: 2_005 }),
NOW,
19
)
).toBeNull()
// Every other bar still fails closed at that same pool pressure.
for (const [overrides, reason] of [
[{ sqlFailures: REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1 }, 'sql_failures'],
[{ controlActivityRecoveryFailures: 1 }, 'control_recovery_failures'],
[
{ reconnects: 19 * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1 },
'elevated_reconnects'
],
[{ observedAt: 0 }, 'monitoring_stale']
] as const) {
expect(
regionalRehomeSafetyFailure(
safety({
databasePoolWaitersMax: 150,
databasePoolWaitMsMax: 2_005,
...overrides
}),
NOW,
19
)
).toBe(reason)
}
})
cloud/apps/relay/src/regional-rehome-safety.ts
+14 -9
View File
@@ -2,12 +2,20 @@ import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
// Limits sit well above the healthy-fleet baseline measured in production on
// 2026-08-28 (peak 2 waiters / 1ms pool waits on every cell; up to ~80
// reconnects per published two-window row on the busiest cell). Sustained
// pool saturation still trips: waiters-max 16 is 8x baseline yet far under a
// backed-up pool, and 250ms peak wait is 1/10 of the incident-monitor alert.
// Instantaneous databasePoolWaiting is not checked separately: it is bounded
// by databasePoolWaitersMax within every published window.
// reconnects per published two-window row on the busiest cell).
export const REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT = 250
// Pool pressure gates one cell, never the fleet. The combined snapshot is a
// Math.max, so a single narrow client pool stops rehoming everywhere: measured
// 2026-09-16, the asia-east2 cells sit at 94-156 waiters and ~2000ms waits
// against 0.2ms server-side execution -- a pool too narrow for a 176ms round
// trip, not database distress -- while us-central1 cells breach in bursts on
// ~33% of polls, and a bar that flaps between the pre-check and the commit
// re-check latches the worker off. regionalRehomeCellSafetyIsClean excludes a
// breaching cell as both source and target; the bars below stay fleet-wide
// because sql failures, control-recovery failures and reconnect storms mean
// database-wide distress. Instantaneous databasePoolWaiting is not checked
// separately: databasePoolWaitersMax bounds it within every published window.
export const REGIONAL_REHOME_POOL_WAITERS_MAX_LIMIT = 16
export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250
@@ -19,7 +27,7 @@ export const REGIONAL_REHOME_POOL_WAIT_MS_MAX_LIMIT = 250
// over four days; genuine database distress produced 395-457. The combined
// snapshot spans up to two 30s windows per process (pathological ambient
// alignment ~164), so 250 stays clear of noise while storms still trip.
// Terminal outages also trip the pool bars and the worker's own
// Terminal outages also trip the per-cell pool bars and the worker's own
// dispatch-failure budget; the sql bar only needs to catch storms.
export const REGIONAL_REHOME_SQL_FAILURES_LIMIT = 250
// Per-cell candidate cleanliness is a soft skip, not a durable latch; the
@@ -47,9 +55,6 @@ export function regionalRehomeSafetyFailure(
return 'monitoring_stale'
}
if (safety.sqlFailures > REGIONAL_REHOME_SQL_FAILURES_LIMIT) return 'sql_failures'
if (regionalRehomePoolPressure(safety)) {
return 'database_pool_pressure'
}
if (safety.controlActivityRecoveryFailures > 0) {
return 'control_recovery_failures'
}
cloud/apps/relay/src/regional-rehome-store.test.ts
+53 -7
View File
@@ -34,8 +34,12 @@ const target = {
connectionHardCap: 1_000 as const,
connectionUnobservedBound: 60
}
// A third general cell in the source region: never a source or target here,
// but it is in the fleet whose safety the gate reads.
const bystander = { ...source, id: 'us-c2', url: 'https://us-c2.relay.example.test' }
const sourceIncarnation = '11111111-1111-4111-8111-111111111111'
const targetIncarnation = '22222222-2222-4222-8222-222222222222'
const bystanderIncarnation = '33333333-3333-4333-8333-333333333333'
describe('regional rehome assignment state', () => {
it('advances past a full candidate page whose destination lacks capacity', async () => {
@@ -424,7 +428,7 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('latches off on sustained pool pressure and logs the disable exactly once', async () => {
it('defers on target pool pressure without disabling, and claims once it clears', async () => {
const context = await setup()
await activatePreferredSource(context, {
userId: 'user-1',
@@ -451,17 +455,59 @@ describe('regional rehome assignment state', () => {
expect(await context.store.commitIdleRegionalRehome(candidate!, safety)).toEqual({
outcome: 'deferred'
})
// Already disabled: the next tick returns before the gate and stays silent.
expect(await context.store.tryIdleRehome()).toBeNull()
} finally {
warnings.restore()
}
// A pool bar crossed between the scan and the commit skips the cell; it must
// not turn the durable switch off, or the worker never comes back.
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
generation: 1,
enabled: true
})
expect(warnings.entries).toMatchObject([
{ reason: 'database_pool_pressure', databasePoolWaitersMax: 17 }
expect(warnings.entries).toEqual([])
await context.database.query(
`UPDATE relay_cell_rehome_safety SET database_pool_waiters_max = 0 WHERE cell_id = ?`,
[target.id]
)
expect(await context.store.tryIdleRehome()).toMatchObject({
userId: 'user-1',
sourceCellId: source.id,
targetCellId: target.id
})
await context.database.close()
})
it('keeps selecting candidates while an unrelated cell is over the pool bar', async () => {
// Production case: the fleet snapshot is a Math.max, so one cell with a
// narrow client pool used to empty every page.
const context = await setup()
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
await context.store.reconcileCells([source, target, bystander])
await heartbeat(context.store, bystander, bystanderIncarnation, 3, 2, {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 150,
databasePoolWaitersMax: 150,
databasePoolWaitMsMax: 2_005
})
const safety: RegionalRehomeSafetySnapshot = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
expect(await context.store.selectIdleRegionalRehomeCandidates(safety)).toMatchObject([
{ sourceCellId: source.id, targetCellId: target.id }
])
await context.database.close()
})
cloud/apps/relay/src/relay-observability.test.ts
+81
View File
@@ -90,6 +90,87 @@ describe('relay observability', () => {
])
})
it('flags a readiness answer served from the last known good probe', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordReadiness({
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_timed_out',
jwksLatencyMs: 2_001,
sqlLatencyMs: 4,
totalLatencyMs: 2_002
})
expect(entries).toEqual([
expect.objectContaining({
severity: 'WARNING',
event: 'orca_relay_readiness_check',
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_timed_out'
})
])
})
it('separates entering the readiness grace window from leaving it', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordReadinessGrace({
dependency: 'sql',
grace: 'entered',
failure: 'sql_failed',
lastSuccessAgeMs: 12_000,
graceMs: 180_000
})
observability.recordReadinessGrace({
dependency: 'sql',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: 180_000
})
expect(entries).toEqual([
{
severity: 'WARNING',
message: 'Orca Relay readiness entered last-known-good grace',
event: 'orca_relay_readiness_grace_entered',
metricVersion: 1,
role: 'cell',
cellId: 'production-gce-c28',
region: 'asia-east2',
dependency: 'sql',
grace: 'entered',
failure: 'sql_failed',
lastSuccessAgeMs: 12_000,
graceMs: 180_000
},
{
severity: 'INFO',
message: 'Orca Relay readiness left last-known-good grace',
event: 'orca_relay_readiness_grace_left',
metricVersion: 1,
role: 'cell',
cellId: 'production-gce-c28',
region: 'asia-east2',
dependency: 'sql',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: 180_000
}
])
})
it('excludes sockets stuck in closing state from observed relay work', () => {
expect(observedRelayRequests(counts)).toBe(7)
})
cloud/apps/relay/src/relay-observability.ts
+38 -2
View File
@@ -1,9 +1,10 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { ControlRenewalFlush } from './control-renewal-batch.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
import type { RelayReadinessObservation } from './relay-readiness.js'
import type { RelayReadinessGraceEvent, RelayReadinessObservation } from './relay-readiness.js'
export type RelayRuntimeCounts = {
totalConnections: number
@@ -53,6 +54,7 @@ export interface RelayRuntimeObserver {
recordReconnect(): void
recordSql(durationMs: number, success: boolean): void
recordControlRenewal?(durationMs: number, outcome: ControlRenewalOutcome): void
recordControlRenewalFlush?(flush: ControlRenewalFlush): void
recordControlActivityRecovery?(success: boolean): void
recordAssignmentAdmission?(outcome: AssignmentAdmissionOutcome): void
recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void
@@ -119,6 +121,8 @@ type RelayMetricDeltas = {
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlRenewalFlushLatenciesMs: number[]
controlRenewalFlushRowsMax: number
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
@@ -164,6 +168,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlRenewalFlushLatenciesMs: [],
controlRenewalFlushRowsMax: 0,
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
@@ -274,6 +280,14 @@ export class RelayObservability implements RelayRuntimeObserver {
(this.deltas.controlRenewalsByOutcome[outcome] ?? 0) + 1
}
recordControlRenewalFlush(flush: ControlRenewalFlush): void {
this.deltas.controlRenewalFlushLatenciesMs.push(flush.durationMs)
this.deltas.controlRenewalFlushRowsMax = Math.max(
this.deltas.controlRenewalFlushRowsMax,
flush.rows
)
}
recordControlActivityRecovery(success: boolean): void {
if (success) this.deltas.controlActivityRecoveries++
else this.deltas.controlActivityRecoveryFailures++
@@ -281,7 +295,7 @@ export class RelayObservability implements RelayRuntimeObserver {
recordReadiness(observation: RelayReadinessObservation): void {
this.write({
severity: observation.ready ? 'INFO' : 'WARNING',
severity: observation.ready && !observation.degraded ? 'INFO' : 'WARNING',
message: 'Orca Relay readiness check',
event: 'orca_relay_readiness_check',
metricVersion: 1,
@@ -290,6 +304,20 @@ export class RelayObservability implements RelayRuntimeObserver {
})
}
recordReadinessGrace(event: RelayReadinessGraceEvent): void {
const entered = event.grace === 'entered'
this.write({
severity: event.grace === 'recovered' ? 'INFO' : 'WARNING',
message: entered
? 'Orca Relay readiness entered last-known-good grace'
: 'Orca Relay readiness left last-known-good grace',
event: entered ? 'orca_relay_readiness_grace_entered' : 'orca_relay_readiness_grace_left',
metricVersion: 1,
...this.identity,
...event
})
}
recordControlClose(code: number): void {
const key = String(code)
this.deltas.controlClosesByCode[key] = (this.deltas.controlClosesByCode[key] ?? 0) + 1
@@ -365,6 +393,7 @@ export class RelayObservability implements RelayRuntimeObserver {
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const controlRenewalFlush = latencySummary(deltas.controlRenewalFlushLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -433,9 +462,16 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
// Meaning changed when renewals began batching: for a batched row this is
// the flush's duration, not that row's own statement latency. The
// per-flush fields below are the ones to read for statement cost.
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
controlRenewalFlushesDelta: deltas.controlRenewalFlushLatenciesMs.length,
controlRenewalFlushRowsMax: deltas.controlRenewalFlushRowsMax,
controlRenewalFlushLatencyMsP95: controlRenewalFlush.p95,
controlRenewalFlushLatencyMsMax: controlRenewalFlush.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
cloud/apps/relay/src/relay-readiness.test.ts
+336 -12
View File
@@ -2,6 +2,9 @@ import { describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import {
createRelayReadiness,
RELAY_READINESS_JWKS_GRACE_MS,
RELAY_READINESS_SQL_GRACE_MS,
type RelayReadinessGraceEvent,
type RelayReadinessObservation
} from './relay-readiness.js'
@@ -31,8 +34,8 @@ describe('relay readiness', () => {
}
)
expect(await jwksFailure()).toBe(false)
expect(await sqlFailure()).toBe(false)
expect(await jwksFailure.check()).toBe(false)
expect(await sqlFailure.check()).toBe(false)
})
it.each([
@@ -68,35 +71,33 @@ describe('relay readiness', () => {
}
])('reports a safe reason for $name', async ({ fetch, query, failure }) => {
const observations: RelayReadinessObservation[] = []
const ready = createRelayReadiness(database(query), 'https://jwks', {
const readiness = createRelayReadiness(database(query), 'https://jwks', {
fetch,
cacheMs: 0,
observe: (observation) => observations.push(observation)
})
expect(await ready()).toBe(false)
expect(observations).toEqual([
expect.objectContaining({ ready: false, failure })
])
expect(await readiness.check()).toBe(false)
expect(observations).toEqual([expect.objectContaining({ ready: false, failure })])
expect(JSON.stringify(observations)).not.toContain('redacted')
if (failure.startsWith('jwks_')) expect(query).not.toHaveBeenCalled()
expect(query).toHaveBeenCalledTimes(1)
})
it('reports the initial success but not healthy repeats or cached reads', async () => {
const observations: RelayReadinessObservation[] = []
let now = 100
const ready = createRelayReadiness(database(async () => [{ ready: 1 }]), 'https://jwks', {
const readiness = createRelayReadiness(database(async () => [{ ready: 1 }]), 'https://jwks', {
fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch,
cacheMs: 10_000,
now: () => now,
observe: (observation) => observations.push(observation)
})
expect(await ready()).toBe(true)
expect(await readiness.check()).toBe(true)
now += 1_000
expect(await ready()).toBe(true)
expect(await readiness.check()).toBe(true)
now += 10_000
expect(await ready()).toBe(true)
expect(await readiness.check()).toBe(true)
expect(observations).toEqual([
{
ready: true,
@@ -107,3 +108,326 @@ describe('relay readiness', () => {
])
})
})
describe('relay readiness last-known-good grace', () => {
function graceProbe(input: {
jwksGraceMs?: number
sqlGraceMs?: number
cacheMs?: number
jwksOk: () => boolean
sqlOk: () => boolean
now: () => number
}) {
const observations: RelayReadinessObservation[] = []
const graceEvents: RelayReadinessGraceEvent[] = []
const query = vi.fn(async () => {
if (!input.sqlOk()) throw new Error('redacted')
return [{ ready: 1 }]
})
const fetchImpl = vi.fn(
async () => new Response('{}', { status: input.jwksOk() ? 200 : 503 })
) as typeof fetch
const readiness = createRelayReadiness(database(query), 'https://jwks', {
fetch: fetchImpl,
cacheMs: input.cacheMs ?? 0,
...(input.jwksGraceMs === undefined ? {} : { jwksGraceMs: input.jwksGraceMs }),
...(input.sqlGraceMs === undefined ? {} : { sqlGraceMs: input.sqlGraceMs }),
now: input.now,
observe: (observation) => observations.push(observation),
observeGrace: (event) => graceEvents.push(event)
})
return { readiness, observations, graceEvents, query, fetchImpl }
}
it('stays ready while a JWKS failure sits inside the default fifteen minute window', async () => {
let now = 1_000
let jwksOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += RELAY_READINESS_JWKS_GRACE_MS - 1
expect(await readiness.check()).toBe(true)
expect(readiness.degradedDependencies()).toEqual(['jwks'])
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_http_failed'
})
)
})
it('drops readiness once the JWKS window expires', async () => {
let now = 1_000
let jwksOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += RELAY_READINESS_JWKS_GRACE_MS
expect(await readiness.check()).toBe(false)
expect(readiness.degradedDependencies()).toEqual([])
expect(observations.at(-1)).toEqual(
expect.objectContaining({ ready: false, failure: 'jwks_http_failed' })
)
expect(observations.at(-1)).not.toHaveProperty('degraded')
})
it('gives SQL a shorter window than JWKS by default', async () => {
let now = 1_000
let sqlOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => true,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
sqlOk = false
now += RELAY_READINESS_SQL_GRACE_MS - 1
expect(await readiness.check()).toBe(true)
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
degraded: true,
degradedDependencies: ['sql'],
failure: 'sql_failed'
})
)
now += 1
expect(await readiness.check()).toBe(false)
expect(RELAY_READINESS_SQL_GRACE_MS).toBeLessThan(RELAY_READINESS_JWKS_GRACE_MS)
})
it('keeps a process that never succeeded out of the grace window', async () => {
let now = 1_000
const { readiness, observations, graceEvents } = graceProbe({
jwksOk: () => false,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(false)
now += 1_000
expect(await readiness.check()).toBe(false)
expect(graceEvents).toEqual([])
expect(observations.every((observation) => observation.degraded === undefined)).toBe(true)
})
it('measures the grace window on the injected clock, not wall time', async () => {
const now = 1_000
let jwksOk = true
const { readiness } = graceProbe({ jwksOk: () => jwksOk, sqlOk: () => true, now: () => now })
expect(await readiness.check()).toBe(true)
jwksOk = false
for (let attempt = 0; attempt < 5; attempt++) expect(await readiness.check()).toBe(true)
})
it('logs once on entering grace and once on leaving it', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = true
now += 1_000
expect(await readiness.check()).toBe(true)
expect(graceEvents).toEqual([
{
dependency: 'jwks',
grace: 'entered',
failure: 'jwks_http_failed',
lastSuccessAgeMs: 1_000,
graceMs: RELAY_READINESS_JWKS_GRACE_MS
},
{
dependency: 'jwks',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: RELAY_READINESS_JWKS_GRACE_MS
}
])
})
it('reports an expired window once when the dependency never comes back', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
now += 20_000
expect(await readiness.check()).toBe(false)
expect(await readiness.check()).toBe(false)
expect(graceEvents.map((event) => event.grace)).toEqual(['entered', 'expired'])
})
it('restarts the grace window from the most recent success', async () => {
let now = 1_000
let jwksOk = true
const { readiness } = graceProbe({
jwksGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 9_000
expect(await readiness.check()).toBe(true)
jwksOk = true
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 9_000
expect(await readiness.check()).toBe(true)
})
it('never serves grace when the window is disabled', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksGraceMs: 0,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
expect(await readiness.check()).toBe(false)
expect(graceEvents).toEqual([])
})
it('keeps one clock per dependency so a healthy JWKS cannot hold SQL open', async () => {
let now = 1_000
let sqlOk = true
const { readiness, graceEvents } = graceProbe({
sqlGraceMs: 10_000,
jwksOk: () => true,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
sqlOk = false
for (let elapsed = 1_000; elapsed <= 11_000; elapsed += 1_000) {
now = 1_000 + elapsed
await readiness.check()
}
expect(await readiness.check()).toBe(false)
expect(graceEvents.map((event) => [event.dependency, event.grace])).toEqual([
['sql', 'entered'],
['sql', 'expired']
])
})
it('logs both sides of an overlap when one dependency recovers as the other fails', async () => {
let now = 1_000
let jwksOk = true
let sqlOk = true
const { readiness, graceEvents } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = true
sqlOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
expect(readiness.degradedDependencies()).toEqual(['sql'])
expect(graceEvents.map((event) => [event.dependency, event.grace])).toEqual([
['jwks', 'entered'],
['jwks', 'recovered'],
['sql', 'entered']
])
})
it('probes SQL on every poll even while JWKS is failing', async () => {
let now = 1_000
let jwksOk = true
let sqlOk = true
const { readiness, observations, query } = graceProbe({
sqlGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
sqlOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
failure: 'jwks_http_failed',
failures: ['jwks_http_failed', 'sql_failed'],
degradedDependencies: ['jwks', 'sql']
})
)
// The SQL clock runs on evidence, so its window opens at the poll that saw SQL fail.
now += 10_000
expect(await readiness.check()).toBe(false)
expect(query).toHaveBeenCalledTimes(4)
})
it('serves a stale ready answer for at most the cache window after grace expires', async () => {
let now = 1_000
let jwksOk = true
const { readiness, fetchImpl } = graceProbe({
cacheMs: 10_000,
jwksGraceMs: 5_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 6_000
expect(await readiness.check()).toBe(true)
now += 3_999
expect(await readiness.check()).toBe(true)
expect(fetchImpl).toHaveBeenCalledTimes(1)
now += 1
expect(await readiness.check()).toBe(false)
expect(fetchImpl).toHaveBeenCalledTimes(2)
})
})
cloud/apps/relay/src/relay-readiness.ts
+133 -33
View File
@@ -6,20 +6,58 @@ export type RelayReadinessFailure =
| 'jwks_timed_out'
| 'sql_failed'
export type RelayReadinessDependency = 'jwks' | 'sql'
export type RelayReadinessObservation = {
ready: boolean
failure?: RelayReadinessFailure
// Both dependencies are probed every poll, so both can fail in the same one.
failures?: RelayReadinessFailure[]
degraded?: true
degradedDependencies?: RelayReadinessDependency[]
jwksLatencyMs: number
sqlLatencyMs: number
totalLatencyMs: number
}
export type RelayReadinessGraceEvent = {
dependency: RelayReadinessDependency
grace: 'entered' | 'recovered' | 'expired'
failure?: RelayReadinessFailure
lastSuccessAgeMs?: number
graceMs: number
}
export type RelayReadinessProbe = {
check: () => Promise<boolean>
degradedDependencies: () => RelayReadinessDependency[]
}
// The token verifier caches keys in process, so a cell keeps verifying tokens right through a JWKS
// outage: the only thing an unreachable JWKS endpoint blocks is a key rotation nobody is running.
export const RELAY_READINESS_JWKS_GRACE_MS = 900_000
// Each cell is its own load balancer backend, so failing readiness never re-routes a host, it only
// makes that hostname unreachable. A host that lands on a SQL-dead cell gets WRONG_CELL and is
// re-placed by the director, and existing control sockets survive a generic Postgres error. Three
// minutes rides a Cloud SQL failover without hiding a per-cell fault for a quarter of an hour.
export const RELAY_READINESS_SQL_GRACE_MS = 180_000
export const RELAY_MAX_READINESS_GRACE_MS = 3_600_000
type RelayReadinessOptions = {
fetch?: typeof fetch
timeoutMs?: number
cacheMs?: number
jwksGraceMs?: number
sqlGraceMs?: number
now?: () => number
observe?: (observation: RelayReadinessObservation) => void
observeGrace?: (event: RelayReadinessGraceEvent) => void
}
type DependencySettlement = {
satisfied: boolean
degraded: boolean
event?: RelayReadinessGraceEvent
}
function fetchFailure(error: unknown): RelayReadinessFailure {
@@ -28,62 +66,124 @@ function fetchFailure(error: unknown): RelayReadinessFailure {
: 'jwks_fetch_failed'
}
function graceTransition(
degraded: boolean,
failure: RelayReadinessFailure | undefined
): RelayReadinessGraceEvent['grace'] {
if (degraded) return 'entered'
return failure === undefined ? 'recovered' : 'expired'
}
// One dependency's own last-known-good clock; collapsing the two would let a healthy JWKS poll keep
// a dead Postgres inside its window forever.
function createDependencyGrace(dependency: RelayReadinessDependency, graceMs: number) {
let lastSuccessAt: number | undefined
let inGrace = false
return (at: number, failure: RelayReadinessFailure | undefined): DependencySettlement => {
if (failure === undefined) lastSuccessAt = at
const lastSuccessAgeMs =
lastSuccessAt === undefined ? undefined : Math.max(0, at - lastSuccessAt)
const degraded =
failure !== undefined && lastSuccessAgeMs !== undefined && lastSuccessAgeMs < graceMs
const crossed = degraded !== inGrace
inGrace = degraded
return {
satisfied: failure === undefined || degraded,
degraded,
...(crossed
? {
event: {
dependency,
grace: graceTransition(degraded, failure),
...(failure ? { failure } : {}),
...(lastSuccessAgeMs === undefined ? {} : { lastSuccessAgeMs }),
graceMs
}
}
: {})
}
}
}
async function timed<T>(
now: () => number,
run: () => Promise<T>
): Promise<{ value: T; latencyMs: number }> {
const startedAt = now()
const value = await run()
return { value, latencyMs: Math.max(0, now() - startedAt) }
}
export function createRelayReadiness(
database: RelayDatabase,
jwksUrl: string,
options: RelayReadinessOptions = {}
): () => Promise<boolean> {
): RelayReadinessProbe {
const fetchImpl = options.fetch ?? fetch
const timeoutMs = options.timeoutMs ?? 2_000
const cacheMs = options.cacheMs ?? 10_000
const now = options.now ?? Date.now
const settleJwks = createDependencyGrace(
'jwks',
options.jwksGraceMs ?? RELAY_READINESS_JWKS_GRACE_MS
)
const settleSql = createDependencyGrace('sql', options.sqlGraceMs ?? RELAY_READINESS_SQL_GRACE_MS)
let cachedAt = Number.NEGATIVE_INFINITY
let cached = false
let lastObservedReady: boolean | undefined
let degraded: RelayReadinessDependency[] = []
return async () => {
const probeJwks = async (): Promise<RelayReadinessFailure | undefined> => {
try {
const response = await fetchImpl(jwksUrl, { signal: AbortSignal.timeout(timeoutMs) })
return response.ok ? undefined : 'jwks_http_failed'
} catch (error) {
return fetchFailure(error)
}
}
const probeSql = async (): Promise<RelayReadinessFailure | undefined> => {
try {
await database.query('SELECT 1 AS ready')
return undefined
} catch {
// The load balancer only needs the boolean; the safe reason is all that is emitted.
return 'sql_failed'
}
}
const check = async (): Promise<boolean> => {
if (now() - cachedAt < cacheMs) return cached
const startedAt = now()
let jwksCompletedAt = startedAt
let sqlStartedAt = startedAt
let failure: RelayReadinessFailure | undefined
try {
let response: Response
try {
response = await fetchImpl(jwksUrl, { signal: AbortSignal.timeout(timeoutMs) })
} catch (error) {
failure = fetchFailure(error)
throw error
} finally {
jwksCompletedAt = now()
}
if (!response.ok) {
failure = 'jwks_http_failed'
throw new Error(failure)
}
sqlStartedAt = now()
try {
await database.query('SELECT 1 AS ready')
} catch (error) {
failure = 'sql_failed'
throw error
}
} catch {
// The load balancer only needs the boolean; the safe reason is emitted below.
}
const [jwks, sql] = await Promise.all([timed(now, probeJwks), timed(now, probeSql)])
const completedAt = now()
cached = failure === undefined
const jwksSettlement = settleJwks(completedAt, jwks.value)
const sqlSettlement = settleSql(completedAt, sql.value)
const failures = [jwks.value, sql.value].filter((value) => value !== undefined)
const failure = failures[0]
degraded = []
if (jwksSettlement.degraded) degraded.push('jwks')
if (sqlSettlement.degraded) degraded.push('sql')
cached = jwksSettlement.satisfied && sqlSettlement.satisfied
cachedAt = completedAt
if (!cached || cached !== lastObservedReady) {
if (failure !== undefined || cached !== lastObservedReady) {
options.observe?.({
ready: cached,
...(failure ? { failure } : {}),
jwksLatencyMs: Math.max(0, jwksCompletedAt - startedAt),
sqlLatencyMs: failure?.startsWith('jwks_') ? 0 : Math.max(0, completedAt - sqlStartedAt),
...(failures.length > 1 ? { failures } : {}),
...(degraded.length > 0 ? { degraded: true, degradedDependencies: [...degraded] } : {}),
jwksLatencyMs: jwks.latencyMs,
sqlLatencyMs: sql.latencyMs,
totalLatencyMs: Math.max(0, completedAt - startedAt)
})
}
for (const event of [jwksSettlement.event, sqlSettlement.event]) {
if (event) options.observeGrace?.(event)
}
lastObservedReady = cached
return cached
}
return { check, degradedDependencies: () => [...degraded] }
}
cloud/apps/relay/src/relay-ready-endpoint.test.ts
+69
View File
@@ -0,0 +1,69 @@
import { describe, expect, it, vi } from 'vitest'
import { createRelayApp } from './app.js'
import type { RelayConfig } from './config.js'
import type { RelayReadinessDependency } from './relay-readiness.js'
function readyApp(input: { ready: boolean; degraded?: RelayReadinessDependency[] }) {
// SAFETY: /ready reads only ready and readinessDegradation, so the rest of the surface, which
// every other app route test also stubs this way, stays unbuilt.
const operations = {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
ready: vi.fn(async () => input.ready),
readinessDegradation: () => input.degraded ?? []
} as Parameters<typeof createRelayApp>[1]
return createRelayApp(config(), operations)
}
describe('relay readiness endpoint', () => {
it('answers a healthy cell with the unchanged body', async () => {
const response = await readyApp({ ready: true }).request('/ready')
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true })
})
it('keeps the 200 the load balancer needs but marks a remembered answer', async () => {
const response = await readyApp({ ready: true, degraded: ['sql'] }).request('/ready')
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, degraded: true, dependency: ['sql'] })
})
it('still fails the cell out of rotation once no window covers the failure', async () => {
const response = await readyApp({ ready: false }).request('/ready')
expect(response.status).toBe(503)
expect(await response.json()).toEqual({ error: 'dependency_unavailable' })
})
})
function config(): RelayConfig {
return {
port: 8080,
publicUrl: 'https://c7.relay.example.test',
cellUrl: 'https://c7.relay.example.test',
region: 'us-central1',
authIssuer: 'https://auth.example.test',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.test/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c7',
cells: [],
adminAudience: 'https://relay.example.test/v1/admin/drain',
deployServiceAccount: 'deploy@example.test',
runtimeServiceAccount: 'relay-cell@example.test',
adminJwksUrl: 'https://auth.example.test/jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './data'
}
}
cloud/apps/relay/src/relay-region-app.test.ts
+34
View File
@@ -265,6 +265,40 @@ describe('Relay region API', () => {
expect(burst.every(({ status }) => status === 200)).toBe(true)
expect(regionCatalog).toHaveBeenCalledOnce()
})
it('answers a pool that cannot hand out a client with a retryable 503', async () => {
const regionCatalog = vi.fn(async () => {
throw new Error('Connection terminated due to connection timeout')
})
const app = createRelayApp(config({ publicAssignmentRetryAfterSeconds: 7 }), {
store: {} as never,
assignments: { regionCatalog } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/regions')
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('7')
expect(await response.json()).toEqual({ error: 'region_catalog_temporarily_unavailable' })
})
it('still fails loudly when the region catalog breaks for a non-transient reason', async () => {
const regionCatalog = vi.fn(async () => {
throw new TypeError('broken invariant')
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: { regionCatalog } as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const response = await app.request('/v1/regions')
expect(response.status).toBe(500)
})
})
function assignmentRequest(relayHostId: string, extra: Record<string, unknown>): RequestInit {
cloud/apps/relay/src/relay-schema-catalog-precheck-postgres.test.ts
+289
View File
@@ -0,0 +1,289 @@
import pg from 'pg'
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import {
applyPostgresSchema,
catalogObjectPresence,
schemaLockTarget,
takesRelationLock
} from '@orca-cloud/postgres-schema'
import {
openRelayDatabase,
POSTGRES_LOCK_TIMEOUT_MS,
relayPostgresSchemaStatements,
type RelayDatabase
} from './database.js'
// The outage this guards against: CREATE INDEX IF NOT EXISTS takes its relation lock before the
// server evaluates the existence test, so on a database that already has the index the boot still
// joins the lock queue, and relation locks are granted in queue order, so every writer queues
// behind it. Only a real server can show that the catalog pre-check removes those statements.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_schema_precheck_test'
// A table the boot would otherwise touch with two CREATE INDEX statements, and the largest table
// in production.
const LOCKED_TABLE = 'relay_control_connection_reservations'
function scopedUrl(): string {
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
return url.toString()
}
async function onAdmin<T>(operation: (client: pg.Client) => Promise<T>): Promise<T> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
return await operation(client)
} finally {
await client.end()
}
}
describePostgres('relay boot-time schema against PostgreSQL', () => {
let url = ''
let pool: pg.Pool
let sent: string[]
const opened: RelayDatabase[] = []
beforeAll(() => {
url = scopedUrl()
})
beforeEach(async () => {
await onAdmin(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
})
// Same lock bound the boot pool uses, so a statement that still queues fails instead of
// hanging the test.
pool = new pg.Pool({ connectionString: url, max: 1, lock_timeout: POSTGRES_LOCK_TIMEOUT_MS })
sent = []
})
afterAll(async () => {
await Promise.all(opened.map((database) => database.close().catch(() => undefined)))
await onAdmin((client) => client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`))
})
async function applyRecording(): Promise<{ ran: number; skipped: number }> {
const record = async (statement: string, params: unknown[] = []): Promise<pg.QueryResult> => {
sent.push(statement)
return await pool.query(statement, params)
}
return await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => record(statement),
{ catalogQuery: async (sql, params) => (await record(sql, params)).rows }
)
}
function lockTaking(): string[] {
return sent.filter(takesRelationLock)
}
it('creates the schema cold, then issues no lock-taking statement on the next boot', async () => {
const cold = await applyRecording()
expect(lockTaking().length).toBeGreaterThan(0)
expect(cold.skipped).toBeGreaterThan(0)
sent = []
const warm = await applyRecording()
// Zero, with no exceptions: every statement that takes a relation lock has a pre-check.
expect(lockTaking()).toEqual([])
const preCheckedCount = relayPostgresSchemaStatements().filter(
(statement) => schemaLockTarget(statement) !== undefined
).length
expect(warm.skipped).toBe(preCheckedCount)
expect(warm.ran).toBe(relayPostgresSchemaStatements().length - preCheckedCount)
// The CREATE TABLEs still run: they resolve a name and take no lock on an existing table.
expect(warm.ran).toBeGreaterThan(0)
await pool.end()
})
it('skips an index a failed concurrent build left invalid instead of rebuilding it', async () => {
await applyRecording()
// A cancelled CREATE INDEX CONCURRENTLY leaves exactly this state, and IF NOT EXISTS skips it
// too, so reading indisvalid as a condition would newly take the lock it used to avoid.
await pool.query(
`UPDATE pg_catalog.pg_index SET indisvalid = false
WHERE indexrelid = to_regclass('${schema}.relay_audit_events_at')`
)
sent = []
await applyRecording()
expect(lockTaking()).toEqual([])
await pool.end()
})
it('re-adds a constraint an operator dropped, then skips it again', async () => {
// The pre-check matches the constraint by name only, so this is the one shape where a changed
// body needs an operator: drop it, and the next boot puts the current definition back.
await applyRecording()
const named = async (): Promise<number> =>
Number(
(
await pool.query(
`SELECT count(*) AS present FROM pg_catalog.pg_constraint
WHERE conrelid = to_regclass('relay_region_rehome_attempts')
AND conname = 'relay_region_rehome_attempts_preferred_region_valid'`
)
).rows[0]?.present
)
expect(await named()).toBe(1)
await pool.query(
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT relay_region_rehome_attempts_preferred_region_valid`
)
sent = []
await applyRecording()
expect(lockTaking()).toHaveLength(1)
expect(await named()).toBe(1)
sent = []
await applyRecording()
expect(lockTaking()).toEqual([])
await pool.end()
})
it('does not let a same-named index on a sibling table answer for this one', async () => {
// Index names are unique per schema, not per table, so a name freed on one table and taken on
// another is reachable. Without tying the index to the table, the pre-check reads that sibling
// as this table's index and skips the real CREATE INDEX for good.
await applyRecording()
const ask = async (table: string): Promise<boolean> =>
(
await catalogObjectPresence(
async (sql, params) => (await pool.query(sql, params)).rows,
{ kind: 'index', table, name: 'relay_audit_events_at', skipWhen: 'present' }
)
).present
expect(await ask('relay_audit_events')).toBe(true)
await pool.query(`DROP INDEX ${schema}.relay_audit_events_at`)
await pool.query(`CREATE TABLE ${schema}.precheck_sibling (at BIGINT)`)
await pool.query(`CREATE INDEX relay_audit_events_at ON ${schema}.precheck_sibling(at)`)
expect(await ask('relay_audit_events')).toBe(false)
expect(await ask('precheck_sibling')).toBe(true)
await pool.end()
})
it('boots while another session holds ACCESS EXCLUSIVE on the largest table', async () => {
// The end-to-end proof through openRelayDatabase: with the lock held, any DDL the boot still
// sent against this table would hit lock_timeout, and 55P03 is no longer retried.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(cold)
await pool.end()
const holder = new pg.Client({ connectionString: url })
await holder.connect()
await holder.query('BEGIN')
await holder.query(`LOCK TABLE ${LOCKED_TABLE} IN ACCESS EXCLUSIVE MODE`)
try {
const warm = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(warm)
} finally {
await holder.query('ROLLBACK')
await holder.end()
}
})
it('defers the activity-lease migrations and still boots while their table is locked', async () => {
// The migration boot, reproduced: the index is there, the table is locked by someone else, and
// all the drop can do is time out. It has to leave the statement for the next boot rather than
// fail, or 28 directors crash-loop through a stall on a table written ~475/s.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(cold)
// Put the database back in its pre-migration shape, which is what makes the drop lock-taking.
await pool.query(
`CREATE INDEX relay_assignment_activity_expiry
ON ${schema}.relay_assignment_activity_leases(expires_at)`
)
await pool.query(`ALTER TABLE ${schema}.relay_assignment_activity_leases RESET (fillfactor)`)
const warned: string[] = []
const warn = vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const holder = new pg.Client({ connectionString: url })
await holder.connect()
await holder.query('BEGIN')
await holder.query(
`LOCK TABLE ${schema}.relay_assignment_activity_leases IN ACCESS EXCLUSIVE MODE`
)
let summary: Awaited<ReturnType<typeof applyPostgresSchema>>
try {
summary = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
} finally {
await holder.query('ROLLBACK')
await holder.end()
warn.mockRestore()
}
// Both statements deferred, and the boot still applied everything else.
expect(summary.deferred).toBe(2)
expect(summary.ran).toBeGreaterThan(0)
const deferred = warned
.map((line) => JSON.parse(line) as { event?: string; name?: string })
.filter((event) => event.event === 'orca_relay_postgres_schema_object_deferred')
expect(deferred.map((event) => event.name)).toEqual([
'relay_assignment_activity_expiry',
'fillfactor=70'
])
// Nothing was applied, so the next boot has the same work to do, not half of it.
const stillThere = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(stillThere.rowCount).toBe(1)
// And the next boot, with the lock gone, finishes the job.
const retry = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
expect(retry.deferred).toBe(0)
const gone = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(gone.rowCount).toBe(0)
const options = await pool.query(
`SELECT reloptions FROM pg_class WHERE oid = to_regclass($1)`,
[`${schema}.relay_assignment_activity_leases`]
)
expect(options.rows[0]?.reloptions).toEqual(['fillfactor=70'])
await pool.end()
})
it('fails that same boot with 55P03 when the pre-check is not wired in', async () => {
// Keeps the test above from passing vacuously: the lock really does block relay's DDL.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(cold)
const holder = new pg.Client({ connectionString: url })
await holder.connect()
await holder.query('BEGIN')
await holder.query(`LOCK TABLE ${LOCKED_TABLE} IN ACCESS EXCLUSIVE MODE`)
try {
await expect(
applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ retryDeadlineMs: 0 }
)
).rejects.toMatchObject({ code: '55P03' })
} finally {
await holder.query('ROLLBACK')
await holder.end()
await pool.end()
}
})
})
cloud/apps/relay/src/relay-schema-lock-targets.test.ts
+289
View File
@@ -0,0 +1,289 @@
import { describe, expect, it } from 'vitest'
import {
requireSchemaLockTarget,
schemaDeferrable,
schemaLockTarget,
sqlWithoutComments,
takesRelationLock,
type SchemaLockTarget
} from '@orca-cloud/postgres-schema'
import { relayPostgresSchemaStatements } from './database.js'
// Golden pin of every boot-time statement that takes a relation lock on Postgres. Each entry with
// a kind is gated by the catalog pre-check, so it costs a catalog read on a migrated database and
// nothing more. An addition to this list is the case the RULE comment beside SCHEMA forbids: a
// brand-new index reports missing on every director at once and each one runs a non-concurrent
// build over the whole table, which is how a boot takes the site down. Build it out of band with
// CREATE INDEX CONCURRENTLY first, then add it to SCHEMA and update this list.
const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_device', skipWhen: 'present' },
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_current_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_grace_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_connection_bases', name: 'relay_connection_bases_active_deadline', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_assignment_region_preferences',
name: 'relay_assignment_region_preferences_observed',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_region_rehome_attempts',
name: 'relay_region_rehome_attempts_pending',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_region_rehome_attempts',
name: 'relay_region_rehome_attempts_host_recency',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_cell_runtime', name: 'relay_cell_runtime_heartbeat', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_cell_connection_runtime',
name: 'relay_cell_connection_runtime_heartbeat',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_cell_connection_snapshots',
name: 'relay_cell_connection_snapshot_freshness',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_cell_fences', name: 'relay_cell_fences_expiry', skipWhen: 'present' },
{ kind: 'index', table: 'relay_cell_committed_fences', name: 'relay_cell_committed_fences_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_cell_legacy_fence_adoptions',
name: 'relay_cell_legacy_fence_adoptions_expiry',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_cell_fence_attempts', name: 'relay_cell_fence_attempts_expiry', skipWhen: 'present' },
{ kind: 'index', table: 'relay_cell_fence_attempts', name: 'relay_cell_fence_attempts_cell', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_cell_fence_apply_invocations',
name: 'relay_cell_fence_apply_invocations_attempt',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_cell_drain_attempt_states',
name: 'relay_cell_drain_attempt_states_cell',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_control_connection_reservations',
name: 'relay_control_connection_reservation_headroom',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_control_connection_reservations',
name: 'relay_control_connection_reservation_assignment',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_rate_windows', name: 'relay_rate_windows_started', skipWhen: 'present' },
{ kind: 'index', table: 'relay_assignment_migrations', name: 'relay_assignment_migrations_active', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_post_drain_migration_pins',
name: 'relay_post_drain_migration_pins_attempt',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_audit_events', name: 'relay_audit_events_at', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_decisions', name: 'last_considered_at', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_decisions', name: 'cohort_bucket', skipWhen: 'present' },
// Constraint swaps are matched by name in pg_constraint, with opposite polarities: nothing to
// drop is nothing to do, and a name already there is nothing to add.
{
kind: 'constraint',
table: 'relay_region_rehome_attempts',
name: 'relay_region_rehome_attempts_preferred_region_check',
skipWhen: 'absent'
},
{
kind: 'constraint',
table: 'relay_region_rehome_attempts',
name: 'relay_region_rehome_attempts_preferred_region_valid',
skipWhen: 'present'
},
{ kind: 'column', table: 'relay_region_rehome_control', name: 'host_cooldown_ms', skipWhen: 'present' },
{ kind: 'column', table: 'relay_control_capabilities', name: 'idle_regional_rehome', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' },
{ kind: 'index-by-name', name: 'relay_assignment_activity_expiry', skipWhen: 'absent' },
{
kind: 'reloption',
table: 'relay_assignment_activity_leases',
name: 'fillfactor=70',
skipWhen: 'present'
}
]
const INDEX_OR_ADD_COLUMN = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|ALTER\s+TABLE\s+[^\s]+\s+ADD\s+COLUMN)/i
function lockTakingStatements(): string[] {
return relayPostgresSchemaStatements().filter(takesRelationLock)
}
describe('relay boot-time lock targets', () => {
it('matches the pinned list of lock-taking statements', () => {
expect(lockTakingStatements().map(schemaLockTarget)).toEqual(GOLDEN_LOCK_TAKING)
})
it('derives a target for every CREATE INDEX and every ALTER TABLE ADD COLUMN', () => {
// A census over the real schema, not two hand-picked cases: a statement that lands here
// without a target is sent on every boot and takes the lock the pre-check exists to avoid.
// requireSchemaLockTarget is what boot calls, so this fails the same way boot would.
for (const statement of relayPostgresSchemaStatements()) {
expect(() => requireSchemaLockTarget(statement)).not.toThrow()
}
const unparsed = relayPostgresSchemaStatements().filter(
(statement) =>
INDEX_OR_ADD_COLUMN.test(sqlWithoutComments(statement)) &&
schemaLockTarget(statement) === undefined
)
expect(unparsed).toEqual([])
})
it('reads every derived name as a bare identifier, never a keyword or a qualified name', () => {
for (const statement of relayPostgresSchemaStatements()) {
const target = schemaLockTarget(statement)
if (!target) continue
// A reloption is the one target whose name is a pair rather than an identifier, because
// pg_class stores reloptions as `name=value` text and the value is half the question.
const shape = target.kind === 'reloption' ? /^[a-z_][a-z0-9_]*=[A-Za-z0-9_.]+$/ : /^[a-z_][a-z0-9_]*$/
expect(target.name).toMatch(shape)
// A DROP INDEX names no table, so there is none to check.
if (target.kind !== 'index-by-name') expect(target.table).toMatch(/^[a-z_][a-z0-9_]*$/)
}
})
it('pre-checks every lock-taking statement, with no exceptions', () => {
// The invariant the rule comment beside SCHEMA depends on: nothing that takes a relation lock
// reaches the server on a warm boot. A statement with no target breaks it.
const unchecked = lockTakingStatements().filter(
(statement) => schemaLockTarget(statement) === undefined
)
expect(unchecked).toEqual([])
expect(lockTakingStatements()).toHaveLength(GOLDEN_LOCK_TAKING.length)
})
it('derives a target through the comment block a split schema glues on', () => {
// Not vacuous: SCHEMA really does carry a comment-prefixed statement, and it is a CREATE INDEX
// on relay_connection_bases. Classifying the raw text would give it no target at all.
const commented = relayPostgresSchemaStatements().filter((statement) =>
statement.startsWith('--')
)
expect(commented.length).toBeGreaterThan(0)
for (const statement of commented) {
if (!takesRelationLock(statement)) continue
expect(schemaLockTarget(statement)).toBeDefined()
}
expect(commented.map(schemaLockTarget)).toContainEqual({
kind: 'index',
table: 'relay_connection_bases',
name: 'relay_connection_bases_active_deadline',
skipWhen: 'present'
})
})
it('leaves the dollar-quoted statement-stats migration byte-identical', () => {
// Its body is a PL/pgSQL block full of commas and parentheses. Reading the tag as anything but
// opaque would change the text classification sees, and it is the only such statement relay has.
const doBlock = relayPostgresSchemaStatements().find((statement) => statement.startsWith('DO '))
expect(doBlock).toBeDefined()
expect(sqlWithoutComments(doBlock!)).toBe(doBlock)
expect(takesRelationLock(doBlock!)).toBe(false)
})
it('leaves every statement classifiable once its leading comments are stripped', () => {
for (const statement of relayPostgresSchemaStatements()) {
expect(sqlWithoutComments(statement)).toMatch(/^(?:CREATE|ALTER|DROP|DO)\s/i)
}
})
it('pre-checks the activity-expiry drop by name, and skips it once the index is gone', () => {
// A DROP INDEX takes ACCESS EXCLUSIVE on the index's table for as long as the index is there,
// so it is in the census like any other lock-taking statement. Its target resolves by name
// alone, because the statement names no table and needs none.
const drops = relayPostgresSchemaStatements().filter((statement) =>
/^DROP\s/i.test(sqlWithoutComments(statement))
)
expect(drops.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
for (const statement of drops) {
expect(takesRelationLock(statement)).toBe(true)
expect(schemaLockTarget(statement)).toEqual({
kind: 'index-by-name',
name: 'relay_assignment_activity_expiry',
skipWhen: 'absent'
})
}
})
it('marks the out-of-band sweep indexes and the activity-lease migrations deferrable, and nothing else', () => {
// The statements a lock timeout must not turn into a crash loop, and the only ones: every
// other statement still fails the boot loudly, which is what keeps the marker meaningful.
const deferrable = relayPostgresSchemaStatements().filter(schemaDeferrable)
expect(deferrable.map((statement) => sqlWithoutComments(statement).replace(/\s+/g, ' '))).toEqual([
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown')",
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation ON relay_invites(reservation_expires_at) WHERE state = 'reserved'",
'CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL',
'CREATE INDEX IF NOT EXISTS relay_rate_windows_started ON relay_rate_windows(window_started_at)',
'DROP INDEX IF EXISTS relay_assignment_activity_expiry',
'ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)'
])
})
it('derives a target for a partial index, WHERE clause and all', () => {
// The pre-check reads the index name and table from the head of the statement, so a trailing
// WHERE is invisible to it. Asserted because the sweep indexes depend on that: a parser that
// gave a partial index no target would send it unchecked on every boot.
const partial = relayPostgresSchemaStatements().filter((statement) =>
/^CREATE\s+INDEX\b[\s\S]*\bWHERE\b/i.test(sqlWithoutComments(statement))
)
expect(partial.map(schemaLockTarget)).toEqual([
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
}
])
})
it('no longer creates an index on the column every control renewal writes', () => {
// The regression this drop exists to prevent: re-adding it would make ~471 renewals/s non-HOT
// again. A CREATE anywhere in the schema naming that index fails here.
const creates = relayPostgresSchemaStatements().filter((statement) =>
/relay_assignment_activity_expiry/i.test(sqlWithoutComments(statement))
)
expect(creates.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
})
})
cloud/apps/relay/src/relay-server.ts
+8 -3
View File
@@ -114,9 +114,13 @@ export function createRelayServer(
recordControlRenewal: (durationMs, outcome) =>
observability.recordControlRenewal?.(durationMs, outcome)
})
const ready = createRelayReadiness(observedDatabase, config.jwksUrl, {
observe: (observation) => observability.recordReadiness(observation)
const readiness = createRelayReadiness(observedDatabase, config.jwksUrl, {
jwksGraceMs: config.readinessJwksGraceMs,
sqlGraceMs: config.readinessSqlGraceMs,
observe: (observation) => observability.recordReadiness(observation),
observeGrace: (event) => observability.recordReadinessGrace(event)
})
const ready = readiness.check
const queuedBytes = new ProcessQueuedByteBudget()
const sessions = new HostSessionRegistry(
config,
@@ -132,7 +136,7 @@ export function createRelayServer(
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drain: (graceMs, options) => sessions.drain(graceMs, options ?? {}),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
@@ -156,6 +160,7 @@ export function createRelayServer(
...readRelayDatabasePoolPressure(database)
}),
ready,
readinessDegradation: () => readiness.degradedDependencies(),
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
recordAssignmentRejectionReason: (lane, reason) =>
observability.recordAssignmentRejectionReason?.(lane, reason),
cloud/apps/relay/src/relay-sweep-schedule.test.ts
+19
View File
@@ -52,4 +52,23 @@ describe('sweep schedule jitter', () => {
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
it('jitters the credential cleanup tick', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const cleanup = /'\[orca-relay\] credential cleanup failed'\s*\),\s*([^\n]*?)\n/.exec(source)
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
// A census, not a list of the timers that happen to be gated today: an ungated sweep runs in
// every cell as well as the director, which multiplies one table scan by the fleet size.
it('gates every periodic sweep in index.ts on the maintenance role', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const timers = source.match(/setInterval\(/g) ?? []
const gated =
source.match(/roleOwnsAssignmentMaintenance\(config\.role\)\s*\?\s*setInterval\(/g) ?? []
expect(timers.length).toBeGreaterThan(0)
expect(gated.length).toBe(timers.length)
})
})
cloud/dev/contracts/production-cloud-sql-app-consumers.json
+2 -2
View File
@@ -4,12 +4,12 @@
"authPoolMax": 10,
"apiInstances": 10,
"apiPoolMax": 5,
"maxConnections": 400,
"maxConnections": 500,
"sources": {
"authInstances": "private apps tfvars: auth service max instances",
"authPoolMax": "private auth service: pg.Pool max",
"apiInstances": "private apps tfvars: API service max instances",
"apiPoolMax": "private API service: pg.Pool max",
"maxConnections": "Cloud SQL tier default; no max_connections flag is set"
"maxConnections": "measured SHOW max_connections = 500 on the live instance 2026-09-16; no max_connections flag is set, so this is the tier default and the previous 400 was an unverified assumption about it"
}
}
cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs
+44 -15
View File
@@ -35,6 +35,9 @@ function cellOrigin(cellId) {
// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
// Matches the cell's own cap on /v1/admin/drain.
const MAX_PACE_WINDOW_MS = 5 * 60 * 1_000
export function parseProductionCapacityCellArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
@@ -64,11 +67,22 @@ export function parseProductionCapacityCellArguments(argv) {
) {
throw new Error('production capacity target origin is not exact')
}
const paceWindowMs = values['pace-window-ms'] === undefined
? 0
: Number(values['pace-window-ms'])
if (
!Number.isSafeInteger(paceWindowMs) ||
paceWindowMs < 0 ||
paceWindowMs > MAX_PACE_WINDOW_MS
) {
throw new Error('--pace-window-ms must be an integer between 0 and 300000')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellOrigin: expectedCellOrigin,
cellId,
mode: values.mode
mode: values.mode,
paceWindowMs
}
}
@@ -82,24 +96,39 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const postAt = async (origin, path, body) =>
await responseJson(
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
),
path
const postRaw = async (origin, path, body) =>
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
)
const postAt = async (origin, path, body) =>
await responseJson(await postRaw(origin, path, body), path)
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
if (config.mode === 'drain') {
const paceWindowMs = config.paceWindowMs ?? 0
if (paceWindowMs > 0) {
const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', {
v: 1,
graceMs: 0,
paceWindowMs
})
if (paced.ok) {
await paced.json().catch(() => ({}))
return { changed: false, drained: true, paceWindowMs }
}
// A cell still on an image without paced drain rejects the unknown field outright.
// An unpaced drain is the behaviour that cell already has, so fall back to it.
if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`)
await paced.json().catch(() => ({}))
}
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
return { changed: false, drained: true }
return { changed: false, drained: true, paceWindowMs: 0 }
}
const before = await inspectAdmissionSelector(post)
const state = selectorCellState(before.selector, config.cellId)
cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs
+77 -7
View File
@@ -90,7 +90,8 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: 'https://c7.relay.onorca.dev',
cellId: 'production-gce-c7',
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -112,8 +113,12 @@ describe('production Relay capacity cell admission', () => {
]), /not approved/)
})
it('admits the same-cap Asia cells only under the same-cap allowlist', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => {
for (const cellId of [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
// Migration-only canaries: the US-only capacity rollout never touches them either.
'production-gce-c17', 'production-gce-c18'
]) {
const hostname = cellId.slice('production-gce-'.length)
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -125,10 +130,11 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname}.relay.onorca.dev`,
cellId,
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
}
for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
for (const cellId of ['production-gce-c12', 'production-gce-c30']) {
const hostname = cellId.slice('production-gce-'.length)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -168,13 +174,77 @@ describe('production Relay capacity cell admission', () => {
{ ...config, mode: 'drain' },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0 }
}])
})
it('paces the drain send when the roll asks for a window', async () => {
const fake = canaryFetch()
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0, paceWindowMs: 120_000 }
}])
})
it('drains unpaced when the cell image rejects the pacing field', async () => {
const bodies = []
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async (url, init) => {
assert.equal(new URL(url).pathname, '/v1/admin/drain')
const body = JSON.parse(init.body)
bodies.push(body)
if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400)
return response({ v: 1, draining: true })
}
}
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(bodies, [
{ v: 1, graceMs: 0, paceWindowMs: 120_000 },
{ v: 1, graceMs: 0 }
])
})
it('fails a paced drain that the cell rejects for any other reason', async () => {
await assert.rejects(
prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async () => response({ error: 'invalid_token' }, 401)
}
),
/returned 401/
)
})
it('refuses a pacing window that is not a bounded integer', () => {
const argv = (value) => [
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c26.relay.onorca.dev',
'--cell-id', 'production-gce-c26',
'--mode', 'drain',
'--pace-window-ms', value
]
for (const value of ['-1', '300001', '1.5', 'soon']) {
assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/)
}
assert.equal(parseProductionCapacityCellArguments(argv('300000')).paceWindowMs, 300_000)
})
it('restores only the selected cell to general admission', async () => {
const fake = canaryFetch()
await prepareProductionCapacityCell(
@@ -228,7 +298,7 @@ describe('production Relay capacity cell admission', () => {
}
)
assert.equal(calls, 2)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
})
it('fails when both drain attempts return a transient 503', async () => {
cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs
+9 -8
View File
@@ -7,11 +7,12 @@ import {
} from './relay-cloud-sql-connection-budget.mjs'
test('production shared consumers keep allowance and reserve below the ceiling', () => {
// cells: 20 pools at 10 (200) + the three asia-east2 pools at 16 (48).
const report = readRelayCloudSqlConnectionBudget()
assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 })
assert.deepEqual(report.asia, { cells: 3, poolMax: 10 })
assert.equal(report.configuredMaximum, 315)
assert.deepEqual(report.consumers, { cells: 248, directors: 15, auth: 20, api: 50 })
assert.deepEqual(report.asia, { cells: 3, poolMax: 16 })
assert.equal(report.configuredMaximum, 333)
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
assert.equal(report.rolloutOverlap.apiCandidate, 65)
assert.equal(report.rolloutOverlap.authCandidate, 35)
@@ -20,11 +21,11 @@ test('production shared consumers keep allowance and reserve below the ceiling',
assert.equal(report.rolloutOverlap.maximum, 65)
assert.equal(report.maintenanceAdminAllowance, 5)
assert.equal(report.explicitReserve, 10)
assert.equal(report.usableCeiling, 390)
assert.equal(report.operatingMaximum, 385)
assert.equal(report.remainingWithinUsableCeiling, 5)
assert.equal(report.budgetedTotal, 395)
assert.equal(report.unallocated, 5)
assert.equal(report.usableCeiling, 490)
assert.equal(report.operatingMaximum, 403)
assert.equal(report.remainingWithinUsableCeiling, 87)
assert.equal(report.budgetedTotal, 413)
assert.equal(report.unallocated, 87)
assert.equal(report.withinBudget, true)
})
cloud/dev/scripts/relay-production-same-cap-wave.mjs
+77 -6
View File
@@ -2,14 +2,29 @@ import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without
// displacing anybody. It enters and must leave migration-only, never general.
export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18']
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
...SAME_CAP_MIGRATION_ONLY_CELLS
]
// A general cell's wave isolates and restores it, advancing the selector twice; a
// migration-only cell's isolate and restore are both no-ops, so its wave advances nothing.
export function selectorWaveDelta(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 0 : 2
}
export function entryAdmission(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 'migration-only' : 'general'
}
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
@@ -23,9 +38,33 @@ function cells(value) {
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
// Every later cell offsets from one per-wave selector delta, and the two classes
// have different ones, so a mixed wave has no single offset any cell could use.
if (new Set(parsed.map(selectorWaveDelta)).size > 1) {
throw new Error('same-cap wave cells must be all general or all migration-only')
}
return parsed
}
// Break-glass: the aggregate 15-minute monitor gate is skipped, nothing else is.
// Returns null when no override was requested, and throws on a partial or
// mismatched one so a malformed override can never reach a mutation.
export function gateOverrideAuthorization(input, targetDigest, mutation) {
const reason = input.gateOverrideReason ?? ''
const confirmation = input.gateOverrideConfirmation ?? ''
if (!reason && !confirmation) return null
if (!mutation) throw new Error('verify does not accept a monitor gate override')
if (confirmation !== `SKIP_RELAY_MONITOR_GATE ${targetDigest}`) {
throw new Error('gate override confirmation does not match the exact target digest')
}
// Printable single-line only: this reason is rendered into the run summary and
// sealed into the canary artifact.
if (!/^[\x20-\x7e]{12,500}$/.test(reason)) {
throw new Error('gate override reason must be 12 to 500 printable characters on one line')
}
return { reason, confirmation }
}
export function validateSameCapWave(input) {
if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) {
throw new Error('same-cap wave mode is invalid')
@@ -53,13 +92,14 @@ export function validateSameCapWave(input) {
throw new Error('same-cap confirmation does not match the exact digest and cells')
}
if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation')
const gateOverride = gateOverrideAuthorization(input, targetDigest, mutation)
if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) {
throw new Error('batch mode requires a canary run ID')
}
if (input.mode !== 'batch-apply' && input.canaryRunId) {
throw new Error('only batch mode accepts a canary run ID')
}
return { cells: selected, targetDigest, rollbackDigest }
return { cells: selected, targetDigest, rollbackDigest, gateOverride }
}
export function canaryAuthority(input) {
@@ -81,13 +121,21 @@ export function canaryAuthority(input) {
cellId: wave.cells[0],
targetDigest: wave.targetDigest,
rollbackDigest: wave.rollbackDigest,
selectorGeneration: selectorGeneration + 2,
rehomeGeneration
selectorGeneration: selectorGeneration + selectorWaveDelta(wave.cells[0]),
rehomeGeneration,
// Audit trail, not authority: a batch reusing this canary is authorized by
// its own confirmation, so verification below neither requires nor forbids it.
gateOverride: wave.gateOverride === null ? null : {
...wave.gateOverride,
actor: input.actor ?? ''
}
}
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
// A mixed wave is already rejected, so the batch's first cell names the whole batch's class.
const batchAdmission = entryAdmission(cells(expected.cellIds ?? '')[0])
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
@@ -101,6 +149,14 @@ export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// A migration-only cell carries no hosts and a different cap, so rolling it proves nothing
// about a general batch, and its wave advances a different selector delta.
if (entryAdmission(authority.cellId) !== batchAdmission) {
throw new Error(
`canary authority cell ${authority.cellId} is ${entryAdmission(authority.cellId)}, ` +
`but this batch is ${batchAdmission}`
)
}
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
@@ -132,7 +188,9 @@ export function main(argv = process.argv.slice(2)) {
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
canaryRunId: input['canary-run-id']
canaryRunId: input['canary-run-id'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation']
})
process.stdout.write(`${JSON.stringify(wave.cells)}\n`)
return
@@ -147,14 +205,27 @@ export function main(argv = process.argv.slice(2)) {
commitSha: input['commit-sha'],
runId: input['run-id'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation']
rehomeGeneration: input['rehome-generation'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation'],
actor: input.actor
}))}\n`)
return
}
if (command === 'cell-class') {
const cellId = input['cell-id']
if (!SAME_CAP_CELLS.includes(cellId)) throw new Error('same-cap wave cells are invalid')
process.stdout.write(`${JSON.stringify({
entryAdmission: entryAdmission(cellId),
selectorWaveDelta: selectorWaveDelta(cellId)
})}\n`)
return
}
if (command === 'verify-canary') {
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
commitSha: input['commit-sha'],
runId: input['run-id'],
cellIds: input['cell-ids'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
selectorGeneration: input['selector-generation'],
cloud/dev/scripts/relay-production-same-cap-wave.test.mjs
+340 -3
View File
@@ -1,14 +1,19 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { execFileSync, spawnSync } from 'node:child_process'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { test } from 'node:test'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
canaryAuthority,
entryAdmission,
main,
validateSameCapWave,
verifyCanaryAuthority
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
const targetDigest = `sha256:${'a'.repeat(64)}`
const rollbackDigest = `sha256:${'b'.repeat(64)}`
@@ -52,6 +57,82 @@ test('requires one canary or a bounded reviewed batch', () => {
}), /cells/)
})
test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)
assert.equal(entryAdmission(cellId), 'migration-only', cellId)
assert.deepEqual(validateSameCapWave({
mode: 'canary-apply',
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`
}).cells, [cellId])
}
const cellIds = 'production-gce-c17,production-gce-c18'
assert.deepEqual(validateSameCapWave({
mode: 'batch-apply',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
canaryRunId: '42'
}).cells, ['production-gce-c17', 'production-gce-c18'])
// A mixed wave has no single selector delta for its later cells to offset from.
const mixed = 'production-gce-c7,production-gce-c17'
assert.throws(() => validateSameCapWave({
mode: 'batch-apply',
cellIds: mixed,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed}`,
canaryRunId: '42'
}), /all general or all migration-only/)
})
test('seals a migration-only canary at the generation its wave leaves behind', () => {
const seal = (cellId) => canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
// Isolate and restore are both no-ops on a migration-only cell, so nothing advances.
assert.equal(seal('production-gce-c17').selectorGeneration, 11)
assert.equal(seal('production-gce-c7').selectorGeneration, 13)
// That canary still authorizes a later batch of its own class; it is evidence about the image.
assert.equal(verifyCanaryAuthority(seal('production-gce-c17'), {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c17,production-gce-c18',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
rehomeGeneration: '4'
}).cellId, 'production-gce-c17')
})
test('reports each approved cell\'s class and selector delta', () => {
const printed = []
const write = process.stdout.write.bind(process.stdout)
process.stdout.write = (chunk) => printed.push(String(chunk))
try {
main(['cell-class', '--cell-id', 'production-gce-c17'])
main(['cell-class', '--cell-id', 'production-gce-c7'])
} finally {
process.stdout.write = write
}
assert.deepEqual(printed.map((line) => JSON.parse(line)), [
{ entryAdmission: 'migration-only', selectorWaveDelta: 0 },
{ entryAdmission: 'general', selectorWaveDelta: 2 }
])
assert.throws(() => main(['cell-class', '--cell-id', 'production-gce-c12']), /cells are invalid/)
})
test('binds rollback confirmation to the exact digest and ordered cells', () => {
assert.throws(() => validateSameCapWave({
mode: 'rollback',
@@ -94,6 +175,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.equal(verifyCanaryAuthority(authority, {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
@@ -102,6 +184,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.throws(() => verifyCanaryAuthority(authority, {
commitSha: 'd'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
@@ -116,8 +199,8 @@ test('reuses a canary across selector advances only within the same control epoc
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9',
targetDigest, rollbackDigest, selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
@@ -191,6 +274,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
commitSha,
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
@@ -206,3 +290,256 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
await rm(repository.root, { recursive: true, force: true })
}
})
// Why: the break-glass override is the one input that removes a safety check, so
// a partial or mismatched one must fail before the gate job reaches a mutation.
test('accepts only a complete digest-bound monitor gate override', () => {
const reason = 'rolling the measured Cloud SQL stall fix'
const confirmation = `SKIP_RELAY_MONITOR_GATE ${targetDigest}`
const wave = {
mode: 'canary-apply',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
}
assert.deepEqual(
validateSameCapWave({
...wave,
gateOverrideReason: reason,
gateOverrideConfirmation: confirmation
}).gateOverride,
{ reason, confirmation }
)
// An ordinary wave carries no override at all.
assert.equal(validateSameCapWave(wave).gateOverride, null)
assert.equal(
validateSameCapWave({ ...wave, gateOverrideReason: '', gateOverrideConfirmation: '' })
.gateOverride,
null
)
assert.throws(
() => validateSameCapWave({ ...wave, gateOverrideConfirmation: confirmation }),
/gate override reason/
)
assert.throws(
() => validateSameCapWave({ ...wave, gateOverrideReason: reason }),
/gate override confirmation/
)
// Bound to the digest this wave installs, not to any digest.
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: reason,
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${rollbackDigest}`
}),
/gate override confirmation/
)
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: 'too short',
gateOverrideConfirmation: confirmation
}),
/gate override reason/
)
// The reason is rendered into the run summary, so it stays printable and single-line.
assert.throws(
() => validateSameCapWave({
...wave,
gateOverrideReason: `${reason}\n| injected | row |`,
gateOverrideConfirmation: confirmation
}),
/gate override reason/
)
assert.throws(
() => validateSameCapWave({
...wave,
mode: 'verify',
confirmation: '',
gateOverrideReason: reason,
gateOverrideConfirmation: confirmation
}),
/verify does not accept a monitor gate override/
)
})
test('a rollback wave may break the glass on its own target digest', () => {
const reason = 'getting off the bad image during an incident'
assert.deepEqual(
validateSameCapWave({
mode: 'rollback',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7`,
gateOverrideReason: reason,
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`
}).gateOverride,
{ reason, confirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}` }
)
})
// Why: the canary authority never carried a monitor run ID, so a batch can reuse
// a canary rolled under an override. Recording it keeps the audit trail in the
// sealed artifact without making it part of what verification demands.
test('seals the override into the canary authority as audit trail only', () => {
const sealed = {
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'f'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
}
const expected = {
commitSha: 'f'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
rehomeGeneration: '4'
}
const overridden = canaryAuthority({
...sealed,
gateOverrideReason: 'rolling the measured Cloud SQL stall fix',
gateOverrideConfirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`,
actor: 'Jinwoo-H'
})
assert.deepEqual(overridden.gateOverride, {
reason: 'rolling the measured Cloud SQL stall fix',
confirmation: `SKIP_RELAY_MONITOR_GATE ${targetDigest}`,
actor: 'Jinwoo-H'
})
assert.equal(canaryAuthority(sealed).gateOverride, null)
// Neither shape changes what a batch verifies.
assert.equal(verifyCanaryAuthority(overridden, expected).cellId, 'production-gce-c7')
assert.equal(
verifyCanaryAuthority(canaryAuthority(sealed), expected).cellId,
'production-gce-c7'
)
})
function sealedCanary(cellId) {
return canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
}
// Why: a migration-only cell holds zero hosts at a different cap and its wave advances no
// selector, so rolling one is no evidence for a general batch, and the reverse is no evidence
// either. Nothing but the sealed cell id says which class a canary actually proved.
test('refuses a canary sealed on a cell of the other admission class', () => {
const expected = {
commitSha: 'c'.repeat(40),
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '99',
rehomeGeneration: '4'
}
const general = 'production-gce-c8,production-gce-c9'
const migrationOnly = SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: general
}),
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: migrationOnly
}),
/canary authority cell production-gce-c7 is general, but this batch is migration-only/
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: general
}).cellId,
'production-gce-c7'
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: migrationOnly
}).cellId,
'production-gce-c17'
)
// A caller that names no batch at all gets no verdict, rather than an unchecked class.
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), expected),
/same-cap wave cells are invalid/
)
})
// The dispatch workflow is the only caller, so the class check only binds anything if that
// step actually hands the batch over; run the step's own shell exactly as written.
function verifyCanaryStepScript() {
const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
const first = ' node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \\\n'
const start = dispatch.indexOf(first)
assert.notEqual(start, -1, 'the dispatch workflow has no verify-canary step')
const last = ' --rehome-generation "${REHOME_GENERATION}"\n'
const end = dispatch.indexOf(last, start)
assert.notEqual(end, -1, 'the verify-canary step does not end at the rehome generation')
return dispatch.slice(start, end + last.length).replace(/^ {10}/gm, '')
}
async function runVerifyCanaryStep(authority, cellIds) {
const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-verify-'))
try {
await mkdir(join(temporary, 'relay-same-cap-canary'), { recursive: true })
await writeFile(
join(temporary, 'relay-same-cap-canary', 'authority.json'),
JSON.stringify(authority)
)
return spawnSync('bash', ['-euo', 'pipefail', '-c', verifyCanaryStepScript()], {
cwd: new URL('../..', import.meta.url),
env: {
...process.env,
RUNNER_TEMP: temporary,
GITHUB_SHA: authority.commitSha,
CANARY_RUN_ID: authority.runId,
CELL_IDS: cellIds,
TARGET_DIGEST: targetDigest,
ROLLBACK_DIGEST: rollbackDigest,
SELECTOR_GENERATION: '99',
REHOME_GENERATION: '4'
},
encoding: 'utf8'
})
} finally {
await rm(temporary, { recursive: true, force: true })
}
}
test('the batch gate hands its own cells to the canary check', async () => {
const accepted = await runVerifyCanaryStep(
sealedCanary('production-gce-c7'),
'production-gce-c8,production-gce-c9'
)
assert.equal(accepted.status, 0, accepted.stderr)
const crossed = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
'production-gce-c8,production-gce-c9'
)
assert.equal(crossed.status, 1, crossed.stdout)
assert.match(
crossed.stderr,
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
const migrationOnly = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
)
assert.equal(migrationOnly.status, 0, migrationOnly.stderr)
})
cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs
+59 -5
View File
@@ -67,15 +67,15 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
)
assert.match(
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(
job,
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
)
assert.match(job, /resume requires the isolated migration-only cell/)
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
@@ -87,7 +87,7 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_RESTORE\}"/)
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
assert.match(job, /FAILSAFE_GENERATION/)
@@ -98,8 +98,62 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
// Wave 0 must retry freshness-only failures too: one Cloud Monitoring publish
// lag at the sample instant is not health evidence, and single-shot wave 0
// failed a whole batch on a series that was fresh again a minute later.
assert.match(job, /dry-run\.state\.json" \\\n --wave-index "\$\{WAVE_INDEX\}" --retry-freshness/)
assert.match(
job,
/dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
assert.doesNotMatch(job, /RETRY_ARGS/)
// Break-glass: the override skips the aggregate 15-minute monitor evidence and
// nothing else. The live per-wave recheck still runs on the override path, off
// the dispatch inputs the rehome inspect below verifies against the director.
assert.match(
job,
/if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,160}?--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
// The override is re-validated here, not trusted from the caller, and it is
// bound to the digest this wave installs.
assert.match(
job,
/test "\$\{GATE_OVERRIDE_CONFIRMATION\}" = \\\n {14}"SKIP_RELAY_MONITOR_GATE \$\{TARGET_IMAGE_DIGEST\}"/
)
assert.match(job, /\[\[ "\$\{GATE_OVERRIDE_REASON\}" =~ \^\[\[:print:\]\]\{12,500\}\$ \]\]/)
// Exactly the aggregate-evidence steps are skipped, and only them: every step
// that reads or spends the sealed monitor artifact carries the override guard.
const overrideSkipped = [
'Require fresh aggregate monitor evidence reference',
'Download private aggregate monitor evidence',
'Verify monitor evidence provenance',
"Download this wave's single-use safety authority",
'Require safety evidence consumed by this workflow'
]
for (const name of overrideSkipped) {
assert.match(
job,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.equal(
job.match(/inputs\.gate-override-confirmation == ''/g).length,
overrideSkipped.length
)
// The wrapper validates the override before anything runs, passes it to every
// cell, seals it into the canary artifact, and prints it in the run summary.
assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
assert.equal(
wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
4
)
assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
for (const name of [
'Reject previously consumed aggregate safety evidence',
'Consume aggregate safety evidence for this exact wave'
]) {
assert.match(
wrapper,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.match(job, /timeout-minutes: 75/)
// Both age gates step by the cell job timeout above; the constant is
// duplicated across the two languages, so pin each copy to it.
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+496 -21
View File
@@ -3,7 +3,12 @@ import { spawnSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { describe, it } from 'node:test'
import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs'
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
entryAdmission,
selectorWaveDelta
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs'
@@ -15,6 +20,7 @@ const production = readFileSync(
)
const REHOME_SOURCE_CELLS = rehomeSourceCells()
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com'
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
@@ -31,10 +37,33 @@ function rehomeSourceCells() {
)
}
function startupScript({ cap, image, trusted }) {
// The job cross-checks its pinned pool against the committed map; model the same read.
function tfvarsDatabasePoolMax(cellId) {
return tfvarsCellBlock(cellId).match(/database_pool_max\s*=\s*(\d+)/)?.[1] ?? '10'
}
function tfvarsHardCap(cellId) {
const cap = /connection_hard_cap\s*=\s*(\d+)/.exec(tfvarsCellBlock(cellId))?.[1]
assert.notEqual(cap, undefined, `${cellId} has no connection_hard_cap`)
return cap
}
function tfvarsCellBlock(cellId) {
const start = production.indexOf(`"${cellId}" = {`)
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
return production.slice(start, production.indexOf('\n }', start))
}
function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) {
return [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacityIdentity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]),
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
...(trusted ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${DIRECTOR_IDENTITY}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${AUDIENCE}'`
@@ -48,7 +77,11 @@ function startupScript({ cap, image, trusted }) {
}
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
function rollPlan({ cellId, cap, protocol }) {
function rollPlan({
cellId, cap, protocol, pool,
beforeCapacityIdentity = CAPACITY_IDENTITY,
afterCapacityIdentity = CAPACITY_IDENTITY
}) {
return {
configuration: {
root_module: {
@@ -77,14 +110,19 @@ function rollPlan({ cellId, cap, protocol }) {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol >= 1
trusted: protocol >= 1,
// The live template predates the reviewed pool raise, as every asia cell's does.
pool: pool === undefined ? undefined : '10',
capacityIdentity: beforeCapacityIdentity
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol >= 1
trusted: protocol >= 1,
pool,
capacityIdentity: afterCapacityIdentity
}),
self_link: null
},
@@ -108,7 +146,8 @@ function hostname(cellId) {
return cellId.slice('production-gce-'.length)
}
// The job resolves cap and region from the cell id before any admin call; run that block alone.
// The job resolves cap, region, and pool from the cell id before any admin call; run that block
// alone. An empty pool is the root default, which the startup template emits no line for.
function resolveCellShape(cellId) {
const start = workflow.indexOf(' TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"')
assert.notEqual(start, -1, 'the same-cap cell shape block is missing')
@@ -119,10 +158,106 @@ function resolveCellShape(cellId) {
'-euo',
'pipefail',
'-c',
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP}"`
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP} pool=\${EXPECTED_DATABASE_POOL_MAX}"`
], { env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
}
function cellShape(cellId) {
const resolved = resolveCellShape(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
const [, cap, pool] = resolved.stdout.trim().split(' ')
return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined }
}
// The class block runs before checkout-independent work and decides the whole wave shape.
function resolveCellClass(cellId) {
return spawnSync('bash', [
'-euo',
'pipefail',
'-c',
`${jobBlock(
' CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \\',
' SELECTOR_WAVE_DELTA="$(jq -er \'.selectorWaveDelta\' <<< "${CELL_CLASS}")"'
)}\necho "\${ENTRY_ADMISSION} \${SELECTOR_WAVE_DELTA}"`
], { cwd: new URL('../..', import.meta.url), env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
}
function drainingBlock() {
return `${jobBlock(
' # Rollback is the documented recovery from a failed canary, which',
' PREDECESSOR_DRAINING_OK=false\n fi'
)}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"`
}
// The three fields gcloud would otherwise default, as the MIG resource declares them.
function migUpdatePolicy() {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const policy = terraform.split(' update_policy {')[1]?.split('\n }')[0] ?? ''
const method = /replacement_method\s+= "([A-Z]+)"/.exec(policy)?.[1]
assert.notEqual(method, undefined, 'the MIG declares no replacement method')
// Both fixed bounds come from the topology locals the MIG resource points at.
const surgeLocal = /max_surge_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
const unavailableLocal =
/max_unavailable_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
assert.notEqual(surgeLocal, undefined, 'the MIG pins no surge local')
assert.notEqual(unavailableLocal, undefined, 'the MIG pins no unavailable local')
const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? ''
const local = (name) => {
const value = new RegExp(`${name}\\s+= (\\d+)`).exec(topology)?.[1]
assert.notEqual(value, undefined, `the topology locals pin no ${name}`)
return value
}
return {
replacementMethod: method.toLowerCase(),
maxSurge: local(surgeLocal),
maxUnavailable: local(unavailableLocal)
}
}
// The stage decides the predecessor, the plan's reviewed rollback image, and whether the
// MIG is rolled explicitly, so run the real block rather than restating its rule.
function stageBlock() {
return `${jobBlock(
' # Two different failures leave the cell on the rollback image, and the image',
' PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"\n fi'
)}\necho "\${ROLLBACK_STAGE} \${ROLLBACK_RESUME} \${PREDECESSOR_IMAGE_DIGEST}` +
` \${PREDECESSOR_REHOME_PROTOCOL} \${PLAN_ROLLBACK_IMAGE}"`
}
function generationBlock() {
return `${jobBlock(
' if test "${DEPLOY_MODE}" = verify; then',
' fi'
)}\necho "\${EFFECTIVE_SELECTOR_GENERATION}"`
}
// The job derives both memberships in one block; run that block alone for each class.
function membership(env) {
const script = `${jobBlock(
' RESTORED_MIGRATION_CELLS="$(jq -rn \\',
' fi'
)}\njq -cn --arg a "\${ISOLATED_MIGRATION_CELLS}" --arg b "\${ISOLATED_GENERAL_CELLS}" \\
--arg c "\${RESTORED_MIGRATION_CELLS}" --arg d "\${RESTORED_GENERAL_CELLS}" \\
'{isolatedMigration:$a,isolatedGeneral:$b,restoredMigration:$c,restoredGeneral:$d}'`
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
env: { ...process.env, ...env },
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
return JSON.parse(resolved.stdout)
}
function jobBlock(firstLine, lastLine) {
const start = workflow.indexOf(`${firstLine}\n`)
assert.notEqual(start, -1, `the job has no ${firstLine.trim()}`)
const end = workflow.indexOf(`\n${lastLine}\n`, start)
assert.notEqual(end, -1, `that block has no ${lastLine.trim()}`)
return workflow.slice(start, end + lastLine.length + 1).replace(/^ {10}/gm, '')
}
describe('same-cap roll scripts accept every same-cap cell', () => {
it('parses every wave cell through the same-cap canary allowlist', () => {
for (const cellId of SAME_CAP_CELLS) {
@@ -137,19 +272,26 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
cellId,
mode
mode,
paceWindowMs: 0
})
}
}
})
it('resolves a cap and region for every wave cell and refuses anything else', () => {
it('resolves a cap, region, and pool for every wave cell and refuses anything else', () => {
for (const cellId of SAME_CAP_CELLS) {
const resolved = resolveCellShape(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.match(resolved.stdout.trim(), /^(us-central1 1000|asia-east2 3000)$/)
assert.match(
resolved.stdout.trim(),
/^(us-central1 1000 pool=|us-central1 600 pool=|asia-east2 3000 pool=16)$/,
cellId
)
assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId)
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
}
assert.equal(resolveCellShape('production-gce-c17').status, 1)
assert.equal(resolveCellShape('production-gce-c12').status, 1)
assert.equal(resolveCellShape('production-gce-c30').status, 1)
})
@@ -161,11 +303,20 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const end = lines.findIndex((line) => !line.endsWith('\\'))
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--approved-cells same-cap/)
assert.match(call, /--mode (isolate|drain|activate)/)
// The restore call picks its mode from the cell's entry admission class.
assert.match(call, /--mode (isolate|drain|activate|"\$\{RESTORE_MODE\}")/)
}
})
it('passes this cell\'s rehome protocol on every plan validation the job runs', () => {
it('paces the drain it sends to the selected cell', () => {
const drain = workflow.split('--mode drain')[1] ?? ''
assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
// The transition wait has to outlast the pacing window on top of the leases it waits on.
assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
})
it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
for (const invocation of invocations) {
@@ -174,25 +325,41 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--mode same-cap-cell/)
assert.match(call, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
assert.match(call, /"\$\{POOL_ARGUMENTS\[@\]\}"/)
}
// Each of those steps must build the flag from the resolved pool, and only when there is one.
const builders = workflow.split(
'if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then\n' +
' POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")'
)
assert.equal(builders.length, 3)
assert.equal(workflow.split('POOL_ARGUMENTS=()').length, 3)
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
// the rest before it plans, and the next test covers them at protocol 0.
assert.deepEqual(
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
SAME_CAP_MIGRATION_ONLY_CELLS
)
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const { cap, pool } = cellShape(cellId)
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: Number(cap),
hardCap: cap,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: String(protocol)
regionalRehomeProtocol: String(protocol),
databasePoolMax: pool
}
const plan = rollPlan({ cellId, cap, protocol })
const plan = rollPlan({ cellId, cap, protocol, pool })
assert.deepEqual(
validateCapacityPlan(plan, config),
{ mode: 'same-cap-cell', changes: 2 },
@@ -207,6 +374,15 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
/reviewed image and capacity/,
cellId
)
// Dropping the pin must reject a pinned cell, and adding one must reject a default cell.
assert.throws(
() => validateCapacityPlan(plan, {
...config,
databasePoolMax: pool === undefined ? '16' : undefined
}),
/reviewed image and capacity/,
cellId
)
}
})
@@ -216,15 +392,16 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
const plan = rollPlan({ cellId, cap: 600, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
@@ -233,6 +410,304 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
)
})
it('resolves the class and selector delta the wave validator declares', () => {
for (const cellId of SAME_CAP_CELLS) {
const resolved = resolveCellClass(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.equal(
resolved.stdout.trim(),
`${entryAdmission(cellId)} ${selectorWaveDelta(cellId)}`,
cellId
)
}
assert.equal(resolveCellClass('production-gce-c12').status, 1)
})
it('offsets a later wave by this cell class\'s own selector delta', () => {
for (const [waveIndex, delta] of [['0', 2], ['3', 2], ['0', 0], ['3', 0]]) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', generationBlock()], {
env: {
...process.env,
DEPLOY_MODE: 'apply',
EXPECTED_SELECTOR_GENERATION: '40',
WAVE_INDEX: waveIndex,
SELECTOR_WAVE_DELTA: String(delta)
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
assert.equal(resolved.stdout.trim(), String(40 + delta * Number(waveIndex)))
}
})
it('hands a migration-only cell back the exact membership it entered with', () => {
const entry = {
EXPECTED_MIGRATION_ONLY_CELLS: 'production-gce-c17,production-gce-c18',
EXPECTED_GENERAL_CELLS: 'production-gce-c7,production-gce-c8'
}
const isolated = membership({
...entry,
TARGET_CELL_ID: 'production-gce-c17',
ENTRY_ADMISSION: 'migration-only'
})
assert.deepEqual(isolated, {
isolatedMigration: 'production-gce-c17,production-gce-c18',
isolatedGeneral: 'production-gce-c7,production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
})
// A general cell still leaves migration-only and returns to general.
assert.deepEqual(
membership({
...entry,
TARGET_CELL_ID: 'production-gce-c7',
ENTRY_ADMISSION: 'general'
}),
{
isolatedMigration: 'production-gce-c17,production-gce-c18,production-gce-c7',
isolatedGeneral: 'production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
}
)
})
it('never activates a migration-only cell and proves its isolate changed nothing', () => {
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /if test "\$\{ENTRY_ADMISSION\}" = migration-only; then\n\s+RESTORE_MODE=isolate/)
assert.match(restore, /--admission "\$\{ENTRY_ADMISSION\}"/)
// The pre-mutation check must demand the class the cell is declared to serve in.
assert.match(workflow, /PRECHECK_ADMISSION="\$\{ENTRY_ADMISSION\}"/)
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /migration-only; then\n\s+jq -e '\.changed == false'/)
})
it('requires rehome source membership exactly when a roll carries trust lines', () => {
const step = workflow
.split('name: Resolve immutable same-cap cell configuration')[1]
.split('\n - name:')[0]
const guard = step.indexOf('jq -e --arg cell "${TARGET_CELL_ID}" \'index($cell) != null\'')
assert.notEqual(guard, -1)
// The guard reads both protocols, so it has to sit after they are resolved.
assert.ok(step.indexOf('DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"') < guard)
assert.match(
step.slice(0, guard),
/test "\$\{DESIRED_REHOME_PROTOCOL\}" != 0 \|\| test "\$\{CURRENT_REHOME_PROTOCOL\}" != 0\n\s+\}; then\s+$/
)
})
it('rolls a template stale enough to predate the pinned capacity identity', () => {
// Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no
// capacity identity line, so the roll adds one. Run 35290908836 failed closed here.
const cellId = 'production-gce-c17'
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null })
assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 })
// The line may only be gained. A roll may not rewrite it,
assert.throws(
() => validateCapacityPlan(stale, {
...config,
capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com'
}),
/reviewed image and capacity/
)
// nor drop it from a template that already carries one.
assert.throws(
() => validateCapacityPlan(
rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }),
config
),
/reviewed image and capacity/
)
// A same-cap roll cannot run without the identity pinned at all.
assert.throws(
() => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }),
/invalid service account/
)
})
it('pins the capacity identity on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
for (const invocation of invocations) {
const lines = invocation.split('\n')
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
assert.match(
lines.slice(0, end + 1).join(' '),
/--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/
)
}
// Both steps must read it from the same repository variable the job already requires.
assert.equal(
workflow.split(
'CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}'
).length,
4
)
})
it('decides the predecessor draining rule from the real block, for both classes', () => {
// A zero-host cell sheds nothing, and a failed canary's own drain leaves the flag set
// with no restart behind it; run 35292335415 stopped on exactly that residue.
const cases = [
// mode, entry class, resume, expected [precheck admission, precheck draining, jq ok]
['apply', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['apply', 'general', 'false', ['general', 'forbidden', 'false']],
['verify', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['verify', 'general', 'false', ['general', 'forbidden', 'false']],
// Every rollback path keeps exactly the behaviour it had.
['rollback', 'general', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'general', 'true', ['general-or-migration-only', 'either', 'false']],
['rollback', 'migration-only', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'migration-only', 'true', ['general-or-migration-only', 'either', 'false']]
]
for (const [mode, entry, resume, expected] of cases) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', drainingBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
ENTRY_ADMISSION: entry,
ROLLBACK_RESUME: resume
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${entry}/${resume}: ${resolved.stderr}`)
assert.deepEqual(
resolved.stdout.trim().split(' '),
expected,
`${mode}/${entry}/${resume}`
)
}
})
it('reads one draining decision in both predecessor checks', () => {
const step = workflow
.split('name: Verify exact current generation, digest, cap, and rollback point')[1]
.split('\n - name:')[0]
// The jq assertion and its diagnostic must not be able to disagree.
assert.equal(step.split('--argjson drainingOk "${PREDECESSOR_DRAINING_OK}"').length, 3)
assert.doesNotMatch(step, /drainingOk "\$\(test/)
// The fresh VM is still required not to be draining, on every path.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /--admission migration-only --draining forbidden/)
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /--draining forbidden --activity allowed/)
})
it('classifies every rollback stage from the real block', () => {
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
const target = `sha256:${'7'.repeat(64)}`
const rollback = `sha256:${'0'.repeat(64)}`
const stage = (mode, live, draining) => {
// Exactly how the job assigns them: rollback swaps desired and current.
const desired = mode === 'rollback' ? rollback : target
const current = mode === 'rollback' ? target : rollback
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', stageBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
CURRENT_RUNTIME: JSON.stringify({ imageDigest: live, draining }),
DESIRED_IMAGE_DIGEST: desired,
CURRENT_IMAGE_DIGEST: current,
DESIRED_IMAGE: `${repository}@${desired}`,
IMAGE_REPOSITORY: repository,
DESIRED_REHOME_PROTOCOL: '1',
CURRENT_REHOME_PROTOCOL: '0'
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${live}/${draining}: ${resolved.stderr}`)
return resolved.stdout.trim().split(' ')
}
const roll = (current, protocol) =>
['roll', 'false', current, protocol, `${repository}@${current}`]
// Only the last row differs from main: it used to read `resume` and wedge, because the
// resume path refuses a draining cell and never restarts one.
assert.deepEqual(stage('apply', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('apply', rollback, true), roll(rollback, '0'))
assert.deepEqual(stage('apply', target, false), roll(rollback, '0'))
assert.deepEqual(stage('verify', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('rollback', target, false), roll(target, '0'))
assert.deepEqual(stage('rollback', target, true), roll(target, '0'))
assert.deepEqual(
stage('rollback', rollback, false),
['resume', 'true', rollback, '1', `${repository}@${target}`]
)
assert.deepEqual(
stage('rollback', rollback, true),
['stranded', 'false', rollback, '1', `${repository}@${rollback}`]
)
// A runtime that reports no drain flag at all must never read as stranded.
const [missing] = stage('rollback', rollback, null)
assert.equal(missing, 'resume')
})
it('rolls the MIG itself when a stranded plan changes nothing', () => {
const apply = workflow
.split('name: Apply only the selected same-cap template and MIG')[1]
.split('\n - id:')[0]
// The plan is reviewed against the image the cell serves, not an assumed predecessor.
assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/)
assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/)
assert.match(
apply,
/test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0/
)
// gcloud persists all three fields into the MIG's update policy and defaults the
// method to substitute here, so every one has to match what Terraform declares or the
// recovery drifts the policy and the next targeted plan is refused as an unreviewed
// MIG change. Read the declared values rather than restating them.
assert.match(apply, /rolling-action replace "\$\{MIG_NAME\}"/)
const declared = migUpdatePolicy()
assert.deepEqual(declared, {
replacementMethod: 'recreate',
maxSurge: '0',
maxUnavailable: '1'
})
assert.match(
apply,
new RegExp(
`--replacement-method ${declared.replacementMethod}` +
` --max-surge ${declared.maxSurge} --max-unavailable ${declared.maxUnavailable}`
)
)
// Nothing else may reach the group, and the roll has to be waited on.
assert.equal(apply.split('rolling-action').length, 2)
assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3)
})
it('waits on the image a stranded cell actually serves', () => {
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /--expected-image-digests "\$\{PREDECESSOR_IMAGE_DIGEST\}"/)
// A stranded cell has to come back on a new process, which is what clears the drain.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /test "\$\{TARGET_INCARNATION\}" != "\$\{SOURCE_INCARNATION\}"/)
assert.match(after, /if test "\$\{ROLLBACK_RESUME\}" = true; then/)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+56 -10
View File
@@ -7,6 +7,8 @@ const SERVICE_ACCOUNT_EMAIL =
const REHOME_CONFIG =
/^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/
const DATABASE_POOL_MAX = /^ printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '[0-9]+'$/
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
@@ -15,6 +17,16 @@ function rehomeProtocol({ regionalRehomeProtocol }) {
return Number(regionalRehomeProtocol)
}
// Only cells off the root pool default get a pool line, so the caller states whether to expect one.
function databasePoolMax({ databasePoolMax: value }) {
if (value === undefined) return undefined
const pool = Number(value)
if (!/^[0-9]+$/.test(String(value)) || pool < 1 || pool > 100) {
throw new Error('same-cap Terraform plan has an invalid database pool max')
}
return String(pool)
}
export function parseCapacityPlanArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
@@ -35,7 +47,10 @@ export function parseCapacityPlanArguments(argv) {
return value
}
if (!values.image) throw new Error('missing --image')
if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) {
if (
['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
!values['capacity-service-account']
) {
throw new Error('missing --capacity-service-account')
}
if (
@@ -48,6 +63,9 @@ export function parseCapacityPlanArguments(argv) {
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
}
if (values.mode !== 'same-cap-cell' && values['database-pool-max'] !== undefined) {
throw new Error('--database-pool-max applies only to same-cap-cell validation')
}
if (values.mode === 'same-cap-image' && !values['rollback-image']) {
throw new Error('same-cap image validation requires a rollback image')
}
@@ -67,7 +85,8 @@ export function parseCapacityPlanArguments(argv) {
rollbackImage: values['rollback-image'],
rehomeDirectorServiceAccount: values['rehome-director-service-account'],
rehomeAudience: values['rehome-audience'],
regionalRehomeProtocol: values['regional-rehome-protocol']
regionalRehomeProtocol: values['regional-rehome-protocol'],
databasePoolMax: values['database-pool-max']
}
}
@@ -182,7 +201,8 @@ function normalizedStartupScript(
script,
stripCapacityIdentity = false,
stripRehomeConfig = false,
preserveCapacity = false
preserveCapacity = false,
stripDatabasePoolMax = false
) {
const image = relayImage(script)
if (!image) throw new Error('cell plan startup script has no Relay image')
@@ -197,7 +217,8 @@ function normalizedStartupScript(
(line) =>
(preserveCapacity || !capacityAssignment.test(line)) &&
(!stripCapacityIdentity || !capacityIdentity.test(line)) &&
(!stripRehomeConfig || !REHOME_CONFIG.test(line))
(!stripRehomeConfig || !REHOME_CONFIG.test(line)) &&
(!stripDatabasePoolMax || !DATABASE_POOL_MAX.test(line))
)
.join('\n')
.replaceAll(image, '<relay-image>')
@@ -221,7 +242,10 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
]
]
if (config.mode === 'bootstrap-cell') {
// A same-cap cell whose template predates this line gains it on its next roll, so the
// before/after comparison ignores it; pinning the exact identity here is what reviews it,
// and what stops a roll dropping or rewriting the line it lets through.
if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
expected.push([
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
@@ -245,10 +269,23 @@ function requireDesiredStartupScript(script, config) {
config.mode === 'same-cap-cell' &&
!rehomeTrusted &&
lines.some((line) => REHOME_CONFIG.test(line))
const pool = config.mode === 'same-cap-cell' ? databasePoolMax(config) : undefined
if (pool !== undefined) {
expected.push([
DATABASE_POOL_MAX,
` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`
])
}
// An unpinned cell sits on the root pool default, so gaining a pool line is real drift.
const unexpectedDatabasePoolMax =
config.mode === 'same-cap-cell' &&
pool === undefined &&
lines.some((line) => DATABASE_POOL_MAX.test(line))
if (
typeof script !== 'string' ||
relayImage(script) !== config.image ||
unexpectedRehome ||
unexpectedDatabasePoolMax ||
expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line))
) {
throw new Error('cell plan does not contain the reviewed image and capacity')
@@ -421,19 +458,27 @@ function cellPlan(plan, changes, config) {
const script = template.change.after?.metadata_startup_script
requireDesiredStartupScript(script, config)
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
// Only a template stale enough to predate the line may move it, and only by gaining it;
// requireDesiredStartupScript holds the after value to the exact reviewed identity.
const stripCapacityIdentity =
['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
if (
typeof beforeScript !== 'string' ||
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
normalizedStartupScript(
beforeScript,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap
sameCap,
stripPool
) !== normalizedStartupScript(
script,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap
sameCap,
stripPool
)
) {
throw new Error('cell plan does not contain the reviewed image and capacity')
@@ -466,13 +511,14 @@ export function validateCapacityPlan(plan, config) {
throw new Error('capacity Terraform plans may change only a cell')
}
if (
config.mode === 'bootstrap-cell' &&
['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
!SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
) {
throw new Error('capacity Terraform plan has an invalid service account')
}
if (config.mode === 'same-cap-cell') {
rehomeProtocol(config)
databasePoolMax(config)
}
if (
config.mode === 'same-cap-cell' &&
cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
+272 -1
View File
@@ -428,10 +428,14 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false }) => [
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
@@ -468,6 +472,7 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
@@ -653,10 +658,12 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
@@ -693,6 +700,7 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
@@ -737,6 +745,133 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
@@ -747,6 +882,7 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
@@ -783,3 +919,138 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
/applies only to same-cap-cell validation/
)
})
test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`,
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const rollPlan = (before, after) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({ selectedImage: rollbackImage, pool: before })
},
after: {
metadata_startup_script: startup({ selectedImage: image, pool: after }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
// The live template still says 10 while the reviewed plan says 16; only the pin bridges that.
assert.deepEqual(
validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: '16' }),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), asiaConfig),
/reviewed image and capacity/
)
assert.throws(
() => validateCapacityPlan(rollPlan('10', '12'), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
// A cell on the root pool default emits no line at all, and gaining one is real drift.
assert.deepEqual(
validateCapacityPlan(rollPlan(undefined, undefined), asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.throws(
() => validateCapacityPlan(rollPlan(undefined, '16'), asiaConfig),
/reviewed image and capacity/
)
// A line already on the live template is still unreviewed without the pin, even standing still.
assert.throws(
() => validateCapacityPlan(rollPlan('16', '16'), asiaConfig),
/reviewed image and capacity/
)
// A pin must also fail closed when the plan drops the line it names.
assert.throws(
() => validateCapacityPlan(rollPlan('16', undefined), { ...asiaConfig, databasePoolMax: '16' }),
/reviewed image and capacity/
)
for (const pool of ['', '0', '101', 'ten']) {
assert.throws(
() => validateCapacityPlan(rollPlan('10', '16'), { ...asiaConfig, databasePoolMax: pool }),
/invalid database pool max/
)
}
})
test('the database pool argument is accepted by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--database-pool-max', '16')).databasePoolMax,
'16'
)
assert.equal(parseCapacityPlanArguments(sameCapArguments()).databasePoolMax, undefined)
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--database-pool-max', '16'
]),
/applies only to same-cap-cell validation/
)
})
cloud/docs/relay-database-failure-diagnostics.md
+20 -6
View File
@@ -8,12 +8,26 @@ covered. These events are diagnostic evidence, not a replacement for total SQL
failure counters.
The event contains only an allowlisted error code, a connection-timeout boolean,
the operation category (`control-renewal` or `other`), total elapsed milliseconds,
and pool total/idle/waiting counts at failure. Total elapsed time includes acquisition.
An acquisition timeout can mean either waiting in the queue or establishing a new
connection; use the pool counts and independent server activity to distinguish them.
Unknown error codes stay `unknown`. Query text, parameters, error messages, and
identifiers are never emitted. Successful queries emit no additional event.
a transient boolean, the operation category (`control-renewal` or `other`), total
elapsed milliseconds, and pool total/idle/waiting counts at failure. Total elapsed
time includes acquisition. An acquisition timeout can mean either waiting in the
queue or establishing a new connection; `connectionTimeout` covers both, and the
pool counts separate them: a queue wait has waiters, a dial does not.
Unknown error codes stay `unknown`.
`transient` is the classification the request routes act on, not a second
opinion: true means retryable, false means terminal. It is not a count of HTTP
responses. Every caller of `PostgresDatabase.query` emits this event, including
background sweeps, startup reconciliation, and admin routes that map a failure
to 409, and none of those produces a 503 or a 500. Counting `transient=false`
therefore over-counts user-facing hard failures; narrow by operation, or join
against the route's own rejection logs, before reading it that way. A pool that
cannot hand out a client carries no error code at all, so `code` stays `unknown`
for that whole class and only these two booleans separate it from a genuine
fault such as a rejected password.
Query text, parameters, error messages, and identifiers are never emitted.
Successful queries emit no additional event.
Use structured GCE logs with `jsonPayload.event="orca_relay_postgres_query_failed"`.
Compare counts by phase, operation, and code with the same cell's renewal outcomes
cloud/docs/relay-incident-monitor.md
+131 -12
View File
@@ -39,13 +39,69 @@ days. No tokens, request bodies, logs, user IDs, host IDs, or relay device IDs a
Reruns keep one stable incident ID, restore the immediately preceding private
artifact, verify its commit/run/attempt provenance and content hashes, and pass
`--restart`. A missing or mismatched artifact fails closed. A missing, stale,
or collector-failed sample is durably recorded and resets the active continuous
window. The next fresh sample starts a new 15- or 90-minute window under the
same incident lineage.
or collector-failed sample is durably recorded. Up to two consecutive such
samples per source are tolerated and the window keeps running; a third resets
the active continuous window, and the next fresh sample starts a new 15- or
90-minute window under the same incident lineage. A pre-drain dry run must reach
a verdict within 35 minutes of its lineage start.
Exit code `2` means the gate froze or a dry run failed. Missing, stale, malformed, unauthorized, or
unavailable telemetry fails closed.
## Gate override (break-glass)
`Deploy Relay Production Same-Cap` can skip this 15-minute dry-run gate. Supply both
`gate-override-reason` and `gate-override-confirmation`, where the confirmation is exactly
`SKIP_RELAY_MONITOR_GATE <target-image-digest>`. Supplying one without the other, a
confirmation bound to any other digest, or a reason shorter than 12 characters fails the
run before it touches production. `verify` mode rejects the override outright.
### When it is legitimate
The gate proves the fleet is healthy before a wave mutates it. That proof is the wrong
question in exactly two situations.
- **The roll is the fix for the measured condition.** When a chronic fault is the reason
the gate freezes, waiting for a green 15-minute window means waiting for the condition
the wave removes. On 2026-09-17 the gate froze 44 consecutive times on the recurring
Cloud SQL stall the rolling image addresses.
- **An incident where the director is healthy.** Rolling back off a bad image should not
wait 15 minutes for aggregate evidence about a fleet the operator is already watching.
It is not a way to move faster on an ordinary wave. Use it when you can name the signal
the gate is freezing on and say why this wave is the answer to it.
### What it does not skip
Only the aggregate 15-minute dry-run and its sealed evidence are skipped. Every other
control still runs, unchanged:
- The live per-wave preflight, against the same thresholds this document lists. With no
sealed state to read, the expected selector comes from the dispatch inputs instead, and
the migration policy is pinned to `strict`. That membership is canonicalised exactly as
the monitor canonicalises its own, so it must still name every configured cell exactly
once and the order you type it in does not matter. A live threshold breach or selector
mismatch still fails the wave before any mutation.
- Durable regional rehome disabled, and the exact selector generation and membership,
verified against the live director.
- The reviewed Terraform plan, the exact image digest served by Artifact Registry, the
predecessor runtime check, and the new-incarnation check.
- One cell at a time behind the Cloud SQL rollout lease, with the failed-wave failsafe
that leaves a cell isolated.
- Single-dispatch mutation: a re-run still cannot replay a wave.
### The audit trail
Three places record it, and none of them depend on the operator writing anything down:
- The workflow run's inputs, kept by GitHub for the life of the run.
- The gate job's run summary: actor, mode, cells, target digest, reason, and confirmation.
- The sealed canary artifact, under `gateOverride`, for a `canary-apply` wave.
The canary authority a later batch verifies never carried a monitor run ID, so a batch can
reuse a canary that was rolled under an override. The override is recorded in that
artifact as audit trail, not as authority: each wave is authorized by its own confirmation.
## Local use
The active `gcloud` identity must be a service account that can mint an ID token for the exact
@@ -83,8 +139,9 @@ freezes as before.
A production candidate or multi-target mutation must download the exact
dry-run artifact by workflow run ID and attempt. It verifies the artifact
hashes and provenance, requires a green completed 15-minute state no older
than five minutes, then rechecks the live selector and one complete fresh
sample of every safety signal immediately before running the mutation command.
than ten minutes (plus 75 minutes per predecessor same-cap wave), then
rechecks the live selector and one complete fresh sample of every safety
signal immediately before running the mutation command.
The signed state binds `strict` evidence to ordinary mutations and
`recover-forward` evidence to the exact recover-forward source; neither can
authorize the other.
@@ -102,17 +159,17 @@ durably marked consumed before mutation and cannot authorize another run.
| Endpoint latency | over 2,000 ms |
| Cloud SQL CPU | over 80% |
| Cloud SQL memory | over 90% |
| Cloud SQL backends | over 250 (62% of the verified 400-connection ceiling) |
| Cloud SQL backends | over 320 (65% of the 490 usable of `max_connections` 500) |
| Cloud SQL waiting backends | over 20 |
| Cloud SQL deadlocks | over 0 |
| Relay pool waiters | over 800 |
| Relay pool wait | over 2,500 ms |
| PostgreSQL retries in five minutes | over 2,000 |
| Exhausted PostgreSQL retries in five minutes | over 300 |
| Director instances | outside 5–6 |
| Director instances | outside 5–6 for more than two consecutive samples |
| Director CPU or memory | over 80% |
| Director concurrency | over 64 |
| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
| Unexpected director 5xx in five minutes (excludes 503) | over 15 |
| Auth 5xx in five minutes | over 0 |
| Connections per cell process | over 500 |
| Queued bytes per cell process | over 48 MiB |
@@ -122,6 +179,23 @@ durably marked consumed before mutation and cannot authorize another run.
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
heartbeats, and matching live admission.
Two readings are the exception to the freeze-on-first-breach rule above.
A cell's endpoint readings are the first. Health, ready and latency are a single HTTP
round trip from one runner, so they must fail more than two consecutive samples before
they freeze the run; the streak is keyed by cell, so one cell's three readings share it.
The director instance count is the second. Cloud Run replaces an instance in place
rather than holding the count, so the reading leaves the band for about one sample
roughly twice a day, and a deploy that briefly serves two revisions raises it the same
way. Neither is an unhealthy fleet. The minimum and maximum share one streak, so a count
that alternates above and below the band still freezes the run.
Absorbed breaches are recorded in the state artifact under `toleratedProbeEvents`. Every
other signal, including the director and auth health probes, freezes on the first bad
sample. The live preflight that runs before each mutating wave re-samples on the same
tolerance.
## Region placement alert policies
Cloud Monitoring alert policies, not monitor freeze bars: these page from
@@ -197,6 +271,49 @@ without its segment is a compile error in relay-contract, not a silent gap.
## Implementation log
- Gave `collector_failed` the same two-consecutive-sample tolerance as an unread
signal and raised the pre-drain lineage cap from 25 to 35 minutes
(2026-09-17). Basis: dry-run 35258662628 sampled a healthy fleet clean for
13 minutes, then a single unreadable Cloud Monitoring sample restarted the
window, and the restarted window ran past the 25-minute cap at 1 500 002 ms,
so a healthy fleet produced no verdict. One failed collector round trip is
evidence about that round trip, not about the fleet, and it cannot freeze the
gate on its own because it carries no threshold breach. `monitor_gap` keeps
zero tolerance: it means the run stopped sampling, so the window has a real
hole. 35 minutes fits a 15-minute window plus one restart: a reset on the
window's last sample restarts at minute 16 and finishes at 31. The job
timeout is already 100 minutes.
- Recalibrated the Cloud SQL backends freeze from 250 to 320, the unexpected
director 5xx freeze from 3 to 15, and gave per-cell endpoint probes a
two-consecutive-sample tolerance (2026-09-17). Basis: the pre-roll dry-run had
frozen 39 times out of 39, every time on a chronic production condition
unrelated to the roll it gates, so it was adding delay rather than safety.
Measured over the 24 h to 2026-09-17 through the Cloud Monitoring API with the
monitor's own aggregation. `cloud_sql.backends` latest-sum per minute: p50 118 /
p90 165 / p95 212 / p99 262 / max 282, so the old bar of 250 sat under the
observed peak and tripped 1.95% of minutes and 21.8% of 15-minute gates; 320
clears every healthy minute with 13% headroom and still fires at 65% of the 490
usable connections, leaving 170 in hand for the runaway that exhaustion actually
is. `director.errors` non-503 5xx per rolling five minutes: p50 0 / p90 3 /
p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze 9.2% of
windows and 29.0% of gates on the chronic `/v1/assign`, `/v1/regions` and
`/v1/resolve` 500 bursts that accompany the recurring Cloud SQL stall; 15 clears
the chronic p99, drops the gate-freeze rate to 1.5%, and deliberately leaves the
exceptional 20-52 bursts detectable. The director serves roughly 50 requests a
minute in 503s alone, so a genuinely broken director lands in the hundreds per
window. For the cell probes, the asia-east2 cells run readiness as `SELECT 1`
against Cloud SQL in us-central1 over a 176 ms round trip behind a 2 s timeout,
so a saturated pool makes the load balancer answer "no healthy upstream" for
about 30 s; 7 of the last 14 freezes were that. It arrives as a real HTTP 503, so
provenance cannot separate it from a cell serving `health=0` and persistence has
to: at the 60 s interval it spans one sample and at worst two. The streak is
keyed by cell rather than by signal, because keyed per signal a cell that
alternates between slow and unanswered holds every streak at one and never
reaches the tolerance. The live preflight before each mutating wave re-samples on
the same tolerance, so a blip cannot fail a wave there either. Thresholds stay
code constants sealed into every checkpoint rather than workflow inputs, so a
green run stays auditable. Re-tighten the backends bar when the auth connection
model lands (#21165).
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
800 waiters / 2,500 ms (2026-08-27). Basis, measured from
`orca_relay_runtime_metrics` (`databasePoolWaitersMax`,
@@ -212,9 +329,11 @@ without its segment is a compile error in relay-contract, not a silent gap.
Basis, measured from `cloudsql.googleapis.com/database/postgresql/num_backends`
latest-sum over 24 healthy hours: mean ~100, 1-minute spikes to 216, with
10 minutes over the old bar of 160 — enough to freeze roughly one in ten
15-minute pre-drain gates on baseline noise. 250 clears measured healthy
peaks and still fires well before the verified 400-connection ceiling;
pool waiters and pool wait latency keep their strict thresholds.
15-minute pre-drain gates on baseline noise. 250 cleared the healthy peaks
measured then and still fired well before the verified 400-connection ceiling;
pool waiters and pool wait latency keep their strict thresholds. Superseded by
the 2026-09-17 entry above, which re-measured a grown baseline against the
490-connection budget.
- Recalibrated the PostgreSQL-retry freeze from 20 to 300 per five minutes
(2026-08-26). Basis, measured from
`jsonPayload.event="orca_relay_postgres_transaction_retry"` in production
@@ -280,4 +399,4 @@ without its segment is a compile error in relay-contract, not a silent gap.
### Director error allowance (2026-09-12)
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance was set to three non-503 director 5xx here, and was superseded by the 2026-09-17 recalibration to 15 recorded above. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
cloud/docs/relay-workflows.md
+85 -1
View File
@@ -374,7 +374,8 @@ remain general at 1,000/60. The workflow lock, single-use evidence marker, exact
targeted Terraform plan, and per-cell heartbeat/admission oracle are unchanged.
`Deploy Relay Production Same-Cap` rolls only the reviewed US 1,000/60 and Asia 3,000/60 serving
sets without changing a cell's connection shape. Use `canary-apply` for exactly one cell. A successful canary
sets and the two migration-only US 600/60 cells, C17 and C18, without changing a cell's connection
shape. Use `canary-apply` for exactly one cell. A successful canary
seals its commit, target and rollback digests, selector generation, and durable rehome generation;
`batch-apply` accepts only that same authority and rolls two to four cells sequentially. Each cell is
isolated, drained to two restart-safe samples, replaced from a targeted saved plan, and restored only
@@ -384,6 +385,89 @@ director; the workflow never receives or mints a director or stamped-cell runtim
keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via
the same workflow's `rollback` mode.
A roll holds the whole startup script identical before and after except the image, so a
template stale enough to predate a pinned line fails closed rather than absorbing the drift.
The one exception is the capacity identity: a cell that predates it gains it on its next roll,
and the plan validator pins the exact reviewed identity instead of comparing that line, so a
roll can never drop or rewrite it. Any other stale line still fails closed and needs a
convergence apply first.
A drained cell is refused before a roll, because draining means something is already
shedding its connections. A migration-only cell has none to shed, so the flag decides nothing
there and is accepted on entry; the replacement VM is still required not to be draining, and
the incarnation check still proves it was replaced. That also unwedges the state a failed
canary leaves behind, where the wave's own drain set the flag and no restart followed.
C17 and C18 hold no hosts and are not general, so rolling one displaces nobody: they are the
zero-displacement canary for a new image. Their wave enters and leaves migration-only, so its
isolate and its restore are both no-ops and the selector generation does not move; a general
cell's wave still advances it by two. One wave may not mix the two classes, because every cell
after the first offsets from a single per-wave delta. Neither cell is a declared regional-rehome
source, so its template carries no rehome trust lines and it may roll only at rehome protocol `0`;
the job refuses a trusted protocol for it before it plans anything.
### Recovering a wave that died after its drain
A cell's drain flag is a one-way latch on the running process. Only a restart clears it, and
the failsafe that isolates a failed cell does not restart anything. So a wave that stopped
any time after its drain step leaves the cell migration-only and draining, and it stays that
way until the cell is rolled.
Read the failed run before dispatching anything. If its log has a
`"event":"relay_production_capacity_canary","mode":"drain"` line for the cell, the cell is
drained. Then read the cell's live runtime image from
`POST https://<hostname>.relay.onorca.dev/v1/admin/runtime-status`.
1. **Do not re-dispatch `apply`.** It requires the cell general and not draining, and a
drained cell is neither. It will fail closed at the predecessor check.
2. **Dispatch `rollback`,** with the same `target-image-digest` and `rollback-image-digest`
the failed wave used, the live selector generation, and the live tri-state membership
with the failed cell listed under migration-only. The confirmation is
`ROLL_BACK_RELAY_SAME_CAP <rollback-digest> <cell-id>`.
3. The job classifies the cell itself and needs no extra input:
- serving the **rollback** image and draining, it is `stranded`. The wave stopped before
or during its template apply. The job re-isolates, re-drains, applies the reviewed
template, and rolls the MIG explicitly if that template was already in place. The cell
comes back on a new instance, so the drain clears, and it is restored to its entry class.
- serving the **target** image, it is `roll`, the ordinary rollback. The template applied
and the instance was replaced.
- serving the **rollback** image and not draining, it is `resume`: a rollback that failed
after its own template apply. Nothing is applied and nothing restarts.
4. Rollback takes exactly one cell per dispatch. Recover the cells one at a time.
5. If the run died inside `wait-until stable`, the MIG is still rolling on its own. Wait for
it to settle and re-read the runtime before dispatching, or the stage will be read off a
state that is about to change.
6. A `stranded` dispatch that fails at plan review means the template already carries the
target image while the old instance is still up. Wait for the MIG to finish replacing it,
then dispatch again; it will classify as `roll`.
A mutating dispatch still needs a fresh aggregate monitor dry-run unless the break-glass
override below is used.
### Gate override (break-glass)
Every mutating same-cap wave normally consumes a fresh 15-minute aggregate monitor dry-run.
`gate-override-reason` plus `gate-override-confirmation`, the latter exactly
`SKIP_RELAY_MONITOR_GATE <target-image-digest>`, skips that aggregate evidence and nothing
else. A partial or mismatched override fails the run before any mutation, and `verify` mode
rejects it outright.
It is legitimate when the roll is the fix for the condition the gate is freezing on, or
during an incident with the director healthy. It is not a way to move faster on an ordinary
wave.
The live per-wave preflight still runs, against the same thresholds, with the expected
selector taken from the dispatch inputs and the migration policy pinned to `strict`. That
membership is canonicalised the same way the monitor canonicalises its own, so it must name
every configured cell exactly once and its order does not matter.
Durable rehome disabled, the exact selector generation and membership, the reviewed
Terraform plan, the predecessor and new-incarnation checks, the rollout lease, the
failed-wave failsafe, and single-dispatch mutation are all unchanged. The actor, reason,
and confirmation are recorded in the gate job's run summary and, for a canary, sealed into
the canary artifact under `gateOverride`; a batch may reuse a canary rolled under an
override, because that authority never carried a monitor run ID. See
[gate override (break-glass)](./relay-incident-monitor.md#gate-override-break-glass).
The first compatible director rollout uses `bootstrap-runtime-identity=true` with
`BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY`. That one-time path requires the exact stamped-cell
predecessor identity, creates both the cold rollback and candidate on the distinct director identity,
cloud/infra/terraform/environments/production.tfvars
+13 -4
View File
@@ -32,7 +32,16 @@ relay_gce_subnetwork_cidr = "10.42.0.0/24"
relay_gce_additional_region_subnetwork_cidrs = {
"asia-east2" = "10.42.1.0/24"
}
relay_gce_fenced_cells = ["production-gce-c1", "production-gce-c2", "production-gce-c3", "production-gce-c6", "production-gce-c11", "production-gce-c12"]
# Fenced cells are retired existing-only capacity: the selector can never place on them again,
# so their MIGs run at zero rather than holding a VM and 10 Postgres connections each.
relay_gce_fenced_cells = [
"production-gce-c1",
"production-gce-c2",
"production-gce-c3",
"production-gce-c6",
"production-gce-c11",
"production-gce-c12"
]
# Initial cells stay admission-disabled until production preflight and go-live approval.
relay_gce_cells = {
"production-gce-c1" = {
@@ -350,7 +359,7 @@ relay_gce_cells = {
boot_disk_gb = 30
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
capacity_requests = 6000
database_pool_max = 10
database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters).
image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563"
initially_enabled = false
connection_hard_cap = 3000
@@ -364,7 +373,7 @@ relay_gce_cells = {
boot_disk_gb = 30
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
capacity_requests = 6000
database_pool_max = 10
database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters).
image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563"
initially_enabled = false
connection_hard_cap = 3000
@@ -378,7 +387,7 @@ relay_gce_cells = {
boot_disk_gb = 30
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
capacity_requests = 6000
database_pool_max = 10
database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters).
image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563"
initially_enabled = false
connection_hard_cap = 3000
cloud/infra/terraform/relay-observability.tf
+9
View File
@@ -60,6 +60,10 @@ locals {
control_renewal_latency_ms_p50 = { field = "controlRenewalLatencyMsP50", description = "Control renewal latency p50 in the interval." }
control_renewal_latency_ms_p95 = { field = "controlRenewalLatencyMsP95", description = "Control renewal latency p95 in the interval." }
control_renewal_latency_ms_max = { field = "controlRenewalLatencyMsMax", description = "Maximum control renewal latency in the interval." }
control_renewal_flushes = { field = "controlRenewalFlushesDelta", description = "Batched control-renewal statements issued in the interval, one per cell per flush window." }
control_renewal_flush_rows_max = { field = "controlRenewalFlushRowsMax", description = "Largest number of hosts renewed by a single statement in the interval; the row ceiling is what bounds how long one flush holds its row locks." }
control_renewal_flush_ms_p95 = { field = "controlRenewalFlushLatencyMsP95", description = "Batched control-renewal statement duration p95 in the interval. Row locks live until the statement commits, so this is the lock hold." }
control_renewal_flush_ms_max = { field = "controlRenewalFlushLatencyMsMax", description = "Maximum batched control-renewal statement duration in the interval." }
control_renewals = { field = "controlRenewalsDelta", description = "Control renewal attempts in the interval." }
control_renewal_successes = { field = "controlRenewalSuccessesDelta", description = "Successful control renewals in the interval." }
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
@@ -93,6 +97,11 @@ locals {
db_waiters_max = { field = "databasePoolWaitersMax", description = "Maximum requests queued for a PostgreSQL connection during the interval." }
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
cell_inventory_hold_ms_max = { field = "cellInventoryHoldMsMax", description = "Longest cell-inventory lock hold in the interval." }
cell_inventory_hold_ms_p95 = { field = "cellInventoryHoldMsP95", description = "Cell-inventory lock hold p95 in the interval; the bound is tuned against this." }
cell_inventory_holds = { field = "cellInventoryHolds", description = "Cell-inventory locks acquired in the interval; the percentiles above summarise these." }
cell_inventory_lock_unavailable = { field = "cellInventoryLockUnavailable", description = "Fail-fast cell-inventory acquisitions that found the lock held. Includes background sweeps, which step aside by design, so this is contention pressure rather than user-visible failure." }
cell_inventory_lock_timeouts = { field = "cellInventoryLockTimeouts", description = "Bounded cell-inventory waits that expired, counted per attempt rather than per request. This is the user-visible lane." }
}
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
cloud/package.json
+1 -1
View File
@@ -28,6 +28,6 @@
"@types/node": "^24.10.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/packages/postgres-schema/package.json
+2 -2
View File
@@ -9,12 +9,12 @@
"build": "tsc -p tsconfig.build.json",
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
"lint": "tsc -p tsconfig.json --noEmit",
"test": "pnpm build",
"test": "pnpm build && vitest run",
"typecheck": "tsc -p tsconfig.json --noEmit"
},
"devDependencies": {
"@types/node": "^24.10.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/packages/postgres-schema/src/apply-postgres-schema.test.ts
+494
View File
@@ -0,0 +1,494 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { applyPostgresSchema } from './apply-postgres-schema.js'
import type { SchemaCatalogRow } from './catalog-object-precheck.js'
function postgresError(code: string, constraint?: string): Error {
return Object.assign(new Error(code), constraint === undefined ? { code } : { code, constraint })
}
const COMMENTED_INDEX = `-- Why the sweep needs this index
CREATE INDEX IF NOT EXISTS relay_bases_active ON relay_connection_bases(active, deadline)`
const COMMENTED_TABLE = `-- Two comment lines, the other shape a split schema carries
-- above a statement
CREATE TABLE IF NOT EXISTS relay_cells (
cell_id TEXT PRIMARY KEY
)`
// Answers absent first and present afterwards, the state a concurrent create leaves behind.
function catalogAnswersInSequence(answers: SchemaCatalogRow[][]): {
catalogQuery: (sql: string, params: unknown[]) => Promise<SchemaCatalogRow[]>
asked: unknown[][]
} {
const asked: unknown[][] = []
return {
asked,
catalogQuery: async (sql, params) => {
asked.push([sql, ...params])
return answers[asked.length - 1] ?? []
}
}
}
function catalogAnswers(rows: SchemaCatalogRow[]): {
catalogQuery: (sql: string, params: unknown[]) => Promise<SchemaCatalogRow[]>
asked: unknown[][]
} {
const asked: unknown[][] = []
return {
asked,
catalogQuery: async (sql, params) => {
asked.push([sql, ...params])
return rows
}
}
}
afterEach(() => {
vi.restoreAllMocks()
})
describe('applyPostgresSchema classification', () => {
it('classifies a comment-prefixed CREATE INDEX by its first SQL keyword', async () => {
let calls = 0
const query = vi.fn(async () => {
calls += 1
if (calls === 1) throw postgresError('42P07')
return undefined
})
await applyPostgresSchema([COMMENTED_INDEX], query, { wait: async () => undefined })
expect(query).toHaveBeenCalledTimes(2)
})
it('classifies a comment-prefixed CREATE TABLE by its own collision codes', async () => {
// pg_type_typname_nsp_index is reached only through the CREATE TABLE branch, so a statement
// misread as unknown would fail the boot on a benign concurrent create instead of retrying.
let calls = 0
const query = vi.fn(async () => {
calls += 1
if (calls === 1) throw postgresError('23505', 'pg_type_typname_nsp_index')
return undefined
})
await applyPostgresSchema([COMMENTED_TABLE], query, { wait: async () => undefined })
expect(query).toHaveBeenCalledTimes(2)
})
it('retries a concurrent index collision until it succeeds', async () => {
let calls = 0
const query = vi.fn(async () => {
calls += 1
if (calls < 3) throw postgresError('23505', 'pg_class_relname_nsp_index')
return undefined
})
const summary = await applyPostgresSchema(['CREATE INDEX IF NOT EXISTS i ON t(c)'], query, {
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(3)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('treats an already-applied constraint as skipped rather than an error', async () => {
// Still the answer for a caller with no pre-check, and for a constraint another director
// committed between this boot's pre-check and its ALTER TABLE.
const query = vi.fn(async () => {
throw postgresError('42710')
})
const summary = await applyPostgresSchema(['ALTER TABLE t ADD CONSTRAINT c CHECK (x > 0)'], query)
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('treats an index another director already dropped as skipped rather than an error', async () => {
// Every director boots at once on a deploy and all of them send the same DROP INDEX IF EXISTS.
// Only one can win; the losers must not fail their boot over a drop that already happened.
const query = vi.fn(async () => {
throw postgresError('42704')
})
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(query).toHaveBeenCalledTimes(1)
})
it('still propagates 42704 from a statement that is not a DROP IF EXISTS', async () => {
// Keeps the case above narrow: an undefined object anywhere else is a real boot failure.
const query = vi.fn(async () => {
throw postgresError('42704')
})
await expect(
applyPostgresSchema(['ALTER TABLE t ADD COLUMN IF NOT EXISTS c BIGINT'], query)
).rejects.toThrow(/42704/)
})
it('leaves a deferrable statement unapplied on a lock timeout instead of failing the boot', async () => {
// The crash loop this prevents: 28 directors reach the same DROP INDEX at once on a table
// under continuous write, all of them time out, and every one restarts to re-queue the same
// DDL behind the same writers.
const warned: string[] = []
vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const query = vi.fn(async () => {
throw postgresError('55P03')
})
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'],
query
)
expect(summary).toEqual({ ran: 0, skipped: 0, deferred: 1 })
expect(query).toHaveBeenCalledTimes(1)
const event = JSON.parse(warned[warned.length - 1] ?? '{}')
expect(event.event).toBe('orca_relay_postgres_schema_object_deferred')
expect(event.code).toBe('55P03')
expect(event.name).toBe('i')
})
it('runs the statements after a deferral, rather than abandoning the boot at that point', async () => {
// A deferral is not a failure, so nothing behind it may be skipped: the schema still has
// tables to create, and a boot that stopped here would come up against a partial schema.
const sent: string[] = []
const query = vi.fn(async (statement: string) => {
sent.push(statement)
if (statement.includes('DROP INDEX')) throw postgresError('55P03')
return undefined
})
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i', 'CREATE TABLE IF NOT EXISTS t (id TEXT)'],
query
)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 1 })
expect(sent).toHaveLength(2)
})
it('still fails the boot on a lock timeout for a statement that is not marked deferrable', async () => {
// Keeps the marker meaningful. An unmarked statement retains the old contract: fail once and
// loudly, because retrying parks every writer behind the same queue again.
vi.spyOn(console, 'error').mockImplementation(() => undefined)
const query = vi.fn(async () => {
throw postgresError('55P03')
})
await expect(applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)).rejects.toMatchObject({
code: '55P03'
})
})
it('defers only on a lock timeout, not on any other error from a deferrable statement', async () => {
// A deferrable statement is not a statement whose failures stop mattering. A permission error
// is still a boot failure.
const query = vi.fn(async () => {
throw postgresError('42501')
})
await expect(
applyPostgresSchema(['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'], query)
).rejects.toThrow(/42501/)
})
it('asks the catalog for a dropped index by name and skips the DROP once it is gone', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
// One parameter, the index name: the statement names no table, and the query references no $2.
expect(asked).toEqual([[expect.stringContaining("relkind = 'i'"), 'i']])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('sends the DROP while the index is still there, which is the boot that has to win', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([{}])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
expect(query).toHaveBeenCalledTimes(1)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('propagates an unrelated error without retrying', async () => {
const query = vi.fn(async () => {
throw postgresError('42501')
})
await expect(
applyPostgresSchema(['CREATE TABLE IF NOT EXISTS t (id TEXT)'], query)
).rejects.toThrow(/42501/)
expect(query).toHaveBeenCalledTimes(1)
})
})
describe('applyPostgresSchema lock timeouts', () => {
it('does not retry a lock timeout', async () => {
vi.spyOn(console, 'error').mockImplementation(() => undefined)
const query = vi.fn(async () => {
throw postgresError('55P03')
})
await expect(
applyPostgresSchema(['CREATE INDEX IF NOT EXISTS i ON t(c)'], query, {
wait: async () => undefined
})
).rejects.toThrow(/55P03/)
expect(query).toHaveBeenCalledTimes(1)
})
it('names the statement that could not take its lock', async () => {
const lines: string[] = []
vi.spyOn(console, 'error').mockImplementation((line: string) => {
lines.push(line)
})
const query = vi.fn(async () => {
throw postgresError('55P03')
})
await expect(applyPostgresSchema([COMMENTED_INDEX], query)).rejects.toThrow(/55P03/)
expect(JSON.parse(lines[0] ?? '{}')).toMatchObject({
event: 'orca_relay_postgres_schema_lock_timeout',
code: '55P03',
statement: 'CREATE INDEX IF NOT EXISTS relay_bases_active ON relay_connection_bases(active, deadline)'
})
})
it('still retries a lock timeout for a caller that opts in', async () => {
// A caller with no catalog pre-check learns nothing from a lock timeout about whether the
// object exists, so its old bounded retry is the correct behaviour there.
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
let calls = 0
const query = vi.fn(async () => {
calls += 1
if (calls < 3) throw postgresError('55P03')
return undefined
})
await applyPostgresSchema(['CREATE INDEX IF NOT EXISTS i ON t(c)'], query, {
retryLockTimeout: true,
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(3)
})
})
describe('applyPostgresSchema catalog pre-check', () => {
it('sends no lock-taking statement when the catalog has the object', async () => {
vi.spyOn(console, 'log').mockImplementation(() => undefined)
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([{ indisvalid: true }])
const summary = await applyPostgresSchema([COMMENTED_TABLE, COMMENTED_INDEX], query, {
catalogQuery
})
expect(query.mock.calls.map(([sql]) => sql)).toEqual([COMMENTED_TABLE])
expect(asked).toEqual([
[expect.stringContaining('pg_catalog.pg_index'), 'relay_connection_bases', 'relay_bases_active']
])
expect(summary).toEqual({ ran: 1, skipped: 1, deferred: 0 })
})
it('skips an index the catalog reports as invalid rather than rebuilding it', async () => {
// A cancelled CREATE INDEX CONCURRENTLY leaves exactly this state, and IF NOT EXISTS skips it
// too, so reading indisvalid as a condition would newly take the lock it used to avoid.
const logged: { event?: string }[] = []
vi.spyOn(console, 'log').mockImplementation((line: string) => {
logged.push(JSON.parse(line))
})
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([{ indisvalid: false }])
await applyPostgresSchema([COMMENTED_INDEX], query, { catalogQuery })
expect(query).not.toHaveBeenCalled()
expect(logged.filter((entry) => entry.event?.endsWith('_object_present'))).toEqual([
{
event: 'orca_relay_postgres_schema_object_present',
kind: 'index',
table: 'relay_connection_bases',
name: 'relay_bases_active',
indisvalid: false
}
])
})
it('reports how many statements ran and how many were skipped', async () => {
const logged: string[] = []
vi.spyOn(console, 'log').mockImplementation((line: string) => {
logged.push(line)
})
const { catalogQuery } = catalogAnswers([{ indisvalid: true }])
await applyPostgresSchema([COMMENTED_TABLE, COMMENTED_INDEX], vi.fn(async () => undefined), {
catalogQuery,
eventPrefix: 'orca_push_postgres_schema'
})
expect(JSON.parse(logged[logged.length - 1] ?? '{}')).toEqual({
event: 'orca_push_postgres_schema_applied',
ran: 1,
skipped: 1,
deferred: 0
})
})
it('asks pg_attribute for a column and sends the ALTER TABLE when no row comes back', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([])
const statement = 'ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle BIGINT'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(asked).toEqual([
[expect.stringContaining('pg_catalog.pg_attribute'), 'relay_control_capabilities', 'idle']
])
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('never probes the catalog for a statement that takes no relation lock', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([{ indisvalid: true }])
await applyPostgresSchema([COMMENTED_TABLE], query, { catalogQuery })
expect(asked).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
})
it('skips an ADD CONSTRAINT the catalog already names', async () => {
vi.spyOn(console, 'log').mockImplementation(() => undefined)
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([{}])
const summary = await applyPostgresSchema(
['ALTER TABLE relay_region_rehome_attempts ADD CONSTRAINT region_valid CHECK (r IN (1))'],
query,
{ catalogQuery }
)
expect(asked).toEqual([
[
expect.stringContaining('pg_catalog.pg_constraint'),
'relay_region_rehome_attempts',
'region_valid'
]
])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', async () => {
// Inverse polarity: an absent constraint is what means there is nothing to drop. Sending it
// anyway takes ACCESS EXCLUSIVE to discover the same thing.
const logged: { event?: string }[] = []
vi.spyOn(console, 'log').mockImplementation((line: string) => {
logged.push(JSON.parse(line))
})
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([])
const summary = await applyPostgresSchema(
['ALTER TABLE relay_region_rehome_attempts DROP CONSTRAINT IF EXISTS region_check'],
query,
{ catalogQuery }
)
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(logged).toContainEqual({
event: 'orca_relay_postgres_schema_object_absent',
kind: 'constraint',
table: 'relay_region_rehome_attempts',
name: 'region_check',
indisvalid: undefined
})
})
it('sends a DROP CONSTRAINT IF EXISTS when the constraint is still there', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([{}])
const statement = 'ALTER TABLE t DROP CONSTRAINT IF EXISTS region_check'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends an ADD CONSTRAINT the catalog does not name yet', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([])
const statement = 'ALTER TABLE t ADD CONSTRAINT region_valid CHECK (r IN (1))'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends every statement when no catalog query is supplied', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const summary = await applyPostgresSchema([COMMENTED_TABLE, COMMENTED_INDEX], query)
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 2, skipped: 0, deferred: 0 })
})
})
describe('applyPostgresSchema concurrent creates', () => {
it('re-asks the catalog on a collision instead of retrying the CREATE INDEX', async () => {
// Another director created the index between the pre-check and this statement. Retrying would
// take SHARE on the table again for an object that is already there.
vi.spyOn(console, 'log').mockImplementation(() => undefined)
const query = vi.fn(async (_statement: string) => {
throw postgresError('42P07')
})
const { catalogQuery, asked } = catalogAnswersInSequence([[], [{ indisvalid: true }]])
const summary = await applyPostgresSchema([COMMENTED_INDEX], query, {
catalogQuery,
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(1)
expect(asked).toHaveLength(2)
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('still retries when the catalog says the object is not there after all', async () => {
let calls = 0
const query = vi.fn(async (_statement: string) => {
calls += 1
if (calls === 1) throw postgresError('23505', 'pg_class_relname_nsp_index')
return undefined
})
const { catalogQuery } = catalogAnswersInSequence([[], []])
const summary = await applyPostgresSchema([COMMENTED_INDEX], query, {
catalogQuery,
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('retries a CREATE TABLE collision without a catalog re-ask, having no target to ask about', async () => {
let calls = 0
const query = vi.fn(async (_statement: string) => {
calls += 1
if (calls === 1) throw postgresError('42710')
return undefined
})
const { catalogQuery, asked } = catalogAnswersInSequence([[], []])
await applyPostgresSchema([COMMENTED_TABLE], query, {
catalogQuery,
wait: async () => undefined
})
expect(asked).toEqual([])
expect(query).toHaveBeenCalledTimes(2)
})
})
describe('applyPostgresSchema unparseable statements', () => {
it('fails the boot rather than sending an index whose target cannot be read', async () => {
const query = vi.fn(async (_statement: string) => undefined)
await expect(applyPostgresSchema(['CREATE INDEX ON t(c)'], query)).rejects.toThrow(
/unparsed_schema_lock_target/
)
expect(query).not.toHaveBeenCalled()
})
it('fails even with no catalog query, because the statement would take the lock either way', async () => {
const query = vi.fn(async (_statement: string) => undefined)
await expect(
applyPostgresSchema(['ALTER TABLE t ADD COLUMN IF NOT EXISTS'], query)
).rejects.toThrow(/unparsed_schema_lock_target/)
expect(query).not.toHaveBeenCalled()
})
})
describe('applyPostgresSchema statement text', () => {
it('sends the original statement, comments included, not the classified form', async () => {
// Classification reads a comment-free copy. Rewriting what the server runs would change the
// DDL itself, and a comment inside a string literal or a quoted name is part of the statement.
const statement = `ALTER TABLE t ADD /* note */ COLUMN c TEXT DEFAULT '-- keep'`
const query = vi.fn(async (_sql: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([])
await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(asked).toEqual([[expect.stringContaining('pg_catalog.pg_attribute'), 't', 'c']])
})
it('sends a comment-prefixed statement unchanged too', async () => {
const query = vi.fn(async (_sql: string) => undefined)
await applyPostgresSchema([COMMENTED_TABLE], query)
expect(query.mock.calls.map(([sql]) => sql)).toEqual([COMMENTED_TABLE])
})
})
cloud/packages/postgres-schema/src/apply-postgres-schema.ts
+216
View File
@@ -0,0 +1,216 @@
import { catalogObjectPresence, type SchemaCatalogQuery } from './catalog-object-precheck.js'
import {
requireSchemaLockTarget,
sqlWithoutComments,
type SchemaLockTarget
} from './schema-lock-target.js'
const RETRYABLE_SCHEMA_CODES = new Set(['57014'])
const LOCK_NOT_AVAILABLE = '55P03'
const DEFAULT_RETRY_DEADLINE_MS = 30_000
const RETRY_BASE_DELAY_MS = 250
const RETRY_MAX_DELAY_MS = 2_000
const DEFAULT_EVENT_PREFIX = 'orca_relay_postgres_schema'
export type SchemaStartupOptions = {
// Enables the catalog pre-check. Without it every lock-taking statement is sent as before.
catalogQuery?: SchemaCatalogQuery
eventPrefix?: string
now?: () => number
random?: () => number
retryDeadlineMs?: number
// Only for a caller with no catalog pre-check, where a lock timeout still says nothing about
// whether the object exists.
retryLockTimeout?: boolean
wait?: (delayMs: number) => Promise<void>
}
export type SchemaApplySummary = { ran: number; skipped: number; deferred: number }
function retryDelayMs(attempt: number, random: () => number): number {
const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
return Math.ceil(ceiling * (0.5 + random() * 0.5))
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs))
}
const CREATE_TABLE_IF_NOT_EXISTS = /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
const CREATE_INDEX_IF_NOT_EXISTS = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
const ALTER_TABLE_ADD_CONSTRAINT = /^ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
const DROP_INDEX_IF_EXISTS = /^DROP\s+INDEX\s+(?:CONCURRENTLY\s+)?IF\s+EXISTS\b/i
// Marked in the schema text, beside the SQL it applies to, and read from the raw statement because
// classification strips comments. Says: this boot may leave the statement unapplied rather than
// fail. Only sound for a statement that is idempotent AND that nothing this boot goes on to do
// depends on, because the database is then simply as it was and the next boot re-sends it.
const DEFERRABLE = /^\s*--[^\n]*\bschema-deferrable\b/
export function schemaDeferrable(statement: string): boolean {
return DEFERRABLE.test(statement)
}
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
function concurrentCreateCollision(
value: { code?: unknown; constraint?: unknown },
sql: string
): boolean {
if (CREATE_TABLE_IF_NOT_EXISTS.test(sql)) {
return (
(value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
value.code === '42710' ||
value.code === '42P07'
)
}
if (CREATE_INDEX_IF_NOT_EXISTS.test(sql)) {
return (
(value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
value.code === '42P07'
)
}
return false
}
function constraintAlreadyApplied(error: unknown, sql: string): boolean {
return (
ALTER_TABLE_ADD_CONSTRAINT.test(sql) && (error as { code?: unknown } | null)?.code === '42710'
)
}
// `IF EXISTS` resolves the name, then locks; between those two steps another director's drop can
// commit and the loser raises 42704 instead of the notice it would have got a moment later. Every
// director boots at once on a deploy, so without this the losers fail their boot over a drop that
// already happened.
function dropAlreadyApplied(error: unknown, sql: string): boolean {
return DROP_INDEX_IF_EXISTS.test(sql) && (error as { code?: unknown } | null)?.code === '42704'
}
function retryableSchemaError(error: unknown, sql: string): boolean {
const value = (error as { code?: unknown; constraint?: unknown } | null) ?? {}
return RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, sql)
}
// Evaluated immediately before each statement, so a pre-check still sees the objects the statements
// ahead of it created in this same boot.
async function nothingToDo(
target: SchemaLockTarget | undefined,
options: SchemaStartupOptions,
eventPrefix: string
): Promise<boolean> {
const catalogQuery = options.catalogQuery
if (!catalogQuery || !target) return false
const presence = await catalogObjectPresence(catalogQuery, target)
if (presence.present !== (target.skipWhen === 'present')) return false
console.log(
JSON.stringify({
event: `${eventPrefix}_object_${target.skipWhen}`,
kind: target.kind,
table: target.kind === 'index-by-name' ? undefined : target.table,
name: target.name,
indisvalid: presence.indisvalid
})
)
return true
}
export async function applyPostgresSchema(
statements: string[],
query: (statement: string) => Promise<unknown>,
options: SchemaStartupOptions = {}
): Promise<SchemaApplySummary> {
const eventPrefix = options.eventPrefix ?? DEFAULT_EVENT_PREFIX
const now = options.now ?? Date.now
const random = options.random ?? Math.random
const pause = options.wait ?? wait
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
const summary: SchemaApplySummary = { ran: 0, skipped: 0, deferred: 0 }
for (const statement of statements) {
// Throws when an index or column statement's target cannot be read, rather than sending it
// unchecked into the lock queue.
const target = requireSchemaLockTarget(statement)
if (await nothingToDo(target, options, eventPrefix)) {
summary.skipped += 1
continue
}
const sql = sqlWithoutComments(statement)
let attempt = 1
while (true) {
try {
await query(statement)
summary.ran += 1
break
} catch (error) {
if (constraintAlreadyApplied(error, sql) || dropAlreadyApplied(error, sql)) {
summary.skipped += 1
break
}
const code = String((error as { code?: unknown } | null)?.code)
// With the pre-check ahead of it a lock timeout means the object is genuinely missing and
// this boot lost the queue. Relation locks are granted in queue order, so each retry parks
// every writer behind it again for another timeout. Fail once, loudly.
if (code === LOCK_NOT_AVAILABLE && !options.retryLockTimeout) {
// A deferrable statement yields the queue instead of crash-looping the instance. Every
// director boots at once on a migration, so a table under continuous write can hand the
// whole fleet a lock timeout on the one statement that has to win once; failing the boot
// for it restarts the instance, which re-queues the same DDL behind the same writers.
if (schemaDeferrable(statement)) {
console.warn(
JSON.stringify({
event: `${eventPrefix}_object_deferred`,
code,
kind: target?.kind,
name: target?.name,
statement: sql.split('\n')[0],
detail: 'could not take its lock; left unapplied for the next boot to retry'
})
)
summary.deferred += 1
break
}
console.error(
JSON.stringify({
event: `${eventPrefix}_lock_timeout`,
code,
statement: sql.split('\n')[0],
detail: 'boot-time DDL could not take its lock; retrying would requeue every writer'
})
)
throw error
}
// The object was created between the pre-check and this statement. Re-asking the catalog
// is the cheap answer; retrying the CREATE INDEX would take SHARE on the table again for
// an object that is already there.
if (
concurrentCreateCollision((error as { code?: unknown; constraint?: unknown }) ?? {}, sql) &&
(await nothingToDo(target, options, eventPrefix))
) {
summary.skipped += 1
break
}
const remainingMs = deadlineAt - now()
const retryable =
retryableSchemaError(error, sql) ||
(code === LOCK_NOT_AVAILABLE && options.retryLockTimeout === true)
if (!retryable || remainingMs <= 0) {
if (retryable) {
console.warn(
JSON.stringify({ event: `${eventPrefix}_retry_exhausted`, code, attempts: attempt })
)
}
throw error
}
const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random))
console.warn(JSON.stringify({ event: `${eventPrefix}_retry`, code, attempt, delayMs }))
await pause(delayMs)
attempt += 1
}
}
}
console.log(JSON.stringify({ event: `${eventPrefix}_applied`, ...summary }))
return summary
}
cloud/packages/postgres-schema/src/catalog-object-precheck.ts
+67
View File
@@ -0,0 +1,67 @@
import type { SchemaLockTarget } from './schema-lock-target.js'
export type SchemaCatalogRow = Record<string, unknown>
// Runs with `$n` placeholders bound to the lock target, on the same connection the DDL would use.
export type SchemaCatalogQuery = (
sql: string,
params: unknown[]
) => Promise<SchemaCatalogRow[]>
// The index name is resolved inside the table's own namespace, and `i.indrelid = t.oid` ties it to
// this table: index names are unique per schema, not per table, so without that condition a
// same-named index on a sibling table answers yes and the real index is skipped forever.
// `to_regclass` returns NULL rather than erroring when the table does not exist yet, which is the
// whole of a cold start.
const INDEX_PRESENT = `SELECT i.indisvalid FROM pg_catalog.pg_class t
JOIN pg_catalog.pg_class c ON c.relnamespace = t.relnamespace AND c.relname = $2
JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid AND i.indrelid = t.oid
WHERE t.oid = to_regclass($1)`
const COLUMN_PRESENT = `SELECT 1 FROM pg_catalog.pg_attribute
WHERE attrelid = to_regclass($1) AND attname = $2 AND attnum > 0 AND NOT attisdropped`
// Name only. The CHECK body is generated from RELAY_REGIONS, so comparing it would re-run the swap
// on every region change, and an ADD CONSTRAINT is the one statement here that scans the table.
const CONSTRAINT_PRESENT = `SELECT 1 FROM pg_catalog.pg_constraint
WHERE conrelid = to_regclass($1) AND conname = $2`
// By name through the search_path, with no table condition, because a DROP INDEX has no table to
// condition on and does not need one: a name that resolves to no visible index is nothing to drop.
// `relkind = 'i'` keeps a same-named table or view from answering for an index. Partitioned indexes
// are 'I', which this deliberately does not match - relay has none, and dropping one is not a
// boot-time operation.
const INDEX_BY_NAME_PRESENT = `SELECT 1 FROM pg_catalog.pg_class c
WHERE c.relname = $1 AND c.relkind = 'i' AND pg_catalog.pg_table_is_visible(c.oid)`
// reloptions is a text[] of `name=value` pairs, absent entirely while the option is at its
// default. Comparing the whole pair is what makes a changed value re-run: `@>` on a different
// value answers no, and the statement runs and overwrites it.
const RELOPTION_PRESENT = `SELECT 1 FROM pg_catalog.pg_class
WHERE oid = to_regclass($1) AND reloptions @> ARRAY[$2]`
const PRESENCE_SQL = {
index: INDEX_PRESENT,
column: COLUMN_PRESENT,
constraint: CONSTRAINT_PRESENT,
reloption: RELOPTION_PRESENT,
'index-by-name': INDEX_BY_NAME_PRESENT
} as const
export type SchemaCatalogPresence = { present: boolean; indisvalid: unknown }
// Row presence is the answer, whatever the row says. An index left invalid by a cancelled
// concurrent build is skipped by `IF NOT EXISTS` today as well, so reading `indisvalid` as a
// condition would newly take the lock for exactly the indexes a failed build left behind.
export async function catalogObjectPresence(
query: SchemaCatalogQuery,
target: SchemaLockTarget
): Promise<SchemaCatalogPresence> {
const sql = PRESENCE_SQL[target.kind]
// The name-only lookup binds one parameter; every other shape binds the table first. Passing a
// parameter the SQL never references is a bind error, not a harmless extra.
const params = target.kind === 'index-by-name' ? [target.name] : [target.table, target.name]
const rows = await query(sql, params)
const row = rows[0]
return row ? { present: true, indisvalid: row.indisvalid } : { present: false, indisvalid: undefined }
}
cloud/packages/postgres-schema/src/index.ts
+19 -113
View File
@@ -1,113 +1,19 @@
const RETRYABLE_SCHEMA_CODES = new Set(['55P03', '57014'])
const DEFAULT_RETRY_DEADLINE_MS = 30_000
const RETRY_BASE_DELAY_MS = 250
const RETRY_MAX_DELAY_MS = 2_000
type SchemaStartupOptions = {
eventPrefix?: string
now?: () => number
random?: () => number
retryDeadlineMs?: number
wait?: (delayMs: number) => Promise<void>
}
function retryDelayMs(attempt: number, random: () => number): number {
const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
return Math.ceil(ceiling * (0.5 + random() * 0.5))
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs))
}
const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
function concurrentCreateCollision(
value: { code?: unknown; constraint?: unknown },
statement: string
): boolean {
if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) {
return (
(value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
value.code === '42710' ||
value.code === '42P07'
)
}
if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) {
return (
(value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
value.code === '42P07'
)
}
return false
}
const ALTER_TABLE_ADD_CONSTRAINT = /^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
return (
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
(error as { code?: unknown }).code === '42710'
)
}
function retryableSchemaError(error: unknown, statement: string): boolean {
const value = error as { code?: unknown; constraint?: unknown }
return (
RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement)
)
}
export async function applyPostgresSchema(
statements: string[],
query: (statement: string) => Promise<unknown>,
options: SchemaStartupOptions = {}
): Promise<void> {
const now = options.now ?? Date.now
const random = options.random ?? Math.random
const pause = options.wait ?? wait
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
for (const statement of statements) {
let attempt = 1
while (true) {
try {
await query(statement)
break
} catch (error) {
if (constraintAlreadyApplied(error, statement)) break
const code = String((error as { code?: unknown }).code)
const remainingMs = deadlineAt - now()
const retryable = retryableSchemaError(error, statement)
if (!retryable || remainingMs <= 0) {
if (retryable) {
console.warn(
JSON.stringify({
event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry_exhausted`,
code,
attempts: attempt
})
)
}
throw error
}
const delayMs = Math.min(remainingMs, retryDelayMs(attempt, random))
console.warn(
JSON.stringify({
event: `${options.eventPrefix ?? 'orca_relay_postgres_schema'}_retry`,
code,
attempt,
delayMs
})
)
await pause(delayMs)
attempt += 1
}
}
}
}
export {
applyPostgresSchema,
schemaDeferrable,
type SchemaApplySummary,
type SchemaStartupOptions
} from './apply-postgres-schema.js'
export {
catalogObjectPresence,
type SchemaCatalogPresence,
type SchemaCatalogQuery,
type SchemaCatalogRow
} from './catalog-object-precheck.js'
export {
requireSchemaLockTarget,
schemaLockTarget,
sqlWithoutComments,
takesRelationLock,
type SchemaLockTarget
} from './schema-lock-target.js'
cloud/packages/postgres-schema/src/schema-lock-target.test.ts
+522
View File
@@ -0,0 +1,522 @@
import { describe, expect, it } from 'vitest'
import { schemaDeferrable } from './apply-postgres-schema.js'
import {
requireSchemaLockTarget,
schemaLockTarget,
sqlWithoutComments,
takesRelationLock
} from './schema-lock-target.js'
// The shape a schema string split on ';' actually produces: the comment written above a statement
// arrives glued to the front of it.
const COMMENTED_INDEX = `-- Why: the maintenance sweep matches (active, deadline) while inactive
-- bases accumulate unboundedly.
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
ON relay_connection_bases(active, deadline)`
const COMMENTED_TABLE = `-- Rehoming is bidirectional, but tables created before that carry the
-- original single-region column check.
CREATE TABLE IF NOT EXISTS relay_cells (
cell_id TEXT PRIMARY KEY
)`
describe('sqlWithoutComments', () => {
it('strips the line comments a split schema glues above a statement', () => {
expect(sqlWithoutComments(COMMENTED_INDEX)).toMatch(/^CREATE INDEX IF NOT EXISTS/)
})
it('strips a leading block comment', () => {
expect(sqlWithoutComments('/* note */\n ALTER TABLE t ADD COLUMN c TEXT')).toBe(
'ALTER TABLE t ADD COLUMN c TEXT'
)
})
it('strips a comment sitting between two keywords', () => {
expect(sqlWithoutComments('ALTER TABLE t ADD /* note */ COLUMN c TEXT')).toBe(
'ALTER TABLE t ADD COLUMN c TEXT'
)
})
it('strips a trailing comment', () => {
expect(sqlWithoutComments('SELECT 1 -- note')).toBe('SELECT 1')
})
it('closes the inner block comment first when they nest', () => {
expect(sqlWithoutComments('ALTER TABLE t /* a /* b */ c */ ADD COLUMN d TEXT')).toBe(
'ALTER TABLE t ADD COLUMN d TEXT'
)
})
it('leaves a comment marker inside a string literal alone', () => {
expect(sqlWithoutComments(`ALTER TABLE t ADD COLUMN c TEXT DEFAULT '-- not a comment'`)).toBe(
`ALTER TABLE t ADD COLUMN c TEXT DEFAULT '-- not a comment'`
)
})
it('leaves a comment marker inside a quoted identifier alone', () => {
expect(sqlWithoutComments('ALTER TABLE t ADD COLUMN "a/* b */c" TEXT')).toBe(
'ALTER TABLE t ADD COLUMN "a/* b */c" TEXT'
)
})
})
describe('comments between keywords', () => {
// Before this, the classification regexes and the must-parse shapes both needed `ADD COLUMN`
// contiguous, so this statement derived NO target and threw NOTHING: the DDL ran with no
// pre-check, taking ACCESS EXCLUSIVE on every boot.
it('derives a column target through a comment between ADD and COLUMN', () => {
expect(requireSchemaLockTarget('ALTER TABLE t ADD /* note */ COLUMN c TEXT')).toEqual({
kind: 'column',
table: 't',
name: 'c',
skipWhen: 'present'
})
})
it('derives an index target through a line comment before ON', () => {
expect(
requireSchemaLockTarget('CREATE INDEX IF NOT EXISTS i\n-- why this index exists\nON t(c)')
).toEqual({ kind: 'index', table: 't', name: 'i', skipWhen: 'present' })
})
it('still counts a commented statement as taking a relation lock', () => {
expect(takesRelationLock('/* note */ ALTER TABLE t ADD COLUMN c TEXT')).toBe(true)
})
it('does not read a comment marker inside a quoted name as a comment', () => {
expect(schemaLockTarget('ALTER TABLE t ADD COLUMN "a--b" TEXT')).toEqual({
kind: 'column',
table: 't',
name: 'a--b',
skipWhen: 'present'
})
})
})
describe('catalog name folding', () => {
it('reads a quoted identifier containing a dot as one name', () => {
expect(schemaLockTarget('CREATE INDEX IF NOT EXISTS "a.b" ON t(c)')).toEqual({
kind: 'index',
table: 't',
name: 'a.b',
skipWhen: 'present'
})
})
it('folds an unquoted name to lower case, the form the catalog stores', () => {
expect(schemaLockTarget('CREATE INDEX IF NOT EXISTS Foo ON Bar(c)')).toEqual({
kind: 'index',
table: 'Bar',
name: 'foo',
skipWhen: 'present'
})
})
it('keeps a quoted name in its written case', () => {
expect(schemaLockTarget('CREATE INDEX IF NOT EXISTS public."Mixed.Name" ON t(c)')).toEqual({
kind: 'index',
table: 't',
name: 'Mixed.Name',
skipWhen: 'present'
})
})
it('unescapes a doubled quote and leaves the qualified table text as written', () => {
expect(schemaLockTarget('ALTER TABLE App."My Table" ADD COLUMN "od""d" TEXT')).toEqual({
kind: 'column',
table: 'App."My Table"',
name: 'od"d',
skipWhen: 'present'
})
})
it('folds an unquoted column name too', () => {
expect(schemaLockTarget('ALTER TABLE t ADD COLUMN IF NOT EXISTS HostCooldownMs BIGINT')).toEqual(
{ kind: 'column', table: 't', name: 'hostcooldownms', skipWhen: 'present' }
)
})
})
describe('square brackets in an ALTER TABLE', () => {
it.each([
['an array type', 'ALTER TABLE t ADD COLUMN c bigint[] DEFAULT ARRAY[1, 2]'],
['a nested array default', "ALTER TABLE t ADD COLUMN c TEXT[] DEFAULT ARRAY['a', 'b']"]
])('does not read a comma inside %s as a second subcommand', (_label, statement) => {
expect(() => requireSchemaLockTarget(statement)).not.toThrow()
})
it('still catches a second subcommand after an array default', () => {
expect(() =>
requireSchemaLockTarget('ALTER TABLE t ADD COLUMN a bigint[] DEFAULT ARRAY[1, 2], ADD COLUMN b TEXT')
).toThrow(/unparsed_schema_lock_target/)
})
})
describe('takesRelationLock', () => {
it('classifies a comment-prefixed CREATE INDEX by its first SQL keyword', () => {
expect(takesRelationLock(COMMENTED_INDEX)).toBe(true)
})
it('classifies a comment-prefixed CREATE TABLE as taking no relation lock', () => {
expect(takesRelationLock(COMMENTED_TABLE)).toBe(false)
})
it('counts every ALTER TABLE, including the constraint swaps', () => {
expect(takesRelationLock('ALTER TABLE t DROP CONSTRAINT IF EXISTS c')).toBe(true)
})
})
describe('schemaLockTarget', () => {
it('derives an index target through the comments above it', () => {
expect(schemaLockTarget(COMMENTED_INDEX)).toEqual({
kind: 'index',
table: 'relay_connection_bases',
name: 'relay_connection_bases_active_deadline',
skipWhen: 'present'
})
})
it('derives an index target across the line break before ON', () => {
expect(
schemaLockTarget(`CREATE INDEX IF NOT EXISTS relay_reservation_assignment
ON relay_control_connection_reservations(
user_id, relay_host_id
)`)
).toEqual({
kind: 'index',
table: 'relay_control_connection_reservations',
name: 'relay_reservation_assignment',
skipWhen: 'present'
})
})
it('derives a unique concurrent index target', () => {
expect(schemaLockTarget('CREATE UNIQUE INDEX CONCURRENTLY i ON t(c)')).toEqual({
kind: 'index',
table: 't',
name: 'i',
skipWhen: 'present'
})
})
it('keeps the schema qualification on the table and drops it from the object name', () => {
// `table` is fed to to_regclass, which needs the qualification; `name` is matched against
// relname, which stores the bare identifier.
expect(schemaLockTarget('CREATE INDEX IF NOT EXISTS app.i ON app.t(c)')).toEqual({
kind: 'index',
table: 'app.t',
name: 'i',
skipWhen: 'present'
})
})
it('unquotes a quoted identifier, doubled quote included', () => {
expect(schemaLockTarget('CREATE INDEX IF NOT EXISTS "od""d" ON "My Table"(c)')).toEqual({
kind: 'index',
table: '"My Table"',
name: 'od"d',
skipWhen: 'present'
})
})
it('derives a column target from a multi-line ADD COLUMN IF NOT EXISTS', () => {
expect(
schemaLockTarget(`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT 604800000`)
).toEqual({
kind: 'column',
table: 'relay_region_rehome_control',
name: 'host_cooldown_ms',
skipWhen: 'present'
})
})
it('derives a column target without IF NOT EXISTS', () => {
expect(schemaLockTarget('ALTER TABLE ONLY t ADD COLUMN c TEXT')).toEqual({
kind: 'column',
table: 't',
name: 'c',
skipWhen: 'present'
})
})
it('skips an ADD CONSTRAINT once the constraint name is there', () => {
expect(schemaLockTarget('ALTER TABLE t ADD CONSTRAINT c CHECK (x > 0)')).toEqual({
kind: 'constraint',
table: 't',
name: 'c',
skipWhen: 'present'
})
})
it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', () => {
// The inverse polarity: nothing to drop is nothing to do.
expect(schemaLockTarget('ALTER TABLE t DROP CONSTRAINT IF EXISTS c')).toEqual({
kind: 'constraint',
table: 't',
name: 'c',
skipWhen: 'absent'
})
})
it('derives a constraint target across a line break', () => {
expect(
schemaLockTarget(`ALTER TABLE relay_region_rehome_attempts
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN ('us-central1'))`)
).toEqual({
kind: 'constraint',
table: 'relay_region_rehome_attempts',
name: 'relay_region_rehome_attempts_preferred_region_valid',
skipWhen: 'present'
})
})
it('gives CREATE TABLE IF NOT EXISTS no target', () => {
expect(schemaLockTarget(COMMENTED_TABLE)).toBeUndefined()
})
})
// Each of these reads as an index or column statement and each fails to yield a target. Letting any
// of them through would send an unchecked lock-taking statement on every boot.
const MALFORMED = [
['an index with no ON clause', 'CREATE INDEX IF NOT EXISTS i'],
['an auto-named index', 'CREATE INDEX ON t(c)'],
['an auto-named unique concurrent index', 'CREATE UNIQUE INDEX CONCURRENTLY ON t(c)'],
['an index whose name ran into a comment', '-- note\nCREATE INDEX IF NOT EXISTS\nON t(c)'],
['an ALTER TABLE with no table', 'ALTER TABLE ADD COLUMN c TEXT'],
['an ADD COLUMN with no column', 'ALTER TABLE t ADD COLUMN'],
['an ADD COLUMN IF NOT EXISTS with no column', 'ALTER TABLE t ADD COLUMN IF NOT EXISTS']
] as const
describe('requireSchemaLockTarget', () => {
it.each(MALFORMED)('throws with the statement text on %s', (_label, statement) => {
expect(() => requireSchemaLockTarget(statement)).toThrow(/unparsed_schema_lock_target/)
})
it('names the offending statement in the error', () => {
expect(() => requireSchemaLockTarget('CREATE INDEX ON t(c)')).toThrow(
'unparsed_schema_lock_target: CREATE INDEX ON t(c)'
)
})
it('returns the target for a statement that parses', () => {
expect(requireSchemaLockTarget(COMMENTED_INDEX)).toEqual({
kind: 'index',
table: 'relay_connection_bases',
name: 'relay_connection_bases_active_deadline',
skipWhen: 'present'
})
})
it.each([
['CREATE TABLE IF NOT EXISTS t (id TEXT)'],
['ALTER TABLE t ALTER COLUMN c SET DEFAULT 0']
])('leaves %s alone, because no target is expected of it', (statement) => {
expect(requireSchemaLockTarget(statement)).toBeUndefined()
})
})
describe('multi-action ALTER TABLE', () => {
it('throws rather than deriving only the first subcommand', () => {
// Deriving `a` and skipping on it would drop `b` for the life of the database, and the first
// subcommand parses fine, so nothing else here would catch it.
const statement =
'ALTER TABLE t ADD COLUMN IF NOT EXISTS a TEXT, ADD COLUMN IF NOT EXISTS b TEXT'
expect(schemaLockTarget(statement)).toEqual({
kind: 'column',
table: 't',
name: 'a',
skipWhen: 'present'
})
expect(() => requireSchemaLockTarget(statement)).toThrow(/unparsed_schema_lock_target/)
})
it('throws on a constraint swap written as one statement', () => {
expect(() =>
requireSchemaLockTarget(
'ALTER TABLE t DROP CONSTRAINT IF EXISTS old, ADD CONSTRAINT new CHECK (x > 0)'
)
).toThrow(/unparsed_schema_lock_target/)
})
it.each([
['a parenthesised type', 'ALTER TABLE t ADD COLUMN IF NOT EXISTS a NUMERIC(10, 2)'],
['a CHECK body', "ALTER TABLE t ADD CONSTRAINT c CHECK (r IN ('us-central1', 'asia-east2'))"],
['a quoted comma', `ALTER TABLE t ADD COLUMN IF NOT EXISTS a TEXT DEFAULT 'x, y'`],
['a doubled quote before a comma', `ALTER TABLE t ADD COLUMN a TEXT DEFAULT 'it''s, fine'`],
['a trailing line comment', 'ALTER TABLE t ADD COLUMN a TEXT -- one, two'],
['a trailing block comment', 'ALTER TABLE t ADD COLUMN a TEXT /* one, two */']
])('does not throw on %s', (_label, statement) => {
expect(() => requireSchemaLockTarget(statement)).not.toThrow()
})
it('derives through a block comment sitting where the column name belongs', () => {
expect(requireSchemaLockTarget('ALTER TABLE t ADD COLUMN /* note */ a TEXT')).toEqual({
kind: 'column',
table: 't',
name: 'a',
skipWhen: 'present'
})
})
it('leaves a multi-column CREATE INDEX alone', () => {
expect(() => requireSchemaLockTarget('CREATE INDEX IF NOT EXISTS i ON t(a, b)')).not.toThrow()
})
})
describe('dollar-quoted bodies', () => {
it('does not read a comment marker inside a dollar-quoted default as a comment', () => {
expect(sqlWithoutComments('ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$--$$')).toBe(
'ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$--$$'
)
expect(requireSchemaLockTarget('ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$--$$')).toEqual({
kind: 'column',
table: 't',
name: 'c',
skipWhen: 'present'
})
})
it('does not count a comma inside a dollar-quoted default as a second subcommand', () => {
expect(() =>
requireSchemaLockTarget('ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$a, b$$')
).not.toThrow()
})
it('reads an inner $$ inside a tagged body as text, not as the close', () => {
// The closing delimiter has to match the opening tag, so the comma and the comment marker
// between the inner $$ pair are still inside the body.
const statement = 'ALTER TABLE t ADD COLUMN c TEXT DEFAULT $tag$ a $$ -- b, c $$ d $tag$'
expect(sqlWithoutComments(statement)).toBe(statement)
expect(() => requireSchemaLockTarget(statement)).not.toThrow()
expect(schemaLockTarget(statement)).toEqual({
kind: 'column',
table: 't',
name: 'c',
skipWhen: 'present'
})
})
it('still catches a second subcommand after a dollar-quoted default', () => {
expect(() =>
requireSchemaLockTarget('ALTER TABLE t ADD COLUMN a TEXT DEFAULT $$x, y$$, ADD COLUMN b TEXT')
).toThrow(/unparsed_schema_lock_target/)
})
it('leaves a numbered placeholder alone, because a tag cannot start with a digit', () => {
expect(sqlWithoutComments('ALTER TABLE t ADD COLUMN c TEXT -- $1 and $2')).toBe(
'ALTER TABLE t ADD COLUMN c TEXT'
)
})
it('treats an unterminated dollar quote as opaque to the end', () => {
expect(sqlWithoutComments('ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$ -- unterminated')).toBe(
'ALTER TABLE t ADD COLUMN c TEXT DEFAULT $$ -- unterminated'
)
})
})
describe('schemaLockTarget storage parameters', () => {
it('reads a storage parameter as a name=value target the catalog can be asked about', () => {
expect(schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')).toEqual({
kind: 'reloption',
table: 't',
name: 'fillfactor=70',
skipWhen: 'present'
})
})
it('folds the option name but keeps the value as written, the way pg_class stores the pair', () => {
expect(schemaLockTarget('ALTER TABLE t SET (FillFactor=70)')?.name).toBe('fillfactor=70')
})
it('makes a changed value a different target, so it re-runs instead of skipping', () => {
// The failure this prevents: matching on the option name alone would read `fillfactor=100` as
// already satisfying `fillfactor = 70` and skip the statement for the life of the database.
const seventy = schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')
const eighty = schemaLockTarget('ALTER TABLE t SET (fillfactor = 80)')
expect(seventy?.name).not.toBe(eighty?.name)
})
it('refuses a multi-option SET rather than skipping on only the first option', () => {
// Same reason a multi-action ALTER TABLE is refused: skipping on one option would silently
// drop the others for good.
expect(() =>
requireSchemaLockTarget('ALTER TABLE t SET (fillfactor = 70, autovacuum_enabled = false)')
).toThrow(/unparsed_schema_lock_target/)
})
it('fails the boot on a SET whose shape it cannot read, rather than sending it unchecked', () => {
// A storage parameter takes a relation lock, so no target means the lock is taken on every
// boot. RESET has no value to compare and is not supported.
expect(() => requireSchemaLockTarget('ALTER TABLE t RESET (fillfactor)')).not.toThrow()
expect(() => requireSchemaLockTarget('ALTER TABLE t SET (fillfactor)')).toThrow(
/unparsed_schema_lock_target/
)
})
it('takes a relation lock, so the census requires it to carry a target', () => {
expect(takesRelationLock('ALTER TABLE t SET (fillfactor = 70)')).toBe(true)
})
})
describe('schemaLockTarget dropped indexes', () => {
it('resolves a dropped index by name, with no table to name', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS i')).toEqual({
kind: 'index-by-name',
name: 'i',
skipWhen: 'absent'
})
})
it('reads CONCURRENTLY as a modifier rather than the index name', () => {
expect(schemaLockTarget('DROP INDEX CONCURRENTLY IF EXISTS i')?.name).toBe('i')
})
it('folds an unquoted name and keeps a quoted one, the way relname stores it', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS MyIndex')?.name).toBe('myindex')
expect(schemaLockTarget('DROP INDEX IF EXISTS "MyIndex"')?.name).toBe('MyIndex')
})
it('takes a relation lock, because the index is there on the boot that has to drop it', () => {
expect(takesRelationLock('DROP INDEX IF EXISTS i')).toBe(true)
})
it('requires IF EXISTS, so a bare DROP fails the boot instead of running unchecked', () => {
// Same contract as DROP CONSTRAINT: a bare DROP on a missing index is an error the server is
// supposed to raise, and a pre-check that skipped it would swallow that.
expect(() => requireSchemaLockTarget('DROP INDEX i')).toThrow(/unparsed_schema_lock_target/)
})
it('refuses a multi-index DROP rather than pre-checking only the first name', () => {
// Skipping on one name would leave the other index in place for the life of the database.
expect(() => requireSchemaLockTarget('DROP INDEX IF EXISTS a, b')).toThrow(
/unparsed_schema_lock_target/
)
})
it('derives the target through a leading deferrable marker', () => {
// The real shape in relay's schema: the marker is a comment, so classification must see past
// it or the statement would reach the server with no pre-check at all.
const statement = '-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'
expect(sqlWithoutComments(statement)).toBe('DROP INDEX IF EXISTS i')
expect(schemaLockTarget(statement)?.name).toBe('i')
})
})
describe('schemaDeferrable', () => {
it('reads the marker only from a leading comment, never from the SQL body', () => {
// A name or a string containing the word must not make a statement deferrable.
expect(schemaDeferrable('-- schema-deferrable: reason\nDROP INDEX IF EXISTS i')).toBe(true)
expect(schemaDeferrable('DROP INDEX IF EXISTS schema_deferrable')).toBe(false)
expect(schemaDeferrable("CREATE TABLE t (c TEXT DEFAULT 'schema-deferrable')")).toBe(false)
})
it('treats an unmarked statement as fatal on a lock timeout, which is the default', () => {
expect(schemaDeferrable('DROP INDEX IF EXISTS i')).toBe(false)
expect(schemaDeferrable('ALTER TABLE t SET (fillfactor = 70)')).toBe(false)
})
})
cloud/packages/postgres-schema/src/schema-lock-target.ts
+305
View File
@@ -0,0 +1,305 @@
// The object a boot-time DDL statement locks on Postgres, so the catalog can be asked whether it
// already exists before the statement joins the lock queue. `table` is kept exactly as the
// statement wrote it, schema qualification and quoting included, because it is fed to
// `to_regclass`; `name` is the bare identifier the catalog stores in `relname`/`attname`.
export type SchemaLockTarget =
| {
kind: 'index' | 'column' | 'constraint' | 'reloption'
table: string
name: string
// The catalog answer that means this statement has nothing left to do. Creating statements
// skip on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is
// done.
skipWhen: 'present' | 'absent'
}
// A `DROP INDEX` names no table, and needs none: an index name that resolves to nothing is
// nothing to drop, whatever table it used to belong to. Resolution is by name through the
// search_path, which is how the DROP itself would resolve it.
| { kind: 'index-by-name'; name: string; skipWhen: 'absent' }
// Keywords that sit in an identifier position when the optional clause before them is absent.
// Without this, `CREATE UNIQUE INDEX CONCURRENTLY ON t(c)` reads CONCURRENTLY as the index name and
// `ADD COLUMN IF NOT EXISTS` with no column reads IF as the column: a silently wrong target, which
// is worse than no target. Excluding them makes both throw instead. A column genuinely named `if`
// has to be quoted to be derivable, which is the safe direction to fail in.
const NOT_KEYWORD = '(?!(?:CONCURRENTLY|IF|NOT|EXISTS|ON|ONLY)\\b)'
const IDENTIFIER = `"(?:[^"]|"")*"|${NOT_KEYWORD}[A-Za-z_][A-Za-z0-9_$]*`
const QUALIFIED = `((?:${IDENTIFIER})(?:\\.(?:${IDENTIFIER}))?)`
// `$$...$$` and `$tag$...$tag$` are opaque: a comment marker, comma, parenthesis or bracket inside
// one is text. The closing delimiter must match the opening tag exactly, so an inner `$$` inside a
// `$tag$` body is more text rather than the end. The tag cannot start with a digit, which is what
// keeps a `$1` placeholder from reading as an opener.
const DOLLAR_QUOTE = /\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/y
function dollarQuoteEnd(sql: string, index: number): number | undefined {
DOLLAR_QUOTE.lastIndex = index
const opener = DOLLAR_QUOTE.exec(sql)?.[0]
if (opener === undefined) return undefined
const close = sql.indexOf(opener, index + opener.length)
return close === -1 ? sql.length : close + opener.length
}
// Every comment, not only the block a ';'-split schema glues above a statement. A comment between
// two keywords (`ADD /* note */ COLUMN`) is invisible to the classification regexes AND to the
// must-parse shapes, so it used to yield no target and no throw: the statement ran with no
// pre-check at all, which is the one direction this must never fail in. Postgres treats a comment
// as whitespace, so each becomes a single space. Only classification reads this; the server is
// always sent the original text.
export function sqlWithoutComments(statement: string): string {
let stripped = ''
let quote: string | undefined
for (let index = 0; index < statement.length; index += 1) {
const character = statement[index]!
if (quote !== undefined) {
stripped += character
if (character !== quote) continue
if (statement[index + 1] === quote) {
stripped += quote
index += 1
} else quote = undefined
continue
}
if (character === "'" || character === '"') {
quote = character
stripped += character
continue
}
if (character === '$') {
const end = dollarQuoteEnd(statement, index)
if (end !== undefined) {
stripped += statement.slice(index, end)
index = end - 1
continue
}
}
if (character === '-' && statement[index + 1] === '-') {
const newline = statement.indexOf('\n', index)
index = newline === -1 ? statement.length : newline
stripped += ' '
continue
}
if (character === '/' && statement[index + 1] === '*') {
// Postgres nests block comments, so a depth counter is what closes the right one.
let depth = 1
index += 2
while (index < statement.length && depth > 0) {
if (statement[index] === '/' && statement[index + 1] === '*') {
depth += 1
index += 2
} else if (statement[index] === '*' && statement[index + 1] === '/') {
depth -= 1
index += 2
} else index += 1
}
index -= 1
stripped += ' '
continue
}
stripped += character
}
return stripped.trim()
}
const CREATE_INDEX = new RegExp(
`^CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+(?:CONCURRENTLY\\s+)?(?:IF\\s+NOT\\s+EXISTS\\s+)?` +
`${QUALIFIED}\\s+ON\\s+(?:ONLY\\s+)?${QUALIFIED}`,
'i'
)
const ADD_COLUMN = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`ADD\\s+COLUMN\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?${QUALIFIED}`,
'i'
)
const ADD_CONSTRAINT = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`ADD\\s+CONSTRAINT\\s+${QUALIFIED}`,
'i'
)
// `IF EXISTS` is required, not optional. A bare `DROP CONSTRAINT` on a missing constraint is an
// error the server is supposed to raise, and skipping it would swallow that. Without a target the
// statement throws at boot instead, which tells the author to write `IF EXISTS`.
const DROP_CONSTRAINT = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`DROP\\s+CONSTRAINT\\s+IF\\s+EXISTS\\s+${QUALIFIED}`,
'i'
)
// `IF EXISTS` is required for the same reason it is on DROP CONSTRAINT: a bare `DROP INDEX` on a
// missing index is an error the server is supposed to raise. Without a target the statement throws
// at boot instead, which tells the author to write `IF EXISTS`.
const DROP_INDEX = new RegExp(`^DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?IF\\s+EXISTS\\s+${QUALIFIED}\\s*$`, 'i')
// One option per statement, and a literal value: the catalog stores reloptions as `name=value`
// text, so the pre-check compares the written pair against that array verbatim. A list of options
// is refused by `hasTopLevelComma` before it reaches here, the same as a multi-action ALTER TABLE.
const SET_RELOPTION = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`SET\\s+\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*([A-Za-z0-9_.]+)\\s*\\)\\s*$`,
'i'
)
// Every statement shape that takes a relation lock before Postgres evaluates its existence test.
// `CREATE TABLE IF NOT EXISTS` is absent on purpose: it resolves a name against the schema and
// takes no lock on an existing table.
// `DROP INDEX` is here because it takes ACCESS EXCLUSIVE on the index's table whenever the index is
// actually there, which is every boot until the first one wins. That it takes no lock once the
// index is gone is what the pre-check turns into the steady state, not a reason to omit it.
const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|ALTER\s+TABLE)\b/i
export function takesRelationLock(statement: string): boolean {
return TAKES_RELATION_LOCK.test(sqlWithoutComments(statement))
}
// Splitting on '.' is not enough: `"a.b"` is one identifier containing a dot, not two parts. Each
// part is read quote-aware, with a doubled quote unescaped to one.
function qualifiedParts(written: string): { text: string; quoted: boolean }[] {
const parts: { text: string; quoted: boolean }[] = []
let text = ''
let quoted = false
let wasQuoted = false
for (let index = 0; index < written.length; index += 1) {
const character = written[index]!
if (quoted) {
if (character !== '"') {
text += character
continue
}
if (written[index + 1] === '"') {
text += '"'
index += 1
} else quoted = false
continue
}
if (character === '"') {
quoted = true
wasQuoted = true
} else if (character === '.') {
parts.push({ text, quoted: wasQuoted })
text = ''
wasQuoted = false
} else text += character
}
parts.push({ text, quoted: wasQuoted })
return parts
}
// Postgres folds an unquoted identifier to lower case before storing it, so `Foo` is `foo` in
// relname, attname and conname. Comparing the written case would miss the row and rebuild the
// object on every boot.
function catalogName(written: string): string {
const last = qualifiedParts(written).pop()
if (!last) return written
return last.quoted ? last.text : last.text.toLowerCase()
}
// Shapes whose lock target the pre-check must be able to derive. Deliberately looser than the
// regexes that parse them, so a statement that reads as one of these but does not parse is caught
// rather than falling through to the lock path.
const MUST_PARSE = [
/^CREATE\s+(?:UNIQUE\s+)?INDEX\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+COLUMN\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bDROP\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bSET\s+\(/i,
/^DROP\s+INDEX\b/i
]
// Derived from the statement itself so a renamed index cannot drift away from its pre-check.
export function schemaLockTarget(statement: string): SchemaLockTarget | undefined {
const sql = sqlWithoutComments(statement)
const index = CREATE_INDEX.exec(sql)
if (index?.[1] && index[2]) {
return { kind: 'index', table: index[2], name: catalogName(index[1]), skipWhen: 'present' }
}
const column = ADD_COLUMN.exec(sql)
if (column?.[1] && column[2]) {
return { kind: 'column', table: column[1], name: catalogName(column[2]), skipWhen: 'present' }
}
const added = ADD_CONSTRAINT.exec(sql)
if (added?.[1] && added[2]) {
return {
kind: 'constraint',
table: added[1],
name: catalogName(added[2]),
skipWhen: 'present'
}
}
const dropped = DROP_CONSTRAINT.exec(sql)
if (dropped?.[1] && dropped[2]) {
return {
kind: 'constraint',
table: dropped[1],
name: catalogName(dropped[2]),
skipWhen: 'absent'
}
}
const droppedIndex = DROP_INDEX.exec(sql)
if (droppedIndex?.[1]) {
return { kind: 'index-by-name', name: catalogName(droppedIndex[1]), skipWhen: 'absent' }
}
const option = SET_RELOPTION.exec(sql)
if (option?.[1] && option[2] && option[3]) {
// Option names are always folded, but the value is stored as written, so only the name goes
// through catalogName. `fillfactor=70` and `fillfactor=80` are different targets, which is
// what makes a changed value re-run rather than skip.
return {
kind: 'reloption',
table: option[1],
name: `${catalogName(option[2])}=${option[3]}`,
skipWhen: 'present'
}
}
return undefined
}
const ALTER_TABLE = /^ALTER\s+TABLE\b/i
// A comma that separates ALTER TABLE subcommands rather than sitting inside a type, a default, a
// CHECK body or a dollar-quoted body. Takes comment-free SQL. Square brackets count as depth too,
// or an array type or `DEFAULT ARRAY[1, 2]` reads as a second subcommand and fails the boot.
function hasTopLevelComma(sql: string): boolean {
let depth = 0
let quote: string | undefined
for (let index = 0; index < sql.length; index += 1) {
const character = sql[index]
if (quote !== undefined) {
if (character !== quote) continue
if (sql[index + 1] === quote) index += 1
else quote = undefined
continue
}
if (character === '$') {
const end = dollarQuoteEnd(sql, index)
if (end !== undefined) {
index = end - 1
continue
}
}
if (character === "'" || character === '"') quote = character
else if (character === '(' || character === '[') depth += 1
else if (character === ')' || character === ']') depth -= 1
else if (character === ',' && depth === 0) return true
}
return false
}
// An index or column statement whose target cannot be read is the dangerous case: it would be sent
// unchecked and take the lock the pre-check exists to avoid, silently and on every boot. An
// auto-named `CREATE INDEX ON t(c)` lands here too, because nothing in the text says what the
// catalog will call it. Fail the boot with the statement instead.
export function requireSchemaLockTarget(statement: string): SchemaLockTarget | undefined {
const sql = sqlWithoutComments(statement)
// A multi-action ALTER TABLE parses to its FIRST subcommand's target only, so skipping on that
// one object would silently drop every later action for the life of the database. One action per
// statement, or no pre-check is possible.
if (ALTER_TABLE.test(sql) && hasTopLevelComma(sql)) {
throw new Error(`unparsed_schema_lock_target: ${sql}`)
}
const target = schemaLockTarget(statement)
if (target) return target
if (MUST_PARSE.some((shape) => shape.test(sql))) {
throw new Error(`unparsed_schema_lock_target: ${sql}`)
}
return undefined
}
cloud/packages/push-contract/package.json
+1 -1
View File
@@ -18,6 +18,6 @@
"devDependencies": {
"@types/node": "^24.10.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/packages/relay-contract/package.json
+1 -1
View File
@@ -18,6 +18,6 @@
"devDependencies": {
"@types/node": "^24.10.0",
"typescript": "^5.9.3",
"vitest": "^4.0.8"
"vitest": "^4.1.11"
}
}
cloud/pnpm-lock.yaml
Generated
+134 -127
View File
@@ -18,8 +18,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
apps/push:
dependencies:
@@ -64,8 +64,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
apps/relay:
dependencies:
@@ -113,8 +113,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
apps/relay-fence-broker:
dependencies:
@@ -138,8 +138,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
apps/relay-ops:
dependencies:
@@ -163,8 +163,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
packages/postgres-schema:
devDependencies:
@@ -175,8 +175,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
packages/push-contract:
dependencies:
@@ -191,8 +191,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
packages/relay-contract:
dependencies:
@@ -207,8 +207,8 @@ importers:
specifier: ^5.9.3
version: 5.9.3
vitest:
specifier: ^4.0.8
version: 4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
specifier: ^4.1.11
version: 4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
packages:
@@ -386,11 +386,12 @@ packages:
'@jridgewell/sourcemap-codec@1.5.5':
resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==}
'@napi-rs/wasm-runtime@1.1.5':
resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==}
'@napi-rs/wasm-runtime@1.2.4':
resolution: {integrity: sha512-AJxoUD2/15ESHbvpcyjU274nsAPLuOtPHCk0vKJM5pj//Fg/B1FXNWjPnXTT9PymCYYiHo4zPj0ZomXBKhoy7g==}
engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0}
peerDependencies:
'@emnapi/core': ^1.7.1
'@emnapi/runtime': ^1.7.1
'@emnapi/core': ^1.7.1 || ^2.0.0-alpha.4
'@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.4
'@oxc-project/types@0.133.0':
resolution: {integrity: sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==}
@@ -490,8 +491,8 @@ packages:
'@standard-schema/spec@1.1.0':
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
'@tybys/wasm-util@0.10.2':
resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==}
'@tybys/wasm-util@0.10.4':
resolution: {integrity: sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==}
'@types/chai@5.2.3':
resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==}
@@ -511,11 +512,11 @@ packages:
'@types/ws@8.18.1':
resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
'@vitest/expect@4.1.9':
resolution: {integrity: sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==}
'@vitest/expect@4.1.11':
resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==}
'@vitest/mocker@4.1.9':
resolution: {integrity: sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==}
'@vitest/mocker@4.1.11':
resolution: {integrity: sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==}
peerDependencies:
msw: ^2.4.9
vite: ^6.0.0 || ^7.0.0 || ^8.0.0
@@ -525,20 +526,20 @@ packages:
vite:
optional: true
'@vitest/pretty-format@4.1.9':
resolution: {integrity: sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==}
'@vitest/pretty-format@4.1.11':
resolution: {integrity: sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==}
'@vitest/runner@4.1.9':
resolution: {integrity: sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==}
'@vitest/runner@4.1.11':
resolution: {integrity: sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==}
'@vitest/snapshot@4.1.9':
resolution: {integrity: sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==}
'@vitest/snapshot@4.1.11':
resolution: {integrity: sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==}
'@vitest/spy@4.1.9':
resolution: {integrity: sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==}
'@vitest/spy@4.1.11':
resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==}
'@vitest/utils@4.1.9':
resolution: {integrity: sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==}
'@vitest/utils@4.1.11':
resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==}
agent-base@7.1.4:
resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}
@@ -660,74 +661,74 @@ packages:
jws@4.0.1:
resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==}
lightningcss-android-arm64@1.32.0:
resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==}
lightningcss-android-arm64@1.33.0:
resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==}
engines: {node: '>= 12.0.0'}
cpu: [arm64]
os: [android]
lightningcss-darwin-arm64@1.32.0:
resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==}
lightningcss-darwin-arm64@1.33.0:
resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==}
engines: {node: '>= 12.0.0'}
cpu: [arm64]
os: [darwin]
lightningcss-darwin-x64@1.32.0:
resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==}
lightningcss-darwin-x64@1.33.0:
resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==}
engines: {node: '>= 12.0.0'}
cpu: [x64]
os: [darwin]
lightningcss-freebsd-x64@1.32.0:
resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==}
lightningcss-freebsd-x64@1.33.0:
resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==}
engines: {node: '>= 12.0.0'}
cpu: [x64]
os: [freebsd]
lightningcss-linux-arm-gnueabihf@1.32.0:
resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==}
lightningcss-linux-arm-gnueabihf@1.33.0:
resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==}
engines: {node: '>= 12.0.0'}
cpu: [arm]
os: [linux]
lightningcss-linux-arm64-gnu@1.32.0:
resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==}
lightningcss-linux-arm64-gnu@1.33.0:
resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==}
engines: {node: '>= 12.0.0'}
cpu: [arm64]
os: [linux]
lightningcss-linux-arm64-musl@1.32.0:
resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==}
lightningcss-linux-arm64-musl@1.33.0:
resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==}
engines: {node: '>= 12.0.0'}
cpu: [arm64]
os: [linux]
lightningcss-linux-x64-gnu@1.32.0:
resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==}
lightningcss-linux-x64-gnu@1.33.0:
resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==}
engines: {node: '>= 12.0.0'}
cpu: [x64]
os: [linux]
lightningcss-linux-x64-musl@1.32.0:
resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==}
lightningcss-linux-x64-musl@1.33.0:
resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==}
engines: {node: '>= 12.0.0'}
cpu: [x64]
os: [linux]
lightningcss-win32-arm64-msvc@1.32.0:
resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==}
lightningcss-win32-arm64-msvc@1.33.0:
resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==}
engines: {node: '>= 12.0.0'}
cpu: [arm64]
os: [win32]
lightningcss-win32-x64-msvc@1.32.0:
resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==}
lightningcss-win32-x64-msvc@1.33.0:
resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==}
engines: {node: '>= 12.0.0'}
cpu: [x64]
os: [win32]
lightningcss@1.32.0:
resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==}
lightningcss@1.33.0:
resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==}
engines: {node: '>= 12.0.0'}
magic-string@0.30.21:
@@ -736,8 +737,8 @@ packages:
ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
nanoid@3.3.13:
resolution: {integrity: sha512-sPdqC6ByMVVGvF1ynvvMo0/o+oD1VX7DaHhijt1bFgjvBkHBib4t49GoNDhf2NDta4oeUNlaGbSt5K7qjZ955Q==}
nanoid@3.3.19:
resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
hasBin: true
@@ -798,8 +799,12 @@ packages:
resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==}
engines: {node: '>=12'}
postcss@8.5.15:
resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==}
picomatch@4.0.7:
resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==}
engines: {node: '>=12'}
postcss@8.5.28:
resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==}
engines: {node: ^10 || ^12 || >=14}
postgres-array@2.0.0:
@@ -920,20 +925,20 @@ packages:
yaml:
optional: true
vitest@4.1.9:
resolution: {integrity: sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==}
vitest@4.1.11:
resolution: {integrity: sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==}
engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0}
hasBin: true
peerDependencies:
'@edge-runtime/vm': '*'
'@opentelemetry/api': ^1.9.0
'@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0
'@vitest/browser-playwright': 4.1.9
'@vitest/browser-preview': 4.1.9
'@vitest/browser-webdriverio': 4.1.9
'@vitest/coverage-istanbul': 4.1.9
'@vitest/coverage-v8': 4.1.9
'@vitest/ui': 4.1.9
'@vitest/browser-playwright': 4.1.11
'@vitest/browser-preview': 4.1.11
'@vitest/browser-webdriverio': 4.1.11
'@vitest/coverage-istanbul': 4.1.11
'@vitest/coverage-v8': 4.1.11
'@vitest/ui': 4.1.11
happy-dom: '*'
jsdom: '*'
vite: ^6.0.0 || ^7.0.0 || ^8.0.0
@@ -1091,11 +1096,11 @@ snapshots:
'@jridgewell/sourcemap-codec@1.5.5': {}
'@napi-rs/wasm-runtime@1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)':
'@napi-rs/wasm-runtime@1.2.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)':
dependencies:
'@emnapi/core': 1.10.0
'@emnapi/runtime': 1.10.0
'@tybys/wasm-util': 0.10.2
'@tybys/wasm-util': 0.10.4
optional: true
'@oxc-project/types@0.133.0': {}
@@ -1140,7 +1145,7 @@ snapshots:
dependencies:
'@emnapi/core': 1.10.0
'@emnapi/runtime': 1.10.0
'@napi-rs/wasm-runtime': 1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)
'@napi-rs/wasm-runtime': 1.2.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)
optional: true
'@rolldown/binding-win32-arm64-msvc@1.0.3':
@@ -1153,7 +1158,7 @@ snapshots:
'@standard-schema/spec@1.1.0': {}
'@tybys/wasm-util@0.10.2':
'@tybys/wasm-util@0.10.4':
dependencies:
tslib: 2.8.1
optional: true
@@ -1181,44 +1186,44 @@ snapshots:
dependencies:
'@types/node': 24.13.2
'@vitest/expect@4.1.9':
'@vitest/expect@4.1.11':
dependencies:
'@standard-schema/spec': 1.1.0
'@types/chai': 5.2.3
'@vitest/spy': 4.1.9
'@vitest/utils': 4.1.9
'@vitest/spy': 4.1.11
'@vitest/utils': 4.1.11
chai: 6.2.2
tinyrainbow: 3.1.0
'@vitest/mocker@4.1.9(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))':
'@vitest/mocker@4.1.11(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))':
dependencies:
'@vitest/spy': 4.1.9
'@vitest/spy': 4.1.11
estree-walker: 3.0.3
magic-string: 0.30.21
optionalDependencies:
vite: 8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)
'@vitest/pretty-format@4.1.9':
'@vitest/pretty-format@4.1.11':
dependencies:
tinyrainbow: 3.1.0
'@vitest/runner@4.1.9':
'@vitest/runner@4.1.11':
dependencies:
'@vitest/utils': 4.1.9
'@vitest/utils': 4.1.11
pathe: 2.0.3
'@vitest/snapshot@4.1.9':
'@vitest/snapshot@4.1.11':
dependencies:
'@vitest/pretty-format': 4.1.9
'@vitest/utils': 4.1.9
'@vitest/pretty-format': 4.1.11
'@vitest/utils': 4.1.11
magic-string: 0.30.21
pathe: 2.0.3
'@vitest/spy@4.1.9': {}
'@vitest/spy@4.1.11': {}
'@vitest/utils@4.1.9':
'@vitest/utils@4.1.11':
dependencies:
'@vitest/pretty-format': 4.1.9
'@vitest/pretty-format': 4.1.11
convert-source-map: 2.0.0
tinyrainbow: 3.1.0
@@ -1358,54 +1363,54 @@ snapshots:
jwa: 2.0.1
safe-buffer: 5.2.1
lightningcss-android-arm64@1.32.0:
lightningcss-android-arm64@1.33.0:
optional: true
lightningcss-darwin-arm64@1.32.0:
lightningcss-darwin-arm64@1.33.0:
optional: true
lightningcss-darwin-x64@1.32.0:
lightningcss-darwin-x64@1.33.0:
optional: true
lightningcss-freebsd-x64@1.32.0:
lightningcss-freebsd-x64@1.33.0:
optional: true
lightningcss-linux-arm-gnueabihf@1.32.0:
lightningcss-linux-arm-gnueabihf@1.33.0:
optional: true
lightningcss-linux-arm64-gnu@1.32.0:
lightningcss-linux-arm64-gnu@1.33.0:
optional: true
lightningcss-linux-arm64-musl@1.32.0:
lightningcss-linux-arm64-musl@1.33.0:
optional: true
lightningcss-linux-x64-gnu@1.32.0:
lightningcss-linux-x64-gnu@1.33.0:
optional: true
lightningcss-linux-x64-musl@1.32.0:
lightningcss-linux-x64-musl@1.33.0:
optional: true
lightningcss-win32-arm64-msvc@1.32.0:
lightningcss-win32-arm64-msvc@1.33.0:
optional: true
lightningcss-win32-x64-msvc@1.32.0:
lightningcss-win32-x64-msvc@1.33.0:
optional: true
lightningcss@1.32.0:
lightningcss@1.33.0:
dependencies:
detect-libc: 2.1.2
optionalDependencies:
lightningcss-android-arm64: 1.32.0
lightningcss-darwin-arm64: 1.32.0
lightningcss-darwin-x64: 1.32.0
lightningcss-freebsd-x64: 1.32.0
lightningcss-linux-arm-gnueabihf: 1.32.0
lightningcss-linux-arm64-gnu: 1.32.0
lightningcss-linux-arm64-musl: 1.32.0
lightningcss-linux-x64-gnu: 1.32.0
lightningcss-linux-x64-musl: 1.32.0
lightningcss-win32-arm64-msvc: 1.32.0
lightningcss-win32-x64-msvc: 1.32.0
lightningcss-android-arm64: 1.33.0
lightningcss-darwin-arm64: 1.33.0
lightningcss-darwin-x64: 1.33.0
lightningcss-freebsd-x64: 1.33.0
lightningcss-linux-arm-gnueabihf: 1.33.0
lightningcss-linux-arm64-gnu: 1.33.0
lightningcss-linux-arm64-musl: 1.33.0
lightningcss-linux-x64-gnu: 1.33.0
lightningcss-linux-x64-musl: 1.33.0
lightningcss-win32-arm64-msvc: 1.33.0
lightningcss-win32-x64-msvc: 1.33.0
magic-string@0.30.21:
dependencies:
@@ -1413,7 +1418,7 @@ snapshots:
ms@2.1.3: {}
nanoid@3.3.13: {}
nanoid@3.3.19: {}
node-domexception@1.0.0: {}
@@ -1466,9 +1471,11 @@ snapshots:
picomatch@4.0.4: {}
postcss@8.5.15:
picomatch@4.0.7: {}
postcss@8.5.28:
dependencies:
nanoid: 3.3.13
nanoid: 3.3.19
picocolors: 1.1.1
source-map-js: 1.2.1
@@ -1543,9 +1550,9 @@ snapshots:
vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4):
dependencies:
lightningcss: 1.32.0
picomatch: 4.0.4
postcss: 8.5.15
lightningcss: 1.33.0
picomatch: 4.0.7
postcss: 8.5.28
rolldown: 1.0.3
tinyglobby: 0.2.17
optionalDependencies:
@@ -1554,15 +1561,15 @@ snapshots:
fsevents: 2.3.3
tsx: 4.22.4
vitest@4.1.9(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)):
vitest@4.1.11(@types/node@24.13.2)(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4)):
dependencies:
'@vitest/expect': 4.1.9
'@vitest/mocker': 4.1.9(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
'@vitest/pretty-format': 4.1.9
'@vitest/runner': 4.1.9
'@vitest/snapshot': 4.1.9
'@vitest/spy': 4.1.9
'@vitest/utils': 4.1.9
'@vitest/expect': 4.1.11
'@vitest/mocker': 4.1.11(vite@8.0.16(@types/node@24.13.2)(esbuild@0.28.1)(tsx@4.22.4))
'@vitest/pretty-format': 4.1.11
'@vitest/runner': 4.1.11
'@vitest/snapshot': 4.1.11
'@vitest/spy': 4.1.11
'@vitest/utils': 4.1.11
es-module-lexer: 2.1.0
expect-type: 1.3.0
magic-string: 0.30.21
config/build-plugins/plain-node-entry-guard.ts
+2 -1
View File
@@ -40,7 +40,8 @@ const WORKER_THREAD_ENTRY_NAMES = [
'session-scanner-opencode-sqlite-worker-entry',
'session-scanner-worker-entry',
'main-thread-hang-watchdog-entry',
'port-scan-command-worker-entry'
'port-scan-command-worker-entry',
'usage-scan-worker-entry'
] as const
export const GUARDED_ENTRY_NAMES = [
config/electron-builder.config.cjs
+13 -7
View File
@@ -14,9 +14,13 @@ const {
} = require('./packaged-runtime-node-modules.cjs')
const { verifyLinuxGlibcFloor } = require('./scripts/verify-linux-glibc-floor.cjs')
const { writeMacBuildCompatibility } = require('./scripts/mac-build-compatibility.cjs')
const {
MOBILE_WEB_BUNDLE_DIR,
assertMobileWebBundleBuilt
} = require('./scripts/verify-packaged-mobile-web-bundle.cjs')
const { verifyPackagedPluginResources } = require('./scripts/verify-packaged-plugin-resources.cjs')
const {
verifyPackagedNodePtyJobOwnership
verifyPackagedWindowsNodePty
} = require('./scripts/verify-packaged-node-pty-job-ownership.cjs')
const { verifySkillsCliRuntime } = require('./scripts/verify-skills-cli-runtime.cjs')
const { verifyStaticAppImagePackage } = require('./scripts/static-appimage-package-contract.cjs')
@@ -177,6 +181,9 @@ module.exports = {
// Why: these repo-only inputs are either bundled into out/ or copied via
// extraResources. Shipping them in app.asar bloats the desktop bundle.
'!src{,/**/*}',
// Redundant under !src above, kept explicit: the built bundle ships from out/mobile-web via the
// out rules exactly as out/web does, and the source tree must never be mistaken for it.
'!src/mobile-web{,/**/*}',
'!config{,/**/*}',
'!docs{,/**/*}',
'!mobile{,/**/*}',
@@ -289,8 +296,11 @@ module.exports = {
verifyStaticAppImagePackage(file, arch)
}
},
beforePack: (context) => {
// electron-builder calls this with the context alone. The second parameter is the bundle root,
// so a test can point the guard at a scratch bundle instead of needing the repo's out/ built.
beforePack: (context, mobileWebBundleDir = MOBILE_WEB_BUNDLE_DIR) => {
assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch)
assertMobileWebBundleBuilt(mobileWebBundleDir)
},
afterPack: async (context) => {
const resourcesDir =
@@ -353,11 +363,7 @@ module.exports = {
const hostArchEnum = archEnumByNodeArch[process.arch]
const canExecuteTargetArch = context.arch === hostArchEnum || context.arch === 4
if (context.electronPlatformName === 'win32') {
if (process.platform === 'win32' && canExecuteTargetArch) {
verifyPackagedNodePtyJobOwnership(resourcesDir)
} else {
console.log('[verify-packaged-node-pty] skipped cross-platform or cross-arch package')
}
verifyPackagedWindowsNodePty(resourcesDir, context.arch, { canExecuteTargetArch })
}
verifySkillsCliRuntime(join(resourcesDir, 'app.asar.unpacked', 'out'), resourcesDir, {
executeCommands: canExecuteTargetArch
config/knip.json
+1
View File
@@ -16,6 +16,7 @@
"src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts",
"src/main/agent-hooks/managed-agent-hook-controls.ts",
"src/main/claude-accounts/keychain.ts",
"src/mobile-web/src/bootstrap.ts",
"src/renderer/src/main.tsx",
"src/renderer/src/popout.tsx",
"src/renderer/src/web/main.tsx",
config/localization-coverage-allowlist.json
+126
View File
@@ -89,5 +89,131 @@
"text": "ghostty",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:title",
"text": "Default shell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:description",
"text": "Shell used for new terminal panes",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "shell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "terminal",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "fish",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "zsh",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "bash",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "nushell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:keywords",
"text": "default",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-attribute:title",
"text": "Terminal shell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-attribute:description",
"text": "Choose what Orca opens for new local terminal panes.",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-attribute:ariaLabel",
"text": "Terminal shell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:label",
"text": "System shell (",
"dynamic": true,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "object-property:label",
"text": "Custom shell",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-attribute:placeholder",
"text": "fish, nu, or /bin/zsh",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-attribute:aria-label",
"text": "Custom shell executable",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-text",
"text": "Enter a shell name on PATH or an executable path. Orca starts it as a login shell.",
"dynamic": false,
"count": 1
},
{
"filePath": "src/renderer/src/components/settings/TerminalPane.tsx",
"kind": "jsx-text",
"text": ". Switch to System shell or choose an executable on this host.",
"dynamic": false,
"count": 1
}
]
config/packaged-runtime-node-modules.cjs
+1
View File
@@ -607,6 +607,7 @@ module.exports = {
createPackagedRuntimeNodeModuleResources,
findAsarEntry,
isPackagedExternalSpecifier,
normalizeNodePtyWindowsArch,
packageNameFromSpecifier,
prunePackagedNodePty,
prunePackagedParcelWatcher,
config/reliability-gates.jsonc
+139
View File
@@ -482,6 +482,145 @@
"demotionRule": "Keep experimental until soak and platform evidence support promotion; do not relax value or identity assertions to hide failures."
},
{
"id": "workspace-session.ssh-host-partition-round-trip",
"title": "An SSH workspace round-trips through its own partition without losing tabs, editor files or browser state",
"maturity": "experimental",
"protection": "partial",
"owner": "workspace-session-persistence",
"layer": "persistence-integration",
"surfaces": [
"workspace session partitions",
"direct SSH remote workspace sync",
"boot session hydration"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["local", "ssh"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local", "ssh"],
"coverageNotes": "Drives the real Store against a temp profile, the real remoteWorkspace:setForConnectedTargets handler, the real session projection and the real pull-side merge. The relay transport is faked at the multiplexer boundary, so no live SSH host or relay is exercised; the partition routing and projection code under test is platform-independent. Runtime (orca environment) partitions, PTY lifecycle and mobile rendering are unaffected.",
"motivatingLinks": [
"https://github.com/stablyai/orca/issues/12721",
"https://github.com/stablyai/orca/issues/18173",
"https://github.com/stablyai/orca/blob/main/src/shared/workspace-session-partition-owner.ts"
],
"invariant": "Every workspace the `ssh:<targetId>` partition names is hydrated at boot and published to the host, whatever kind of state it holds - terminal tabs, open editor files with unsaved hot-exit drafts, browser workspaces, tab groups, or host-qualified visit recency. An empty tab row is read as a gap, never as evidence the tabs were closed, so a replace-session upload can never delete a populated host list. A workspace the local partition already holds terminal tabs for is left untouched, and every workspace routes back to the partition that owns it. Adoption never destroys what it was not told about: a host row the partition holds nothing for cannot replace a populated base row, and a bare id the read found contested is gap-filled rather than replaced, because that is exactly the id where `local` and `ssh:<targetId>` are not one workspace written twice. Tab-, pane- and file-keyed rows are recovered through the same indexes the split routed them by. An explicit empty tab row written by this build survives the round trip as a present empty row and is declined by the shared seeding predicate. Boot enumerates the partitions persistence actually holds rather than inferring them from the repo catalog, so a target whose only workspace is a folder is read like any other; a folder workspace routes to the same partition main's RuntimeWorkspaceSessionController writes it to, so one workspace is never written into two stores; and a workspace adopted out of a partition routes back to that partition even on a boot whose repo catalog cannot name the host, so the write cannot re-strand what the read just reunited. `contested` means a repo id the catalog registers on more than one host, or two SSH partitions naming one id - never the mere co-presence of 'local' and `ssh:<targetId>`, which is the repair's own input.",
"oracle": "Assert the renderer routes an SSH folder workspace to `ssh:<targetId>`, that its save carries the folder row to that partition and leaves none behind in 'local' (main applies a patch field-wise, so a partition write that omits the row erases it), and that an SSH partition no repo names is still read. Assert a bare id whose repo the catalog registers on two hosts keeps the local workspace's unsaved dirtyDraftContent and never names the rival partition as its write target, while two SSH partitions naming one id resolve to the same winner on every boot. Seed a real Store the way shipping builds leave it: the local blob holds the worktree key with an empty list while `ssh:<targetId>` holds the real one, with a second populated SSH partition present. Publish through the IPC handler with no session argument (the path the debounced writer takes) and assert the host snapshot carries the runtime-authored tabs rather than []. Separately drive the shipping split (buildWorkspaceSessionHostSnapshots) and feed its own output back through the real boot read, pinning the write and read halves to each other rather than to a hand-built fixture: an SSH workspace with open editor files, an unsaved dirtyDraftContent and no terminal tabs must come back intact, as must one with no tabsByWorktree key at all. Export and re-import through the real projection and merge through mergeDirectSshRemoteWorkspaceSession, asserting tabs survive a publish, the next pull, and an older client publishing an empty list for them. Assert the reunited workspace routes to `ssh:<targetId>`, that a workspace the base holds tabs for is not modified, and that a contested id claimed by an SSH and a runtime host does not send the SSH rows into the rotating runtime partition.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-host-partition-session-export.test.ts src/renderer/src/lib/workspace-session-ssh-partition-round-trip.test.ts src/renderer/src/lib/workspace-session-ssh-partition-ownership.test.ts src/renderer/src/lib/workspace-session-host-contention.test.ts src/shared/workspace-session-partition-owner.test.ts"
],
"testFiles": [
"src/main/ipc/ssh-host-partition-session-export.test.ts",
"src/renderer/src/lib/workspace-session-ssh-partition-round-trip.test.ts",
"src/renderer/src/lib/workspace-session-ssh-partition-ownership.test.ts",
"src/renderer/src/lib/workspace-session-host-contention.test.ts",
"src/shared/workspace-session-partition-owner.test.ts"
],
"assertionRefs": [
{
"file": "src/main/ipc/ssh-host-partition-session-export.test.ts",
"assertions": [
"publishes tabs the runtime persisted into the target ssh partition",
"never replaces the host snapshot with an empty list for a worktree that has tabs"
]
},
{
"file": "src/renderer/src/lib/workspace-session-ssh-partition-round-trip.test.ts",
"assertions": [
"hydrates tabs the runtime persisted into the ssh partition",
"adopts the stranded workspace rows alongside its tabs",
"adopts a hibernated agent record the host partition alone holds",
"leaves a workspace the local partition already holds tabs for untouched",
"routes the reunited workspace back to the partition that owns it",
"does not delete the worktree tabs across a publish and the next pull",
"publishes the stranded tabs rather than an empty list",
"does not let an empty host row destroy an unsaved draft the base alone holds",
"still adopts a populated host row over the base leftovers",
"adopts the layout of a tab the host slice names only in unifiedTabs",
"does not overwrite a contested workspace's own rows with the ssh workspace's",
"still fills a gap on a contested id",
"does not let a legacy bare recency key move another host workspace of the same id",
"still adopts a host-qualified recency key, which names its own owner",
"writes an SSH workspace emptied by this build into the partition that owns it",
"restores that tombstone as an explicit empty row, not a deleted key",
"leaves the restored workspace un-seeded by the shared seeding predicate",
"does not adopt a stale populated ssh row over a tombstone in the owning partition",
"resurrects the legacy-transition shape exactly once and not again",
"publishes the tombstone rather than a row the host can read as unknown"
]
},
{
"file": "src/renderer/src/lib/workspace-session-ssh-partition-ownership.test.ts",
"assertions": [
"routes a folder workspace to the partition main writes it to",
"keeps a folder row out of the local partition so a save cannot erase main\u2019s copy",
"reads an ssh partition that only a folder workspace owns",
"returns adopted rows to the partition they were read from with no repo catalog",
"gap-fills instead of replacing, so the local workspace keeps its unsaved draft",
"never names the rival partition as the write target for a contested id",
"gap-fills a contested id from the same partition on every boot",
"does not adopt a partition the catalog says does not own the workspace",
"does not adopt a contested id the assembled session holds no row for"
]
},
{
"file": "src/renderer/src/lib/workspace-session-host-contention.test.ts",
"assertions": [
"keeps an SSH claimant out of the rotating runtime partition",
"does not strand the runtime co-claimant when the SSH row is written"
]
},
{
"file": "src/shared/workspace-session-partition-owner.test.ts",
"assertions": [
"gives an SSH host its own partition, matching what the runtime already writes"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-15",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-host-partition-session-export.test.ts src/renderer/src/lib/workspace-session-ssh-partition-round-trip.test.ts src/renderer/src/lib/workspace-session-ssh-partition-ownership.test.ts src/renderer/src/lib/workspace-session-host-contention.test.ts src/shared/workspace-session-partition-owner.test.ts",
"result": "passed",
"durationSeconds": 7.13,
"summary": "64 tests passed across 5 files: the real Store publish, the shipping write/read split round trip, the older-client empty-publish skew direction, the contested-bare-id and empty-host-row guards, the closed-last-terminal tombstone boundary, and the partition-ownership suite covering folder-workspace routing, the persistence-side partition census, read-source write-back, and the catalog attribution that keeps a residue partition from winning the read."
}
],
"runtimeBudget": {
"p95Seconds": 30,
"scope": "Real Store on a temp profile plus renderer partition units; no launched app and no relay."
},
"flakeHistory": {
"status": "not-started",
"evidence": "Deterministic local validation only; no CI soak yet. The tests have no timers, network or real relay."
},
"redGreenEvidence": {
"status": "complete",
"evidence": "Against pristine main 12f53da542d, 9 of the 11 assertions that target the original defect fail: the boot read returns no tabs, the export publishes an empty tabsByWorktreePath row, the round trip ends with the tabs deleted, routing answers local instead of ssh:target-1, and an older client's empty publish deletes the tabs on pull. The remaining assertions guard the fix itself rather than main's bug: review found that gating adoption on terminal tabs stranded editor-only and browser-only SSH workspaces, destroying unsaved hot-exit drafts no other channel can recover. The gate now discovers a workspace through an exhaustive switch over the field-ownership table, and reintroducing the tabs-only gate fails 6 assertions including the write/read round trip."
},
"performanceBudget": {
"required": false,
"evidence": "Boot adds one session:get per SSH host that owns a repo, issued in parallel with the existing runtime partition reads and served from already-loaded main state. The publish fallback adds one partition read and a shallow keyed-record merge per target, which returns its input unchanged when nothing is stranded and stays the same order as the projection it feeds."
},
"knownGaps": [
"No live SSH host or relay is exercised; the multiplexer is faked at the request boundary.",
"The publish fallback reads only 'local' and the target's own partition, so it cannot see the local/runtime rivalry or a second SSH partition the renderer's read can. Bounded rather than closed: a key either side would judge differently is one whose repo id resolves ambiguously or to another host, and the export already publishes those to nobody.",
"A lingering `ssh:<targetId>` partition for a connection the user removed is now read by the census. Its rows stay in that partition rather than spilling into 'local', and the deregistered-repo residue sweep already clears repo-backed ones, but a folder workspace left behind by a removed connection is not swept.",
"Rows stranded beside a workspace the local partition already holds terminal tabs for are deliberately not recovered, and no assertion claims they are.",
"One-shot resurrection in the legacy-transition shape: where an older build left an empty local row while the runtime partition still holds that workspace's tabs, boot adopts them back once. Non-destructive, and non-recurrence is now asserted rather than argued - a workspace this build empties writes its empty row to the owning partition and leaves no local row behind, so there is nothing left to resurrect from.",
"A contested bare id is gap-filled rather than replaced, so rows stranded beside a contested workspace stay stranded. Separating them needs host-qualified keys through the tab store, which is the same open gap `workspace-session-host-contention.ts` records.",
"A `local` row left by a repo registration the user removed, whose repo id the catalog now resolves to the SSH host alone, is neither ambiguous nor foreign, so a populated host row still replaces it - including an unsaved `dirtyDraftContent`. Unchanged from main; closing it needs host-qualified keys, the same gap `workspace-session-host-contention.ts` records.",
"A bare `lastVisitedAtByWorktreeId` stamp left in 'local' for a workspace now owned by an SSH partition gap-fills rather than replaces, so the workspace can keep an older Cmd+J position for one boot after the migration.",
"`lastVisitedAtByWorktreeId` still discovers a workspace as adoptable from a recency entry alone. Harmless now that a recency row cannot replace another host's entry for the same bare id, but it means recency is a discovery trigger and no other field of its kind is."
],
"promotionCriteria": [
"Complete the CI soak requirement with no unexplained flakes.",
"Add coverage for a live SSH target before claiming the ssh provider is exercised end to end."
],
"demotionRule": "Keep experimental until CI soak completes. Investigate any failure without weakening the empty-row-is-a-gap oracle, which is the assertion the data-loss fix rests on."
},
{
"id": "agent-session.history-forward-read-budget",
"title": "Journal catch-up reads only the next page and one lookahead row",
config/scripts/build-mobile-web-bundle.mjs
+230
View File
@@ -0,0 +1,230 @@
import { createHash } from 'node:crypto'
import { realpathSync } from 'node:fs'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import * as esbuild from 'esbuild'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const sourceDir = join(projectDir, 'src', 'mobile-web')
const defaultOutDir = join(projectDir, 'out', 'mobile-web')
export const MOBILE_WEB_BUNDLE_SCHEMA_VERSION = 1
export const MOBILE_WEB_BUNDLE_ENTRYPOINT = 'index.html'
const CONTENT_TYPE_BY_EXTENSION = {
css: 'text/css; charset=utf-8',
html: 'text/html; charset=utf-8',
js: 'text/javascript; charset=utf-8',
png: 'image/png'
}
/**
* Canonical serialization the buildId hashes. Key order is fixed and the list is sorted by path,
* so the id is a pure function of content. Must stay byte-identical to the contract module's
* serializer in src/shared/mobile-web-bundle/.
*/
export function serializeMobileWebBundleAssets(assets) {
return JSON.stringify(
[...assets]
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
.map(({ path, sha256, byteLength, contentType }) => ({
path,
sha256,
byteLength,
contentType
}))
)
}
export function computeMobileWebBundleBuildId(assets) {
return createHash('sha256').update(serializeMobileWebBundleAssets(assets), 'utf8').digest('hex')
}
function sha256Hex(bytes) {
return createHash('sha256').update(bytes).digest('hex')
}
function contentTypeForExtension(extension) {
const contentType = CONTENT_TYPE_BY_EXTENSION[extension]
if (!contentType) {
throw new Error(`[build-mobile-web-bundle] no content type registered for .${extension}`)
}
return contentType
}
function readIntegerConstant(source, name) {
const match = new RegExp(`export const ${name} = (\\d+)`).exec(source)
if (!match) {
throw new Error(`[build-mobile-web-bundle] ${name} not found in src/shared/protocol-version.ts`)
}
return Number.parseInt(match[1], 10)
}
/**
* Parsed rather than imported because protocol-version.ts is TypeScript and this script runs on
* bare node during packaging, before any build output exists.
*/
async function readProtocolWindow() {
const source = await readFile(join(projectDir, 'src', 'shared', 'protocol-version.ts'), 'utf8')
return {
runtimeProtocolVersion: readIntegerConstant(source, 'RUNTIME_PROTOCOL_VERSION'),
// The bundle is a client: the floor it cares about is the oldest host protocol it can talk to.
minCompatibleRuntimeProtocolVersion: readIntegerConstant(
source,
'MIN_COMPATIBLE_RUNTIME_SERVER_VERSION'
)
}
}
async function readDesktopVersion() {
const packageJson = JSON.parse(await readFile(join(projectDir, 'package.json'), 'utf8'))
if (typeof packageJson.version !== 'string' || packageJson.version.length === 0) {
throw new Error('[build-mobile-web-bundle] root package.json has no version')
}
return packageJson.version
}
async function transformEntries(protocolWindow, desktopVersion) {
const result = await esbuild.build({
absWorkingDir: sourceDir,
entryPoints: [join(sourceDir, 'src', 'bootstrap.ts'), join(sourceDir, 'src', 'bootstrap.css')],
bundle: true,
minify: true,
// Virtual: write is false, so outdir only names the emitted files esbuild hands back.
outdir: 'dist',
write: false,
format: 'iife',
target: ['es2022'],
charset: 'utf8',
legalComments: 'none',
// Why no sourcemap and no metafile: both embed absolute paths, which would break reproducibility.
sourcemap: false,
logLevel: 'silent',
define: {
ORCA_MOBILE_WEB_DESKTOP_VERSION: JSON.stringify(desktopVersion),
ORCA_MOBILE_WEB_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.runtimeProtocolVersion
),
ORCA_MOBILE_WEB_MIN_COMPATIBLE_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.minCompatibleRuntimeProtocolVersion
)
}
})
const byExtension = new Map()
for (const file of result.outputFiles) {
const extension = file.path.endsWith('.css') ? 'css' : 'js'
byExtension.set(extension, Buffer.from(file.contents))
}
const script = byExtension.get('js')
const stylesheet = byExtension.get('css')
if (!script || !stylesheet) {
throw new Error('[build-mobile-web-bundle] esbuild did not emit both a script and a stylesheet')
}
return { script, stylesheet }
}
function hashedAsset(bytes, extension) {
const sha256 = sha256Hex(bytes)
return {
bytes,
path: `assets/${sha256}.${extension}`,
sha256,
byteLength: bytes.byteLength,
contentType: contentTypeForExtension(extension)
}
}
export async function buildMobileWebBundle({ outDir = defaultOutDir } = {}) {
const [desktopVersion, protocolWindow] = await Promise.all([
readDesktopVersion(),
readProtocolWindow()
])
const { script, stylesheet } = await transformEntries(protocolWindow, desktopVersion)
const mark = await readFile(join(sourceDir, 'src', 'orca-mark.png'))
const hashed = [
hashedAsset(script, 'js'),
hashedAsset(stylesheet, 'css'),
hashedAsset(mark, 'png')
]
const [scriptAsset, stylesheetAsset, markAsset] = hashed
const template = await readFile(join(sourceDir, MOBILE_WEB_BUNDLE_ENTRYPOINT), 'utf8')
const substitutions = {
__ORCA_BOOTSTRAP_JS__: scriptAsset.path,
__ORCA_BOOTSTRAP_CSS__: stylesheetAsset.path,
__ORCA_MARK_PNG__: markAsset.path
}
let html = template
for (const [token, value] of Object.entries(substitutions)) {
if (!html.includes(token)) {
throw new Error(`[build-mobile-web-bundle] ${MOBILE_WEB_BUNDLE_ENTRYPOINT} lacks ${token}`)
}
html = html.replaceAll(token, value)
}
const indexBytes = Buffer.from(html, 'utf8')
const indexAsset = {
bytes: indexBytes,
path: MOBILE_WEB_BUNDLE_ENTRYPOINT,
sha256: sha256Hex(indexBytes),
byteLength: indexBytes.byteLength,
contentType: contentTypeForExtension('html')
}
const written = [indexAsset, ...hashed]
const assets = written
.map(({ path, sha256, byteLength, contentType }) => ({ path, sha256, byteLength, contentType }))
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
const manifest = {
schemaVersion: MOBILE_WEB_BUNDLE_SCHEMA_VERSION,
buildId: computeMobileWebBundleBuildId(assets),
desktopVersion,
minCompatibleRuntimeProtocolVersion: protocolWindow.minCompatibleRuntimeProtocolVersion,
runtimeProtocolVersion: protocolWindow.runtimeProtocolVersion,
entrypoint: MOBILE_WEB_BUNDLE_ENTRYPOINT,
totalBytes: assets.reduce((total, asset) => total + asset.byteLength, 0),
assets
}
// Why a full clear: a stale asset left from an earlier build would ship unreferenced inside asar.
await rm(outDir, { recursive: true, force: true })
await mkdir(join(outDir, 'assets'), { recursive: true })
for (const asset of written) {
await writeFile(join(outDir, asset.path), asset.bytes)
}
await writeFile(join(outDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
return { manifest, outDir }
}
/**
* Whether this module was run as the entry script. Two ways to get this wrong, both of which end
* with the builder exiting 0 having written nothing: `file://${path}` never matches on Windows,
* where import.meta.url is `file:///C:/...`; and Node resolves symlinks in import.meta.url but not
* in argv[1], so `node /tmp/...` against a /private/tmp realpath compares two different strings.
* Both seams are injectable so win32 and a missing path can be exercised from a posix runner.
*/
export function isDirectInvocation(
moduleUrl,
scriptPath,
{ toFileUrl = pathToFileURL, realpath = realpathSync } = {}
) {
if (!scriptPath) {
return false
}
let resolved = scriptPath
try {
resolved = realpath(scriptPath)
} catch {
// A path that cannot be resolved cannot be this module; fall through to the literal compare.
}
return moduleUrl === toFileUrl(resolved).href
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
const { manifest, outDir } = await buildMobileWebBundle()
console.log(
`[build-mobile-web-bundle] OK — ${String(manifest.assets.length)} asset(s), ` +
`${String(manifest.totalBytes)} bytes, buildId ${manifest.buildId} -> ${outDir}`
)
}
config/scripts/build-mobile-web-bundle.test.mjs
+261
View File
@@ -0,0 +1,261 @@
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
buildMobileWebBundle,
computeMobileWebBundleBuildId,
isDirectInvocation,
serializeMobileWebBundleAssets
} from './build-mobile-web-bundle.mjs'
import {
MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS,
MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES,
assertNoCarriageReturnsInSource
} from './verify-mobile-web-bundle.mjs'
async function buildIntoScratch() {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-build-'))
const bundleDir = join(scratch, 'mobile-web')
const { manifest } = await buildMobileWebBundle({ outDir: bundleDir })
return { scratch, bundleDir, manifest }
}
describe('buildMobileWebBundle', () => {
it('emits a content-addressed bundle whose only stable name is the entrypoint', async () => {
const { scratch, bundleDir, manifest } = await buildIntoScratch()
try {
const root = await readdir(bundleDir)
expect(root.sort()).toEqual(['assets', 'index.html', 'manifest.json'])
for (const name of await readdir(join(bundleDir, 'assets'))) {
const [digest, extension] = name.split('.')
expect(digest).toMatch(/^[0-9a-f]{64}$/)
const bytes = await readFile(join(bundleDir, 'assets', name))
expect(createHash('sha256').update(bytes).digest('hex')).toBe(digest)
expect(extension).toMatch(/^(js|css|png)$/)
}
const html = await readFile(join(bundleDir, 'index.html'), 'utf8')
for (const asset of manifest.assets) {
if (asset.path !== 'index.html') {
expect(html).toContain(asset.path)
}
}
expect(html).not.toContain('__ORCA_')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('carries every manifest field the Phase A contract names', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(Object.keys(manifest)).toEqual([
'schemaVersion',
'buildId',
'desktopVersion',
'minCompatibleRuntimeProtocolVersion',
'runtimeProtocolVersion',
'entrypoint',
'totalBytes',
'assets'
])
expect(manifest.schemaVersion).toBe(1)
expect(manifest.entrypoint).toBe('index.html')
const packageJson = JSON.parse(
await readFile(new URL('../../package.json', import.meta.url), 'utf8')
)
expect(manifest.desktopVersion).toBe(packageJson.version)
const protocolSource = await readFile(
new URL('../../src/shared/protocol-version.ts', import.meta.url),
'utf8'
)
expect(protocolSource).toContain(
`export const RUNTIME_PROTOCOL_VERSION = ${String(manifest.runtimeProtocolVersion)}`
)
expect(protocolSource).toContain(
`export const MIN_COMPATIBLE_RUNTIME_SERVER_VERSION = ${String(manifest.minCompatibleRuntimeProtocolVersion)}`
)
expect(manifest.totalBytes).toBe(
manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('produces the same buildId from two independent builds', async () => {
const first = await buildIntoScratch()
const second = await buildIntoScratch()
try {
expect(second.manifest.buildId).toBe(first.manifest.buildId)
expect(second.manifest).toEqual(first.manifest)
} finally {
await rm(first.scratch, { recursive: true, force: true })
await rm(second.scratch, { recursive: true, force: true })
}
})
it('embeds no absolute path from the machine that built it', async () => {
const { scratch, bundleDir } = await buildIntoScratch()
try {
const names = [
'index.html',
'manifest.json',
...(await readdir(join(bundleDir, 'assets'))).map((name) => join('assets', name))
]
for (const name of names) {
const text = (await readFile(join(bundleDir, name))).toString('latin1')
expect(text).not.toContain(scratch)
expect(text).not.toContain(process.cwd())
}
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('stays inside the Phase A budget', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(manifest.assets.length).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS)
expect(manifest.totalBytes).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
describe('computeMobileWebBundleBuildId', () => {
const assets = [
{ path: 'index.html', sha256: 'a'.repeat(64), byteLength: 3, contentType: 'text/html' },
{ path: 'assets/b.js', sha256: 'b'.repeat(64), byteLength: 5, contentType: 'text/javascript' }
]
it('sorts by path, so input order cannot change the id', () => {
expect(computeMobileWebBundleBuildId(assets.toReversed())).toBe(
computeMobileWebBundleBuildId(assets)
)
})
it('serializes a fixed key order regardless of the input object key order', () => {
const reordered = assets.map(({ contentType, byteLength, sha256, path }) => ({
contentType,
byteLength,
sha256,
path
}))
expect(serializeMobileWebBundleAssets(reordered)).toBe(serializeMobileWebBundleAssets(assets))
})
it('changes when any hashed field changes', () => {
const baseline = computeMobileWebBundleBuildId(assets)
for (const field of ['sha256', 'byteLength', 'contentType', 'path']) {
const mutated = assets.map((asset, index) =>
index === 0 ? { ...asset, [field]: field === 'byteLength' ? 4 : `${asset[field]}x` } : asset
)
expect(computeMobileWebBundleBuildId(mutated)).not.toBe(baseline)
}
})
})
describe('isDirectInvocation', () => {
const thisFile = import.meta.filename
it('matches the path this module was loaded from', () => {
expect(isDirectInvocation(import.meta.url, thisFile)).toBe(true)
})
it('does not match a different script', () => {
expect(isDirectInvocation(import.meta.url, join(thisFile, '..', 'other.mjs'))).toBe(false)
})
it('tolerates an absent argv[1]', () => {
expect(isDirectInvocation(import.meta.url, undefined)).toBe(false)
expect(isDirectInvocation(import.meta.url, '')).toBe(false)
})
// Why an injected converter: a win32 path cannot be exercised through node:url's pathToFileURL
// on a posix runner, and CI is ubuntu.
const toWin32FileUrl = (windowsPath) => new URL(`file:///${windowsPath.replaceAll('\\', '/')}`)
it('matches a Windows entry path, which the file:// template form never does', () => {
const scriptPath = 'C:\\orca\\config\\scripts\\build-mobile-web-bundle.mjs'
const moduleUrl = 'file:///C:/orca/config/scripts/build-mobile-web-bundle.mjs'
const keepAsIs = (path) => path
expect(
isDirectInvocation(moduleUrl, scriptPath, {
toFileUrl: toWin32FileUrl,
realpath: keepAsIs
})
).toBe(true)
// The regression this guards: `file://${argv[1]}` yields file://C:\orca\... on Windows,
// so the builder exited 0 having written nothing and packaging failed downstream.
expect(`file://${scriptPath}`).not.toBe(moduleUrl)
})
it('is not written with the file:// template form', async () => {
const source = await readFile(new URL('./build-mobile-web-bundle.mjs', import.meta.url), 'utf8')
expect(source).not.toMatch(/file:\/\/\$\{process\.argv\[1\]\}/)
expect(source).toContain('pathToFileURL')
})
})
describe('mobile web source line endings', () => {
it('accepts the committed source tree', async () => {
await expect(assertNoCarriageReturnsInSource()).resolves.toBeUndefined()
})
it('rejects a CRLF source file, because CRLF changes every asset hash and the buildId', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-eol-'))
try {
await writeFile(join(scratch, 'bootstrap.ts'), 'const a = 1\r\nconst b = 2\r\n', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).rejects.toThrow(
/CRLF in mobile web source/
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('pins eol=lf for every committed text source and -text for the binary', () => {
const files = execFileSync('git', ['ls-files', 'src/mobile-web'], { encoding: 'utf8' })
.split('\n')
.filter(Boolean)
expect(files.length).toBeGreaterThanOrEqual(4)
for (const file of files) {
const attributes = execFileSync('git', ['check-attr', 'text', 'eol', '--', file], {
encoding: 'utf8'
})
if (file.endsWith('.png')) {
expect(attributes).toContain('text: unset')
} else {
expect(attributes).toContain('eol: lf')
}
}
})
})
describe('running the builder through a symlink', () => {
// Node resolves symlinks in import.meta.url but not in argv[1]. Before the guard realpath'd the
// entry path, `node /tmp/<link>` compared /tmp against /private/tmp and the builder exited 0
// having written nothing — a green packaging job with no bundle in it.
it('still recognises the entry module', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-link-'))
try {
const builderUrl = new URL('./build-mobile-web-bundle.mjs', import.meta.url).href
const real = join(scratch, 'entry.mjs')
await writeFile(
real,
`import { isDirectInvocation } from ${JSON.stringify(builderUrl)}\n` +
'process.stdout.write(String(isDirectInvocation(import.meta.url, process.argv[1])))\n',
'utf8'
)
const link = join(scratch, 'entry-link.mjs')
await symlink(real, link)
expect(execFileSync(process.execPath, [link], { encoding: 'utf8' })).toBe('true')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
config/scripts/build-windows-process-tree-relay-addon.mjs
+11 -30
View File
@@ -19,16 +19,8 @@
* node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=arm64
*/
import { execFileSync } from 'node:child_process'
import {
closeSync,
copyFileSync,
existsSync,
mkdirSync,
openSync,
readFileSync,
readSync,
writeFileSync
} from 'node:fs'
import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { createRequire } from 'node:module'
import { join, resolve } from 'node:path'
import { RELAY_WINDOWS_PROCESS_TREE_FILENAME } from '../../src/shared/relay-artifacts.ts'
import {
@@ -39,12 +31,13 @@ import {
WINDOWS_PROCESS_TREE_PACKAGE_DIR as PACKAGE_DIR
} from './windows-process-tree-gyp-rebuild.mjs'
const { PE_MACHINE, describePeMachine, readPeMachine } = createRequire(import.meta.url)(
'./windows-pe-machine.cjs'
)
const ROOT = resolve(import.meta.dirname, '..', '..')
const SUPPORTED_ARCHES = ['x64', 'arm64']
/** PE `IMAGE_FILE_HEADER.Machine` values, so a cross-build cannot silently emit host arch. */
const PE_MACHINE = { x64: 0x8664, arm64: 0xaa64 }
function parseArgs(argv) {
const arch = argv.find((a) => a.startsWith('--arch='))?.slice('--arch='.length) ?? process.arch
const outDir = argv.find((a) => a.startsWith('--out='))?.slice('--out='.length)
@@ -363,21 +356,6 @@ function applyWindowsProcessTreeBuildFixes() {
}
}
/** Read the PE machine field, so an arm64 request cannot ship an x64 binary. */
function readPeMachine(binaryPath) {
const fd = openSync(binaryPath, 'r')
try {
const header = Buffer.alloc(4)
readSync(fd, header, 0, 4, 0x3c)
const peOffset = header.readUInt32LE(0)
const machine = Buffer.alloc(2)
readSync(fd, machine, 0, 2, peOffset + 4)
return machine.readUInt16LE(0)
} finally {
closeSync(fd)
}
}
function main() {
const { arch, outDir } = parseArgs(process.argv.slice(2))
if (process.platform !== 'win32') {
@@ -410,9 +388,12 @@ function main() {
}
const machine = readPeMachine(built)
if (machine !== PE_MACHINE[arch]) {
const cause =
machine === null
? 'A truncated or quarantined build output looks like this; a relay would get a binary no host can load.'
: 'node-gyp ignored --arch; a relay would get a binary its host cannot load.'
throw new Error(
`Built binary is machine 0x${machine.toString(16)}, expected 0x${PE_MACHINE[arch].toString(16)} for ${arch}. ` +
'node-gyp ignored --arch; a relay would get a binary its host cannot load.'
`Built binary is ${describePeMachine(machine)}, expected 0x${PE_MACHINE[arch].toString(16)} for ${arch}. ${cause}`
)
}
config/scripts/check-changed-code-quality.mjs
+11
View File
@@ -24,6 +24,17 @@ export const OXLINT_SCANS = [
label: 'casting code quality',
args: ['--config', 'config/oxlint-code-quality-casting.json']
},
{
// Why the allow: CI's `audit:code-quality:native` runs before the mobile install, so it can
// never see a cycle inside mobile/ — locally, where mobile/node_modules exists, it would.
label: 'focused plugins',
args: [
'--config',
'config/oxlint-code-quality-native-plugins.json',
'--allow',
'import/no-cycle'
]
},
{
label: 'type-aware code quality',
args: ['--type-aware', '--config', 'config/oxlint-code-quality-type-aware.json']
config/scripts/check-changed-code-quality.test.mjs
+10
View File
@@ -57,6 +57,16 @@ describe('changed-code quality line matching', () => {
expect(scan.args).not.toContain('--disable-nested-config')
})
// Why: import/no-duplicates was reachable only through the repo-wide CI audit, so it first
// surfaced after push. The cycle rule stays out because CI's audit runs before the mobile install.
it('runs the focused plugin config the repo-wide audit enforces, minus the cycle rule', () => {
const scan = OXLINT_SCANS.find((candidate) => candidate.label === 'focused plugins')
expect(scan.args).toContain('config/oxlint-code-quality-native-plugins.json')
expect(scan.args).toContain('import/no-cycle')
expect(scan.args[scan.args.indexOf('import/no-cycle') - 1]).toBe('--allow')
})
it('leaves Cloud source to the independent Cloud quality checks', () => {
expect(isRootCodeQualityPath('cloud/apps/relay/src/index.ts')).toBe(false)
expect(isRootCodeQualityPath('src/main/index.ts')).toBe(true)

Some files were not shown because too many files have changed in this diff Show More