mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
Merge refreshed contrast-cache base and regenerate terminal cell cleanup patches
This commit is contained in:
@@ -23,6 +23,9 @@
|
||||
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
|
||||
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
|
||||
/config/patches/*.patch -text
|
||||
# Same reason, and pnpm parses these too: a CRLF checkout makes the mobile
|
||||
# patches unparseable, so Windows packaging dies on ERR_PNPM_INVALID_PATCH.
|
||||
/mobile/patches/*.patch -text
|
||||
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
|
||||
# source patch under xterm-src/ instead. The sibling patches stay diffable.
|
||||
/config/patches/@xterm__xterm@*.patch -diff
|
||||
@@ -41,3 +44,48 @@
|
||||
# Generated method->params catalog: compared byte-for-byte by
|
||||
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
|
||||
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
|
||||
# Mobile web bundle source. Every text byte here is hashed into an asset digest and
|
||||
# from there into buildId, so a CRLF checkout produces a different bundle id for the
|
||||
# same commit (91af2897 vs 9d78435e). The PNG is -text because it must not be touched.
|
||||
/src/mobile-web/index.html text eol=lf
|
||||
/src/mobile-web/src/*.ts text eol=lf
|
||||
/src/mobile-web/src/*.css text eol=lf
|
||||
/src/mobile-web/src/*.png -text
|
||||
# Mobile web page source. Same buildId hazard as src/mobile-web above: these bytes are
|
||||
# hashed into the Phase C bundle, so a CRLF Windows checkout would ship a different
|
||||
# buildId for identical source. web-entry/ does not exist yet; the pin lands ahead of it.
|
||||
/mobile/src/** text eol=lf
|
||||
/mobile/app/** text eol=lf
|
||||
/mobile/web-entry/** text eol=lf
|
||||
# The blanket pin above would mark a future binary as text; exempt the asset types an
|
||||
# RN page actually carries, the same way src/mobile-web exempts its PNG.
|
||||
/mobile/src/**/*.png -text
|
||||
/mobile/src/**/*.jpg -text
|
||||
/mobile/src/**/*.jpeg -text
|
||||
/mobile/src/**/*.gif -text
|
||||
/mobile/src/**/*.ico -text
|
||||
/mobile/src/**/*.webp -text
|
||||
/mobile/src/**/*.ttf -text
|
||||
/mobile/src/**/*.otf -text
|
||||
/mobile/src/**/*.woff -text
|
||||
/mobile/src/**/*.woff2 -text
|
||||
/mobile/app/**/*.png -text
|
||||
/mobile/app/**/*.jpg -text
|
||||
/mobile/app/**/*.jpeg -text
|
||||
/mobile/app/**/*.gif -text
|
||||
/mobile/app/**/*.ico -text
|
||||
/mobile/app/**/*.webp -text
|
||||
/mobile/app/**/*.ttf -text
|
||||
/mobile/app/**/*.otf -text
|
||||
/mobile/app/**/*.woff -text
|
||||
/mobile/app/**/*.woff2 -text
|
||||
/mobile/web-entry/**/*.png -text
|
||||
/mobile/web-entry/**/*.jpg -text
|
||||
/mobile/web-entry/**/*.jpeg -text
|
||||
/mobile/web-entry/**/*.gif -text
|
||||
/mobile/web-entry/**/*.ico -text
|
||||
/mobile/web-entry/**/*.webp -text
|
||||
/mobile/web-entry/**/*.ttf -text
|
||||
/mobile/web-entry/**/*.otf -text
|
||||
/mobile/web-entry/**/*.woff -text
|
||||
/mobile/web-entry/**/*.woff2 -text
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Install mobile dependencies
|
||||
description: Frozen pnpm install for the mobile/ project, whose node_modules the mobile web bundle build and the mobile-aware lint passes resolve React Native and Expo from.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# Why a separate install: mobile is its own pnpm project, so the root install leaves
|
||||
# mobile/node_modules empty and every mobile import resolves to nothing.
|
||||
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the
|
||||
# gitignored terminal/mermaid webview engine modules that tracked source imports.
|
||||
# The drift guard mirrors the root install so a stale mobile lockfile fails by name --
|
||||
# mobile's lockfile carries patchedDependencies that a silent rewrite would drop.
|
||||
- name: Install mobile dependencies
|
||||
shell: bash
|
||||
working-directory: mobile
|
||||
run: |
|
||||
pnpm install --frozen-lockfile
|
||||
# Job containers can run composite steps from a source mirror without .git.
|
||||
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
|
||||
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
|
||||
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
|
||||
fi
|
||||
@@ -184,6 +184,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
- name: Cache electron-builder downloads
|
||||
uses: actions/cache@v5
|
||||
@@ -205,6 +208,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why: signing is what makes an adhoc build installable over an existing
|
||||
# Orca, so a missing cert must fail here rather than after a 20-minute build.
|
||||
- name: Verify macOS signing environment
|
||||
|
||||
@@ -16,6 +16,8 @@ on:
|
||||
expected-rehome-generation: { required: true, type: string }
|
||||
monitor-run-id: { required: true, type: string }
|
||||
monitor-run-attempt: { required: true, type: string }
|
||||
gate-override-reason: { required: false, type: string, default: '' }
|
||||
gate-override-confirmation: { required: false, type: string, default: '' }
|
||||
wave-index: { required: true, type: string }
|
||||
|
||||
permissions:
|
||||
@@ -52,7 +54,12 @@ jobs:
|
||||
WAVE_INDEX: ${{ inputs.wave-index }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
|
||||
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
||||
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
||||
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
|
||||
# ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
|
||||
# 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
|
||||
DRAIN_PACE_WINDOW_MS: '120000'
|
||||
steps:
|
||||
- name: Require exact reusable-workflow configuration
|
||||
working-directory: .
|
||||
@@ -72,12 +79,14 @@ jobs:
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
|
||||
if test "${DEPLOY_MODE}" = verify; then
|
||||
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
|
||||
else
|
||||
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
|
||||
# The caller validated this too; re-check here so a malformed override
|
||||
# can never reach a mutation through this reusable workflow.
|
||||
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
|
||||
test "${DEPLOY_MODE}" != verify
|
||||
test "${GATE_OVERRIDE_CONFIRMATION}" = \
|
||||
"SKIP_RELAY_MONITOR_GATE ${TARGET_IMAGE_DIGEST}"
|
||||
[[ "${GATE_OVERRIDE_REASON}" =~ ^[[:print:]]{12,500}$ ]]
|
||||
fi
|
||||
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
|
||||
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
|
||||
echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
|
||||
echo "so recover each remaining cell with its own fresh monitor" >&2
|
||||
@@ -109,14 +118,34 @@ jobs:
|
||||
- uses: hashicorp/setup-terraform@v3
|
||||
with: { terraform_wrapper: false }
|
||||
|
||||
# One approved-cell table, in the wave validator the dispatch gate already uses, so
|
||||
# a cell's class and its wave's selector delta cannot drift apart between the two.
|
||||
- name: Resolve this cell's admission class and wave selector delta
|
||||
run: |
|
||||
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
|
||||
--cell-id "${TARGET_CELL_ID}")"
|
||||
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
|
||||
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
|
||||
if test "${DEPLOY_MODE}" = verify; then
|
||||
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
|
||||
else
|
||||
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
|
||||
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
|
||||
fi
|
||||
{
|
||||
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
|
||||
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
|
||||
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Require fresh aggregate monitor evidence reference
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
run: |
|
||||
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
- name: Download private aggregate monitor evidence
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
@@ -125,7 +154,7 @@ jobs:
|
||||
run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
- name: Verify monitor evidence provenance
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
run: |
|
||||
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
|
||||
--directory "${OUTPUT_DIRECTORY}" \
|
||||
@@ -138,7 +167,7 @@ jobs:
|
||||
--wave-index "${WAVE_INDEX}"
|
||||
|
||||
- name: Download this wave's single-use safety authority
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
@@ -147,7 +176,7 @@ jobs:
|
||||
run-id: ${{ github.run_id }}
|
||||
|
||||
- name: Require safety evidence consumed by this workflow
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
run: |
|
||||
# Mutations are single-dispatch: a fresh dispatch cannot resume a
|
||||
# partial batch (the canary authority binds the batch-entry selector
|
||||
@@ -184,9 +213,32 @@ jobs:
|
||||
# Freshness-only failures are publish lag, not health, on every wave
|
||||
# including the first; the CLI still caps the retry at the wave's
|
||||
# evidence-age budget, so this cannot mutate on aged evidence.
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-index "${WAVE_INDEX}" --retry-freshness
|
||||
#
|
||||
# This live recheck runs on every mutating wave, including a
|
||||
# break-glass one. With the aggregate gate overridden there is no
|
||||
# sealed state to read, so the expected selector comes from the
|
||||
# dispatch inputs the rehome inspect below verifies against the live
|
||||
# director; every threshold the sample is judged against is unchanged.
|
||||
if test -n "${GATE_OVERRIDE_CONFIRMATION}"; then
|
||||
jq -n \
|
||||
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
|
||||
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
|
||||
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
|
||||
| map_values(split(",") | map(select(length > 0)))' \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
|
||||
pnpm incident:relay-preflight -- \
|
||||
--no-monitor-state \
|
||||
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
|
||||
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
|
||||
else
|
||||
pnpm incident:relay-preflight -- \
|
||||
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
|
||||
--wave-index "${WAVE_INDEX}" \
|
||||
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
|
||||
fi
|
||||
|
||||
- name: Require durable rehome disabled and exact selector
|
||||
env:
|
||||
@@ -215,6 +267,11 @@ jobs:
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c17|c18)
|
||||
EXPECTED_HARD_CAP=600
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c27|c28|c29)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=asia-east2
|
||||
@@ -230,10 +287,6 @@ jobs:
|
||||
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
|
||||
-var-file=environments/production.tfvars \
|
||||
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
|
||||
if test "${EXPECTED_REGION}" = us-central1; then
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
|
||||
<<< "${SOURCE_CELLS}" >/dev/null
|
||||
fi
|
||||
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
|
||||
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
|
||||
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
|
||||
@@ -255,6 +308,14 @@ jobs:
|
||||
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
|
||||
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
|
||||
fi
|
||||
# The startup template emits rehome trust lines only for a declared source cell, so
|
||||
# require membership exactly when either side of this roll expects those lines.
|
||||
if test "${EXPECTED_REGION}" = us-central1 && {
|
||||
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
|
||||
}; then
|
||||
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
|
||||
<<< "${SOURCE_CELLS}" >/dev/null
|
||||
fi
|
||||
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
|
||||
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
|
||||
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
|
||||
@@ -299,20 +360,66 @@ jobs:
|
||||
}
|
||||
CURRENT_RUNTIME="$(admin_post current-runtime \
|
||||
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
|
||||
# A rollback that failed between template apply and admission restore
|
||||
# leaves the cell already on the rollback image; resume from that
|
||||
# state instead of demanding the pre-rollback predecessor.
|
||||
# Two different failures leave the cell on the rollback image, and the image
|
||||
# alone cannot tell them apart. A rollback that failed between its template
|
||||
# apply and its admission restore restarted the cell, so that cell is not
|
||||
# draining and resumes. A wave that stopped after its drain and before its
|
||||
# template apply never restarted anything, so its cell is still draining and
|
||||
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
|
||||
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
|
||||
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
|
||||
ROLLBACK_RESUME=true
|
||||
if test "${LIVE_DRAINING}" = true; then
|
||||
ROLLBACK_STAGE=stranded
|
||||
else
|
||||
ROLLBACK_STAGE=resume
|
||||
fi
|
||||
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
|
||||
else
|
||||
ROLLBACK_RESUME=false
|
||||
ROLLBACK_STAGE=roll
|
||||
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
|
||||
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
|
||||
fi
|
||||
if test "${ROLLBACK_STAGE}" = resume; then
|
||||
ROLLBACK_RESUME=true
|
||||
else
|
||||
ROLLBACK_RESUME=false
|
||||
fi
|
||||
# A stranded cell's template still carries the image the cell is serving, so that
|
||||
# is the predecessor its plan is reviewed against. A template already moved on to
|
||||
# the target is refused here rather than rolled backwards under a stale review.
|
||||
if test "${ROLLBACK_STAGE}" = stranded; then
|
||||
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
|
||||
else
|
||||
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
|
||||
fi
|
||||
# Rollback is the documented recovery from a failed canary, which
|
||||
# leaves the cell migration-only (and possibly still marked
|
||||
# draining); apply and verify still require the cell pristine in the
|
||||
# class it is declared to serve in.
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
PRECHECK_ADMISSION=general-or-migration-only
|
||||
else
|
||||
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
|
||||
fi
|
||||
# Draining sheds connections, and a migration-only cell holds none, so the flag
|
||||
# carries no precondition there. It also outlives a failed wave, because the drain
|
||||
# that set it is followed by no restart, which is the state a failed canary leaves.
|
||||
if test "${DEPLOY_MODE}" = rollback \
|
||||
|| test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
PRECHECK_DRAINING=either
|
||||
else
|
||||
PRECHECK_DRAINING=forbidden
|
||||
fi
|
||||
# A resumed rollback already restarted, so its cell has to come back not draining;
|
||||
# that is what separates it from a wave that stopped before its template apply.
|
||||
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
|
||||
PREDECESSOR_DRAINING_OK=true
|
||||
else
|
||||
PREDECESSOR_DRAINING_OK=false
|
||||
fi
|
||||
RESTORED_MIGRATION_CELLS="$(jq -rn \
|
||||
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
|
||||
--arg target "${TARGET_CELL_ID}" \
|
||||
@@ -333,8 +440,19 @@ jobs:
|
||||
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
|
||||
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
|
||||
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
|
||||
# A migration-only cell is already isolated and is handed back isolated, so both
|
||||
# halves of its wave see exactly the membership it entered with.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
|
||||
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
|
||||
fi
|
||||
{
|
||||
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
|
||||
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
|
||||
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
|
||||
# The drain wait and the plan review both read the image this cell actually
|
||||
# serves, which is the rollback image on a stranded cell and not the current one.
|
||||
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
|
||||
# The failsafe consumes these; deriving them here keeps them
|
||||
# defined for a failure in any later step.
|
||||
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
|
||||
@@ -353,8 +471,7 @@ jobs:
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
|
||||
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
|
||||
(.region == $region or
|
||||
($region == "us-central1" and $protocol == 0 and .region == null)) and
|
||||
@@ -370,8 +487,7 @@ jobs:
|
||||
--argjson hardCap "${EXPECTED_HARD_CAP}" \
|
||||
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
|
||||
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
|
||||
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
|
||||
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
|
||||
'[
|
||||
if .role != "cell" then "role" else empty end,
|
||||
if .cellId != $cell then "cellId" else empty end,
|
||||
@@ -407,16 +523,6 @@ jobs:
|
||||
fi
|
||||
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
|
||||
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
|
||||
# Rollback is the documented recovery from a failed canary, which
|
||||
# leaves the cell migration-only (and possibly still marked
|
||||
# draining); apply and verify still require a pristine general cell.
|
||||
if test "${DEPLOY_MODE}" = rollback; then
|
||||
PRECHECK_ADMISSION=general-or-migration-only
|
||||
PRECHECK_DRAINING=either
|
||||
else
|
||||
PRECHECK_ADMISSION=general
|
||||
PRECHECK_DRAINING=forbidden
|
||||
fi
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
@@ -442,18 +548,25 @@ jobs:
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
|
||||
echo "${ISOLATE_RESULT}"
|
||||
# Isolating a migration-only cell must be a read-only no-op; a change here would
|
||||
# mean the live class is not the one this wave planned around.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
|
||||
fi
|
||||
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
|
||||
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
|
||||
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
|
||||
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
|
||||
# The wait has to outlast the pacing window as well as the leases it waits on.
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat either --admission migration-only --draining required \
|
||||
--activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
|
||||
--timeout-ms 900000
|
||||
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
|
||||
--timeout-ms 1020000
|
||||
|
||||
- id: capacity-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
@@ -466,6 +579,7 @@ jobs:
|
||||
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
|
||||
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
|
||||
run: |
|
||||
# A cell on the root pool default emits no pool line, so pin one only where it exists.
|
||||
@@ -511,6 +625,7 @@ jobs:
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${DESIRED_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
@@ -538,20 +653,37 @@ jobs:
|
||||
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
|
||||
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
|
||||
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
|
||||
| node dev/scripts/validate-relay-capacity-plan.mjs \
|
||||
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
|
||||
--hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
|
||||
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
|
||||
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
|
||||
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
|
||||
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
|
||||
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
|
||||
"${POOL_ARGUMENTS[@]}"
|
||||
"${POOL_ARGUMENTS[@]}")"
|
||||
echo "${PLAN_REVIEW}"
|
||||
terraform -chdir=infra/terraform apply -auto-approve \
|
||||
"${RUNNER_TEMP}/relay-same-cap.tfplan"
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
# A stranded cell already runs the reviewed template, so the apply above replaces
|
||||
# no instance and the drain flag, which only a restart clears, would survive the
|
||||
# whole wave. Roll the MIG explicitly on exactly the policy a template change uses.
|
||||
# Every field is passed: gcloud persists these into the MIG's update policy, and it
|
||||
# defaults the method to substitute on a group with no stateful config, so omitting
|
||||
# one drifts the policy off the reviewed one and fails every later targeted plan.
|
||||
if test "${ROLLBACK_STAGE}" = stranded \
|
||||
&& test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then
|
||||
gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
|
||||
--replacement-method recreate --max-surge 0 --max-unavailable 1
|
||||
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
|
||||
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
|
||||
fi
|
||||
|
||||
- id: post-auth
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
@@ -626,28 +758,36 @@ jobs:
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
|
||||
--cell-incarnation "${TARGET_INCARNATION}"
|
||||
|
||||
- name: Restore only the verified selected cell to general admission
|
||||
- name: Restore only the verified selected cell to its entry admission
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
env:
|
||||
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
|
||||
run: |
|
||||
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
|
||||
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
# Activating a migration-only cell would promote it to a serving cell for good, so
|
||||
# restore it with the idempotent isolate that reports the authoritative generation.
|
||||
if test "${ENTRY_ADMISSION}" = migration-only; then
|
||||
RESTORE_MODE=isolate
|
||||
else
|
||||
RESTORE_MODE=activate
|
||||
fi
|
||||
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
|
||||
echo "${ACTIVATE_RESULT}"
|
||||
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
|
||||
<<< "${ACTIVATE_RESULT}")"
|
||||
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
|
||||
echo "${RESTORE_RESULT}"
|
||||
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
|
||||
<<< "${RESTORE_RESULT}")"
|
||||
node dev/scripts/verify-relay-capacity-transition.mjs \
|
||||
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
|
||||
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
|
||||
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
|
||||
--heartbeat fresh --admission general --draining forbidden --activity allowed \
|
||||
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
|
||||
--draining forbidden --activity allowed \
|
||||
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
|
||||
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
|
||||
node dev/scripts/operate-relay-regional-rehome.mjs \
|
||||
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
|
||||
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
|
||||
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
|
||||
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
|
||||
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
|
||||
|
||||
@@ -69,6 +69,17 @@ on:
|
||||
description: Exact digest-and-cell-bound mutation confirmation
|
||||
required: false
|
||||
type: string
|
||||
gate-override-reason:
|
||||
description: >-
|
||||
Break-glass only: why this wave may skip the aggregate 15-minute monitor
|
||||
dry-run gate. The live per-wave preflight still runs.
|
||||
required: false
|
||||
type: string
|
||||
gate-override-confirmation:
|
||||
description: >-
|
||||
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
@@ -111,11 +122,16 @@ jobs:
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
||||
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
||||
run: |
|
||||
# Fails closed on a partial or mismatched override, before any mutation.
|
||||
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
|
||||
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
|
||||
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
|
||||
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
||||
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
|
||||
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
|
||||
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
|
||||
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
|
||||
@@ -123,6 +139,35 @@ jobs:
|
||||
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Record the monitor gate override in the run summary
|
||||
if: ${{ inputs.gate-override-confirmation != '' }}
|
||||
env:
|
||||
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
||||
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
||||
ACTOR: ${{ github.actor }}
|
||||
MODE: ${{ inputs.mode }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
run: |
|
||||
{
|
||||
echo '## Aggregate monitor gate overridden (break-glass)'
|
||||
echo
|
||||
echo '| field | value |'
|
||||
echo '| --- | --- |'
|
||||
echo "| actor | ${ACTOR} |"
|
||||
echo "| mode | ${MODE} |"
|
||||
echo "| cells | ${CELL_IDS} |"
|
||||
echo "| target digest | \`${TARGET_DIGEST}\` |"
|
||||
echo "| reason | ${GATE_OVERRIDE_REASON} |"
|
||||
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
|
||||
echo
|
||||
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
|
||||
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
|
||||
echo 'durable rehome disabled, the exact selector generation and membership, the'
|
||||
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
|
||||
echo 'single-dispatch mutation.'
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Download exact prior canary authority
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -136,17 +181,23 @@ jobs:
|
||||
if: ${{ inputs.mode == 'batch-apply' }}
|
||||
env:
|
||||
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
|
||||
CELL_IDS: ${{ inputs.cell-ids }}
|
||||
TARGET_DIGEST: ${{ inputs.target-image-digest }}
|
||||
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
|
||||
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
|
||||
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
|
||||
run: |
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
|
||||
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
|
||||
--target-digest "${{ inputs.target-image-digest }}" \
|
||||
--rollback-digest "${{ inputs.rollback-image-digest }}" \
|
||||
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
||||
--rehome-generation "${{ inputs.expected-rehome-generation }}"
|
||||
--cell-ids "${CELL_IDS}" \
|
||||
--target-digest "${TARGET_DIGEST}" \
|
||||
--rollback-digest "${ROLLBACK_DIGEST}" \
|
||||
--selector-generation "${SELECTOR_GENERATION}" \
|
||||
--rehome-generation "${REHOME_GENERATION}"
|
||||
|
||||
- name: Reject previously consumed aggregate safety evidence
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
@@ -163,7 +214,7 @@ jobs:
|
||||
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
|
||||
|
||||
- name: Consume aggregate safety evidence for this exact wave
|
||||
if: ${{ inputs.mode != 'verify' }}
|
||||
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
|
||||
@@ -188,6 +239,8 @@ jobs:
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '0'
|
||||
secrets: inherit
|
||||
|
||||
@@ -209,6 +262,8 @@ jobs:
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '1'
|
||||
secrets: inherit
|
||||
|
||||
@@ -230,6 +285,8 @@ jobs:
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '2'
|
||||
secrets: inherit
|
||||
|
||||
@@ -251,6 +308,8 @@ jobs:
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '3'
|
||||
secrets: inherit
|
||||
|
||||
@@ -266,6 +325,10 @@ jobs:
|
||||
with: { node-version: 24 }
|
||||
|
||||
- name: Seal exact successful canary authority
|
||||
env:
|
||||
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
|
||||
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
|
||||
ACTOR: ${{ github.actor }}
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
|
||||
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
|
||||
@@ -276,6 +339,9 @@ jobs:
|
||||
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
|
||||
--selector-generation "${{ inputs.expected-selector-generation }}" \
|
||||
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
|
||||
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
|
||||
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
|
||||
--actor "${ACTOR}" \
|
||||
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -57,7 +57,28 @@ jobs:
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: dist/win-unpacked
|
||||
key: win-unpacked-${{ hashFiles('src/**', 'config/**', 'package.json', 'pnpm-lock.yaml') }}
|
||||
# mobile/ is in the key because beforePack requires out/mobile-web, whose bytes come from
|
||||
# the mobile install and, once Phase C flips the bundle, from the page trees below; a
|
||||
# mobile-only change must miss this cache, not reuse a stale installer. src/** and
|
||||
# config/** already cover src/mobile-web and the two bundle builders.
|
||||
key: >-
|
||||
win-unpacked-${{ hashFiles(
|
||||
'src/**',
|
||||
'config/**',
|
||||
'package.json',
|
||||
'pnpm-lock.yaml',
|
||||
'mobile/package.json',
|
||||
'mobile/pnpm-lock.yaml',
|
||||
'mobile/app/**',
|
||||
'mobile/src/**',
|
||||
'mobile/web-entry/**'
|
||||
) }}
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules. Gated with the
|
||||
# build it feeds, so a cache hit does not pay for an install nothing consumes.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
if: steps.cache-unpacked.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Build unpacked app
|
||||
if: steps.cache-unpacked.outputs.cache-hit != 'true'
|
||||
|
||||
@@ -156,6 +156,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
- name: Cache electron-builder downloads
|
||||
if: steps.freshness.outputs.should_build == 'true'
|
||||
@@ -179,6 +182,11 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
if: steps.freshness.outputs.should_build == 'true'
|
||||
|
||||
# Why: signing is what makes a daily installable over an existing Orca, so
|
||||
# a missing cert must fail here rather than after a 20-minute build.
|
||||
- name: Verify macOS signing environment
|
||||
|
||||
@@ -203,6 +203,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# Caches the Electron binary and electron-builder's tool downloads (nsis,
|
||||
# winCodeSign). Same key shape as release-cut's Windows leg.
|
||||
@@ -229,6 +232,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why the packaging check runs before the 20-minute build: it only needs
|
||||
# node_modules, and a stale config should cost seconds rather than a build.
|
||||
- name: Verify dev-channel packaging identity
|
||||
|
||||
@@ -164,6 +164,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
- name: Cache electron-builder downloads
|
||||
uses: actions/cache@v5
|
||||
@@ -185,6 +188,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why: signing is what makes an hourly installable over an existing Orca, so
|
||||
# a missing cert must fail here rather than after a 20-minute build.
|
||||
- name: Verify macOS signing environment
|
||||
|
||||
@@ -9,6 +9,18 @@ on:
|
||||
- ready_for_review
|
||||
paths:
|
||||
- 'mobile/**'
|
||||
# Mobile launch contracts exercise the real host dispatcher and durable receipt store.
|
||||
- 'src/main/agent-launch/**'
|
||||
- 'src/main/runtime/rpc/**'
|
||||
- 'src/main/runtime/runtime-rpc/**'
|
||||
- 'src/main/runtime/runtime-rpc.ts'
|
||||
- 'src/main/runtime/device-registry.ts'
|
||||
- 'src/main/runtime/orca-runtime.ts'
|
||||
- 'src/main/runtime/agent-session-*.ts'
|
||||
- 'src/main/native-chat/agent-session-wire/**'
|
||||
- 'src/shared/agent-launch-*.ts'
|
||||
- 'src/shared/agent-session-*.ts'
|
||||
- 'src/shared/new-workspace/worktree-create-collision.ts'
|
||||
# Why: the mobile terminal link parsers are conformance-tested against
|
||||
# these shared fixtures; desktop-side fixture edits must re-run this suite.
|
||||
- 'src/shared/terminal-file-link-conformance.ts'
|
||||
@@ -21,11 +33,6 @@ on:
|
||||
# schema edit anywhere under here changes mobile's types, so a desktop-only
|
||||
# change can break mobile's typecheck with no other mobile signal.
|
||||
- 'src/shared/rpc-contract/**'
|
||||
# Why: the catalog above holds params only. This file is the sole holder of
|
||||
# the agent.launch RESULT shape, and mobile imports it as a value, not just
|
||||
# a type. CROSS_VERSION_WIRE_PREFIXES already treats it as wire-critical, so
|
||||
# without this one gate classes it that way while this one cannot see it.
|
||||
- 'src/shared/agent-launch-intent.ts'
|
||||
# Why: this job holds the only checks that load the Fastfile, so edits to
|
||||
# it or to the release workflow it guards must re-run them.
|
||||
- '.github/workflows/mobile.yml'
|
||||
@@ -92,6 +99,19 @@ jobs:
|
||||
- name: Typecheck
|
||||
run: pnpm typecheck
|
||||
|
||||
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
|
||||
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
|
||||
# drifted. This fails when a test file that checks today stops checking, when a test leaves
|
||||
# the program, and on @ts-nocheck; the baseline may only shrink.
|
||||
- name: Typecheck tests (ratchet)
|
||||
run: pnpm run check:tests-typecheck
|
||||
|
||||
# This includes the bridged replay of the whole recording corpus, which used to be a second
|
||||
# step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point:
|
||||
# it fails when a divergence class grows, when a divergence lands in no class at all, or when
|
||||
# one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is
|
||||
# ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite,
|
||||
# so folding it in costs a fraction of that in wall time and one step less to skip.
|
||||
- name: Test
|
||||
run: pnpm test
|
||||
|
||||
|
||||
+89
-17
@@ -29,6 +29,7 @@ jobs:
|
||||
should_run: ${{ steps.filter.outputs.should_run }}
|
||||
native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
|
||||
mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
|
||||
mobile_web_app: ${{ steps.filter.outputs.mobile_web_app }}
|
||||
static_analysis: ${{ steps.filter.outputs.static_analysis }}
|
||||
typecheck: ${{ steps.filter.outputs.typecheck }}
|
||||
git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
|
||||
@@ -142,24 +143,11 @@ jobs:
|
||||
- name: Enforce type-aware code-quality baseline
|
||||
run: pnpm run audit:code-quality:type-aware
|
||||
|
||||
# Why: the changed-code gate lints mobile files too, and its type-aware pass
|
||||
# resolves types from mobile/node_modules. Mobile is a separate pnpm project,
|
||||
# so the root install above leaves it empty and every mobile type degrades to
|
||||
# an `error` type — reported as phantom findings against the changed lines.
|
||||
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
|
||||
# the gitignored terminal/mermaid webview engine modules that tracked source imports,
|
||||
# and skipping it degrades those very types the step exists to resolve. The drift
|
||||
# guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
|
||||
# lockfile carries patchedDependencies that a silent rewrite would drop.
|
||||
- name: Install mobile dependencies
|
||||
# Why here: the changed-code gate lints mobile files too, and its type-aware pass
|
||||
# resolves types from mobile/node_modules. Without the install every mobile type
|
||||
# degrades to an `error` type — reported as phantom findings against the changed lines.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
if: needs.code_paths.outputs.mobile_dependencies == 'true'
|
||||
working-directory: mobile
|
||||
run: |
|
||||
pnpm install --frozen-lockfile
|
||||
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
|
||||
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
|
||||
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
|
||||
fi
|
||||
|
||||
- name: Enforce changed-code quality
|
||||
run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
|
||||
@@ -663,6 +651,64 @@ jobs:
|
||||
pnpm exec vitest run --config config/vitest.config.ts \
|
||||
src/main/orcad/external-chromium-browser-process.integration.test.ts
|
||||
|
||||
# Why its own job: it needs mobile/node_modules and a real browser, and the sharded `test`
|
||||
# matrix would pay for both on every shard to run two files. Dark through Phase C: this proves
|
||||
# `build:mobile-web:app` on every PR that touches the page, and ships nothing -- packaging still
|
||||
# builds the Phase A bootstrap via build:mobile-web.
|
||||
mobile_web_app:
|
||||
name: mobile web app bundle
|
||||
needs: [code_paths]
|
||||
if: needs.code_paths.outputs.mobile_web_app == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Why no native-runtime: the builder is esbuild and the render check is a browser. Nothing
|
||||
# in this job loads node-pty.
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# The entry lives in mobile/ so one React resolves; without this every RN import is nothing.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why the runner's Google Chrome and not a downloaded chromium: same reason as the orcad
|
||||
# browser job -- Ubuntu 24.04 only ships an AppArmor userns profile for the Chrome .deb.
|
||||
# Why fail instead of skip: a silently skipped render check is the failure this job exists
|
||||
# to prevent.
|
||||
- name: Resolve Chrome for the render check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chrome="$(command -v google-chrome || command -v google-chrome-stable || true)"
|
||||
if [ -z "$chrome" ]; then
|
||||
echo "::error::No Google Chrome on the runner; the render check would silently skip."
|
||||
exit 1
|
||||
fi
|
||||
"$chrome" --version
|
||||
echo "ORCA_MOBILE_WEB_RENDER_BROWSER=$chrome" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build and verify the app bundle
|
||||
run: pnpm run build:mobile-web:app
|
||||
|
||||
# The bundling tests skip themselves where mobile dependencies are absent, which is how they
|
||||
# stay green in the sharded `test` job. This is the job that installs them, so here a missing
|
||||
# install has to fail rather than skip everything the job exists to run.
|
||||
- name: Builder, override census and render check
|
||||
env:
|
||||
ORCA_MOBILE_WEB_APP_DEPS_REQUIRED: '1'
|
||||
run: |
|
||||
pnpm exec vitest run --config config/vitest.config.ts \
|
||||
config/scripts/build-mobile-web-app-bundle.test.mjs \
|
||||
config/scripts/mobile-web-app-web-overrides.test.mjs \
|
||||
config/scripts/mobile-web-app-render.test.mjs
|
||||
|
||||
cross-version-wire:
|
||||
name: cross-version wire compatibility
|
||||
needs: [code_paths]
|
||||
@@ -697,6 +743,8 @@ jobs:
|
||||
tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts
|
||||
tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts
|
||||
tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts
|
||||
tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts
|
||||
tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts
|
||||
|
||||
managed_hook_node18:
|
||||
name: managed hooks on Node 18
|
||||
@@ -748,6 +796,13 @@ jobs:
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why --no-file-parallelism: every file here launches a full Electron stack twice, and each
|
||||
# probe carries its own in-process deadline. Four at once on a 4-vCPU runner starve each other
|
||||
@@ -782,6 +837,12 @@ jobs:
|
||||
- name: Project web client from renderer build
|
||||
run: pnpm run build:web-from-renderer
|
||||
|
||||
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
|
||||
# step instead of calling build:release, and electron-builder's beforePack guard hard-fails
|
||||
# without out/mobile-web.
|
||||
- name: Build mobile web bundle
|
||||
run: pnpm run build:mobile-web
|
||||
|
||||
- name: Build native components
|
||||
run: pnpm run build:native
|
||||
|
||||
@@ -855,6 +916,13 @@ jobs:
|
||||
with:
|
||||
native-runtime: node
|
||||
persist-native-cache: 'false'
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
- name: Save compiled Node native modules
|
||||
if: steps.deps.outputs.native-cache-hit != 'true'
|
||||
@@ -1013,6 +1081,7 @@ jobs:
|
||||
- shell_contracts
|
||||
- test
|
||||
- orcad_browser
|
||||
- mobile_web_app
|
||||
- cross-version-wire
|
||||
- managed_hook_node18
|
||||
- package
|
||||
@@ -1049,6 +1118,8 @@ jobs:
|
||||
TEST_SHOULD_RUN: ${{ needs.code_paths.outputs.test }}
|
||||
ORCAD_BROWSER: ${{ needs.orcad_browser.result }}
|
||||
ORCAD_BROWSER_SHOULD_RUN: ${{ needs.code_paths.outputs.orcad_browser }}
|
||||
MOBILE_WEB_APP: ${{ needs.mobile_web_app.result }}
|
||||
MOBILE_WEB_APP_SHOULD_RUN: ${{ needs.code_paths.outputs.mobile_web_app }}
|
||||
CROSS_VERSION_WIRE: ${{ needs.cross-version-wire.result }}
|
||||
CROSS_VERSION_WIRE_SHOULD_RUN: ${{ needs.code_paths.outputs.cross-version-wire }}
|
||||
MANAGED_HOOK_NODE18: ${{ needs.managed_hook_node18.result }}
|
||||
@@ -1091,6 +1162,7 @@ jobs:
|
||||
check_job shell_contracts "$SHELL_CONTRACTS" "$SHELL_CONTRACTS_SHOULD_RUN"
|
||||
check_job test "$TEST" "$TEST_SHOULD_RUN"
|
||||
check_job orcad_browser "$ORCAD_BROWSER" "$ORCAD_BROWSER_SHOULD_RUN"
|
||||
check_job mobile_web_app "$MOBILE_WEB_APP" "$MOBILE_WEB_APP_SHOULD_RUN"
|
||||
check_job cross-version-wire "$CROSS_VERSION_WIRE" "$CROSS_VERSION_WIRE_SHOULD_RUN"
|
||||
check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN"
|
||||
check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN"
|
||||
|
||||
@@ -871,8 +871,17 @@ jobs:
|
||||
npm install -g node-gyp@11.5.0
|
||||
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
|
||||
|
||||
# Why: this install runs lifecycle scripts, so node-gyp rebuilds
|
||||
# native/windows-registry and fetches that Node version's headers from
|
||||
# nodejs.org. One `read ECONNRESET` there failed this blocking gate and the
|
||||
# whole cut. Retry like the release build's install below.
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
uses: nick-fields/retry@v4
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build Electron app for platform golden
|
||||
run: npx electron-vite build --mode e2e
|
||||
@@ -1088,8 +1097,14 @@ jobs:
|
||||
npm install -g node-gyp@11.5.0
|
||||
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
|
||||
|
||||
# Same node-gyp header fetch as the blocking golden gate above.
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
uses: nick-fields/retry@v4
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build Electron app for terminal rendering evidence
|
||||
run: npx electron-vite build --mode e2e
|
||||
@@ -1204,22 +1219,33 @@ jobs:
|
||||
# ref, so cutting from an older/off-main ref whose tree predates a composite
|
||||
# action would fail the step with "Can't find 'action.yml'". Restore the
|
||||
# actions directory from the commit this workflow file itself came from.
|
||||
# Not Windows-only: every platform now consumes install-mobile-dependencies, so
|
||||
# any of them can be the one whose cut ref predates the action.
|
||||
- name: Restore composite actions from the workflow ref
|
||||
if: matrix.platform == 'win' && github.run_attempt == 1
|
||||
shell: bash
|
||||
env:
|
||||
WORKFLOW_SHA: ${{ github.workflow_sha }}
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
action_path=".github/actions/install-signpath-module/action.yml"
|
||||
if [ -f "$action_path" ]; then
|
||||
required=(.github/actions/install-mobile-dependencies/action.yml)
|
||||
if [ "$PLATFORM" = win ] && [ "$GITHUB_RUN_ATTEMPT" = 1 ]; then
|
||||
required+=(.github/actions/install-signpath-module/action.yml)
|
||||
fi
|
||||
missing=()
|
||||
for action_path in "${required[@]}"; do
|
||||
[ -f "$action_path" ] || missing+=("$action_path")
|
||||
done
|
||||
if [ "${#missing[@]}" -eq 0 ]; then
|
||||
echo "Composite actions already present at the cut ref."
|
||||
exit 0
|
||||
fi
|
||||
echo "Cut ref predates $action_path; restoring it from $WORKFLOW_SHA."
|
||||
echo "Cut ref predates ${missing[*]}; restoring from $WORKFLOW_SHA."
|
||||
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
|
||||
git checkout "$WORKFLOW_SHA" -- .github/actions
|
||||
test -f "$action_path"
|
||||
for action_path in "${required[@]}"; do
|
||||
test -f "$action_path"
|
||||
done
|
||||
|
||||
# pnpm must be on PATH before setup-node so setup-node can locate the store for caching.
|
||||
- name: Setup pnpm
|
||||
@@ -1232,6 +1258,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# Why: release builds hit the same native-module postinstall path as
|
||||
# PR CI, so keep the pinned node-gyp override here too instead of
|
||||
@@ -1272,6 +1301,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why: `pnpm build:release` verifies the Linux computer-use provider by
|
||||
# importing AT-SPI bindings, which are runtime package deps but are not
|
||||
# present on stock GitHub Ubuntu release runners.
|
||||
|
||||
@@ -47,6 +47,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
# Cache the Electron binary + electron-builder tool downloads (notarytool,
|
||||
# winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job, incl. mac.
|
||||
@@ -74,6 +77,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
- name: Verify macOS signing environment
|
||||
run: node config/scripts/verify-macos-release-env.mjs
|
||||
env:
|
||||
|
||||
@@ -55,6 +55,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
@@ -67,6 +70,9 @@ jobs:
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: dist/orca-windows-setup.exe
|
||||
# The mobile page trees are in the key because beforePack builds the mobile web bundle
|
||||
# into the installer; src/** and config/** already cover src/mobile-web and the two
|
||||
# bundle builders. A mobile-only change must miss this cache, not reuse a stale exe.
|
||||
key: >-
|
||||
crash-survival-installer-${{ hashFiles(
|
||||
'src/**',
|
||||
@@ -85,7 +91,12 @@ jobs:
|
||||
'.npmrc',
|
||||
'package.json',
|
||||
'pnpm-lock.yaml',
|
||||
'pnpm-workspace.yaml'
|
||||
'pnpm-workspace.yaml',
|
||||
'mobile/package.json',
|
||||
'mobile/pnpm-lock.yaml',
|
||||
'mobile/app/**',
|
||||
'mobile/src/**',
|
||||
'mobile/web-entry/**'
|
||||
) }}
|
||||
|
||||
# Why: production edits miss the installer cache by design, but Electron
|
||||
@@ -101,6 +112,12 @@ jobs:
|
||||
restore-keys: |
|
||||
crash-survival-electron-builder-
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules. Gated with the
|
||||
# build it feeds, so a cache hit does not pay for an install nothing consumes.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
if: steps.cache-installer.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Build Windows installer (unsigned)
|
||||
if: steps.cache-installer.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
|
||||
@@ -75,6 +75,12 @@ jobs:
|
||||
path: dist/orca-windows-setup.exe
|
||||
key: branch-installer-${{ hashFiles('src/**', 'config/**', 'native/**', 'resources/win32/**', 'package.json', 'pnpm-lock.yaml') }}
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules. Gated with the
|
||||
# build it feeds, so a cache hit does not pay for an install nothing consumes.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
if: steps.cache-installer.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Build Windows installer (unsigned)
|
||||
if: steps.cache-installer.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
|
||||
@@ -57,6 +57,9 @@ jobs:
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: |
|
||||
pnpm-lock.yaml
|
||||
mobile/pnpm-lock.yaml
|
||||
|
||||
- name: Cache electron-builder downloads
|
||||
uses: actions/cache@v5
|
||||
@@ -78,6 +81,10 @@ jobs:
|
||||
retry_wait_seconds: 30
|
||||
command: pnpm install --frozen-lockfile
|
||||
|
||||
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
|
||||
# build resolves React Native and Expo from mobile/node_modules.
|
||||
- uses: ./.github/actions/install-mobile-dependencies
|
||||
|
||||
# Why: rehearsal builds are never published, so the official-build
|
||||
# secrets (telemetry key, diagnostics URL) are intentionally omitted.
|
||||
- name: Build app
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import type { AdmissionSelector } from './incident-selector.js'
|
||||
|
||||
const directories: string[] = []
|
||||
@@ -63,6 +64,38 @@ function stateFile(
|
||||
return path
|
||||
}
|
||||
|
||||
// Every configured production cell, in the lexicographic order
|
||||
// normalizeSelectorMembership canonicalises to. Derived from the same durable
|
||||
// Terraform config the override path reads, so a new cell cannot strand these.
|
||||
const configuredCellIds = relayOpsEnvironment('production').cells.map(
|
||||
(cell) => cell.cellId
|
||||
)
|
||||
const canonicalCellIds = [...configuredCellIds].sort()
|
||||
const canonicalMembership = {
|
||||
existingOnly: canonicalCellIds,
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
|
||||
// What the director reports: a normalised selector, never an echo of what the
|
||||
// caller expected. An order-sensitive comparison only holds if the override path
|
||||
// canonicalises its own input the same way.
|
||||
function canonicalSample(generation = 1): IncidentSample {
|
||||
const next = sample()
|
||||
next.selector = { generation, membership: canonicalMembership }
|
||||
return next
|
||||
}
|
||||
|
||||
function membershipFile(
|
||||
membership: Record<string, string[]> = canonicalMembership
|
||||
): string {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-selector-'))
|
||||
directories.push(directory)
|
||||
const path = join(directory, 'selector.json')
|
||||
writeFileSync(path, JSON.stringify(membership))
|
||||
return path
|
||||
}
|
||||
|
||||
function sample(): IncidentSample {
|
||||
const observedAt = new Date(now).toISOString()
|
||||
const signal = (value: number) => ({ value, observedAt })
|
||||
@@ -154,9 +187,9 @@ describe('relay incident live preflight', () => {
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('rejects monitor evidence beyond the 25-minute lineage bound', async () => {
|
||||
it('rejects monitor evidence beyond the 35-minute lineage bound', async () => {
|
||||
const path = stateFile('strict', {
|
||||
startedAt: new Date(now - 26 * 60_000 - 1).toISOString()
|
||||
startedAt: new Date(now - 36 * 60_000 - 1).toISOString()
|
||||
})
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', path],
|
||||
@@ -183,24 +216,31 @@ describe('relay incident live preflight', () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', oneRollOld, '--wave-index', '1'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
// Both edges of one predecessor job timeout: 5min + 75min exactly.
|
||||
// Wave 0 edges: the 10-minute bound covers same-cap job start-up latency.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(80 * 60_000), '--wave-index', '1'], deps
|
||||
['--state-file', agedState(10 * 60_000), '--wave-index', '0'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(80 * 60_000 + 1), '--wave-index', '1'], deps
|
||||
['--state-file', agedState(10 * 60_000 + 1), '--wave-index', '0'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
// Both edges of one predecessor job timeout: 10min + 75min exactly.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(85 * 60_000), '--wave-index', '1'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(85 * 60_000 + 1), '--wave-index', '1'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(155 * 60_000), '--wave-index', '2'], deps
|
||||
['--state-file', agedState(160 * 60_000), '--wave-index', '2'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(155 * 60_000 + 1), '--wave-index', '2'], deps
|
||||
['--state-file', agedState(160 * 60_000 + 1), '--wave-index', '2'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(230 * 60_000), '--wave-index', '3'], deps
|
||||
['--state-file', agedState(235 * 60_000), '--wave-index', '3'], deps
|
||||
)).resolves.toBeUndefined()
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedState(230 * 60_000 + 1), '--wave-index', '3'], deps
|
||||
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
|
||||
)).rejects.toThrow('monitor evidence is incomplete or stale')
|
||||
// The wave index is a strict single-use 0-3 argument.
|
||||
await expect(runIncidentLivePreflight(
|
||||
@@ -260,9 +300,11 @@ describe('relay incident live preflight', () => {
|
||||
slowCell.sources['active-probe']!.signals[
|
||||
'cell.production-gce-c1.latency_ms'
|
||||
]!.value = 2_568
|
||||
// A cell probe now re-samples before it fails a wave, so the wait is injected;
|
||||
// this cell stays slow on every sample and still names what stopped it.
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: async () => slowCell }
|
||||
{ now: () => now, collect: async () => slowCell, wait: async () => {} }
|
||||
)).rejects.toThrow(
|
||||
'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
|
||||
)
|
||||
@@ -278,6 +320,150 @@ describe('relay incident live preflight', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Why: this one sample decides a mutating wave, so an Asia cell's ~30 s
|
||||
// "no healthy upstream" window could still fail a wave here even after the
|
||||
// 15-minute gate learned to ride it out.
|
||||
describe('cell probe tolerance', () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
|
||||
// Serves `badSamples` unhealthy cell readings, then healthy ones.
|
||||
const downThen = (badSamples: number) => {
|
||||
let index = 0
|
||||
return async () => {
|
||||
const next = sample()
|
||||
if (index++ < badSamples) {
|
||||
next.sources['active-probe']!.signals['cell.production-gce-c1.health']!
|
||||
.value = 0
|
||||
next.sources['active-probe']!.signals['cell.production-gce-c1.ready']!
|
||||
.value = 0
|
||||
}
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
it('re-samples through a probe outage within the tolerance', async () => {
|
||||
const waits: number[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: downThen(tolerance),
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(waits).toHaveLength(tolerance)
|
||||
})
|
||||
|
||||
it('fails the wave once the probe outage outlasts the tolerance', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: downThen(tolerance + 1),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('active-probe/threshold_equal cell.production-gce-c1.health')
|
||||
})
|
||||
|
||||
it('does not re-sample a director probe failure', async () => {
|
||||
let samples = 0
|
||||
const down = async () => {
|
||||
samples++
|
||||
const next = sample()
|
||||
next.sources['active-probe']!.signals['director.health']!.value = 0
|
||||
return next
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: down, wait: async () => {} }
|
||||
)).rejects.toThrow('active-probe/threshold_equal director.health')
|
||||
expect(samples).toBe(1)
|
||||
})
|
||||
|
||||
it('does not re-sample a non-probe threshold failure', async () => {
|
||||
let samples = 0
|
||||
const hot = async () => {
|
||||
samples++
|
||||
const next = sample()
|
||||
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
|
||||
return next
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{ now: () => now, collect: hot, wait: async () => {} }
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max')
|
||||
expect(samples).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// Why: on 2026-09-17 a canary wave died because one director admin read
|
||||
// returned 404 for a 2 s Cloud SQL pool timeout. Collecting the sample is not
|
||||
// a health verdict, so a thrown collector spends an attempt instead.
|
||||
describe('collector failures', () => {
|
||||
it('re-samples after a thrown collector and then passes', async () => {
|
||||
const waits: number[] = []
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
if (attempts === 1) throw new Error('Relay admin telemetry returned 404')
|
||||
return sample()
|
||||
},
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(attempts).toBe(2)
|
||||
expect(waits).toEqual([15_000])
|
||||
})
|
||||
|
||||
it('fails the wave when every attempt throws, naming the collector', async () => {
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', stateFile()],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
throw new Error('Relay admin telemetry returned 404')
|
||||
},
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow(
|
||||
'relay live preflight failed: collector: Relay admin telemetry returned 404'
|
||||
)
|
||||
expect(attempts).toBe(1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples)
|
||||
})
|
||||
|
||||
it('does not re-sample a collector failure past the evidence-age budget', async () => {
|
||||
const agedPath = stateFile('strict', {
|
||||
startedAt: new Date(now - 26 * 60_000).toISOString(),
|
||||
windowStartedAt: new Date(now - 25 * 60_000).toISOString(),
|
||||
lastSampleAt: new Date(now - 10 * 60_000 + 7).toISOString(),
|
||||
completedAt: new Date(now - 10 * 60_000 + 14).toISOString()
|
||||
})
|
||||
let attempts = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
['--state-file', agedPath],
|
||||
{
|
||||
now: () => now,
|
||||
collect: async () => {
|
||||
attempts++
|
||||
throw new Error('Relay admin telemetry returned 404')
|
||||
},
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('relay live preflight failed: collector:')
|
||||
expect(attempts).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
it('enforces the signed migration policy', async () => {
|
||||
const inactiveTarget = sample()
|
||||
inactiveTarget.sources['director-admin']!.signals[
|
||||
@@ -374,7 +560,7 @@ describe('relay incident live preflight', () => {
|
||||
})
|
||||
|
||||
it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
|
||||
const completedAt = now - 290_000
|
||||
const completedAt = now - 590_000
|
||||
const stale = sample()
|
||||
stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
|
||||
const collect = vi.fn(async () => stale)
|
||||
@@ -420,6 +606,253 @@ describe('relay incident live preflight', () => {
|
||||
expect(wait).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
|
||||
|
||||
// Why: the same-cap break-glass skips the sealed 15-minute aggregate evidence,
|
||||
// so this live recheck is the only thing left standing between the dispatch and
|
||||
// a mutation. It must judge the fleet exactly as it does with evidence, and it
|
||||
// must never accept a half-specified override.
|
||||
describe('break-glass without monitor state', () => {
|
||||
const overrideArgs = (extra: string[] = [], membership = canonicalMembership) => [
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '1',
|
||||
'--selector-membership-file', membershipFile(membership),
|
||||
...extra
|
||||
]
|
||||
|
||||
// The director's reading, plus whatever the override path decided to expect.
|
||||
const liveCollect = (
|
||||
mutate: (next: IncidentSample) => IncidentSample = (next) => next
|
||||
) => async (expected: AdmissionSelector) => {
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return mutate(next)
|
||||
}
|
||||
|
||||
it('accepts one complete fresh green sample with no sealed evidence', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
// Why: the live selector is normalised and the comparison is an ordered
|
||||
// stringify, so an operator's unsorted membership must canonicalise here or
|
||||
// every override wave reads as selector drift on a healthy fleet.
|
||||
it('canonicalises an unsorted operator membership', async () => {
|
||||
const shuffled = {
|
||||
existingOnly: [...canonicalCellIds].reverse(),
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
expect(shuffled.existingOnly).not.toEqual(canonicalCellIds)
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], shuffled),
|
||||
{
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(seen[0]!.membership.existingOnly).toEqual(canonicalCellIds)
|
||||
})
|
||||
|
||||
// Why: normalising is also what enforces every configured cell exactly once,
|
||||
// which a bare schema parse would have dropped.
|
||||
it('rejects a membership that is not every configured cell exactly once', async () => {
|
||||
const duplicated = {
|
||||
existingOnly: [...canonicalCellIds, canonicalCellIds[0] as string],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], duplicated),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
const missing = {
|
||||
existingOnly: canonicalCellIds.slice(1),
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], missing),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
const unknown = {
|
||||
existingOnly: [...canonicalCellIds.slice(1), 'production-gce-c999'],
|
||||
migrationOnly: [],
|
||||
general: []
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs([], unknown),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('every configured cell exactly once')
|
||||
})
|
||||
|
||||
it('fails closed on a live threshold breach', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.99
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('cloud-monitoring/threshold_max cloud_sql.cpu')
|
||||
})
|
||||
|
||||
it('fails closed on a live selector mismatch', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.selector = { ...next.selector, generation: 7 }
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('selector_mismatch')
|
||||
})
|
||||
|
||||
it('expects the wave-adjusted live selector generation', async () => {
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--wave-index', '2']),
|
||||
{
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(seen[0]!.generation).toBe(5)
|
||||
})
|
||||
|
||||
it('offsets by the wave delta the cell class declares', async () => {
|
||||
const generationFor = async (args: string[]) => {
|
||||
const seen: AdmissionSelector[] = []
|
||||
await expect(runIncidentLivePreflight(args, {
|
||||
now: () => now,
|
||||
collect: async (expected) => {
|
||||
seen.push(expected)
|
||||
const next = canonicalSample(expected.generation)
|
||||
next.expectedSelector = expected
|
||||
return next
|
||||
}
|
||||
})).resolves.toBeUndefined()
|
||||
return seen[0]!.generation
|
||||
}
|
||||
// A migration-only cell's wave isolates and restores nothing, so no predecessor moved it.
|
||||
expect(await generationFor(
|
||||
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0'])
|
||||
)).toBe(1)
|
||||
expect(await generationFor(
|
||||
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2'])
|
||||
)).toBe(5)
|
||||
})
|
||||
|
||||
it('rejects a selector wave delta no cell class produces', async () => {
|
||||
for (const delta of ['1', '3', '4', '', '-0', '02']) {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--selector-wave-delta', delta]),
|
||||
{ now: () => now }
|
||||
)).rejects.toThrow('usage:')
|
||||
}
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']),
|
||||
{ now: () => now }
|
||||
)).rejects.toThrow('usage:')
|
||||
})
|
||||
|
||||
it('pins the strictest migration policy', async () => {
|
||||
// An inactive migration target is tolerable only under recover-forward,
|
||||
// and an override cannot elect that policy, so this must still fail.
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
next.sources['director-admin']!.signals[
|
||||
'cell.production-gce-c1.migration_target_inactive'
|
||||
]!.value = 30
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).rejects.toThrow('director-admin/threshold_max')
|
||||
})
|
||||
|
||||
it('rejects a half-specified override', async () => {
|
||||
const cases: string[][] = [
|
||||
['--no-monitor-state'],
|
||||
['--no-monitor-state', '--expected-selector-generation', '1'],
|
||||
['--no-monitor-state', '--selector-membership-file', membershipFile()],
|
||||
// Mixing the two sources would let a caller pass sealed evidence it
|
||||
// never wants read.
|
||||
[
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '1',
|
||||
'--selector-membership-file', membershipFile(),
|
||||
'--state-file', stateFile()
|
||||
],
|
||||
// Override arguments without the flag must not be silently ignored.
|
||||
['--state-file', stateFile(), '--expected-selector-generation', '1'],
|
||||
['--no-monitor-state', '--no-monitor-state'],
|
||||
['--expected-selector-generation', '1']
|
||||
]
|
||||
for (const args of cases) {
|
||||
await expect(runIncidentLivePreflight(
|
||||
args,
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('usage:')
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects an unknown option and a negative generation', async () => {
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(['--skip-everything']),
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow('usage:')
|
||||
await expect(runIncidentLivePreflight(
|
||||
[
|
||||
'--no-monitor-state',
|
||||
'--expected-selector-generation', '-1',
|
||||
'--selector-membership-file', membershipFile()
|
||||
],
|
||||
{ now: () => now, collect: liveCollect() }
|
||||
)).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('re-samples a tolerable failure and then passes', async () => {
|
||||
let samples = 0
|
||||
await expect(runIncidentLivePreflight(
|
||||
overrideArgs(),
|
||||
{
|
||||
now: () => now,
|
||||
collect: liveCollect((next) => {
|
||||
samples++
|
||||
if (samples === 1) {
|
||||
next.sources['cloud-monitoring']!.signals['director.instances']!.value = 2
|
||||
}
|
||||
return next
|
||||
}),
|
||||
wait: async () => {}
|
||||
}
|
||||
)).resolves.toBeUndefined()
|
||||
expect(samples).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
it('uses the supplied admin token without minting through gcloud', async () => {
|
||||
const identityToken = vi.fn(async () => 'minted.token.value')
|
||||
const gcloud = livePreflightGcloud(
|
||||
|
||||
@@ -2,13 +2,21 @@ import { readFile } from 'node:fs/promises'
|
||||
import { resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { z } from 'zod'
|
||||
import { relayOpsEnvironment } from './environment-config.js'
|
||||
import { createGcloudClient } from './gcloud-client.js'
|
||||
import { suppliedIdentityToken } from './incident-monitor-cli.js'
|
||||
import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
|
||||
import {
|
||||
AdmissionSelectorSchema,
|
||||
normalizeSelectorMembership,
|
||||
SelectorMembershipSchema,
|
||||
type AdmissionSelector
|
||||
} from './incident-selector.js'
|
||||
import {
|
||||
evaluateIncidentSample,
|
||||
FRESHNESS_FAILURE_CODES,
|
||||
INCIDENT_MONITOR_THRESHOLDS,
|
||||
preDrainDryRunPassed,
|
||||
toleratedStreakKey,
|
||||
type IncidentFailure,
|
||||
type IncidentSample
|
||||
} from './incident-monitor.js'
|
||||
@@ -16,10 +24,15 @@ import { createIncidentSampleCollector } from './incident-monitor-sources.js'
|
||||
|
||||
const FRESHNESS_RETRY_ATTEMPTS = 5
|
||||
const FRESHNESS_RETRY_INTERVAL_MS = 15_000
|
||||
const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
|
||||
// 10 min, not 5: the same-cap job reaches this check ~5 min after the monitor
|
||||
// completes (runner queue ~2 min, gate job ~80 s, checkout ~60 s); on 2026-09-17
|
||||
// a green gate died at 302 s. The live samples below hold every wave to now.
|
||||
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
|
||||
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
|
||||
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
|
||||
const WAVE_INDEX_PATTERN = /^[0-3]$/
|
||||
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
|
||||
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
|
||||
|
||||
export function livePreflightGcloud(
|
||||
gcloud: ReturnType<typeof createGcloudClient>,
|
||||
@@ -81,51 +94,80 @@ function describeFailure(failure: IncidentFailure): string {
|
||||
return [`${failure.source}/${failure.code}`, ...detail].join(' ')
|
||||
}
|
||||
|
||||
export async function runIncidentLivePreflight(
|
||||
argv: string[],
|
||||
dependencies: {
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
|
||||
gcloud?: ReturnType<typeof createGcloudClient>
|
||||
environment?: NodeJS.ProcessEnv
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
const PREFLIGHT_USAGE =
|
||||
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>]' +
|
||||
' [--selector-wave-delta <0|2>] [--retry-freshness]' +
|
||||
' | --no-monitor-state --expected-selector-generation <n>' +
|
||||
' --selector-membership-file <json> [--wave-index <0-3>]' +
|
||||
' [--selector-wave-delta <0|2>]'
|
||||
|
||||
const VALUE_OPTIONS = new Set([
|
||||
'--state-file',
|
||||
'--wave-index',
|
||||
'--selector-wave-delta',
|
||||
'--expected-selector-generation',
|
||||
'--selector-membership-file'
|
||||
])
|
||||
const FLAG_OPTIONS = new Set(['--retry-freshness', '--no-monitor-state'])
|
||||
|
||||
// Rejects an unknown option and a repeated one, so a typo can never silently
|
||||
// widen what this check accepts.
|
||||
export function parsePreflightArgs(argv: string[]): {
|
||||
options: Map<string, string>
|
||||
flags: Set<string>
|
||||
} {
|
||||
const args = argv[0] === '--' ? argv.slice(1) : argv
|
||||
const freshnessRetryCount = args.filter((arg) => arg === '--retry-freshness').length
|
||||
const rest = args.filter((arg) => arg !== '--retry-freshness')
|
||||
const stateArgs: string[] = []
|
||||
let waveIndex = '0'
|
||||
let waveIndexCount = 0
|
||||
for (let index = 0; index < rest.length; index += 1) {
|
||||
if (rest[index] === '--wave-index') {
|
||||
waveIndexCount += 1
|
||||
waveIndex = rest[index + 1] ?? ''
|
||||
index += 1
|
||||
} else {
|
||||
stateArgs.push(rest[index] as string)
|
||||
const options = new Map<string, string>()
|
||||
const flags = new Set<string>()
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index] as string
|
||||
if (FLAG_OPTIONS.has(arg)) {
|
||||
if (flags.has(arg)) throw new Error(PREFLIGHT_USAGE)
|
||||
flags.add(arg)
|
||||
continue
|
||||
}
|
||||
const value = args[index + 1]
|
||||
if (!VALUE_OPTIONS.has(arg) || options.has(arg) || !value) {
|
||||
throw new Error(PREFLIGHT_USAGE)
|
||||
}
|
||||
options.set(arg, value)
|
||||
index += 1
|
||||
}
|
||||
return { options, flags }
|
||||
}
|
||||
|
||||
// What the live recheck measures the fleet against. Either source supplies the
|
||||
// exact same fields; only where they come from and how they can go stale differs.
|
||||
type PreflightPlan = {
|
||||
environment: 'production'
|
||||
expectedSelector: AdmissionSelector
|
||||
migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
|
||||
recoverySourceCellId: string | null
|
||||
capacityCellId: string | null
|
||||
// The instant the live samples age from. Monitor evidence ages from the moment
|
||||
// the 15-minute window closed; an override has no evidence to age, so its
|
||||
// retry budget starts when this process does.
|
||||
evidenceAnchorMs: number
|
||||
}
|
||||
|
||||
async function monitorEvidencePreflightPlan(
|
||||
options: Map<string, string>,
|
||||
nowMs: number,
|
||||
waveIndex: string
|
||||
): Promise<PreflightPlan> {
|
||||
const stateFile = options.get('--state-file')
|
||||
if (
|
||||
freshnessRetryCount > 1 ||
|
||||
waveIndexCount > 1 ||
|
||||
!WAVE_INDEX_PATTERN.test(waveIndex) ||
|
||||
stateArgs.length !== 2 ||
|
||||
stateArgs[0] !== '--state-file' ||
|
||||
!stateArgs[1]
|
||||
) {
|
||||
throw new Error(
|
||||
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>] [--retry-freshness]'
|
||||
)
|
||||
}
|
||||
!stateFile ||
|
||||
options.has('--expected-selector-generation') ||
|
||||
options.has('--selector-membership-file')
|
||||
) throw new Error(PREFLIGHT_USAGE)
|
||||
const state = PreflightStateSchema.parse(
|
||||
JSON.parse(await readFile(resolve(stateArgs[1]), 'utf8'))
|
||||
JSON.parse(await readFile(resolve(stateFile), 'utf8'))
|
||||
)
|
||||
const now = dependencies.now ?? Date.now
|
||||
const completedAt = Date.parse(state.completedAt)
|
||||
const windowStartedAt = Date.parse(state.windowStartedAt)
|
||||
const lastSampleAt = Date.parse(state.lastSampleAt)
|
||||
const evidenceAgeMs = now() - completedAt
|
||||
const evidenceAgeMs = nowMs - completedAt
|
||||
// Later same-cap waves start after sequential predecessor cell rolls, so the
|
||||
// freshness bound grows by one cell-job timeout per predecessor; the live
|
||||
// samples collected below still hold every wave to current health.
|
||||
@@ -144,19 +186,89 @@ export async function runIncidentLivePreflight(
|
||||
) {
|
||||
throw new Error('relay live preflight monitor evidence is incomplete or stale')
|
||||
}
|
||||
return {
|
||||
environment: state.environment,
|
||||
expectedSelector: state.expectedSelector,
|
||||
migrationPolicy: state.migrationPolicy,
|
||||
recoverySourceCellId: state.recoverySourceCellId,
|
||||
capacityCellId: state.capacityCellId,
|
||||
evidenceAnchorMs: completedAt
|
||||
}
|
||||
}
|
||||
|
||||
// Break-glass: the caller authorized skipping the aggregate 15-minute monitor
|
||||
// gate, so the expected selector comes straight from the dispatch inputs instead
|
||||
// of sealed evidence. Nothing about this weakens the live sample below, and the
|
||||
// policy is pinned to strict -- the only one the same-cap rollout ever verifies.
|
||||
async function overridePreflightPlan(
|
||||
options: Map<string, string>,
|
||||
nowMs: number
|
||||
): Promise<PreflightPlan> {
|
||||
const generation = options.get('--expected-selector-generation')
|
||||
const membershipFile = options.get('--selector-membership-file')
|
||||
if (!generation || !membershipFile || options.has('--state-file')) {
|
||||
throw new Error(PREFLIGHT_USAGE)
|
||||
}
|
||||
// Canonicalise exactly as the monitor CLI does when it seals evidence. The live
|
||||
// selector read from the director is normalised too and the comparison is an
|
||||
// ordered stringify, so unsorted operator input would read as selector drift on
|
||||
// a healthy fleet; normalising is also what enforces every configured cell
|
||||
// exactly once.
|
||||
const membership = normalizeSelectorMembership(
|
||||
SelectorMembershipSchema.parse(
|
||||
JSON.parse(await readFile(resolve(membershipFile), 'utf8'))
|
||||
),
|
||||
new Set(relayOpsEnvironment('production').cells.map((cell) => cell.cellId))
|
||||
)
|
||||
return {
|
||||
environment: 'production',
|
||||
expectedSelector: AdmissionSelectorSchema.parse({
|
||||
generation: Number(generation),
|
||||
membership
|
||||
}),
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
evidenceAnchorMs: nowMs
|
||||
}
|
||||
}
|
||||
|
||||
export async function runIncidentLivePreflight(
|
||||
argv: string[],
|
||||
dependencies: {
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
collect?: (expectedSelector: AdmissionSelector) => Promise<IncidentSample>
|
||||
gcloud?: ReturnType<typeof createGcloudClient>
|
||||
environment?: NodeJS.ProcessEnv
|
||||
} = {}
|
||||
): Promise<void> {
|
||||
const parsed = parsePreflightArgs(argv)
|
||||
const waveIndex = parsed.options.get('--wave-index') ?? '0'
|
||||
if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE)
|
||||
const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2'
|
||||
if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE)
|
||||
const now = dependencies.now ?? Date.now
|
||||
const plan = parsed.flags.has('--no-monitor-state')
|
||||
? await overridePreflightPlan(parsed.options, now())
|
||||
: await monitorEvidencePreflightPlan(parsed.options, now(), waveIndex)
|
||||
const maxEvidenceAgeMs =
|
||||
MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
|
||||
const gcloud = livePreflightGcloud(
|
||||
dependencies.gcloud ?? createGcloudClient(),
|
||||
dependencies.environment
|
||||
)
|
||||
// Each predecessor same-cap apply wave reversibly isolates and restores its
|
||||
// cell, advancing the selector generation by exactly 2 with membership
|
||||
// unchanged (rollback is single-cell, so it never reaches a later wave), so
|
||||
// the live selector comparison must expect the wave-adjusted generation.
|
||||
// cell with membership unchanged (rollback is single-cell, so it never reaches
|
||||
// a later wave), so the live selector comparison must expect the wave-adjusted
|
||||
// generation. A general cell advances it by 2; a migration-only cell is already
|
||||
// isolated and stays that way, so its wave advances it by 0. A wave is never
|
||||
// mixed, so one delta covers every predecessor.
|
||||
const collectOptions = {
|
||||
environment: state.environment,
|
||||
environment: plan.environment,
|
||||
expectedSelector: {
|
||||
...state.expectedSelector,
|
||||
generation: state.expectedSelector.generation + 2 * Number(waveIndex)
|
||||
...plan.expectedSelector,
|
||||
generation: plan.expectedSelector.generation + Number(selectorWaveDelta) * Number(waveIndex)
|
||||
},
|
||||
...(dependencies.now ? { now: dependencies.now } : {})
|
||||
}
|
||||
@@ -167,32 +279,71 @@ export async function runIncidentLivePreflight(
|
||||
const wait = dependencies.wait ?? ((ms: number) => new Promise<void>((resolveWait) => {
|
||||
setTimeout(resolveWait, ms)
|
||||
}))
|
||||
const attempts = freshnessRetryCount === 1 ? FRESHNESS_RETRY_ATTEMPTS : 1
|
||||
const freshnessAttempts = parsed.flags.has('--retry-freshness') ? FRESHNESS_RETRY_ATTEMPTS : 1
|
||||
// Why: this single sample decides a mutating wave, so an Asia cell's ~30 s
|
||||
// "no healthy upstream" window, or a one-minute director instance-replacement
|
||||
// dip, could fail a wave here even after the 15-minute gate learned to ride it
|
||||
// out. Hold the two to the same tolerance. Unlike the freshness retry this
|
||||
// needs no flag, because a tolerated breach is never the operator's call to
|
||||
// waive.
|
||||
const cellProbeAttempts = 1 + INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const attempts = Math.max(freshnessAttempts, cellProbeAttempts)
|
||||
let freshnessRetries = freshnessAttempts - 1
|
||||
let cellProbeRetries = cellProbeAttempts - 1
|
||||
// Waiting must never carry the mutation past the same evidence-age bound the
|
||||
// entry check enforces, so the wave budget also caps the retry window.
|
||||
const budgetExhausted = (): boolean =>
|
||||
now() + FRESHNESS_RETRY_INTERVAL_MS - plan.evidenceAnchorMs > maxEvidenceAgeMs
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
// A director admin read can fail on its own (its handler maps a Cloud SQL
|
||||
// pool timeout onto 404), which says nothing about relay health; spend an
|
||||
// attempt on it rather than failing the wave on one unlucky sample.
|
||||
let sample: IncidentSample
|
||||
try {
|
||||
sample = await collect()
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'sample collection failed'
|
||||
if (attempt === attempts || budgetExhausted()) {
|
||||
throw new Error(`relay live preflight failed: collector: ${message}`)
|
||||
}
|
||||
console.warn(
|
||||
`relay live preflight re-sampling after collector failure (${attempt}/${attempts - 1})`
|
||||
)
|
||||
await wait(FRESHNESS_RETRY_INTERVAL_MS)
|
||||
continue
|
||||
}
|
||||
const evaluation = evaluateIncidentSample(
|
||||
await collect(),
|
||||
sample,
|
||||
now(),
|
||||
state.migrationPolicy,
|
||||
state.recoverySourceCellId,
|
||||
state.capacityCellId
|
||||
plan.migrationPolicy,
|
||||
plan.recoverySourceCellId,
|
||||
plan.capacityCellId
|
||||
)
|
||||
if (evaluation.status === 'green') return
|
||||
const freshnessOnly = evaluation.failures.every((failure) =>
|
||||
const freshnessFailures = evaluation.failures.filter((failure) =>
|
||||
FRESHNESS_FAILURE_CODES.has(failure.code)
|
||||
)
|
||||
// Waiting must never carry the mutation past the same evidence-age bound
|
||||
// the entry check enforces, so the wave budget also caps the retry window.
|
||||
const budgetExhausted =
|
||||
now() + FRESHNESS_RETRY_INTERVAL_MS - completedAt > maxEvidenceAgeMs
|
||||
if (!freshnessOnly || attempt === attempts || budgetExhausted) {
|
||||
// Tolerated readings only (per-cell probes and the director instance count).
|
||||
// The director and auth health probes are absent here on purpose and fail
|
||||
// the wave on their first bad sample.
|
||||
const cellProbeFailures = evaluation.failures.filter((failure) =>
|
||||
!FRESHNESS_FAILURE_CODES.has(failure.code) && toleratedStreakKey(failure) !== null
|
||||
)
|
||||
const retryable =
|
||||
freshnessFailures.length + cellProbeFailures.length === evaluation.failures.length &&
|
||||
(freshnessFailures.length === 0 || freshnessRetries > 0) &&
|
||||
(cellProbeFailures.length === 0 || cellProbeRetries > 0)
|
||||
if (!retryable || attempt === attempts || budgetExhausted()) {
|
||||
throw new Error(
|
||||
`relay live preflight failed: ${evaluation.failures
|
||||
.map(describeFailure)
|
||||
.join(',')}`
|
||||
)
|
||||
}
|
||||
if (freshnessFailures.length > 0) freshnessRetries--
|
||||
if (cellProbeFailures.length > 0) cellProbeRetries--
|
||||
console.warn(
|
||||
`relay live preflight awaiting fresh evidence (${attempt}/${attempts - 1})`
|
||||
`relay live preflight re-sampling after tolerable failure (${attempt}/${attempts - 1})`
|
||||
)
|
||||
await wait(FRESHNESS_RETRY_INTERVAL_MS)
|
||||
}
|
||||
|
||||
@@ -68,6 +68,20 @@ const StateSchema = z.object({
|
||||
observed: z.number().optional(),
|
||||
threshold: z.number().optional()
|
||||
})),
|
||||
// Pre-2026-09-17 state files predate cell-probe tolerance; a resumed run that
|
||||
// carries no streak is one that also restarts its window, so it earns nothing.
|
||||
probeStreaks: z.record(z.string(), z.number().int().nonnegative()).default({}),
|
||||
toleratedProbeEvents: z.array(z.object({
|
||||
recordedAt: z.string(),
|
||||
windowSequence: z.number().int().nonnegative(),
|
||||
failures: z.array(z.object({
|
||||
code: z.string(),
|
||||
source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
|
||||
signal: z.string().optional(),
|
||||
observed: z.number().optional(),
|
||||
threshold: z.number().optional()
|
||||
}))
|
||||
})).default([]),
|
||||
completedAt: z.string().nullable()
|
||||
})
|
||||
|
||||
@@ -463,6 +477,7 @@ export async function runIncidentMonitorCli(
|
||||
await wait(ms)
|
||||
},
|
||||
collect: segmentedCollect,
|
||||
warn: (message) => console.warn(message),
|
||||
persist: async (nextState) => await persistState(options.stateFile, nextState),
|
||||
checkpoint: async (checkpoint) => {
|
||||
await appendCheckpoint(options.summaryFile, checkpoint)
|
||||
|
||||
@@ -458,4 +458,112 @@ describe('incident monitor sources', () => {
|
||||
expect(serialized).not.toContain(identityToken)
|
||||
expect(serialized).not.toContain(sensitiveIdentity)
|
||||
})
|
||||
|
||||
// Why: the director maps a Cloud SQL pool connect timeout in cell-status onto
|
||||
// a 404, so without this the whole sample dies on one 2 s database stall.
|
||||
it('retries a transient director admin failure and then reports the cell', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
|
||||
}
|
||||
const waits: number[] = []
|
||||
let cellStatusCalls = 0
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
|
||||
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
|
||||
if (body.cellId) {
|
||||
cellStatusCalls++
|
||||
if (cellStatusCalls === 1) {
|
||||
return Response.json(
|
||||
{ error: 'timeout exceeded when trying to connect' },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
return Response.json({
|
||||
status: {
|
||||
enabled: true,
|
||||
connectionCapacity: { hardCap: 600 },
|
||||
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
|
||||
}
|
||||
})
|
||||
}
|
||||
return Response.json({
|
||||
blocked: 0,
|
||||
blockedExpiredUnregistered: 0,
|
||||
registeredTargetInactive: 0
|
||||
})
|
||||
}
|
||||
const result = await directorSignals(
|
||||
'production',
|
||||
selector,
|
||||
gcloud,
|
||||
now,
|
||||
fetchImpl,
|
||||
async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
)
|
||||
expect(waits).toEqual([5_000])
|
||||
expect(cellStatusCalls).toBe(productionCells.length + 1)
|
||||
expect(result.cells).toHaveLength(productionCells.length)
|
||||
expect(
|
||||
result.source.signals['cell.production-gce-c1.connection_hard_cap']
|
||||
).toMatchObject({ value: 600 })
|
||||
})
|
||||
|
||||
// A rejected admin token is a decision, not weather: retrying it only burns
|
||||
// the sample budget and hides the misconfiguration.
|
||||
it('fails immediately on an unauthorized director admin response', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
const selector = {
|
||||
generation: 1,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: productionCells }
|
||||
}
|
||||
const waits: number[] = []
|
||||
let requestCount = 0
|
||||
const fetchImpl: typeof fetch = async (_input, init) => {
|
||||
requestCount++
|
||||
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
|
||||
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
|
||||
return Response.json({ error: 'invalid_token' }, { status: 401 })
|
||||
}
|
||||
await expect(
|
||||
directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
|
||||
waits.push(ms)
|
||||
})
|
||||
).rejects.toThrow('Relay admin telemetry returned 401')
|
||||
expect(requestCount).toBe(2)
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
// A 404 the director means (wrong role) must not be retried either.
|
||||
it('does not retry a director-only 404', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => 'unused',
|
||||
identityToken: async () => 'unused'
|
||||
}
|
||||
let requestCount = 0
|
||||
const fetchImpl: typeof fetch = async () => {
|
||||
requestCount++
|
||||
return Response.json({ error: 'director_only' }, { status: 404 })
|
||||
}
|
||||
await expect(
|
||||
directorSignals(
|
||||
'production',
|
||||
{ generation: 1, membership: { existingOnly: [], migrationOnly: [], general: productionCells } },
|
||||
gcloud,
|
||||
now,
|
||||
fetchImpl,
|
||||
async () => {}
|
||||
)
|
||||
).rejects.toThrow('Relay admin telemetry returned 404')
|
||||
expect(requestCount).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -414,21 +414,96 @@ function relaySignals(
|
||||
return { observedAt, signals }
|
||||
}
|
||||
|
||||
const ADMIN_RETRY_ATTEMPTS = 3
|
||||
const ADMIN_RETRY_INTERVAL_MS = 5_000
|
||||
const ADMIN_RETRYABLE_STATUSES = new Set([500, 502, 503, 504])
|
||||
const AdminErrorBodySchema = z.object({ error: z.string() })
|
||||
// The director's /v1/admin/cell-status maps any thrown operation error onto 404
|
||||
// with an {error: message} body, so a Cloud SQL pool connect timeout arrives
|
||||
// here as a 404. These are the pool-acquire and transient SQLSTATE messages the
|
||||
// relay app's own isRelayDatabaseTransientError treats as re-runnable.
|
||||
const ADMIN_TRANSIENT_ERROR_MESSAGES = [
|
||||
'timeout exceeded when trying to connect',
|
||||
'Connection terminated due to connection timeout',
|
||||
'Connection terminated unexpectedly',
|
||||
'database_temporarily_unavailable',
|
||||
'deadlock detected',
|
||||
'canceling statement due to',
|
||||
'too many connections',
|
||||
'the database system is shutting down',
|
||||
'the database system is starting up'
|
||||
]
|
||||
|
||||
const defaultWait = async (ms: number): Promise<void> =>
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, ms))
|
||||
|
||||
// A dropped socket or the 30 s AbortSignal firing; both leave the request with
|
||||
// no verdict, and every admin read here is a plain query.
|
||||
function transientFetchFailure(error: unknown): boolean {
|
||||
if (error instanceof TypeError) return true
|
||||
const name = (error as { name?: unknown } | null)?.name
|
||||
return name === 'AbortError' || name === 'TimeoutError'
|
||||
}
|
||||
|
||||
function retryableAdminStatus(status: number, body: string): boolean {
|
||||
if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
|
||||
// 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
|
||||
if (status !== 404) return false
|
||||
const parsed = AdminErrorBodySchema.safeParse(
|
||||
((): unknown => {
|
||||
try {
|
||||
return JSON.parse(body)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
})()
|
||||
)
|
||||
return (
|
||||
parsed.success &&
|
||||
ADMIN_TRANSIENT_ERROR_MESSAGES.some((message) => parsed.data.error.includes(message))
|
||||
)
|
||||
}
|
||||
|
||||
async function adminPost(
|
||||
fetchImpl: typeof fetch,
|
||||
origin: string,
|
||||
token: string,
|
||||
path: string,
|
||||
body: unknown
|
||||
body: unknown,
|
||||
wait: (ms: number) => Promise<void> = defaultWait
|
||||
): Promise<unknown> {
|
||||
const response = await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Relay admin telemetry returned ${response.status}`)
|
||||
return await response.json()
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
const lastAttempt = attempt >= ADMIN_RETRY_ATTEMPTS
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchImpl(`${origin}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
} catch (error) {
|
||||
if (lastAttempt || !transientFetchFailure(error)) throw error
|
||||
console.warn(
|
||||
`relay admin telemetry retrying ${path} after network failure` +
|
||||
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
|
||||
)
|
||||
await wait(ADMIN_RETRY_INTERVAL_MS)
|
||||
continue
|
||||
}
|
||||
if (response.ok) return await response.json()
|
||||
// Status only: the body carries a director error message, which must not
|
||||
// reach the log the way an identity would.
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) {
|
||||
throw new Error(`Relay admin telemetry returned ${response.status}`)
|
||||
}
|
||||
console.warn(
|
||||
`relay admin telemetry retrying ${path} after ${response.status}` +
|
||||
` (attempt ${attempt}/${ADMIN_RETRY_ATTEMPTS})`
|
||||
)
|
||||
await wait(ADMIN_RETRY_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
export async function directorSignals(
|
||||
@@ -436,7 +511,8 @@ export async function directorSignals(
|
||||
expectedSelector: AdmissionSelector,
|
||||
gcloud: GcloudClient,
|
||||
nowMs: number,
|
||||
fetchImpl: typeof fetch
|
||||
fetchImpl: typeof fetch,
|
||||
wait: (ms: number) => Promise<void> = defaultWait
|
||||
): Promise<{
|
||||
source: IncidentSource
|
||||
selector: AdmissionSelector
|
||||
@@ -452,7 +528,8 @@ export async function directorSignals(
|
||||
environment.directorOrigin,
|
||||
token,
|
||||
'/v1/admin/admission-selector/status',
|
||||
{ v: 1 }
|
||||
{ v: 1 },
|
||||
wait
|
||||
)
|
||||
).selector
|
||||
const selector = {
|
||||
@@ -467,10 +544,14 @@ export async function directorSignals(
|
||||
statuses.push({
|
||||
cell,
|
||||
status: CellStatusSchema.parse(
|
||||
await adminPost(fetchImpl, environment.directorOrigin, token, '/v1/admin/cell-status', {
|
||||
v: 1,
|
||||
cellId: cell.cellId
|
||||
})
|
||||
await adminPost(
|
||||
fetchImpl,
|
||||
environment.directorOrigin,
|
||||
token,
|
||||
'/v1/admin/cell-status',
|
||||
{ v: 1, cellId: cell.cellId },
|
||||
wait
|
||||
)
|
||||
).status
|
||||
})
|
||||
}
|
||||
@@ -495,7 +576,8 @@ export async function directorSignals(
|
||||
sourceCellId: source.cellId,
|
||||
targetCellId: target.cellId,
|
||||
completeReady: false
|
||||
}
|
||||
},
|
||||
wait
|
||||
)
|
||||
)
|
||||
})
|
||||
@@ -571,6 +653,7 @@ export type IncidentSampleCollectorOptions = {
|
||||
expectedSelector: AdmissionSelector
|
||||
fetchImpl?: typeof fetch
|
||||
now?: () => number
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export function createIncidentSampleCollector(
|
||||
@@ -610,7 +693,8 @@ export function createIncidentSampleCollector(
|
||||
options.expectedSelector,
|
||||
gcloud,
|
||||
nowMs,
|
||||
fetchImpl
|
||||
fetchImpl,
|
||||
options.wait ?? defaultWait
|
||||
),
|
||||
cloudMetricEntries
|
||||
])
|
||||
|
||||
@@ -273,16 +273,19 @@ describe('incident monitor evaluator', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
|
||||
for (const errors of [1, 2, 3]) {
|
||||
// Why: measured non-503 5xx per rolling five minutes over the 24 h to 2026-09-17
|
||||
// was p90 3 / p95 5 / p99 9 / max 52, so the old bar of 3 sat on the p90 and froze
|
||||
// 29% of 15-minute gates on chronic director 500 bursts.
|
||||
it('tolerates the measured chronic director error rate without relaxing other gates', () => {
|
||||
for (const errors of [1, 3, 5, 9, 15]) {
|
||||
const sample = healthySample()
|
||||
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
|
||||
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
|
||||
}
|
||||
const excess = healthySample()
|
||||
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
|
||||
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(16)
|
||||
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
|
||||
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
|
||||
expect.objectContaining({ signal: 'director.errors', observed: 16, threshold: 15 })
|
||||
)
|
||||
const auth = healthySample()
|
||||
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
|
||||
@@ -358,17 +361,20 @@ describe('incident monitor evaluator', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// Why: measured latest-sum over the 24 h to 2026-09-17 was p95 212 / p99 262 /
|
||||
// max 282, so the old bar of 250 sat under the observed peak and froze 21.8% of
|
||||
// 15-minute gates. 320 still fires at 65% of the 490 usable connections.
|
||||
it('bounds Cloud SQL backends above measured healthy peaks', () => {
|
||||
const sample = healthySample()
|
||||
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(250)
|
||||
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(282)
|
||||
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
|
||||
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(251)
|
||||
sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(321)
|
||||
expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
|
||||
code: 'threshold_max',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'cloud_sql.backends',
|
||||
observed: 251,
|
||||
threshold: 250
|
||||
observed: 321,
|
||||
threshold: 320
|
||||
})
|
||||
})
|
||||
|
||||
@@ -807,6 +813,150 @@ describe('incident monitor lifecycle', () => {
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
// Why: dry-run 35258662628 read a healthy fleet clean for 13 minutes, then one
|
||||
// unreadable Cloud Monitoring sample restarted the window and the restart blew
|
||||
// the lineage cap, so a green fleet produced no verdict.
|
||||
it('carries a 15-minute window through a single collector failure', async () => {
|
||||
let now = startedAt
|
||||
const warnings: string[] = []
|
||||
const state = initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
environment: 'production',
|
||||
expectedSelector: selector,
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
const result = await runIncidentMonitor(state, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
if (now === startedAt + 10 * 60_000) {
|
||||
throw new Error('cloud monitoring read failed')
|
||||
}
|
||||
return healthySample(now)
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {},
|
||||
warn: (message) => {
|
||||
warnings.push(message)
|
||||
}
|
||||
})
|
||||
expect(result.windowSequence).toBe(0)
|
||||
expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
|
||||
expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
|
||||
expect(result.sampleCount).toBe(16)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(result.failures).toEqual([])
|
||||
expect(result.continuityEvents).toEqual([{
|
||||
recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
|
||||
windowSequence: 0,
|
||||
tolerated: true,
|
||||
failures: [{ code: 'collector_failed', source: 'cloud-monitoring' }]
|
||||
}])
|
||||
expect(warnings).toEqual([
|
||||
'incident monitor collector failed: cloud monitoring read failed'
|
||||
])
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
it('restarts the window after three consecutive collector failures', async () => {
|
||||
let now = startedAt
|
||||
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
|
||||
const state = initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
environment: 'production',
|
||||
expectedSelector: selector,
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
const result = await runIncidentMonitor(state, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
if (failures > 0 && now >= startedAt + 10 * 60_000) {
|
||||
failures--
|
||||
throw new Error('cloud monitoring read failed')
|
||||
}
|
||||
return healthySample(now)
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {},
|
||||
warn: () => {}
|
||||
})
|
||||
const restartMinute = 10 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
|
||||
expect(result.windowSequence).toBe(1)
|
||||
expect(result.windowStartedAt).toBe(
|
||||
new Date(startedAt + restartMinute * 60_000).toISOString()
|
||||
)
|
||||
expect(result.completedAt).toBe(
|
||||
new Date(startedAt + (restartMinute + 15) * 60_000).toISOString()
|
||||
)
|
||||
expect(result.sampleCount).toBe(16)
|
||||
expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
|
||||
...Array<boolean>(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
|
||||
false
|
||||
])
|
||||
expect(result.continuityEvents.at(-1)!.failures).toEqual([
|
||||
{ code: 'collector_failed', source: 'cloud-monitoring' }
|
||||
])
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
it('still reaches a dry-run verdict after a restart on the last sample', async () => {
|
||||
let now = startedAt
|
||||
let failures = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
|
||||
const state = initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
environment: 'production',
|
||||
expectedSelector: selector,
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
const result = await runIncidentMonitor(state, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
if (failures > 0 && now >= startedAt + 13 * 60_000) {
|
||||
failures--
|
||||
throw new Error('cloud monitoring read failed')
|
||||
}
|
||||
return healthySample(now)
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {},
|
||||
warn: () => {}
|
||||
})
|
||||
expect(result.windowSequence).toBe(1)
|
||||
expect(result.windowStartedAt).toBe(new Date(startedAt + 16 * 60_000).toISOString())
|
||||
expect(result.completedAt).toBe(new Date(startedAt + 31 * 60_000).toISOString())
|
||||
expect(result.sampleCount).toBe(16)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(result.failures).toEqual([])
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
it('gives a signal a fresh budget only after it reads fresh again', async () => {
|
||||
let now = startedAt
|
||||
const staleMinutes = new Set([3, 5, 6, 9, 10])
|
||||
@@ -971,7 +1121,7 @@ describe('incident monitor lifecycle', () => {
|
||||
expect(result.sampleCount).toBe(16)
|
||||
})
|
||||
|
||||
it('fails a dry run after 25 total minutes of continuity resets', async () => {
|
||||
it('fails a dry run after 35 total minutes of continuity resets', async () => {
|
||||
let now = startedAt
|
||||
const state = initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
@@ -985,15 +1135,16 @@ describe('incident monitor lifecycle', () => {
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
|
||||
// Two restarts: the first on the window's last sample, the second far enough
|
||||
// into the replacement window that no third window can finish in the lineage.
|
||||
const staleMinutes = new Set([13, 14, 15, 24, 25, 26])
|
||||
const result = await runIncidentMonitor(state, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
|
||||
staleSamples--
|
||||
if (staleMinutes.has((now - startedAt) / 60_000)) {
|
||||
return healthySample(
|
||||
now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
|
||||
)
|
||||
@@ -1008,8 +1159,8 @@ describe('incident monitor lifecycle', () => {
|
||||
new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString()
|
||||
)
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(result.windowSequence).toBe(1)
|
||||
expect(result.sampleCount).toBe(13)
|
||||
expect(result.windowSequence).toBe(2)
|
||||
expect(result.sampleCount).toBe(9)
|
||||
expect(result.failures).toContainEqual({
|
||||
code: 'continuity_deadline_exceeded',
|
||||
source: 'active-probe',
|
||||
@@ -1114,3 +1265,371 @@ describe('incident monitor lifecycle', () => {
|
||||
expect(waits[0]).toBe(45_000)
|
||||
})
|
||||
})
|
||||
|
||||
// Why: the asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
|
||||
// us-central1 behind a 2 s timeout, so a saturated pool makes the load balancer
|
||||
// answer "no healthy upstream" for about 30 s. Every one of 39 pre-roll gates froze
|
||||
// on that, and 7 of the last 14 froze on this signal alone.
|
||||
describe('incident monitor cell probe tolerance', () => {
|
||||
const dryRunState = () =>
|
||||
initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
environment: 'production',
|
||||
expectedSelector: selector,
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
|
||||
// Returns the finished state of a 15-minute dry-run whose cell probe reads
|
||||
// health=0 and ready=0 on the sample indexes in `badSamples`.
|
||||
const runWithCellProbeGaps = async (badSamples: Set<number>) => {
|
||||
let now = startedAt
|
||||
let index = -1
|
||||
return await runIncidentMonitor(dryRunState(), {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
index++
|
||||
const sample = healthySample(now)
|
||||
if (badSamples.has(index)) {
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
|
||||
signal(0, now)
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
|
||||
signal(0, now)
|
||||
}
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
}
|
||||
|
||||
it('passes a dry-run through a probe outage no longer than the tolerance', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const result = await runWithCellProbeGaps(
|
||||
new Set(Array.from({ length: tolerance }, (_, offset) => 3 + offset))
|
||||
)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(result.failures).toEqual([])
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
// The blip is absorbed, not hidden: the sealed state still carries it.
|
||||
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
|
||||
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
source: 'active-probe',
|
||||
signal: 'cell.production-gce-c1.health',
|
||||
observed: 0,
|
||||
threshold: 1
|
||||
})
|
||||
)
|
||||
// A recovered probe hands back the full budget rather than a partial one.
|
||||
expect(result.probeStreaks).toEqual({})
|
||||
})
|
||||
|
||||
it('freezes once a cell probe fails past the tolerance', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const result = await runWithCellProbeGaps(
|
||||
new Set(Array.from({ length: tolerance + 1 }, (_, offset) => 3 + offset))
|
||||
)
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(false)
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
source: 'active-probe',
|
||||
signal: 'cell.production-gce-c1.health',
|
||||
observed: 0,
|
||||
threshold: 1
|
||||
})
|
||||
)
|
||||
// Only the samples past the tolerance freeze; the first two are still absorbed.
|
||||
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
|
||||
})
|
||||
|
||||
it('does not accumulate a streak across a recovered sample', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
// Repeated single-sample outages, each separated by a healthy sample, never
|
||||
// reach the tolerance however many times they recur.
|
||||
const spaced = new Set([2, 4, 6, 8, 10])
|
||||
expect(spaced.size).toBeGreaterThan(tolerance)
|
||||
const result = await runWithCellProbeGaps(spaced)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
// Why: health, ready and latency_ms all describe the same round trip, so the streak
|
||||
// is keyed by cell. Keyed per signal, this cell holds every individual streak at one
|
||||
// and never reaches the tolerance, yet it is unhealthy without a break from sample 2.
|
||||
it('freezes on a cell that alternates between slow and unanswered', async () => {
|
||||
let now = startedAt
|
||||
let index = -1
|
||||
const result = await runIncidentMonitor(dryRunState(), {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
index++
|
||||
const sample = healthySample(now)
|
||||
if (index < 2) return sample
|
||||
// Two samples slow, then two samples down, repeating: never the same signal
|
||||
// twice in a row beyond the tolerance, but never healthy either.
|
||||
if (Math.floor((index - 2) / 2) % 2 === 0) {
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
|
||||
signal(9_000, now)
|
||||
} else {
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
|
||||
signal(0, now)
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] =
|
||||
signal(0, now)
|
||||
}
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(false)
|
||||
})
|
||||
|
||||
// Why: the streak lives in the state file, so a resumed run must not hand a cell
|
||||
// that was already failing a fresh budget.
|
||||
it('freezes immediately when a resumed state carries a full streak', async () => {
|
||||
let now = startedAt
|
||||
const resumed = {
|
||||
...dryRunState(),
|
||||
lastSampleAt: new Date(startedAt).toISOString(),
|
||||
probeStreaks: {
|
||||
'active-probe/cell.production-gce-c1':
|
||||
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
}
|
||||
}
|
||||
const result = await runIncidentMonitor(resumed, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
const sample = healthySample(now)
|
||||
sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] =
|
||||
signal(0, now)
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
|
||||
expect(result.toleratedProbeEvents).toEqual([])
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
source: 'active-probe',
|
||||
signal: 'cell.production-gce-c1.health'
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('gives the director and auth probes no tolerance', async () => {
|
||||
let now = startedAt
|
||||
let index = -1
|
||||
const result = await runIncidentMonitor(dryRunState(), {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
index++
|
||||
const sample = healthySample(now)
|
||||
if (index === 3) {
|
||||
sample.sources['active-probe']!.signals['director.health'] = signal(0, now)
|
||||
}
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(result.toleratedProbeEvents).toEqual([])
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({ source: 'active-probe', signal: 'director.health' })
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// Why: Cloud Run replaces director instances in place, so the count leaves the
|
||||
// [5, 6] band for about one sample roughly twice a day, and a deploy overlap
|
||||
// raises it the same way. Neither is an unhealthy fleet, and freezing on it
|
||||
// blocks the roll that fixes the measured condition.
|
||||
describe('incident monitor director instance tolerance', () => {
|
||||
const dryRunState = () =>
|
||||
initialIncidentMonitorState({
|
||||
incidentId: 'incident-1',
|
||||
environment: 'production',
|
||||
expectedSelector: selector,
|
||||
preDrainDryRun: true,
|
||||
migrationPolicy: 'strict',
|
||||
recoverySourceCellId: null,
|
||||
capacityCellId: null,
|
||||
startedAt: new Date(startedAt).toISOString(),
|
||||
durationMinutes: 15,
|
||||
intervalMs: 60_000
|
||||
})
|
||||
|
||||
// Finished state of a 15-minute dry run whose director instance count reads
|
||||
// `counts[index]` on the sample indexes that map has, and 5 everywhere else.
|
||||
const runWithInstanceCounts = async (
|
||||
counts: Map<number, number>,
|
||||
state = dryRunState()
|
||||
) => {
|
||||
let now = startedAt
|
||||
let index = -1
|
||||
return await runIncidentMonitor(state, {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
index++
|
||||
const sample = healthySample(now)
|
||||
const count = counts.get(index)
|
||||
if (count !== undefined) {
|
||||
sample.sources['cloud-monitoring']!.signals['director.instances'] =
|
||||
signal(count, now)
|
||||
}
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
}
|
||||
|
||||
it('passes a dry run through an instance dip no longer than the tolerance', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const result = await runWithInstanceCounts(
|
||||
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 2]))
|
||||
)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(result.failures).toEqual([])
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
// Absorbed, not hidden: the sealed state still carries the dip.
|
||||
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
|
||||
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
code: 'threshold_min',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'director.instances',
|
||||
observed: 2,
|
||||
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin
|
||||
})
|
||||
)
|
||||
expect(result.probeStreaks).toEqual({})
|
||||
})
|
||||
|
||||
it('passes a dry run through a deploy-overlap overshoot within the tolerance', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const result = await runWithInstanceCounts(
|
||||
new Map(Array.from({ length: tolerance }, (_, offset) => [3 + offset, 9]))
|
||||
)
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
expect(result.toleratedProbeEvents[0]!.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
code: 'threshold_max',
|
||||
signal: 'director.instances',
|
||||
observed: 9,
|
||||
threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('freezes once the instance count stays out of band past the tolerance', async () => {
|
||||
const tolerance = INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
const result = await runWithInstanceCounts(
|
||||
new Map(Array.from({ length: tolerance + 1 }, (_, offset) => [3 + offset, 2]))
|
||||
)
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(false)
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({
|
||||
code: 'threshold_min',
|
||||
source: 'cloud-monitoring',
|
||||
signal: 'director.instances',
|
||||
observed: 2
|
||||
})
|
||||
)
|
||||
expect(result.toleratedProbeEvents).toHaveLength(tolerance)
|
||||
})
|
||||
|
||||
it('does not accumulate a streak across a recovered sample', async () => {
|
||||
const spaced = new Map([2, 4, 6, 8, 10].map((index) => [index, 2] as const))
|
||||
expect(spaced.size).toBeGreaterThan(
|
||||
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
)
|
||||
const result = await runWithInstanceCounts(new Map(spaced))
|
||||
expect(result.frozenAt).toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(true)
|
||||
})
|
||||
|
||||
// Why min and max share one streak: a count that alternates above and below
|
||||
// the band would otherwise hold each streak at one and never freeze, yet the
|
||||
// director is never at its configured size.
|
||||
it('freezes on a count that alternates above and below the band', async () => {
|
||||
const counts = new Map<number, number>()
|
||||
for (let index = 2; index < 12; index += 1) {
|
||||
counts.set(index, index % 2 === 0 ? 2 : 9)
|
||||
}
|
||||
const result = await runWithInstanceCounts(counts)
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(preDrainDryRunPassed(result)).toBe(false)
|
||||
})
|
||||
|
||||
// Why: the streak lives in the state file, so a resumed run must not hand a
|
||||
// director that was already out of band a fresh budget.
|
||||
it('freezes immediately when a resumed state carries a full streak', async () => {
|
||||
const result = await runWithInstanceCounts(new Map([[0, 2]]), {
|
||||
...dryRunState(),
|
||||
lastSampleAt: new Date(startedAt).toISOString(),
|
||||
probeStreaks: {
|
||||
'cloud-monitoring/director.instances':
|
||||
INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
}
|
||||
})
|
||||
expect(result.frozenAt).toBe(new Date(startedAt).toISOString())
|
||||
expect(result.toleratedProbeEvents).toEqual([])
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({ signal: 'director.instances' })
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps every other cloud-monitoring signal at zero tolerance', async () => {
|
||||
let now = startedAt
|
||||
let index = -1
|
||||
const result = await runIncidentMonitor(dryRunState(), {
|
||||
now: () => now,
|
||||
wait: async (ms) => {
|
||||
now += ms
|
||||
},
|
||||
collect: async () => {
|
||||
index++
|
||||
const sample = healthySample(now)
|
||||
if (index === 3) {
|
||||
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.99, now)
|
||||
}
|
||||
return sample
|
||||
},
|
||||
persist: async () => {},
|
||||
checkpoint: async () => {}
|
||||
})
|
||||
expect(result.frozenAt).not.toBeNull()
|
||||
expect(result.toleratedProbeEvents).toEqual([])
|
||||
expect(result.failures).toContainEqual(
|
||||
expect.objectContaining({ source: 'cloud-monitoring', signal: 'cloud_sql.cpu' })
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -42,12 +42,18 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
|
||||
} as const satisfies Record<RelayOpsRegion, number>,
|
||||
cloudSqlCpuUtilization: 0.8,
|
||||
cloudSqlMemoryUtilization: 0.9,
|
||||
// Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
|
||||
// bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a
|
||||
// pre-drain gate on baseline noise). 250 clears measured healthy peaks while
|
||||
// firing well before the verified 400-connection ceiling; the retry signals
|
||||
// below discriminate incident-class contention.
|
||||
cloudSqlBackends: 250,
|
||||
// Why: 320, recalibrated 2026-09-17 from 250. The auth instance sums seven
|
||||
// databases, and its steady load has grown past the 2026-08-26 measurement the
|
||||
// old bar came from. Measured latest-sum over the 24 h to 2026-09-17, aligned
|
||||
// per minute exactly as this signal reads it: p50 118 / p90 165 / p95 212 /
|
||||
// p99 262 / max 282. 250 was under the observed max, so 1.95% of minutes and
|
||||
// 21.8% of 15-minute pre-drain gates froze on ordinary load. 320 clears every
|
||||
// measured healthy minute with 13% of headroom above the peak and still fires
|
||||
// at 65% of the 490 connections the budget work (#21165) treats as usable out
|
||||
// of max_connections 500, so exhaustion-class growth is caught with 170
|
||||
// connections still in hand. The retry signals below discriminate incident-class
|
||||
// contention. Re-tighten when the auth connection model lands (#21165).
|
||||
cloudSqlBackends: 320,
|
||||
// Bound the observed recovery load; deadlocks remain zero-tolerance.
|
||||
cloudSqlLockWaits: 20,
|
||||
cloudSqlDeadlocks: 0,
|
||||
@@ -101,18 +107,48 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
|
||||
directorCpuUtilization: 0.8,
|
||||
directorMemoryUtilization: 0.8,
|
||||
directorConcurrency: 64,
|
||||
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
|
||||
directorErrors: 3,
|
||||
// Why: 15, recalibrated 2026-09-17 from 3. This counts non-503 5xx answers from
|
||||
// the director over the rolling five-minute query window; 503s are excluded
|
||||
// because they are the documented back-pressure answer a client retries.
|
||||
// Measured over the 24 h to 2026-09-17 (272 non-503 5xx against 71 941 503s):
|
||||
// p50 0 / p90 3 / p95 5 / p99 9 / max 52. A bar of 3 sits at the p90, so 9.2%
|
||||
// of windows and 29.0% of 15-minute pre-drain gates froze on the chronic 500
|
||||
// bursts that /v1/assign, /v1/regions and /v1/resolve emit alongside the
|
||||
// recurring Cloud SQL stall. 15 clears the chronic p99 with margin and drops
|
||||
// the baseline gate-freeze rate to 1.5%, while leaving the exceptional 20-52
|
||||
// bursts detectable. A director that is actually broken answers 5xx on a large
|
||||
// share of its traffic: it serves ~50 requests a minute in 503s alone, so a
|
||||
// real fault lands in the hundreds per window, an order of magnitude clear of
|
||||
// this bar.
|
||||
directorErrors: 15,
|
||||
authErrors: 0,
|
||||
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
|
||||
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
|
||||
cellConnections: 500,
|
||||
cellQueuedBytes: 48 * 1024 * 1024,
|
||||
migrationBlocked: 0
|
||||
migrationBlocked: 0,
|
||||
// Why: one sample is one HTTP round trip from one GitHub runner to one cell, so
|
||||
// a single bad reading is evidence about that round trip, not about the fleet.
|
||||
// The asia-east2 cells' readiness probe runs SELECT 1 against Cloud SQL in
|
||||
// us-central1 over a 176 ms round trip behind a 2 s statement timeout, so a
|
||||
// saturated pool makes the load balancer answer "no healthy upstream" for about
|
||||
// 30 s. That answer is an HTTP 503, not a transport failure, so provenance
|
||||
// cannot separate it from a cell that genuinely serves health=0 -- persistence
|
||||
// can. At the 60 s sample interval a 30 s outage shows up in one sample and at
|
||||
// worst two, so a cell's probe must fail more than this many consecutive samples
|
||||
// before it freezes the run. The streak is per cell, not per signal, so a cell
|
||||
// that alternates between slow and unanswered still accumulates one. This applies
|
||||
// to per-cell probes and the director instance count; the director and auth health
|
||||
// probes stay at zero tolerance.
|
||||
cellProbeToleranceSamples: 2
|
||||
} as const
|
||||
|
||||
export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const
|
||||
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000
|
||||
// Why: 35 minutes, raised 2026-09-17 from 25. A 15-minute window plus one
|
||||
// restart must fit: a continuity reset on the window's last sample restarts at
|
||||
// minute 16 and finishes at 31. Under 25 a reset past minute 9 cost the whole
|
||||
// verdict, which is what run 35258662628 hit on a healthy fleet.
|
||||
export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 35 * 60_000
|
||||
|
||||
export type IncidentSourceName =
|
||||
| 'active-probe'
|
||||
@@ -212,6 +248,17 @@ export type IncidentMonitorState = {
|
||||
}[]
|
||||
frozenAt: string | null
|
||||
failures: IncidentFailure[]
|
||||
// Consecutive samples each tolerated reading has currently been failing for,
|
||||
// keyed by cell so its health, ready and latency readings share one streak, and
|
||||
// by signal for the director instance count.
|
||||
probeStreaks: Record<string, number>
|
||||
// Cell-probe breaches absorbed by the tolerance, kept so a green artifact still
|
||||
// shows what the gate chose not to freeze on.
|
||||
toleratedProbeEvents: {
|
||||
recordedAt: string
|
||||
windowSequence: number
|
||||
failures: IncidentFailure[]
|
||||
}[]
|
||||
completedAt: string | null
|
||||
}
|
||||
|
||||
@@ -625,6 +672,8 @@ export function initialIncidentMonitorState(input: {
|
||||
continuityEvents: [],
|
||||
frozenAt: null,
|
||||
failures: [],
|
||||
probeStreaks: {},
|
||||
toleratedProbeEvents: [],
|
||||
completedAt: null
|
||||
}
|
||||
}
|
||||
@@ -635,6 +684,7 @@ export type IncidentMonitorDependencies = {
|
||||
collect(): Promise<IncidentSample>
|
||||
persist(state: IncidentMonitorState): Promise<void>
|
||||
checkpoint(summary: IncidentCheckpoint): Promise<void>
|
||||
warn?(message: string): void
|
||||
}
|
||||
|
||||
function checkpointMinutes(durationMinutes: number): number[] {
|
||||
@@ -656,10 +706,19 @@ const CONTINUITY_FAILURE_CODES = new Set([
|
||||
...FRESHNESS_FAILURE_CODES
|
||||
])
|
||||
|
||||
// A whole sample we could not read gets the same consecutive-sample budget as an
|
||||
// unread signal, for the same reason: one failed collector round trip is evidence
|
||||
// about that round trip, not about the fleet. `monitor_gap` is excluded because it
|
||||
// means the run itself stopped sampling, so the window genuinely has a hole.
|
||||
const TOLERABLE_CONTINUITY_FAILURE_CODES = new Set([
|
||||
'collector_failed',
|
||||
...FRESHNESS_FAILURE_CODES
|
||||
])
|
||||
|
||||
// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
|
||||
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
|
||||
// past minute 10 costs the entire verdict, so a healthy fleet produced none on
|
||||
// 2026-09-05. A signal may miss this many consecutive samples before the window
|
||||
// not evidence of an unhealthy fleet. Under the 25-minute lineage cap in force
|
||||
// then, a restart past minute 10 cost the entire verdict, so a healthy fleet
|
||||
// produced none on 2026-09-05. A signal may miss this many consecutive samples before the window
|
||||
// restarts; the sample is still evaluated against every threshold it can read,
|
||||
// and a threshold breach still freezes the run outright.
|
||||
export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
|
||||
@@ -668,6 +727,44 @@ function freshnessKey(failure: IncidentFailure): string {
|
||||
return `${failure.source}/${failure.signal ?? '*'}`
|
||||
}
|
||||
|
||||
// The streak key for a per-cell active-probe reading, or null if the failure is not
|
||||
// one. A cell probe is a single HTTP round trip and so is subject to
|
||||
// cellProbeToleranceSamples; director and auth probes return null on purpose,
|
||||
// because they are the single points of failure this gate exists to catch.
|
||||
//
|
||||
// Why the key is the cell and not the signal: health, ready and latency_ms all
|
||||
// describe the same round trip. Keyed per signal, a cell that alternates between
|
||||
// answering slowly and not answering at all holds every individual streak at one and
|
||||
// never reaches the tolerance, so a continuously unhealthy cell passes the gate.
|
||||
export function cellProbeStreakKey(failure: IncidentFailure): string | null {
|
||||
const signal = failure.signal
|
||||
if (failure.source !== 'active-probe' || signal === undefined) return null
|
||||
if (!signal.startsWith('cell.')) return null
|
||||
const lastDot = signal.lastIndexOf('.')
|
||||
if (lastDot < 'cell.'.length) return null
|
||||
return `${failure.source}/${signal.slice(0, lastDot)}`
|
||||
}
|
||||
|
||||
// Cloud Run replaces director instances in place rather than holding the count,
|
||||
// so the reading leaves [min, max] for about one sample roughly twice a day, and a
|
||||
// deploy that briefly serves two revisions raises it the same way. Neither is an
|
||||
// unhealthy fleet, and on 2026-09-17 this was one of the signals freezing the
|
||||
// pre-drain gate on a condition the roll exists to fix. Min and max share one
|
||||
// streak on purpose: a count that alternates above and below the band would
|
||||
// otherwise hold each individual streak at one and never reach the tolerance.
|
||||
export function directorInstancesStreakKey(failure: IncidentFailure): string | null {
|
||||
if (failure.source !== 'cloud-monitoring' || failure.signal !== 'director.instances') {
|
||||
return null
|
||||
}
|
||||
return `${failure.source}/${failure.signal}`
|
||||
}
|
||||
|
||||
// The streak key for any reading subject to cellProbeToleranceSamples, or null
|
||||
// for a reading that freezes the run on its first bad sample.
|
||||
export function toleratedStreakKey(failure: IncidentFailure): string | null {
|
||||
return cellProbeStreakKey(failure) ?? directorInstancesStreakKey(failure)
|
||||
}
|
||||
|
||||
// Rebuild the per-signal tolerated streak from the trailing continuity events so a
|
||||
// resumed monitor cannot hand a signal a fresh budget.
|
||||
function resumeFreshnessStreaks(
|
||||
@@ -777,7 +874,12 @@ export async function runIncidentMonitor(
|
||||
state.recoverySourceCellId,
|
||||
state.capacityCellId
|
||||
)
|
||||
} catch {
|
||||
} catch (error) {
|
||||
dependencies.warn?.(
|
||||
`incident monitor collector failed: ${
|
||||
error instanceof Error ? error.message : 'unknown error'
|
||||
}`
|
||||
)
|
||||
evaluation = {
|
||||
status: 'freeze',
|
||||
evaluatedAt: new Date(dependencies.now()).toISOString(),
|
||||
@@ -798,7 +900,8 @@ export async function runIncidentMonitor(
|
||||
const toleratedKeys = new Set(
|
||||
state.windowStartedAt !== null &&
|
||||
continuityFailures.length > 0 &&
|
||||
continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
|
||||
continuityFailures.every((failure) =>
|
||||
TOLERABLE_CONTINUITY_FAILURE_CODES.has(failure.code))
|
||||
? continuityFailures.map(freshnessKey)
|
||||
: []
|
||||
)
|
||||
@@ -828,9 +931,40 @@ export async function runIncidentMonitor(
|
||||
}
|
||||
state.sampleCount++
|
||||
}
|
||||
if (thresholdFailures.length > 0) {
|
||||
const probeFailures = new Map<string, IncidentFailure[]>()
|
||||
for (const failure of thresholdFailures) {
|
||||
const key = toleratedStreakKey(failure)
|
||||
if (key === null) continue
|
||||
probeFailures.set(key, [...(probeFailures.get(key) ?? []), failure])
|
||||
}
|
||||
for (const key of Object.keys(state.probeStreaks)) {
|
||||
if (!probeFailures.has(key)) delete state.probeStreaks[key]
|
||||
}
|
||||
const sustainedProbeFailures: IncidentFailure[] = []
|
||||
const toleratedProbeFailures: IncidentFailure[] = []
|
||||
for (const [key, entries] of probeFailures) {
|
||||
const streak = (state.probeStreaks[key] ?? 0) + 1
|
||||
state.probeStreaks[key] = streak
|
||||
const target =
|
||||
streak > INCIDENT_MONITOR_THRESHOLDS.cellProbeToleranceSamples
|
||||
? sustainedProbeFailures
|
||||
: toleratedProbeFailures
|
||||
target.push(...entries)
|
||||
}
|
||||
if (toleratedProbeFailures.length > 0) {
|
||||
state.toleratedProbeEvents.push({
|
||||
recordedAt: evaluation.evaluatedAt,
|
||||
windowSequence: state.windowSequence,
|
||||
failures: toleratedProbeFailures
|
||||
})
|
||||
}
|
||||
const freezingFailures = [
|
||||
...thresholdFailures.filter((failure) => toleratedStreakKey(failure) === null),
|
||||
...sustainedProbeFailures
|
||||
]
|
||||
if (freezingFailures.length > 0) {
|
||||
state.frozenAt ??= evaluation.evaluatedAt
|
||||
state.failures = [...state.failures, ...thresholdFailures]
|
||||
state.failures = [...state.failures, ...freezingFailures]
|
||||
}
|
||||
if (state.windowStartedAt === null) {
|
||||
if (dependencies.now() >= lineageDeadlineMs) {
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayConfig } from './config.js'
|
||||
|
||||
vi.mock('./admin-token-verifier.js', () => ({
|
||||
createAdminTokenVerifier: () => async (token: string) => token === 'deploy-token',
|
||||
createReadOnlyAdminTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
|
||||
createRegionalRehomeTokenVerifier: () => async () => false,
|
||||
createRuntimeTokenVerifier: () => async () => false
|
||||
}))
|
||||
|
||||
vi.mock('./relay-token-verifier.js', () => ({
|
||||
createRelayTokenVerifier: () => async () => null,
|
||||
readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
|
||||
}))
|
||||
|
||||
import { createRelayApp } from './app.js'
|
||||
|
||||
// The message the pool raises when its own dial outruns connectionTimeoutMillis.
|
||||
// This is the shape that failed a rollout wave as an HTTP 404.
|
||||
const poolTimeout = () => new Error('Connection terminated due to connection timeout')
|
||||
|
||||
function adminRequest(body: unknown): RequestInit {
|
||||
return {
|
||||
method: 'POST',
|
||||
headers: { authorization: 'Bearer deploy-token', 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
}
|
||||
}
|
||||
|
||||
function appWith(assignments: Record<string, unknown>, overrides: Partial<RelayConfig> = {}) {
|
||||
return createRelayApp(config(overrides), {
|
||||
store: {} as never,
|
||||
assignments: assignments as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
}
|
||||
|
||||
describe('admin routes under a database that is briefly out of reach', () => {
|
||||
it('answers cell-status with a retryable 503 instead of a not-found', async () => {
|
||||
const cellDeploymentStatus = vi.fn(async () => {
|
||||
throw poolTimeout()
|
||||
})
|
||||
const app = appWith({ cellDeploymentStatus }, { publicAssignmentRetryAfterSeconds: 7 })
|
||||
|
||||
const response = await app.request(
|
||||
'/v1/admin/cell-status',
|
||||
adminRequest({ v: 1, cellId: 'production-gce-c7' })
|
||||
)
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('Retry-After')).toBe('7')
|
||||
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
|
||||
})
|
||||
|
||||
it('keeps the not-found mapping for a real cell-status failure', async () => {
|
||||
const cellDeploymentStatus = vi.fn(async () => {
|
||||
throw new Error('unknown_cell')
|
||||
})
|
||||
const app = appWith({ cellDeploymentStatus })
|
||||
|
||||
const response = await app.request(
|
||||
'/v1/admin/cell-status',
|
||||
adminRequest({ v: 1, cellId: 'production-gce-c7' })
|
||||
)
|
||||
|
||||
expect(response.status).toBe(404)
|
||||
expect(await response.json()).toEqual({ error: 'unknown_cell' })
|
||||
})
|
||||
|
||||
it('answers admission-selector/status with a retryable 503 instead of a conflict', async () => {
|
||||
const inspectCellAdmissionSelector = vi.fn(async () => {
|
||||
throw poolTimeout()
|
||||
})
|
||||
const app = appWith({ inspectCellAdmissionSelector })
|
||||
|
||||
const response = await app.request(
|
||||
'/v1/admin/admission-selector/status',
|
||||
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
|
||||
)
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(response.headers.get('Retry-After')).toBe('5')
|
||||
expect(await response.json()).toEqual({ error: 'database_temporarily_unavailable' })
|
||||
})
|
||||
|
||||
it('keeps the conflict mapping for a real admission-selector/status failure', async () => {
|
||||
const inspectCellAdmissionSelector = vi.fn(async () => {
|
||||
throw new Error('admission_selector_attempt_not_found')
|
||||
})
|
||||
const app = appWith({ inspectCellAdmissionSelector })
|
||||
|
||||
const response = await app.request(
|
||||
'/v1/admin/admission-selector/status',
|
||||
adminRequest({ v: 1, attemptId: '22222222-2222-4222-8222-222222222222' })
|
||||
)
|
||||
|
||||
expect(response.status).toBe(409)
|
||||
expect(await response.json()).toEqual({ error: 'admission_selector_attempt_not_found' })
|
||||
})
|
||||
|
||||
it('leaves the pre-request guards ahead of the mapping alone', async () => {
|
||||
const cellDeploymentStatus = vi.fn(async () => {
|
||||
throw poolTimeout()
|
||||
})
|
||||
const app = appWith({ cellDeploymentStatus })
|
||||
|
||||
// An unauthenticated caller must never learn the database is struggling.
|
||||
const unauthenticated = await app.request('/v1/admin/cell-status', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ v: 1, cellId: 'production-gce-c7' })
|
||||
})
|
||||
expect(unauthenticated.status).toBe(401)
|
||||
|
||||
const invalid = await app.request('/v1/admin/cell-status', adminRequest({ v: 1 }))
|
||||
expect(invalid.status).toBe(400)
|
||||
expect(cellDeploymentStatus).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
|
||||
return {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay.example.test',
|
||||
cellUrl: 'https://relay.example.test',
|
||||
region: 'us-central1',
|
||||
authIssuer: 'https://auth.example.test',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.test/jwks',
|
||||
assignmentSigningKey: new Uint8Array(32),
|
||||
role: 'director',
|
||||
cellId: 'director',
|
||||
cells: [],
|
||||
adminAudience: 'https://relay.example.test/v1/admin/drain',
|
||||
deployServiceAccount: 'deploy@example.test',
|
||||
runtimeServiceAccount: 'relay-cell@example.test',
|
||||
adminJwksUrl: 'https://auth.example.test/jwks',
|
||||
databasePoolMax: 10,
|
||||
publicAssignmentsEnabled: true,
|
||||
publicAssignmentConcurrency: 2,
|
||||
publicAssignmentQueueMax: 128,
|
||||
publicAssignmentWaitMs: 4_000,
|
||||
publicResolveConcurrency: 1,
|
||||
publicResolveWaitMs: 5_000,
|
||||
publicAssignmentRetryAfterSeconds: 5,
|
||||
dataDir: './data',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
+118
-61
@@ -27,10 +27,11 @@ import {
|
||||
createRegionalRehomeTokenVerifier,
|
||||
createRuntimeTokenVerifier
|
||||
} from './admin-token-verifier.js'
|
||||
import type {
|
||||
CellFenceAttemptEvidence,
|
||||
RelayAssignment,
|
||||
RelayAssignmentStore
|
||||
import {
|
||||
RelayHomeCellUnavailableError,
|
||||
type CellFenceAttemptEvidence,
|
||||
type RelayAssignment,
|
||||
type RelayAssignmentStore
|
||||
} from './assignment-store.js'
|
||||
import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
|
||||
import { CELL_ADMISSION_STATES } from './cell-admission-selector.js'
|
||||
@@ -58,6 +59,8 @@ const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
|
||||
|
||||
const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
|
||||
const REGION_CATALOG_CACHE_MS = 30_000
|
||||
// A drain that outlives the roll step it belongs to is an outage, not a pacing win.
|
||||
const DRAIN_PACE_WINDOW_MAX_MS = 5 * 60 * 1_000
|
||||
|
||||
type AdmissionRejectionLogEntry = {
|
||||
route: 'assign' | 'resolve'
|
||||
@@ -72,7 +75,7 @@ export function createRelayApp(
|
||||
operations: {
|
||||
store: RelayCredentialStore
|
||||
assignments: RelayAssignmentStore
|
||||
drain: (graceMs: number) => void
|
||||
drain: (graceMs: number, options?: { paceWindowMs?: number }) => void
|
||||
idleRehome?: (input: IdleRegionalRehomeRequest & {
|
||||
cohortPercent: number
|
||||
directorSafety: RegionalRehomeSafetySnapshot
|
||||
@@ -176,6 +179,21 @@ export function createRelayApp(
|
||||
context.header('Retry-After', String(stickyRetryAfterSeconds))
|
||||
return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
|
||||
}
|
||||
// An admin route that collapses every failure into one status cannot tell a
|
||||
// real conflict from a database that was briefly out of reach, and the rollout
|
||||
// tooling retries on 503 only. Transient failures get the answer the public
|
||||
// routes already give; everything else keeps the route's own mapping.
|
||||
const rejectAdminOperation = (
|
||||
context: Context,
|
||||
error: unknown,
|
||||
status: 404 | 409
|
||||
): Response => {
|
||||
if (!isRelayDatabaseTransientError(error)) {
|
||||
return context.json({ error: operationError(error) }, status)
|
||||
}
|
||||
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
|
||||
return context.json({ error: 'database_temporarily_unavailable' }, 503)
|
||||
}
|
||||
// Aggregate counters cannot separate a handful of pathological hosts from a broad
|
||||
// population, so every admission rejection names its host and reason. Keyed on
|
||||
// route:lane:reason rather than host, the log stays bounded under load.
|
||||
@@ -223,7 +241,15 @@ export function createRelayApp(
|
||||
})
|
||||
app.get('/v1/regions', async (context) => {
|
||||
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
|
||||
return context.json({ v: 1, regions: await regionCatalog() })
|
||||
try {
|
||||
return context.json({ v: 1, regions: await regionCatalog() })
|
||||
} catch (error) {
|
||||
if (!isRelayDatabaseTransientError(error)) throw error
|
||||
// Same contract as the assignment routes: a database that is briefly out
|
||||
// of reach is a retry, not a director fault.
|
||||
context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
|
||||
return context.json({ error: 'region_catalog_temporarily_unavailable' }, 503)
|
||||
}
|
||||
})
|
||||
app.post('/v1/assign', async (context) => {
|
||||
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
|
||||
@@ -336,16 +362,17 @@ export function createRelayApp(
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
|
||||
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
|
||||
logAssignmentRejection({
|
||||
route: 'assign',
|
||||
lane,
|
||||
hinted: Boolean(body.data.reconnect),
|
||||
relayHostId: claims.relayHostId,
|
||||
reason: operationError(error)
|
||||
reason: operationError(error),
|
||||
...homeCellRejectionDetail(error)
|
||||
})
|
||||
}
|
||||
if (isRelayAssignmentCapacityError(error)) {
|
||||
if (isRelayAssignmentUnavailableError(error)) {
|
||||
if (lane === 'placement') {
|
||||
operations.recordRegionSelection?.({ targetRegion, fallback: false })
|
||||
}
|
||||
@@ -364,11 +391,13 @@ export function createRelayApp(
|
||||
fallback: lane === 'placement' && assignment.region !== targetRegion
|
||||
})
|
||||
// Grant-side counterpart of the rejection log: reconnect grants are rare
|
||||
// enough to log and make "which cell is this host on" answerable.
|
||||
if (lane === 'sticky') {
|
||||
// enough to log and make "which cell is this host on" answerable. The
|
||||
// placement-lane ones matter most — they are the only record that a host
|
||||
// whose sticky lane failed verification landed anywhere at all.
|
||||
if (body.data.reconnect) {
|
||||
console.warn(
|
||||
`[orca-relay] assignment granted lane=sticky host=${relayHostLogDigest(claims.relayHostId)}` +
|
||||
` cell=${assignment.cellId}`
|
||||
`[orca-relay] assignment granted lane=${lane} hinted=true` +
|
||||
` host=${relayHostLogDigest(claims.relayHostId)} cell=${assignment.cellId}`
|
||||
)
|
||||
}
|
||||
const lease = await new SignJWT({
|
||||
@@ -441,16 +470,17 @@ export function createRelayApp(
|
||||
leaseExpiresAt: assignment.leaseExpiresAt
|
||||
})
|
||||
} catch (error) {
|
||||
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
|
||||
if (isRelayAssignmentUnavailableError(error) || isRelayDatabaseTransientError(error)) {
|
||||
logAssignmentRejection({
|
||||
route: 'resolve',
|
||||
lane: 'none',
|
||||
hinted: false,
|
||||
relayHostId: body.data.relayHostId,
|
||||
reason: operationError(error)
|
||||
reason: operationError(error),
|
||||
...homeCellRejectionDetail(error)
|
||||
})
|
||||
}
|
||||
if (isRelayAssignmentCapacityError(error)) {
|
||||
if (isRelayAssignmentUnavailableError(error)) {
|
||||
return context.json({ error: operationError(error) }, 503)
|
||||
}
|
||||
if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context)
|
||||
@@ -465,12 +495,18 @@ export function createRelayApp(
|
||||
return context.json({ error: 'invalid_token' }, 401)
|
||||
}
|
||||
const body = z
|
||||
.object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) })
|
||||
.object({
|
||||
v: z.literal(1),
|
||||
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
|
||||
// Spreads the drain sends, and so the re-dials, over this window.
|
||||
paceWindowMs: z.number().int().nonnegative().max(DRAIN_PACE_WINDOW_MAX_MS).optional()
|
||||
})
|
||||
.strict()
|
||||
.safeParse(await context.req.json().catch(() => null))
|
||||
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
|
||||
operations.drain(body.data.graceMs)
|
||||
return context.json({ ok: true })
|
||||
const paceWindowMs = body.data.paceWindowMs ?? 0
|
||||
operations.drain(body.data.graceMs, { paceWindowMs })
|
||||
return context.json({ ok: true, paceWindowMs })
|
||||
})
|
||||
app.post('/v1/admin/host-idle-rehome', async (context) => {
|
||||
if (config.role !== 'cell' || !operations.idleRehome) {
|
||||
@@ -497,7 +533,7 @@ export function createRelayApp(
|
||||
try {
|
||||
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/host-drain', async (context) => {
|
||||
@@ -548,7 +584,7 @@ export function createRelayApp(
|
||||
...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {})
|
||||
})
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/runtime-status', async (context) => {
|
||||
@@ -602,7 +638,7 @@ export function createRelayApp(
|
||||
await operations.assignments.recordCellHeartbeat(body.data)
|
||||
return context.json({ ok: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-rehome-status', async (context) => {
|
||||
@@ -622,7 +658,7 @@ export function createRelayApp(
|
||||
await operations.assignments.recordCellRegionalRehomeStatus(body.data)
|
||||
return context.json({ ok: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.get('/v1/admin/regional-rehome-preview', async (context) => {
|
||||
@@ -661,7 +697,7 @@ export function createRelayApp(
|
||||
const control = await operations.assignments.applyRegionalRehomeControl(body.data)
|
||||
return context.json({ v: 1, control })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/regional-rehome-trust-probe', async (context) => {
|
||||
@@ -705,7 +741,7 @@ export function createRelayApp(
|
||||
})
|
||||
return context.json(result)
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/evacuate', async (context) => {
|
||||
@@ -726,7 +762,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, migration })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/migration-complete', async (context) => {
|
||||
@@ -747,7 +783,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ ok: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/migration-supersede-cell', async (context) => {
|
||||
@@ -772,7 +808,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, superseded })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/rebalance-dormant', async (context) => {
|
||||
@@ -793,7 +829,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, assignment })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/admission-selector/apply', async (context) => {
|
||||
@@ -813,7 +849,7 @@ export function createRelayApp(
|
||||
const result = await operations.assignments.applyCellAdmissionSelector(body.data)
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => {
|
||||
@@ -838,7 +874,7 @@ export function createRelayApp(
|
||||
})
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/admission-selector/status', async (context) => {
|
||||
@@ -860,7 +896,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => {
|
||||
@@ -891,7 +927,7 @@ export function createRelayApp(
|
||||
})
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-state', async (context) => {
|
||||
@@ -916,7 +952,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ ok: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => {
|
||||
@@ -939,7 +975,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, cellId: body.data.cellId, expiresAt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => {
|
||||
@@ -962,7 +998,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, cellId: body.data.cellId, committed: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attest', async (context) => {
|
||||
@@ -991,7 +1027,7 @@ export function createRelayApp(
|
||||
attempt: result.attempt
|
||||
})
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => {
|
||||
@@ -1012,7 +1048,7 @@ export function createRelayApp(
|
||||
const attempt = await operations.assignments.prepareCellFenceAttempt(evidence)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-start', async (context) => {
|
||||
@@ -1037,7 +1073,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-plan', async (context) => {
|
||||
@@ -1061,7 +1097,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-operation', async (context) => {
|
||||
@@ -1087,7 +1123,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-status', async (context) => {
|
||||
@@ -1107,7 +1143,7 @@ export function createRelayApp(
|
||||
const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-fence-attempt-abort', async (context) => {
|
||||
@@ -1128,7 +1164,7 @@ export function createRelayApp(
|
||||
const attempt = await operations.assignments.abortCellFenceAttempt(evidence)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/drain-attempt-prepare', async (context) => {
|
||||
@@ -1154,7 +1190,7 @@ export function createRelayApp(
|
||||
})
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/drain-attempt-send', async (context) => {
|
||||
@@ -1174,7 +1210,7 @@ export function createRelayApp(
|
||||
const attempt = await operations.assignments.beginCellDrainSend(body.data)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/drain-attempt-receipt', async (context) => {
|
||||
@@ -1196,7 +1232,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, attempt })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/drain-attempt-recover-forward', async (context) => {
|
||||
@@ -1216,7 +1252,7 @@ export function createRelayApp(
|
||||
const result = await operations.assignments.prepareCellDrainRecovery(body.data)
|
||||
return context.json({ v: 1, ...result })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-config', async (context) => {
|
||||
@@ -1243,7 +1279,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ ok: true })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/evacuate-cell', async (context) => {
|
||||
@@ -1265,7 +1301,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, started })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/evacuation-capacity', async (context) => {
|
||||
@@ -1288,7 +1324,7 @@ export function createRelayApp(
|
||||
)
|
||||
return context.json({ v: 1, ...capacity })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 409)
|
||||
return rejectAdminOperation(context, error, 409)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/evacuation-status', async (context) => {
|
||||
@@ -1307,12 +1343,17 @@ export function createRelayApp(
|
||||
if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) {
|
||||
return context.json({ error: 'insufficient_permission' }, 403)
|
||||
}
|
||||
const status = await operations.assignments.cellEvacuationStatus(
|
||||
body.data.sourceCellId,
|
||||
body.data.targetCellId,
|
||||
body.data.completeReady
|
||||
)
|
||||
return context.json({ v: 1, ...status })
|
||||
try {
|
||||
const status = await operations.assignments.cellEvacuationStatus(
|
||||
body.data.sourceCellId,
|
||||
body.data.targetCellId,
|
||||
body.data.completeReady
|
||||
)
|
||||
return context.json({ v: 1, ...status })
|
||||
} catch (error) {
|
||||
if (!isRelayDatabaseTransientError(error)) throw error
|
||||
return context.json({ error: 'database_temporarily_unavailable' }, 503)
|
||||
}
|
||||
})
|
||||
app.post('/v1/admin/cell-status', async (context) => {
|
||||
const bearer = readBearer(context.req.header('authorization'))
|
||||
@@ -1329,7 +1370,7 @@ export function createRelayApp(
|
||||
const status = await operations.assignments.cellDeploymentStatus(body.data.cellId)
|
||||
return context.json({ v: 1, status })
|
||||
} catch (error) {
|
||||
return context.json({ error: operationError(error) }, 404)
|
||||
return rejectAdminOperation(context, error, 404)
|
||||
}
|
||||
})
|
||||
return app
|
||||
@@ -1912,25 +1953,41 @@ function logAssignmentRejection(input: {
|
||||
hinted: boolean
|
||||
relayHostId: string
|
||||
reason: string
|
||||
cause?: string
|
||||
cell?: string
|
||||
suppressed?: number
|
||||
}): void {
|
||||
console.warn(
|
||||
`[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` +
|
||||
` hinted=${input.hinted} reason=${input.reason}` +
|
||||
` host=${relayHostLogDigest(input.relayHostId)}` +
|
||||
(input.cause === undefined ? '' : ` cause=${input.cause}`) +
|
||||
(input.cell === undefined ? '' : ` cell=${input.cell}`) +
|
||||
(input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`)
|
||||
)
|
||||
}
|
||||
|
||||
function isRelayAssignmentCapacityError(error: unknown): boolean {
|
||||
// The home-cell reason is not capacity, but it is the same answer to the client:
|
||||
// retry, the director cannot place you right now.
|
||||
function isRelayAssignmentUnavailableError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
['relay_capacity_exhausted', 'relay_connection_headroom_exhausted'].includes(
|
||||
error.message
|
||||
)
|
||||
[
|
||||
'relay_capacity_exhausted',
|
||||
'relay_connection_headroom_exhausted',
|
||||
'relay_home_cell_unavailable'
|
||||
].includes(error.message)
|
||||
)
|
||||
}
|
||||
|
||||
function homeCellRejectionDetail(
|
||||
error: unknown
|
||||
): { cause: string; cell: string } | Record<string, never> {
|
||||
return error instanceof RelayHomeCellUnavailableError
|
||||
? { cause: error.unavailableCause, cell: error.cellId }
|
||||
: {}
|
||||
}
|
||||
|
||||
function isCanonicalRelayOrigin(value: string): boolean {
|
||||
const url = new URL(value)
|
||||
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
|
||||
import type { RelayCellConfig } from './config.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const HEARTBEAT_TTL_MS = 45_000
|
||||
const START_MS = 100
|
||||
const IDENTITY = { userId: 'user-1', relayHostId: 'host000000000001' }
|
||||
|
||||
// A connection-limited cell is what makes the committed fence mandatory, and
|
||||
// that is the branch which used to answer "capacity exhausted".
|
||||
const FENCED_CELL: RelayCellConfig = {
|
||||
id: 'home',
|
||||
url: 'https://home.example.com',
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const database of databases.splice(0)) await database.close()
|
||||
})
|
||||
|
||||
interface Harness {
|
||||
store: RelayAssignmentStore
|
||||
heartbeat: (cell: RelayCellConfig, ready: boolean) => Promise<void>
|
||||
setNow: (value: number) => void
|
||||
}
|
||||
|
||||
async function setup(cells: RelayCellConfig[] = [FENCED_CELL]): Promise<Harness> {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
databases.push(database)
|
||||
let now = START_MS
|
||||
const store = new RelayAssignmentStore(database, () => now, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: HEARTBEAT_TTL_MS
|
||||
})
|
||||
await store.reconcileCells(cells, true)
|
||||
const heartbeat = async (cell: RelayCellConfig, ready: boolean): Promise<void> => {
|
||||
await store.recordCellHeartbeat({
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: `1111111${cells.indexOf(cell)}-1111-4111-8111-111111111111`,
|
||||
startedAt: 50,
|
||||
ready,
|
||||
observedRequests: 0,
|
||||
...(cell.connectionHardCap === undefined
|
||||
? {}
|
||||
: {
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionHardCap: cell.connectionHardCap,
|
||||
connectionUnobservedBound: cell.connectionUnobservedBound
|
||||
})
|
||||
})
|
||||
}
|
||||
for (const cell of cells) await heartbeat(cell, true)
|
||||
return { store, heartbeat, setNow: (value: number) => (now = value) }
|
||||
}
|
||||
|
||||
async function assignFailure(store: RelayAssignmentStore): Promise<unknown> {
|
||||
return await store.assign(IDENTITY).then(
|
||||
() => new Error('assign unexpectedly succeeded'),
|
||||
(error: unknown) => error
|
||||
)
|
||||
}
|
||||
|
||||
function homeCellError(error: unknown): RelayHomeCellUnavailableError {
|
||||
expect(error).toBeInstanceOf(RelayHomeCellUnavailableError)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the assertion above.
|
||||
return error as RelayHomeCellUnavailableError
|
||||
}
|
||||
|
||||
describe('home cell unavailable', () => {
|
||||
it('names a readiness failure rather than reporting fleet capacity', async () => {
|
||||
const { store, heartbeat, setNow } = await setup()
|
||||
await store.assign(IDENTITY)
|
||||
setNow(START_MS + 1_000)
|
||||
await heartbeat(FENCED_CELL, false)
|
||||
|
||||
const error = homeCellError(await assignFailure(store))
|
||||
|
||||
expect(error.message).toBe('relay_home_cell_unavailable')
|
||||
expect(error.unavailableCause).toBe('not_ready')
|
||||
expect(error.cellId).toBe(FENCED_CELL.id)
|
||||
})
|
||||
|
||||
it('names a heartbeat gap as unheard even though the cell last reported ready', async () => {
|
||||
const { store, setNow } = await setup()
|
||||
await store.assign(IDENTITY)
|
||||
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
|
||||
|
||||
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('unheard')
|
||||
})
|
||||
|
||||
it('names a drained cell as draining ahead of its heartbeat gap', async () => {
|
||||
const { store, setNow } = await setup()
|
||||
await store.assign(IDENTITY)
|
||||
await store.configureCell(FENCED_CELL, false)
|
||||
setNow(START_MS + HEARTBEAT_TTL_MS + 1)
|
||||
|
||||
expect(homeCellError(await assignFailure(store)).unavailableCause).toBe('draining')
|
||||
})
|
||||
|
||||
it('still reports capacity exhaustion when the fleet has no headroom', async () => {
|
||||
const { store } = await setup([{ ...FENCED_CELL, capacityRequests: 1 }])
|
||||
await store.assign(IDENTITY)
|
||||
|
||||
await expect(
|
||||
store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
|
||||
).rejects.toThrow('relay_capacity_exhausted')
|
||||
})
|
||||
|
||||
it('rehomes instead of rejecting when the unavailable cell needs no fence', async () => {
|
||||
const home: RelayCellConfig = {
|
||||
id: 'home',
|
||||
url: 'https://home.example.com',
|
||||
capacityRequests: 1_000
|
||||
}
|
||||
const spare: RelayCellConfig = {
|
||||
id: 'spare',
|
||||
url: 'https://spare.example.com',
|
||||
capacityRequests: 1_000
|
||||
}
|
||||
const { store, heartbeat, setNow } = await setup([home, spare])
|
||||
expect((await store.assign(IDENTITY)).cellId).toBe(home.id)
|
||||
setNow(START_MS + 1_000)
|
||||
await heartbeat(home, false)
|
||||
|
||||
expect((await store.assign(IDENTITY)).cellId).toBe(spare.id)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayAssignment } from './assignment-store.js'
|
||||
import { RelayHomeCellUnavailableError, type RelayAssignment } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
@@ -52,6 +52,39 @@ describe('assignment rejection logging', () => {
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('separates an unavailable home cell from capacity and names its cause', async () => {
|
||||
const host = 'cccccccccccccccc'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
assign: vi.fn(async () => {
|
||||
throw new RelayHomeCellUnavailableError('cell-asia-1', 'not_ready')
|
||||
}),
|
||||
// The sticky lane refuses a host whose home cell is not live, so this
|
||||
// arrives hinted on the placement lane.
|
||||
resolve: vi.fn(async () => null)
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(await response.json()).toEqual({ error: 'relay_home_cell_unavailable' })
|
||||
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
|
||||
entry.includes('assignment rejected')
|
||||
)
|
||||
expect(line).toContain('lane=placement')
|
||||
expect(line).toContain('hinted=true')
|
||||
expect(line).toContain('reason=relay_home_cell_unavailable')
|
||||
expect(line).toContain('cause=not_ready')
|
||||
expect(line).toContain('cell=cell-asia-1')
|
||||
expect(line).not.toContain('relay_capacity_exhausted')
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('logs an unhinted placement rejection without the raw host id', async () => {
|
||||
const host = 'gggggggggggggggg'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
@@ -221,6 +254,31 @@ describe('assignment grant logging', () => {
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('logs a hinted grant served by the placement lane', async () => {
|
||||
const host = 'rrrrrrrrrrrrrrrr'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const app = createRelayApp(config(), {
|
||||
store: {} as never,
|
||||
assignments: {
|
||||
assign: vi.fn(async () => assignment('cell-new', host)),
|
||||
resolve: vi.fn(async () => null)
|
||||
} as never,
|
||||
drain: vi.fn(),
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
|
||||
entry.includes('assignment granted')
|
||||
)
|
||||
expect(line).toContain('lane=placement')
|
||||
expect(line).toContain('hinted=true')
|
||||
expect(line).toContain('cell=cell-new')
|
||||
expect(line).not.toContain(host)
|
||||
})
|
||||
|
||||
it('does not log unhinted placement grants', async () => {
|
||||
const host = 'pppppppppppppppp'
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
import { createDrainMigrationRowLookup } from './drain-migration-row-lookup.js'
|
||||
import { IDLE_REHOME_PAGE_SIZE, selectIdleRegionalRehomes } from './idle-regional-rehome-selection.js'
|
||||
import {
|
||||
selectIdleRegionalRehomes,
|
||||
type IdleRegionalRehomeCandidate,
|
||||
type IdleRehomeHostCursor
|
||||
} from './idle-regional-rehome-selection.js'
|
||||
import {
|
||||
RegionalRehomePollTelemetry,
|
||||
type RegionalRehomePollGate
|
||||
} from './regional-rehome-poll-telemetry.js'
|
||||
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
|
||||
import {
|
||||
previewRegionalRehomeEligibility,
|
||||
@@ -45,6 +53,15 @@ import {
|
||||
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
|
||||
} from './assignment-connection-headroom-query.js'
|
||||
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
|
||||
import {
|
||||
CONTROL_RENEWAL_BATCH_SQL,
|
||||
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
|
||||
controlRenewalBatchParams,
|
||||
orderedControlRenewalRows,
|
||||
readControlRenewalOutcomes,
|
||||
type ControlRenewalOutcome,
|
||||
type ControlRenewalRequest
|
||||
} from './control-renewal-statement.js'
|
||||
import {
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
||||
} from './database.js'
|
||||
@@ -101,21 +118,8 @@ type RelayAssignmentStoreOptions = {
|
||||
recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
|
||||
}
|
||||
|
||||
export type ControlRenewalOutcome =
|
||||
| 'renewed'
|
||||
| 'assignment_not_found'
|
||||
| 'activity_cell_not_authoritative'
|
||||
| 'control_activity_not_found'
|
||||
| 'control_activity_moved'
|
||||
| 'database_error'
|
||||
export type { ControlRenewalOutcome, ControlRenewalRequest }
|
||||
|
||||
const CONTROL_RENEWAL_OUTCOMES = new Set<ControlRenewalOutcome>([
|
||||
'renewed',
|
||||
'assignment_not_found',
|
||||
'activity_cell_not_authoritative',
|
||||
'control_activity_not_found',
|
||||
'control_activity_moved'
|
||||
])
|
||||
export type RelayAssignment = AssignmentIdentity & {
|
||||
cellId: string
|
||||
cellUrl: string
|
||||
@@ -395,6 +399,25 @@ class AssignmentInventoryScopeChanged extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
// Why a reason of its own: a host whose home cell is fenced-but-unattested is
|
||||
// refused regardless of fleet headroom, so reporting it as capacity sends
|
||||
// operators after capacity that was never short. Every cell boot and every
|
||||
// readiness dip produces these.
|
||||
export type RelayHomeCellUnavailableCause =
|
||||
| 'draining'
|
||||
| 'booting'
|
||||
| 'unheard'
|
||||
| 'not_ready'
|
||||
|
||||
export class RelayHomeCellUnavailableError extends Error {
|
||||
constructor(
|
||||
readonly cellId: string,
|
||||
readonly unavailableCause: RelayHomeCellUnavailableCause
|
||||
) {
|
||||
super('relay_home_cell_unavailable')
|
||||
}
|
||||
}
|
||||
|
||||
// Debt holds connection headroom for a control that may still arrive shortly
|
||||
// after its director-side timeout. Nothing legitimately arrives minutes late
|
||||
// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that
|
||||
@@ -925,7 +948,10 @@ export class RelayAssignmentStore {
|
||||
)) &&
|
||||
!(await this.cellHasCommittedFence(transaction, current.cellId, now))
|
||||
) {
|
||||
throw new Error('relay_capacity_exhausted')
|
||||
throw new RelayHomeCellUnavailableError(
|
||||
current.cellId,
|
||||
await this.homeCellUnavailableCause(transaction, current.cellId, now)
|
||||
)
|
||||
}
|
||||
forcedDeadReassignment = true
|
||||
}
|
||||
@@ -2677,7 +2703,13 @@ export class RelayAssignmentStore {
|
||||
)
|
||||
if (assignment.cellId !== text(row, 'cell_id')) moved++
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error
|
||||
// One unplaceable host must not end the sweep for the rest.
|
||||
if (
|
||||
!(error instanceof RelayHomeCellUnavailableError) &&
|
||||
!(error instanceof Error && error.message === 'relay_capacity_exhausted')
|
||||
) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
return moved
|
||||
@@ -3333,28 +3365,57 @@ export class RelayAssignmentStore {
|
||||
return previewRegionCorrection(this.database, this.now())
|
||||
}
|
||||
|
||||
private idleRegionalCandidateOffset = 0
|
||||
private idleRegionalCandidateCursor: IdleRehomeHostCursor = null
|
||||
private readonly regionalRehomePollTelemetry = new RegionalRehomePollTelemetry()
|
||||
|
||||
async selectIdleRegionalRehomeCandidates(
|
||||
processSafety?: RegionalRehomeSafetySnapshot
|
||||
): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
|
||||
): Promise<IdleRegionalRehomeCandidate[]> {
|
||||
const now = this.now()
|
||||
if (!processSafety || this.regionalRehomeCohortPercent === 0) return []
|
||||
const gated = (gate: RegionalRehomePollGate): IdleRegionalRehomeCandidate[] => {
|
||||
this.regionalRehomePollTelemetry.record({ now, gate, candidates: 0 })
|
||||
return []
|
||||
}
|
||||
if (!processSafety) return gated('process-safety-unavailable')
|
||||
if (this.regionalRehomeCohortPercent === 0) return gated('cohort-zero')
|
||||
const control = (await this.database.query(
|
||||
"SELECT enabled, not_before FROM relay_region_rehome_control WHERE control_id = 'global'"
|
||||
`SELECT enabled, not_before, preference_max_age_ms, host_cooldown_ms
|
||||
FROM relay_region_rehome_control WHERE control_id = 'global'`
|
||||
))[0]
|
||||
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) return []
|
||||
if (!control || Number(control.enabled) !== 1 || Number(control.not_before) > now) {
|
||||
return gated('control-closed')
|
||||
}
|
||||
// The dispatch budget is durable and global, but until now only
|
||||
// `commitIdleRegionalRehome` consulted it -- after the join had already run and
|
||||
// the worker had already POSTed every candidate to its source cell. An absent
|
||||
// row means the budget has never been spent, so it opens the gate.
|
||||
const worker = (await this.database.query(
|
||||
`SELECT paused_until, next_dispatch_at FROM relay_region_rehome_worker_state
|
||||
WHERE worker_id = 'global'`
|
||||
))[0]
|
||||
if (worker && (Number(worker.paused_until) > now || Number(worker.next_dispatch_at) > now)) {
|
||||
return gated('budget-closed')
|
||||
}
|
||||
const fleetSafety = await this.readRegionalRehomeFleetSafety(this.database, now)
|
||||
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return []
|
||||
const candidates = await selectIdleRegionalRehomes({
|
||||
if (regionalRehomeFleetSafetyFailure(processSafety, fleetSafety, now)) return gated('fleet-safety')
|
||||
const startedAt = performance.now()
|
||||
const selection = await selectIdleRegionalRehomes({
|
||||
database: this.database, now, heartbeatTtlMs: this.heartbeatTtlMs,
|
||||
cohortPercent: this.regionalRehomeCohortPercent, offset: this.idleRegionalCandidateOffset,
|
||||
cohortPercent: this.regionalRehomeCohortPercent,
|
||||
preferenceMaxAgeMs: Number(control.preference_max_age_ms),
|
||||
hostCooldownMs: Number(control.host_cooldown_ms),
|
||||
cursor: this.idleRegionalCandidateCursor,
|
||||
connectionHeadroom: await this.connectionHeadroomByCell(this.database),
|
||||
cellIsClean: regionalRehomeCellSafetyIsClean
|
||||
})
|
||||
this.idleRegionalCandidateOffset = candidates.length < IDLE_REHOME_PAGE_SIZE
|
||||
? 0 : this.idleRegionalCandidateOffset + candidates.length
|
||||
return candidates
|
||||
this.idleRegionalCandidateCursor = selection.cursor
|
||||
this.regionalRehomePollTelemetry.record({
|
||||
now,
|
||||
gate: 'open',
|
||||
candidates: selection.candidates.length,
|
||||
selectionMs: performance.now() - startedAt
|
||||
})
|
||||
return selection.candidates
|
||||
}
|
||||
|
||||
async commitIdleRegionalRehome(
|
||||
@@ -3478,132 +3539,149 @@ export class RelayAssignmentStore {
|
||||
})
|
||||
}
|
||||
|
||||
// Kept as the single-row contract for callers and tests: resolves on a
|
||||
// renewal and throws the outcome (or the driver's own error) otherwise.
|
||||
async renewControlActivity(
|
||||
identity: AssignmentIdentity,
|
||||
input: { activityId: string; cellId: string; expiresAt: number }
|
||||
): Promise<void> {
|
||||
validateActivityId(input.activityId)
|
||||
const now = this.now()
|
||||
const maximumExpiresAt =
|
||||
now +
|
||||
ASSIGNMENT_LIMITS.activityLeaseMs +
|
||||
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
|
||||
if (
|
||||
!Number.isSafeInteger(input.expiresAt) ||
|
||||
input.expiresAt <= now ||
|
||||
input.expiresAt > maximumExpiresAt
|
||||
) {
|
||||
if (!controlRenewalExpiryIsValid(input.expiresAt, now)) {
|
||||
throw new Error('invalid_activity_expiry')
|
||||
}
|
||||
const startedAt = performance.now()
|
||||
let outcome: ControlRenewalOutcome = 'database_error'
|
||||
try {
|
||||
outcome =
|
||||
this.database.dialect === 'postgres'
|
||||
? await this.renewPostgresControlActivity(identity, input, now)
|
||||
: await this.renewTransactionalControlActivity(identity, input, now)
|
||||
outcome = await this.renewOneControlActivity({ identity, ...input }, now)
|
||||
if (outcome !== 'renewed') throw new Error(outcome)
|
||||
} catch (error) {
|
||||
const message = String((error as { message?: unknown }).message)
|
||||
if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) {
|
||||
outcome = message as ControlRenewalOutcome
|
||||
}
|
||||
outcome = controlRenewalOutcomeOfError(error)
|
||||
throw error
|
||||
} finally {
|
||||
this.recordControlRenewal?.(performance.now() - startedAt, outcome)
|
||||
}
|
||||
}
|
||||
|
||||
private async renewPostgresControlActivity(
|
||||
identity: AssignmentIdentity,
|
||||
input: { activityId: string; cellId: string; expiresAt: number },
|
||||
// Renews every due control lease on a cell in one write transaction, returning
|
||||
// one outcome per input row in input order. Never throws for a multi-row batch:
|
||||
// a caller routes its own session on its own outcome.
|
||||
async renewControlActivities(
|
||||
rows: readonly ControlRenewalRequest[]
|
||||
): Promise<ControlRenewalOutcome[]> {
|
||||
const now = this.now()
|
||||
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
|
||||
const accepted: Array<ControlRenewalRequest & { index: number }> = []
|
||||
for (const [index, row] of rows.entries()) {
|
||||
const rejection = controlRenewalRejection(row, now)
|
||||
if (rejection) outcomes[index] = rejection
|
||||
else accepted.push({ ...row, index })
|
||||
}
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
if (accepted.length === 0) return outcomes
|
||||
let results: ControlRenewalOutcome[]
|
||||
try {
|
||||
results = await this.executeControlRenewals(accepted, now)
|
||||
} catch (error) {
|
||||
if (rows.length === 1) {
|
||||
outcomes[accepted[0]!.index] = controlRenewalOutcomeOfError(error)
|
||||
throw error
|
||||
}
|
||||
results = accepted.map(() => 'database_error')
|
||||
}
|
||||
for (const [position, row] of accepted.entries()) outcomes[row.index] = results[position]!
|
||||
return outcomes
|
||||
} finally {
|
||||
// Every path, so a rethrown lone renewal and an all-invalid batch are
|
||||
// counted the same as a batch that reached PostgreSQL.
|
||||
const durationMs = performance.now() - startedAt
|
||||
for (const outcome of outcomes) this.recordControlRenewal?.(durationMs, outcome)
|
||||
}
|
||||
}
|
||||
|
||||
private async executeControlRenewals(
|
||||
rows: readonly ControlRenewalRequest[],
|
||||
now: number
|
||||
): Promise<ControlRenewalOutcome[]> {
|
||||
if (rows.length === 1) return [await this.renewOneControlActivity(rows[0]!, now)]
|
||||
if (this.database.dialect !== 'postgres') {
|
||||
// Correctness over throughput: the SQLite writer is serialized anyway, and
|
||||
// this is the dialect the unit suites run on.
|
||||
return await this.renewControlActivitiesInSeries(rows, now)
|
||||
}
|
||||
const ordered = orderedControlRenewalRows(rows.map((row, index) => ({ ...row, index })))
|
||||
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
|
||||
try {
|
||||
const parsed = readControlRenewalOutcomes(
|
||||
await this.database.query(
|
||||
CONTROL_RENEWAL_BATCH_SQL,
|
||||
controlRenewalBatchParams(ordered, now)
|
||||
),
|
||||
ordered.length
|
||||
)
|
||||
for (const [position, row] of ordered.entries()) outcomes[row.index] = parsed[position]!
|
||||
return outcomes
|
||||
} catch (error) {
|
||||
// One statement means one contended assignment row can fail the whole
|
||||
// batch, so a failure degrades to the per-host statements this replaced
|
||||
// rather than costing every other host on the cell its renewal.
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_control_renewal_batch_failed',
|
||||
rows: ordered.length,
|
||||
message: String((error as { message?: unknown }).message)
|
||||
})
|
||||
)
|
||||
await Promise.all(
|
||||
ordered.map(async (row) => {
|
||||
try {
|
||||
outcomes[row.index] = await this.renewOneControlActivity(row, now)
|
||||
} catch {
|
||||
outcomes[row.index] = 'database_error'
|
||||
}
|
||||
})
|
||||
)
|
||||
return outcomes
|
||||
}
|
||||
}
|
||||
|
||||
private async renewControlActivitiesInSeries(
|
||||
rows: readonly ControlRenewalRequest[],
|
||||
now: number
|
||||
): Promise<ControlRenewalOutcome[]> {
|
||||
const outcomes: ControlRenewalOutcome[] = []
|
||||
for (const row of rows) {
|
||||
try {
|
||||
outcomes.push(await this.renewOneControlActivity(row, now))
|
||||
} catch {
|
||||
outcomes.push('database_error')
|
||||
}
|
||||
}
|
||||
return outcomes
|
||||
}
|
||||
|
||||
// Returns the outcome; a driver or pool failure reaches the caller unchanged.
|
||||
private async renewOneControlActivity(
|
||||
row: ControlRenewalRequest,
|
||||
now: number
|
||||
): Promise<ControlRenewalOutcome> {
|
||||
const row = (
|
||||
await this.database.query(
|
||||
`WITH assignment_state AS MATERIALIZED (
|
||||
SELECT cell_id, assignment_epoch
|
||||
FROM relay_assignments
|
||||
WHERE user_id = ? AND relay_host_id = ?
|
||||
FOR UPDATE
|
||||
), migration_state AS MATERIALIZED (
|
||||
SELECT migration.assignment_epoch
|
||||
FROM relay_assignment_migrations migration
|
||||
JOIN assignment_state assignment
|
||||
ON migration.target_cell_id = assignment.cell_id
|
||||
AND migration.assignment_epoch = assignment.assignment_epoch
|
||||
WHERE migration.user_id = ? AND migration.relay_host_id = ?
|
||||
AND migration.source_cell_id = ?
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
|
||||
FOR UPDATE OF migration
|
||||
), authorization_state AS MATERIALIZED (
|
||||
SELECT 1 AS authorized
|
||||
FROM assignment_state assignment
|
||||
WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state)
|
||||
), lease_state AS MATERIALIZED (
|
||||
SELECT lease.activity_kind, lease.cell_id
|
||||
FROM relay_assignment_activity_leases lease
|
||||
CROSS JOIN authorization_state
|
||||
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
|
||||
FOR UPDATE OF lease
|
||||
), renewed_lease AS (
|
||||
UPDATE relay_assignment_activity_leases lease
|
||||
SET expires_at = GREATEST(lease.expires_at, ?),
|
||||
updated_at = GREATEST(lease.updated_at, ?)
|
||||
FROM lease_state state
|
||||
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
|
||||
AND state.activity_kind = 'control' AND state.cell_id = ?
|
||||
RETURNING 1
|
||||
), renewed_assignment AS (
|
||||
UPDATE relay_assignments assignment
|
||||
SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?),
|
||||
last_activity_at = GREATEST(assignment.last_activity_at, ?)
|
||||
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
|
||||
AND EXISTS (SELECT 1 FROM renewed_lease)
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT CASE
|
||||
WHEN NOT EXISTS (SELECT 1 FROM assignment_state)
|
||||
THEN 'assignment_not_found'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM authorization_state)
|
||||
THEN 'activity_cell_not_authoritative'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM lease_state)
|
||||
THEN 'control_activity_not_found'
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM lease_state
|
||||
WHERE activity_kind <> 'control' OR cell_id <> ?
|
||||
) THEN 'control_activity_moved'
|
||||
WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed'
|
||||
ELSE 'control_activity_not_found'
|
||||
END AS outcome`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
input.cellId,
|
||||
input.cellId,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
input.activityId,
|
||||
input.expiresAt,
|
||||
now,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
input.activityId,
|
||||
input.cellId,
|
||||
input.expiresAt,
|
||||
now,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
input.cellId
|
||||
]
|
||||
)
|
||||
)[0]
|
||||
if (!row) throw new Error('missing_control_renewal_outcome')
|
||||
const outcome = text(row, 'outcome') as ControlRenewalOutcome
|
||||
if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome')
|
||||
return outcome
|
||||
if (this.database.dialect === 'postgres') {
|
||||
return readControlRenewalOutcomes(
|
||||
await this.database.query(
|
||||
CONTROL_RENEWAL_BATCH_SQL,
|
||||
controlRenewalBatchParams([row], now)
|
||||
),
|
||||
1
|
||||
)[0]!
|
||||
}
|
||||
try {
|
||||
return await this.renewTransactionalControlActivity(row.identity, row, now)
|
||||
} catch (error) {
|
||||
const message = String((error as { message?: unknown }).message)
|
||||
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) throw error
|
||||
return message as ControlRenewalOutcome
|
||||
}
|
||||
}
|
||||
|
||||
private async renewTransactionalControlActivity(
|
||||
@@ -7111,6 +7189,31 @@ export class RelayAssignmentStore {
|
||||
return rows.length === 1
|
||||
}
|
||||
|
||||
// Reports which of `cellIsLive`'s conditions failed, so the rejection log
|
||||
// separates an expected drain or boot from a cell whose readiness went out
|
||||
// from under its hosts.
|
||||
private async homeCellUnavailableCause(
|
||||
database: RelayDatabase,
|
||||
cellId: string,
|
||||
now: number
|
||||
): Promise<RelayHomeCellUnavailableCause> {
|
||||
const row = (
|
||||
await database.query(
|
||||
`SELECT cell.enabled, runtime.last_heartbeat_at
|
||||
FROM relay_cells cell
|
||||
LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
|
||||
WHERE cell.cell_id = ?`,
|
||||
[cellId]
|
||||
)
|
||||
)[0]
|
||||
if (!row) return 'booting'
|
||||
if (integer(row, 'enabled') === 0) return 'draining'
|
||||
const heartbeatAt = optionalInteger(row, 'last_heartbeat_at')
|
||||
if (heartbeatAt === undefined) return 'booting'
|
||||
// Readiness is all that is left: `cellIsLive` already refused this cell.
|
||||
return heartbeatAt <= now - this.heartbeatTtlMs ? 'unheard' : 'not_ready'
|
||||
}
|
||||
|
||||
private async cellHasActiveFence(cellId: string): Promise<boolean> {
|
||||
const rows = await this.database.query(
|
||||
`SELECT fence.cell_id FROM relay_cell_fences fence
|
||||
@@ -7900,6 +8003,33 @@ function validateActivityId(activityId: string): void {
|
||||
if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id')
|
||||
}
|
||||
|
||||
function controlRenewalExpiryIsValid(expiresAt: number, now: number): boolean {
|
||||
const maximumExpiresAt =
|
||||
now + ASSIGNMENT_LIMITS.activityLeaseMs + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
|
||||
return Number.isSafeInteger(expiresAt) && expiresAt > now && expiresAt <= maximumExpiresAt
|
||||
}
|
||||
|
||||
// A renewal that threw still owes the metric an outcome: the message carries one
|
||||
// when the statement decided it, and anything else is the driver failing.
|
||||
function controlRenewalOutcomeOfError(error: unknown): ControlRenewalOutcome {
|
||||
const message = String((error as { message?: unknown }).message)
|
||||
return CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)
|
||||
? (message as ControlRenewalOutcome)
|
||||
: 'database_error'
|
||||
}
|
||||
|
||||
function controlRenewalRejection(
|
||||
row: ControlRenewalRequest,
|
||||
now: number
|
||||
): ControlRenewalOutcome | null {
|
||||
try {
|
||||
validateActivityId(row.activityId)
|
||||
} catch {
|
||||
return 'invalid_activity_id'
|
||||
}
|
||||
return controlRenewalExpiryIsValid(row.expiresAt, now) ? null : 'invalid_activity_expiry'
|
||||
}
|
||||
|
||||
function activityKind(row: SqlRow): AssignmentActivityKind {
|
||||
const value = text(row, 'activity_kind')
|
||||
if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind')
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { afterEach, describe, expect, it, vi, type MockInstance } from 'vitest'
|
||||
import { openRelayDatabaseAtBoot } from './boot-database-open.js'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
|
||||
const input = { dataDir: '/tmp/orca-relay-boot', databaseUrl: 'postgres://relay@localhost/relay' }
|
||||
// The message the fleet actually saw: pg-pool reports the connect timeout with
|
||||
// no SQLSTATE, so the classifier has only this text to go on.
|
||||
const connectTimeout = (): Error => new Error('Connection terminated due to connection timeout')
|
||||
const database = {} as RelayDatabase
|
||||
|
||||
function loggedEvents(warn: MockInstance<typeof console.warn>): string[] {
|
||||
return warn.mock.calls.map((call) => String(JSON.parse(String(call[0])).event))
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('relay boot database open', () => {
|
||||
it('waits out a cold proxy instead of failing the boot', async () => {
|
||||
vi.useFakeTimers()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const open = vi
|
||||
.fn<() => Promise<RelayDatabase>>()
|
||||
.mockRejectedValueOnce(connectTimeout())
|
||||
.mockRejectedValueOnce(connectTimeout())
|
||||
.mockResolvedValue(database)
|
||||
|
||||
const opening = openRelayDatabaseAtBoot(input, open)
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
expect(await opening).toBe(database)
|
||||
expect(open).toHaveBeenCalledTimes(3)
|
||||
expect(open).toHaveBeenCalledWith(input)
|
||||
expect(loggedEvents(warn)).toEqual([
|
||||
'orca_relay_boot_database_retry',
|
||||
'orca_relay_boot_database_retry',
|
||||
'orca_relay_boot_database_recovered'
|
||||
])
|
||||
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
|
||||
attempt: 1,
|
||||
delayMs: expect.any(Number),
|
||||
code: 'unknown',
|
||||
connectionTimeout: true
|
||||
})
|
||||
})
|
||||
|
||||
it('fails the boot immediately when the database rejects the relay', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const denied = Object.assign(new Error('password authentication failed'), { code: '28P01' })
|
||||
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(denied)
|
||||
|
||||
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(denied)
|
||||
expect(open).toHaveBeenCalledTimes(1)
|
||||
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
|
||||
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
|
||||
attempts: 1,
|
||||
retryable: false,
|
||||
code: 'unknown'
|
||||
})
|
||||
})
|
||||
|
||||
// A retry re-runs the schema apply, which must never re-queue a boot DDL
|
||||
// behind the writers that beat it; the request path treats these as transient.
|
||||
it.each(['55P03', '57014', '53300'])(
|
||||
'refuses to re-queue the schema apply after SQLSTATE %s',
|
||||
async (code) => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const contention = Object.assign(new Error('lock unavailable'), { code })
|
||||
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(contention)
|
||||
|
||||
await expect(openRelayDatabaseAtBoot(input, open)).rejects.toBe(contention)
|
||||
expect(open).toHaveBeenCalledTimes(1)
|
||||
expect(loggedEvents(warn)).toEqual(['orca_relay_boot_database_failed'])
|
||||
expect(JSON.parse(String(warn.mock.calls[0]?.[0]))).toMatchObject({
|
||||
attempts: 1,
|
||||
retryable: false,
|
||||
code
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it('waits out a connection failure the driver does report a SQLSTATE for', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const unreachable = Object.assign(new Error('connection refused'), { code: '08006' })
|
||||
const open = vi
|
||||
.fn<() => Promise<RelayDatabase>>()
|
||||
.mockRejectedValueOnce(unreachable)
|
||||
.mockResolvedValue(database)
|
||||
|
||||
const opening = openRelayDatabaseAtBoot(input, open)
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
expect(await opening).toBe(database)
|
||||
expect(open).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('gives up once the retry budget is spent', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
vi.spyOn(Math, 'random').mockReturnValue(0)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const failure = connectTimeout()
|
||||
const open = vi.fn<() => Promise<RelayDatabase>>().mockRejectedValue(failure)
|
||||
|
||||
const opening = openRelayDatabaseAtBoot(input, open)
|
||||
const rejection = expect(opening).rejects.toBe(failure)
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
|
||||
expect(Date.now()).toBeLessThanOrEqual(45_000)
|
||||
expect(open.mock.calls.length).toBeGreaterThan(1)
|
||||
const events = loggedEvents(warn)
|
||||
expect(events.at(-1)).toBe('orca_relay_boot_database_failed')
|
||||
expect(events.filter((event) => event === 'orca_relay_boot_database_retry')).toHaveLength(
|
||||
open.mock.calls.length - 1
|
||||
)
|
||||
expect(JSON.parse(String(warn.mock.calls.at(-1)?.[0]))).toMatchObject({
|
||||
attempts: open.mock.calls.length,
|
||||
retryable: true,
|
||||
connectionTimeout: true
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,71 @@
|
||||
import { openRelayDatabase, type RelayDatabase, type RelayDatabaseOpenInput } from './database.js'
|
||||
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
|
||||
import {
|
||||
isPostgresPoolConnectFailure,
|
||||
isPostgresPoolConnectTimeout
|
||||
} from './postgres-pool-pressure.js'
|
||||
import { postgresErrorCodeCategory } from './postgres-query-failure.js'
|
||||
|
||||
// Only a failure to reach Postgres at all. A retry here re-runs the schema
|
||||
// apply, and applyPostgresSchema refuses to repeat a DDL lock timeout on
|
||||
// purpose: relation locks are granted in queue order, so a repeat parks every
|
||||
// writer behind the same statement again. 55P03, 57014 and 53300 therefore stay
|
||||
// terminal at boot even though the request path calls them transient.
|
||||
function isBootDatabaseUnreachable(error: unknown): boolean {
|
||||
const code = postgresErrorCodeCategory(error)
|
||||
return isPostgresPoolConnectFailure(error) || code === '08001' || code === '08006'
|
||||
}
|
||||
|
||||
// A cell boots beside a cloud-sql-proxy that is itself still dialling, so the
|
||||
// first pool acquire can outrun the 2s connect timeout that protects the
|
||||
// request path. The window is longer than a proxy cold start and shorter than
|
||||
// the restart loop it replaces.
|
||||
const BOOT_OPEN_RETRY = {
|
||||
attempts: 20,
|
||||
windowMs: 45_000,
|
||||
baseDelayMs: 250,
|
||||
maxDelayMs: 4_000,
|
||||
jitterMs: 250,
|
||||
isRetryable: isBootDatabaseUnreachable
|
||||
}
|
||||
|
||||
function bootDatabaseErrorFields(error: unknown): Record<string, unknown> {
|
||||
return {
|
||||
code: postgresErrorCodeCategory(error),
|
||||
connectionTimeout: isPostgresPoolConnectTimeout(error)
|
||||
}
|
||||
}
|
||||
|
||||
export async function openRelayDatabaseAtBoot(
|
||||
input: RelayDatabaseOpenInput,
|
||||
open: (input: RelayDatabaseOpenInput) => Promise<RelayDatabase> = openRelayDatabase
|
||||
): Promise<RelayDatabase> {
|
||||
return await retryTransientDatabaseStartup(
|
||||
async () => await open(input),
|
||||
BOOT_OPEN_RETRY,
|
||||
{
|
||||
onRetry: ({ attempt, delayMs, error }) =>
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_boot_database_retry',
|
||||
attempt,
|
||||
delayMs,
|
||||
...bootDatabaseErrorFields(error)
|
||||
})
|
||||
),
|
||||
onRecovered: ({ attempts }) =>
|
||||
console.warn(
|
||||
JSON.stringify({ event: 'orca_relay_boot_database_recovered', attempts })
|
||||
),
|
||||
onGaveUp: ({ attempts, error, retryable }) =>
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_boot_database_failed',
|
||||
attempts,
|
||||
retryable,
|
||||
...bootDatabaseErrorFields(error)
|
||||
})
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -1,14 +1,19 @@
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { isRelayDatabaseTransientError } from './database.js'
|
||||
import { retryTransientDatabaseStartup } from './database-startup-retry.js'
|
||||
|
||||
type CellAdmissionStartupConfig = Pick<RelayConfig, 'role' | 'cells'>
|
||||
type CellAdmissionStore = Pick<RelayAssignmentStore, 'reconcileCellsAtStartup'>
|
||||
|
||||
const STARTUP_RECONCILE_ATTEMPTS = 20
|
||||
const STARTUP_RECONCILE_RETRY_WINDOW_MS = 45_000
|
||||
const STARTUP_RECONCILE_RETRY_BASE_MS = 250
|
||||
const STARTUP_RECONCILE_RETRY_JITTER_MS = 250
|
||||
const STARTUP_RECONCILE_RETRY = {
|
||||
attempts: 20,
|
||||
windowMs: 45_000,
|
||||
// Flat: the contention this waits out is another director's schema lock, which
|
||||
// clears on its own schedule rather than easing as the wait grows.
|
||||
baseDelayMs: 250,
|
||||
maxDelayMs: 250,
|
||||
jitterMs: 250
|
||||
}
|
||||
|
||||
export function roleOwnsAssignmentMaintenance(role: RelayConfig['role']): boolean {
|
||||
// Cell workers share the database but the director is the sole authority
|
||||
@@ -23,34 +28,21 @@ export async function reconcileCellAdmissionAtStartup(
|
||||
// Admission is operator/director state. A new worker must not enable itself
|
||||
// before its distinct candidate has passed production preflight.
|
||||
if (config.role === 'cell') return
|
||||
const retryDeadline = Date.now() + STARTUP_RECONCILE_RETRY_WINDOW_MS
|
||||
for (let attempt = 1; attempt <= STARTUP_RECONCILE_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
await assignments.reconcileCellsAtStartup(config.cells)
|
||||
if (attempt > 1) {
|
||||
await retryTransientDatabaseStartup(
|
||||
async () => await assignments.reconcileCellsAtStartup(config.cells),
|
||||
STARTUP_RECONCILE_RETRY,
|
||||
{
|
||||
onRecovered: ({ attempts }) =>
|
||||
console.warn(
|
||||
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts: attempt })
|
||||
)
|
||||
}
|
||||
return
|
||||
} catch (error) {
|
||||
const remainingMs = retryDeadline - Date.now()
|
||||
if (
|
||||
attempt === STARTUP_RECONCILE_ATTEMPTS ||
|
||||
remainingMs <= 0 ||
|
||||
!isRelayDatabaseTransientError(error)
|
||||
) {
|
||||
if (isRelayDatabaseTransientError(error)) {
|
||||
JSON.stringify({ event: 'orca_relay_startup_reconcile_recovered', attempts })
|
||||
),
|
||||
onGaveUp: ({ attempts, retryable }) => {
|
||||
if (retryable) {
|
||||
console.warn(
|
||||
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts: attempt })
|
||||
JSON.stringify({ event: 'orca_relay_startup_reconcile_exhausted', attempts })
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const delayMs =
|
||||
STARTUP_RECONCILE_RETRY_BASE_MS +
|
||||
Math.floor(Math.random() * (STARTUP_RECONCILE_RETRY_JITTER_MS + 1))
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs)))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ASSIGNMENT_LIMITS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayCredentialStore } from './credential-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
@@ -133,6 +134,10 @@ describePostgres('expired control lease after a database outage', () => {
|
||||
internals.heartbeat(session)
|
||||
}
|
||||
|
||||
// A due renewal leaves the heartbeat as a batch enqueue, so a poll has to
|
||||
// outlast the batch window before it can call the renewal missing.
|
||||
const renewalPoll = { timeout: CONTROL_RENEWAL_BATCH_INTERVAL_MS + 4_000 }
|
||||
|
||||
const leaseRows = async (relayHostId: string) =>
|
||||
await database.query(
|
||||
`SELECT activity_id, cell_id FROM relay_assignment_activity_leases
|
||||
@@ -149,7 +154,8 @@ describePostgres('expired control lease after a database outage', () => {
|
||||
.poll(
|
||||
async () =>
|
||||
socket.close.mock.calls.length > 0 ||
|
||||
(await leaseRows(relayHostId)).length === expectedRows
|
||||
(await leaseRows(relayHostId)).length === expectedRows,
|
||||
renewalPoll
|
||||
)
|
||||
.toBe(true)
|
||||
}
|
||||
@@ -212,7 +218,7 @@ describePostgres('expired control lease after a database outage', () => {
|
||||
).rejects.toThrow('control_activity_moved')
|
||||
|
||||
heartbeat(registry, session)
|
||||
await expect.poll(() => socket.close.mock.calls.length).toBe(1)
|
||||
await expect.poll(() => socket.close.mock.calls.length, renewalPoll).toBe(1)
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
|
||||
expect(await leaseRows(identity.relayHostId)).toEqual([
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
|
||||
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
type RelayDatabase,
|
||||
type SqlRow
|
||||
} from './database.js'
|
||||
|
||||
const now = 1_900_000_000_000
|
||||
const expiresAt = now + 105_000
|
||||
|
||||
function renewal(userId: string, relayHostId: string, expiry = expiresAt) {
|
||||
return {
|
||||
identity: { userId, relayHostId },
|
||||
activityId: 'control:cell-a:1',
|
||||
cellId: 'cell-a',
|
||||
expiresAt: expiry
|
||||
}
|
||||
}
|
||||
|
||||
// A PostgreSQL-dialect database that answers the renewal statement without a
|
||||
// server, so the statement count and its parameter arrays are observable.
|
||||
class RenewalStatementProbe implements RelayDatabase {
|
||||
readonly dialect = 'postgres' as const
|
||||
readonly statements: Array<{ sql: string; params: unknown[] }> = []
|
||||
failuresRemaining = 0
|
||||
|
||||
constructor(private readonly outcomeFor: (userId: string) => ControlRenewalOutcome) {}
|
||||
|
||||
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
|
||||
this.statements.push({ sql, params })
|
||||
if (this.failuresRemaining > 0) {
|
||||
this.failuresRemaining -= 1
|
||||
throw new Error('canceling statement due to statement timeout')
|
||||
}
|
||||
const userIds = params[0] as string[]
|
||||
return userIds.map((userId, index) => ({
|
||||
row_index: String(index + 1),
|
||||
outcome: this.outcomeFor(userId)
|
||||
}))
|
||||
}
|
||||
|
||||
async queryLocked(): Promise<SqlRow[]> {
|
||||
throw new Error('unexpected_locked_query')
|
||||
}
|
||||
|
||||
async transaction<T>(): Promise<T> {
|
||||
// Renewals must never open one: that is the write transaction per host this
|
||||
// batch exists to remove.
|
||||
throw new Error('unexpected_transaction')
|
||||
}
|
||||
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
|
||||
describe('batched control renewals on PostgreSQL', () => {
|
||||
it('spends one statement on every host that came due', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-a', 'host000000000002'),
|
||||
renewal('user-b', 'host000000000003')
|
||||
])
|
||||
|
||||
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
|
||||
expect(probe.statements).toHaveLength(1)
|
||||
expect(probe.statements[0]!.sql).toBe(CONTROL_RENEWAL_BATCH_SQL)
|
||||
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
|
||||
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
|
||||
})
|
||||
|
||||
it('locks assignment rows in primary-key order and still answers in input order', async () => {
|
||||
const probe = new RenewalStatementProbe((userId) =>
|
||||
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
|
||||
)
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-c', 'host000000000003'),
|
||||
renewal('user-a', 'host000000000002'),
|
||||
renewal('user-b', 'host000000000001'),
|
||||
renewal('user-a', 'host000000000001')
|
||||
])
|
||||
|
||||
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
|
||||
// statement's ORDER BY repeats it: no batch can queue against another in a
|
||||
// different sequence.
|
||||
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
|
||||
expect(probe.statements[0]!.params[1]).toEqual([
|
||||
'host000000000001',
|
||||
'host000000000002',
|
||||
'host000000000001',
|
||||
'host000000000003'
|
||||
])
|
||||
expect(outcomes).toEqual([
|
||||
'renewed',
|
||||
'renewed',
|
||||
'control_activity_moved',
|
||||
'renewed'
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps a malformed request out of the statement and fails only that row', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-a', 'host000000000002', now + ASSIGNMENT_LIMITS.activityLeaseMs * 10),
|
||||
{ ...renewal('user-a', 'host000000000003'), activityId: '' },
|
||||
renewal('user-a', 'host000000000004')
|
||||
])
|
||||
|
||||
expect(outcomes).toEqual([
|
||||
'renewed',
|
||||
'invalid_activity_expiry',
|
||||
'invalid_activity_id',
|
||||
'renewed'
|
||||
])
|
||||
expect(probe.statements[0]!.params[1]).toEqual(['host000000000001', 'host000000000004'])
|
||||
})
|
||||
|
||||
it('degrades to one statement per host when the batch statement fails', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
probe.failuresRemaining = 1
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-a', 'host000000000002')
|
||||
])
|
||||
|
||||
expect(outcomes).toEqual(['renewed', 'renewed'])
|
||||
expect(probe.statements).toHaveLength(3)
|
||||
expect(probe.statements[1]!.params[1]).toEqual(['host000000000001'])
|
||||
expect(probe.statements[2]!.params[1]).toEqual(['host000000000002'])
|
||||
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
|
||||
event: 'orca_relay_control_renewal_batch_failed',
|
||||
rows: 2
|
||||
})
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('reports a host that fails its own fallback statement without touching the rest', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
probe.failuresRemaining = 2
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-a', 'host000000000002')
|
||||
])
|
||||
|
||||
expect(outcomes.filter((outcome) => outcome === 'renewed')).toHaveLength(1)
|
||||
expect(outcomes.filter((outcome) => outcome === 'database_error')).toHaveLength(1)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('reports a contended assignment row apart from a missing one', async () => {
|
||||
const probe = new RenewalStatementProbe((userId) =>
|
||||
userId === 'user-b' ? 'assignment_lock_unavailable' : 'renewed'
|
||||
)
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-b', 'host000000000002')
|
||||
])
|
||||
|
||||
// Retryable: SKIP LOCKED passed over the row rather than queueing the whole
|
||||
// flush behind whoever held it.
|
||||
expect(outcomes).toEqual(['renewed', 'assignment_lock_unavailable'])
|
||||
})
|
||||
|
||||
it('counts a lone renewal that threw before rethrowing it', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
probe.failuresRemaining = 1
|
||||
const recordControlRenewal = vi.fn()
|
||||
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
|
||||
|
||||
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
|
||||
// still owes an outcome for the attempt.
|
||||
await expect(
|
||||
store.renewControlActivities([renewal('user-a', 'host000000000001')])
|
||||
).rejects.toThrow('statement timeout')
|
||||
|
||||
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
|
||||
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
|
||||
})
|
||||
|
||||
it('counts a batch in which every row was rejected before the statement', async () => {
|
||||
const probe = new RenewalStatementProbe(() => 'renewed')
|
||||
const recordControlRenewal = vi.fn()
|
||||
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
|
||||
renewal('user-a', 'host000000000002', now - 1)
|
||||
])
|
||||
|
||||
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
|
||||
expect(probe.statements).toHaveLength(0)
|
||||
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
|
||||
'invalid_activity_id',
|
||||
'invalid_activity_expiry'
|
||||
])
|
||||
})
|
||||
|
||||
it('counts one renewal metric per row against the flush latency', async () => {
|
||||
const probe = new RenewalStatementProbe((userId) =>
|
||||
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
|
||||
)
|
||||
const recordControlRenewal = vi.fn()
|
||||
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
|
||||
|
||||
await store.renewControlActivities([
|
||||
renewal('user-a', 'host000000000001'),
|
||||
renewal('user-b', 'host000000000002')
|
||||
])
|
||||
|
||||
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
|
||||
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
|
||||
'renewed',
|
||||
'assignment_not_found'
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('batched control renewals on SQLite', () => {
|
||||
it('renews every host through the transactional path', async () => {
|
||||
let clock = now
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
try {
|
||||
const store = new RelayAssignmentStore(database, () => clock)
|
||||
await store.reconcileCells([
|
||||
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
|
||||
])
|
||||
const hosts = ['host000000000001', 'host000000000002']
|
||||
const requests = []
|
||||
for (const relayHostId of hosts) {
|
||||
const identity = { userId: 'user-a', relayHostId }
|
||||
const assignment = await store.assign(identity)
|
||||
await store.activateControl(identity, {
|
||||
cellId: assignment.cellId,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
requests.push({
|
||||
identity,
|
||||
activityId: `control:${assignment.cellId}:1`,
|
||||
cellId: assignment.cellId,
|
||||
expiresAt: clock + 105_000
|
||||
})
|
||||
}
|
||||
// A host with no assignment at all must not cost the others their renewal.
|
||||
requests.push({
|
||||
identity: { userId: 'user-a', relayHostId: 'host000000000009' },
|
||||
activityId: 'control:cell-a:1',
|
||||
cellId: 'cell-a',
|
||||
expiresAt: clock + 105_000
|
||||
})
|
||||
clock += 1_000
|
||||
|
||||
const outcomes = await store.renewControlActivities(requests)
|
||||
|
||||
expect(outcomes).toEqual(['renewed', 'renewed', 'assignment_not_found'])
|
||||
const leases = await database.query(
|
||||
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? ORDER BY relay_host_id ASC`,
|
||||
['user-a']
|
||||
)
|
||||
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
|
||||
now + 105_000,
|
||||
now + 105_000
|
||||
])
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,231 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
CONTROL_RENEWAL_BATCH_INTERVAL_MS,
|
||||
CONTROL_RENEWAL_BATCH_MAX_ROWS,
|
||||
ControlRenewalBatch,
|
||||
type ControlRenewalFlush
|
||||
} from './control-renewal-batch.js'
|
||||
import type {
|
||||
ControlRenewalOutcome,
|
||||
ControlRenewalRequest
|
||||
} from './control-renewal-statement.js'
|
||||
|
||||
// Settles into the outcome the caller saw, attached at enqueue so a rejection is
|
||||
// never momentarily unhandled.
|
||||
function outcomeOf(renewal: Promise<void>): Promise<string> {
|
||||
return renewal.then(
|
||||
() => 'renewed',
|
||||
(error: unknown) => String((error as { message?: unknown }).message)
|
||||
)
|
||||
}
|
||||
|
||||
function request(
|
||||
host: string,
|
||||
expiresAt = 1_000,
|
||||
activityId = 'control:cell-a:1'
|
||||
): ControlRenewalRequest {
|
||||
return {
|
||||
identity: { userId: 'user-a', relayHostId: host },
|
||||
activityId,
|
||||
cellId: 'cell-a',
|
||||
expiresAt
|
||||
}
|
||||
}
|
||||
|
||||
describe('control renewal batch', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('spends one call on every renewal that came due in the window', async () => {
|
||||
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
|
||||
rows.map((): ControlRenewalOutcome => 'renewed')
|
||||
)
|
||||
const batch = new ControlRenewalBatch(renew)
|
||||
const settled = [
|
||||
batch.enqueue(request('host0000000000a1')),
|
||||
batch.enqueue(request('host0000000000a2')),
|
||||
batch.enqueue(request('host0000000000a3'))
|
||||
]
|
||||
|
||||
expect(renew).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
await expect(Promise.all(settled)).resolves.toEqual([undefined, undefined, undefined])
|
||||
expect(renew).toHaveBeenCalledOnce()
|
||||
expect(renew.mock.calls[0]![0].map((row) => row.identity.relayHostId)).toEqual([
|
||||
'host0000000000a1',
|
||||
'host0000000000a2',
|
||||
'host0000000000a3'
|
||||
])
|
||||
})
|
||||
|
||||
it('routes each outcome back to the caller that asked for it', async () => {
|
||||
const outcomes: ControlRenewalOutcome[] = [
|
||||
'renewed',
|
||||
'assignment_not_found',
|
||||
'control_activity_moved'
|
||||
]
|
||||
const batch = new ControlRenewalBatch(async () => outcomes)
|
||||
const first = outcomeOf(batch.enqueue(request('host0000000000b1')))
|
||||
const second = outcomeOf(batch.enqueue(request('host0000000000b2')))
|
||||
const third = outcomeOf(batch.enqueue(request('host0000000000b3')))
|
||||
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
await expect(Promise.all([first, second, third])).resolves.toEqual([
|
||||
'renewed',
|
||||
'assignment_not_found',
|
||||
'control_activity_moved'
|
||||
])
|
||||
})
|
||||
|
||||
it('flushes on reaching the row ceiling instead of waiting out the window', async () => {
|
||||
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
|
||||
rows.map((): ControlRenewalOutcome => 'renewed')
|
||||
)
|
||||
const batch = new ControlRenewalBatch(renew)
|
||||
for (let row = 0; row < CONTROL_RENEWAL_BATCH_MAX_ROWS - 1; row++) {
|
||||
void batch.enqueue(request(`host${String(row).padStart(12, '0')}`))
|
||||
}
|
||||
expect(renew).not.toHaveBeenCalled()
|
||||
|
||||
void batch.enqueue(request('host0000000000zz'))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(renew).toHaveBeenCalledOnce()
|
||||
expect(renew.mock.calls[0]![0]).toHaveLength(CONTROL_RENEWAL_BATCH_MAX_ROWS)
|
||||
// The window timer must not fire a second, empty statement.
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
expect(renew).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('does not hold a new window behind a statement still in PostgreSQL', async () => {
|
||||
let release!: (outcomes: ControlRenewalOutcome[]) => void
|
||||
const renew = vi
|
||||
.fn<(rows: readonly ControlRenewalRequest[]) => Promise<ControlRenewalOutcome[]>>()
|
||||
.mockImplementationOnce(
|
||||
async () => await new Promise<ControlRenewalOutcome[]>((resolve) => (release = resolve))
|
||||
)
|
||||
.mockResolvedValue(['renewed'])
|
||||
const batch = new ControlRenewalBatch(renew)
|
||||
const stalled = batch.enqueue(request('host0000000000c1'))
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
const next = batch.enqueue(request('host0000000000c2'))
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
expect(renew).toHaveBeenCalledTimes(2)
|
||||
await expect(next).resolves.toBeUndefined()
|
||||
release(['renewed'])
|
||||
await expect(stalled).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports the driver failure to every caller in the flush', async () => {
|
||||
const batch = new ControlRenewalBatch(async () => {
|
||||
throw new Error('pool timeout')
|
||||
})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
const first = outcomeOf(batch.enqueue(request('host0000000000d1')))
|
||||
const second = outcomeOf(batch.enqueue(request('host0000000000d2')))
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
await expect(Promise.all([first, second])).resolves.toEqual([
|
||||
'pool timeout',
|
||||
'pool timeout'
|
||||
])
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('supersedes a second attempt for one lease and answers both callers', async () => {
|
||||
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
|
||||
rows.map((): ControlRenewalOutcome => 'renewed')
|
||||
)
|
||||
const batch = new ControlRenewalBatch(renew)
|
||||
const earlier = batch.enqueue(request('host0000000000e1', 1_000))
|
||||
const later = batch.enqueue(request('host0000000000e1', 2_000))
|
||||
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
expect(renew.mock.calls[0]![0]).toEqual([
|
||||
expect.objectContaining({ expiresAt: 2_000 })
|
||||
])
|
||||
await expect(earlier).resolves.toBeUndefined()
|
||||
await expect(later).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('holds a second activity for one host back to the next flush', async () => {
|
||||
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
|
||||
rows.map((): ControlRenewalOutcome => 'renewed')
|
||||
)
|
||||
const batch = new ControlRenewalBatch(renew)
|
||||
const first = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:1'))
|
||||
const second = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:2'))
|
||||
const other = batch.enqueue(request('host0000000000h2'))
|
||||
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
// One statement updates a host's assignment row once, so the host appears in
|
||||
// one flush only; the newer generation leads the next one.
|
||||
expect(renew.mock.calls[0]![0].map((row) => row.activityId)).toEqual([
|
||||
'control:cell-a:1',
|
||||
'control:cell-a:1'
|
||||
])
|
||||
await expect(Promise.all([first, other])).resolves.toEqual([undefined, undefined])
|
||||
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
expect(renew).toHaveBeenCalledTimes(2)
|
||||
expect(renew.mock.calls[1]![0].map((row) => row.activityId)).toEqual(['control:cell-a:2'])
|
||||
await expect(second).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('stays quiet for a fast flush that renewed everything', async () => {
|
||||
const flushes: ControlRenewalFlush[] = []
|
||||
const batch = new ControlRenewalBatch(
|
||||
async () => ['renewed'],
|
||||
() => ({ cellId: 'cell-a' }),
|
||||
(flush) => flushes.push(flush)
|
||||
)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
void batch.enqueue(request('host0000000000f1'))
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
|
||||
expect(warn).not.toHaveBeenCalled()
|
||||
expect(flushes).toEqual([
|
||||
{ rows: 1, durationMs: expect.any(Number), outcomes: { renewed: 1 } }
|
||||
])
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('logs one line with the outcome counts when a flush did not renew everything', async () => {
|
||||
const batch = new ControlRenewalBatch(
|
||||
async () => ['renewed', 'control_activity_not_found'],
|
||||
() => ({ cellId: 'cell-a' })
|
||||
)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
void outcomeOf(batch.enqueue(request('host0000000000g1')))
|
||||
const missing = outcomeOf(batch.enqueue(request('host0000000000g2')))
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
await expect(missing).resolves.toBe('control_activity_not_found')
|
||||
|
||||
expect(warn).toHaveBeenCalledOnce()
|
||||
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
|
||||
event: 'orca_relay_control_renewal_flush',
|
||||
cellId: 'cell-a',
|
||||
rows: 2,
|
||||
outcomes: { renewed: 1, control_activity_not_found: 1 }
|
||||
})
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,148 @@
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import type {
|
||||
ControlRenewalOutcome,
|
||||
ControlRenewalRequest
|
||||
} from './control-renewal-statement.js'
|
||||
|
||||
// One flush per second turns the fleet's control-lease write rate into a
|
||||
// function of the cell count rather than the host count: a cell's ~10 due
|
||||
// renewals per second become one write transaction instead of ten. Well inside
|
||||
// the 105s lease runway, so a host that misses a window is never at risk.
|
||||
export const CONTROL_RENEWAL_BATCH_INTERVAL_MS = 1_000
|
||||
// Ceiling on the parameter arrays. Row locks live until the statement commits,
|
||||
// so this is what bounds how long one flush holds them: measured at 11.5ms for
|
||||
// 200 rows against a 20,000-row table, and 9.4ms with a host wedged in a
|
||||
// per-host transaction.
|
||||
export const CONTROL_RENEWAL_BATCH_MAX_ROWS = 200
|
||||
// A flush slower than this is the only latency worth a line; the metrics event
|
||||
// carries the distribution.
|
||||
const CONTROL_RENEWAL_SLOW_FLUSH_MS = 250
|
||||
|
||||
export type ControlRenewalFlush = {
|
||||
rows: number
|
||||
durationMs: number
|
||||
outcomes: Record<string, number>
|
||||
}
|
||||
|
||||
type PendingWaiter = { resolve: () => void; reject: (error: unknown) => void }
|
||||
|
||||
type PendingRenewal = { request: ControlRenewalRequest; waiters: PendingWaiter[] }
|
||||
|
||||
type QueuedRenewal = { request: ControlRenewalRequest; waiter: PendingWaiter }
|
||||
|
||||
// Per host, not per activity: one statement updates a host's assignment row
|
||||
// once, so two activities for the same host must not share a flush.
|
||||
function pendingKey(request: ControlRenewalRequest): string {
|
||||
return [request.identity.userId, request.identity.relayHostId].join('\u0000')
|
||||
}
|
||||
|
||||
// Collects the control-lease renewals a cell owes and spends one statement on
|
||||
// them. Each caller still gets the single-renewal contract: the promise resolves
|
||||
// on `renewed` and rejects with the outcome as its message otherwise, so callers
|
||||
// keep their per-session error routing unchanged.
|
||||
export class ControlRenewalBatch {
|
||||
private pending = new Map<string, PendingRenewal>()
|
||||
// Renewals a host cannot contribute to the flush being built; they open the
|
||||
// next one.
|
||||
private deferred: QueuedRenewal[] = []
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
constructor(
|
||||
private readonly renew: (
|
||||
rows: readonly ControlRenewalRequest[]
|
||||
) => Promise<ControlRenewalOutcome[]>,
|
||||
private readonly logFields: () => Record<string, unknown> = () => ({}),
|
||||
private readonly observe?: (flush: ControlRenewalFlush) => void
|
||||
) {}
|
||||
|
||||
enqueue(request: ControlRenewalRequest): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
this.admit({ request, waiter: { resolve, reject } })
|
||||
})
|
||||
}
|
||||
|
||||
private admit(queued: QueuedRenewal): void {
|
||||
const key = pendingKey(queued.request)
|
||||
const existing = this.pending.get(key)
|
||||
if (existing && existing.request.activityId !== queued.request.activityId) {
|
||||
this.deferred.push(queued)
|
||||
this.scheduleFlush()
|
||||
return
|
||||
}
|
||||
if (existing) {
|
||||
// A second attempt at the same lease inside one window supersedes the
|
||||
// first expiry; both callers still hear the outcome they waited for.
|
||||
existing.request = {
|
||||
...queued.request,
|
||||
expiresAt: Math.max(existing.request.expiresAt, queued.request.expiresAt)
|
||||
}
|
||||
existing.waiters.push(queued.waiter)
|
||||
return
|
||||
}
|
||||
this.pending.set(key, { request: queued.request, waiters: [queued.waiter] })
|
||||
if (this.pending.size >= CONTROL_RENEWAL_BATCH_MAX_ROWS) {
|
||||
void this.flush()
|
||||
return
|
||||
}
|
||||
this.scheduleFlush()
|
||||
}
|
||||
|
||||
private scheduleFlush(): void {
|
||||
this.timer ??= setTimeout(() => {
|
||||
this.timer = null
|
||||
void this.flush()
|
||||
}, CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
|
||||
// Flushes run concurrently on purpose: a statement stalled in PostgreSQL must
|
||||
// not hold back the renewals that came due while it was waiting.
|
||||
async flush(): Promise<void> {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
const batch = [...this.pending.values()]
|
||||
this.pending = new Map()
|
||||
// Re-admitted against the empty map, so a host deferred out of this flush
|
||||
// leads the next one.
|
||||
const deferred = this.deferred
|
||||
this.deferred = []
|
||||
for (const queued of deferred) this.admit(queued)
|
||||
if (batch.length === 0) return
|
||||
const startedAt = performance.now()
|
||||
let outcomes: ControlRenewalOutcome[]
|
||||
try {
|
||||
outcomes = await this.renew(batch.map((entry) => entry.request))
|
||||
} catch (error) {
|
||||
for (const entry of batch) for (const waiter of entry.waiters) waiter.reject(error)
|
||||
this.report(batch.length, performance.now() - startedAt, { flush_failed: batch.length })
|
||||
return
|
||||
}
|
||||
const counts: Record<string, number> = {}
|
||||
for (const [index, entry] of batch.entries()) {
|
||||
const outcome = outcomes[index] ?? 'database_error'
|
||||
counts[outcome] = (counts[outcome] ?? 0) + 1
|
||||
for (const waiter of entry.waiters) {
|
||||
if (outcome === 'renewed') waiter.resolve()
|
||||
else waiter.reject(new Error(outcome))
|
||||
}
|
||||
}
|
||||
this.report(batch.length, performance.now() - startedAt, counts)
|
||||
}
|
||||
|
||||
private report(rows: number, durationMs: number, outcomes: Record<string, number>): void {
|
||||
this.observe?.({ rows, durationMs, outcomes })
|
||||
const renewed = outcomes.renewed ?? 0
|
||||
if (durationMs <= CONTROL_RENEWAL_SLOW_FLUSH_MS && renewed === rows) return
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_control_renewal_flush',
|
||||
...this.logFields(),
|
||||
rows,
|
||||
durationMs: Math.round(durationMs),
|
||||
outcomes
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { ASSIGNMENT_LIMITS, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
POSTGRES_LOCK_TIMEOUT_MS,
|
||||
type RelayDatabase,
|
||||
type RelayLockOptions,
|
||||
type SqlRow
|
||||
@@ -22,7 +25,9 @@ const targetCell = {
|
||||
capacityRequests: 100
|
||||
}
|
||||
const userId = 'control-renewal-postgres-user'
|
||||
const identities = Array.from({ length: 6 }, (_, index) => ({
|
||||
// Indexes 0-5 belong to the single-renewal cases below, which mutate their
|
||||
// host's migration and lease state; the batch cases own 6-13.
|
||||
const identities = Array.from({ length: 14 }, (_, index) => ({
|
||||
userId,
|
||||
relayHostId: `controlrenewal${index + 1}`
|
||||
}))
|
||||
@@ -72,7 +77,7 @@ class StallFirstRenewalQueryDatabase implements RelayDatabase {
|
||||
constructor(private readonly database: RelayDatabase) {}
|
||||
|
||||
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
||||
if (this.stallNext && sql.includes('WITH assignment_state AS MATERIALIZED')) {
|
||||
if (this.stallNext && sql === CONTROL_RENEWAL_BATCH_SQL) {
|
||||
this.stallNext = false
|
||||
this.stalled.resolve()
|
||||
await this.continue.promise
|
||||
@@ -103,7 +108,7 @@ class RenewalQueryProbeDatabase implements RelayDatabase {
|
||||
constructor(private readonly database: RelayDatabase) {}
|
||||
|
||||
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
||||
if (sql.includes('WITH assignment_state AS MATERIALIZED')) this.renewalQueries++
|
||||
if (sql === CONTROL_RENEWAL_BATCH_SQL) this.renewalQueries++
|
||||
return await this.database.query(sql, params)
|
||||
}
|
||||
|
||||
@@ -332,6 +337,164 @@ describePostgres('PostgreSQL control renewal', () => {
|
||||
).rejects.toThrow('invalid_activity_expiry')
|
||||
})
|
||||
|
||||
it('renews every due host in one autocommitted statement', async () => {
|
||||
const probe = new RenewalQueryProbeDatabase(database)
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
const batch = identities.slice(6, 10)
|
||||
now += 30_000
|
||||
const expiresAt = now + 105_000
|
||||
|
||||
const outcomes = await store.renewControlActivities(
|
||||
batch.map((identity) => ({
|
||||
identity,
|
||||
activityId: controlId(sourceCell.id),
|
||||
cellId: sourceCell.id,
|
||||
expiresAt
|
||||
}))
|
||||
)
|
||||
|
||||
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed', 'renewed'])
|
||||
expect(probe.renewalQueries).toBe(1)
|
||||
expect(probe.transactions).toBe(0)
|
||||
const leases = await database.query(
|
||||
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND activity_id = ? ORDER BY relay_host_id ASC`,
|
||||
[userId, controlId(sourceCell.id)]
|
||||
)
|
||||
expect(
|
||||
leases
|
||||
.filter((lease) =>
|
||||
batch.some((identity) => identity.relayHostId === lease.relay_host_id)
|
||||
)
|
||||
.map((lease) => Number(lease.expires_at))
|
||||
).toEqual([expiresAt, expiresAt, expiresAt, expiresAt])
|
||||
})
|
||||
|
||||
it('reports each host its own verdict inside one batch', async () => {
|
||||
const store = new RelayAssignmentStore(database, () => now)
|
||||
now += 30_000
|
||||
const expiresAt = now + 105_000
|
||||
const live = identities[10]!
|
||||
const absent = { userId, relayHostId: 'controlrenewalgone' }
|
||||
|
||||
const outcomes = await store.renewControlActivities([
|
||||
{ identity: absent, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
|
||||
{ identity: live, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
|
||||
{
|
||||
identity: live,
|
||||
activityId: controlId(targetCell.id),
|
||||
cellId: targetCell.id,
|
||||
expiresAt
|
||||
}
|
||||
])
|
||||
|
||||
expect(outcomes).toEqual([
|
||||
'assignment_not_found',
|
||||
'renewed',
|
||||
'activity_cell_not_authoritative'
|
||||
])
|
||||
})
|
||||
|
||||
it('passes over a host whose assignment row is held and renews the rest', async () => {
|
||||
const store = new RelayAssignmentStore(database, () => now)
|
||||
now += 30_000
|
||||
const expiresAt = now + 105_000
|
||||
const held = identities[11]!
|
||||
const free = identities[12]!
|
||||
const locked = signal()
|
||||
const release = signal()
|
||||
// Holds the row the way every per-host transactional path does.
|
||||
const holder = database.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(
|
||||
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[held.userId, held.relayHostId]
|
||||
)
|
||||
locked.resolve()
|
||||
await release.promise
|
||||
})
|
||||
await locked.promise
|
||||
|
||||
const startedAt = performance.now()
|
||||
const outcomes = await store.renewControlActivities(
|
||||
[held, free].map((identity) => ({
|
||||
identity,
|
||||
activityId: controlId(sourceCell.id),
|
||||
cellId: sourceCell.id,
|
||||
expiresAt
|
||||
}))
|
||||
)
|
||||
const elapsedMs = performance.now() - startedAt
|
||||
release.resolve()
|
||||
await holder
|
||||
|
||||
expect(outcomes).toEqual(['assignment_lock_unavailable', 'renewed'])
|
||||
// It skipped rather than queued: a blocking FOR UPDATE would have spent the
|
||||
// pool's whole lock_timeout here and failed the free host too.
|
||||
expect(elapsedMs).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS)
|
||||
const lease = (
|
||||
await database.query(
|
||||
`SELECT expires_at FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
|
||||
[free.userId, free.relayHostId, controlId(sourceCell.id)]
|
||||
)
|
||||
)[0]
|
||||
expect(Number(lease!.expires_at)).toBe(expiresAt)
|
||||
})
|
||||
|
||||
it('renews both of one host\u2019s control leases in a single batch', async () => {
|
||||
const store = new RelayAssignmentStore(database, () => now)
|
||||
const identity = identities[13]!
|
||||
// Two live control leases on one host. Written directly because
|
||||
// activateControl retires the prior generation, and what is under test is the
|
||||
// statement's row-wise behaviour, not how the second lease came to exist.
|
||||
await database.query(
|
||||
`INSERT INTO relay_assignment_activity_leases
|
||||
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
|
||||
request_units, expires_at, updated_at)
|
||||
VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`control:${sourceCell.id}:2`,
|
||||
sourceCell.id,
|
||||
now,
|
||||
now
|
||||
]
|
||||
)
|
||||
now += 30_000
|
||||
const expiresAt = now + 105_000
|
||||
|
||||
const outcomes = await store.renewControlActivities(
|
||||
[1, 2].map((generation) => ({
|
||||
identity,
|
||||
activityId: `control:${sourceCell.id}:${generation}`,
|
||||
cellId: sourceCell.id,
|
||||
expiresAt: expiresAt - generation
|
||||
}))
|
||||
)
|
||||
|
||||
expect(outcomes).toEqual(['renewed', 'renewed'])
|
||||
const leases = await database.query(
|
||||
`SELECT activity_id, expires_at FROM relay_assignment_activity_leases
|
||||
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
|
||||
expiresAt - 1,
|
||||
expiresAt - 2
|
||||
])
|
||||
// The assignment row is written once, carrying the later of the two.
|
||||
const row = (
|
||||
await database.query(
|
||||
`SELECT lease_expires_at, last_activity_at FROM relay_assignments
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
expect(Number(row!.lease_expires_at)).toBe(expiresAt - 1)
|
||||
expect(Number(row!.last_activity_at)).toBe(now)
|
||||
})
|
||||
|
||||
it('uses one autocommitted PostgreSQL statement for a steady renewal', async () => {
|
||||
const probe = new RenewalQueryProbeDatabase(database)
|
||||
const store = new RelayAssignmentStore(probe, () => now)
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import type { AssignmentIdentity } from './assignment-identity-queue.js'
|
||||
import type { SqlRow } from './database.js'
|
||||
|
||||
export type ControlRenewalOutcome =
|
||||
| 'renewed'
|
||||
| 'assignment_not_found'
|
||||
| 'activity_cell_not_authoritative'
|
||||
| 'control_activity_not_found'
|
||||
| 'control_activity_moved'
|
||||
// The host's assignment row was already locked by one of the per-host
|
||||
// transactional paths. Retryable, and never a reason to close a control: the
|
||||
// next tick is 15s away and the lease has 105s on it.
|
||||
| 'assignment_lock_unavailable'
|
||||
// Decided per row before the statement runs, so one malformed request cannot
|
||||
// cost the rest of the batch its renewal.
|
||||
| 'invalid_activity_id'
|
||||
| 'invalid_activity_expiry'
|
||||
| 'database_error'
|
||||
|
||||
// Outcomes the statement itself can report. `database_error` is raised by the
|
||||
// driver, and `invalid_activity_expiry` is decided per row before the statement
|
||||
// is built, so neither can come back as a row.
|
||||
export const CONTROL_RENEWAL_STATEMENT_OUTCOMES = new Set<ControlRenewalOutcome>([
|
||||
'renewed',
|
||||
'assignment_not_found',
|
||||
'activity_cell_not_authoritative',
|
||||
'control_activity_not_found',
|
||||
'control_activity_moved',
|
||||
'assignment_lock_unavailable'
|
||||
])
|
||||
|
||||
export type ControlRenewalRequest = {
|
||||
identity: AssignmentIdentity
|
||||
activityId: string
|
||||
cellId: string
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
// LOCK ORDER - (user_id, relay_host_id), the primary key of relay_assignments,
|
||||
// applied here and repeated as the statement's ORDER BY so it holds whether the
|
||||
// planner walks the primary-key index or sorts under the LockRows node.
|
||||
//
|
||||
// The batch never waits for an assignment row: SKIP LOCKED reports a contended
|
||||
// host separately instead. That is what bounds how long a flush holds its locks
|
||||
// to its own execution time, because row locks live until the statement commits,
|
||||
// and it is why one host wedged in a per-host transaction cannot stall the
|
||||
// renewals of every other host sharing the flush.
|
||||
//
|
||||
// With no wait on the assignment pass, the deadlock question reduces to the two
|
||||
// later passes. Every writer in this store locks a host's assignment row before
|
||||
// that host's lease rows (`assignmentRow` then `lockAssignmentActivities`), and
|
||||
// a host whose assignment row is held was skipped, so the batch never reaches
|
||||
// that host's lease: the lease pass cannot wait either.
|
||||
// `markMigrationTargetRegistered` is the one writer that locks a migration row
|
||||
// without the assignment row first. It takes no further locks, so it can delay a
|
||||
// mid-migration row by up to the pool's lock_timeout but cannot close a cycle.
|
||||
export function orderedControlRenewalRows<Row extends { identity: AssignmentIdentity }>(
|
||||
rows: readonly Row[]
|
||||
): Row[] {
|
||||
return [...rows].sort(
|
||||
(left, right) =>
|
||||
left.identity.userId.localeCompare(right.identity.userId) ||
|
||||
left.identity.relayHostId.localeCompare(right.identity.relayHostId)
|
||||
)
|
||||
}
|
||||
|
||||
// One statement renewing every due control lease on this cell, row-wise over the
|
||||
// unnested parameter arrays. Logic per row is what the single-row predecessor
|
||||
// did: lock the assignment, admit the caller's cell either as the current cell or
|
||||
// as the source of an active forward migration, lock that host's control lease,
|
||||
// push both expiries forward, and report one outcome. The one addition is
|
||||
// `present_assignment`, an unlocked probe that separates a host with no
|
||||
// assignment row at all from one whose row SKIP LOCKED passed over - the first
|
||||
// closes the control, the second retries.
|
||||
export const CONTROL_RENEWAL_BATCH_SQL = `WITH renewal_input AS MATERIALIZED (
|
||||
SELECT
|
||||
renewal.ordinality AS row_index,
|
||||
renewal.user_id,
|
||||
renewal.relay_host_id,
|
||||
renewal.activity_id,
|
||||
renewal.cell_id,
|
||||
renewal.expires_at
|
||||
FROM unnest(?::text[], ?::text[], ?::text[], ?::text[], ?::bigint[])
|
||||
WITH ORDINALITY AS renewal(
|
||||
user_id, relay_host_id, activity_id, cell_id, expires_at, ordinality
|
||||
)
|
||||
), present_assignment AS MATERIALIZED (
|
||||
SELECT input.row_index
|
||||
FROM renewal_input input
|
||||
JOIN relay_assignments assignment
|
||||
ON assignment.user_id = input.user_id
|
||||
AND assignment.relay_host_id = input.relay_host_id
|
||||
), assignment_state AS MATERIALIZED (
|
||||
SELECT input.row_index, assignment.cell_id, assignment.assignment_epoch
|
||||
FROM renewal_input input
|
||||
JOIN relay_assignments assignment
|
||||
ON assignment.user_id = input.user_id
|
||||
AND assignment.relay_host_id = input.relay_host_id
|
||||
ORDER BY assignment.user_id, assignment.relay_host_id
|
||||
FOR UPDATE OF assignment SKIP LOCKED
|
||||
), migration_state AS MATERIALIZED (
|
||||
SELECT locked.row_index
|
||||
FROM assignment_state locked
|
||||
JOIN renewal_input input ON input.row_index = locked.row_index
|
||||
JOIN relay_assignment_migrations migration
|
||||
ON migration.user_id = input.user_id
|
||||
AND migration.relay_host_id = input.relay_host_id
|
||||
AND migration.source_cell_id = input.cell_id
|
||||
AND migration.target_cell_id = locked.cell_id
|
||||
AND migration.assignment_epoch = locked.assignment_epoch
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
|
||||
ORDER BY migration.user_id, migration.relay_host_id
|
||||
FOR UPDATE OF migration
|
||||
), authorization_state AS MATERIALIZED (
|
||||
SELECT locked.row_index
|
||||
FROM assignment_state locked
|
||||
JOIN renewal_input input ON input.row_index = locked.row_index
|
||||
WHERE locked.cell_id = input.cell_id
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM migration_state moving
|
||||
WHERE moving.row_index = locked.row_index
|
||||
)
|
||||
), lease_state AS MATERIALIZED (
|
||||
SELECT authorized.row_index, lease.activity_kind, lease.cell_id
|
||||
FROM authorization_state authorized
|
||||
JOIN renewal_input input ON input.row_index = authorized.row_index
|
||||
JOIN relay_assignment_activity_leases lease
|
||||
ON lease.user_id = input.user_id
|
||||
AND lease.relay_host_id = input.relay_host_id
|
||||
AND lease.activity_id = input.activity_id
|
||||
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id
|
||||
FOR UPDATE OF lease
|
||||
), renewed_lease AS (
|
||||
UPDATE relay_assignment_activity_leases lease
|
||||
SET expires_at = GREATEST(lease.expires_at, input.expires_at),
|
||||
updated_at = GREATEST(lease.updated_at, ?)
|
||||
FROM lease_state state
|
||||
JOIN renewal_input input ON input.row_index = state.row_index
|
||||
WHERE lease.user_id = input.user_id
|
||||
AND lease.relay_host_id = input.relay_host_id
|
||||
AND lease.activity_id = input.activity_id
|
||||
AND state.activity_kind = 'control' AND state.cell_id = input.cell_id
|
||||
RETURNING state.row_index
|
||||
), renewed_assignment AS (
|
||||
-- Grouped per host: an UPDATE whose FROM offers a target row more than
|
||||
-- once applies one source row and returns one, so two leases on one
|
||||
-- host would leave the assignment carrying the wrong expiry. The
|
||||
-- aggregate hands it exactly one row, carrying the later expiry.
|
||||
UPDATE relay_assignments assignment
|
||||
SET lease_expires_at = GREATEST(assignment.lease_expires_at, renewed.expires_at),
|
||||
last_activity_at = GREATEST(assignment.last_activity_at, ?)
|
||||
FROM (
|
||||
SELECT input.user_id, input.relay_host_id, MAX(input.expires_at) AS expires_at
|
||||
FROM renewed_lease renewed
|
||||
JOIN renewal_input input ON input.row_index = renewed.row_index
|
||||
GROUP BY input.user_id, input.relay_host_id
|
||||
) renewed
|
||||
WHERE assignment.user_id = renewed.user_id
|
||||
AND assignment.relay_host_id = renewed.relay_host_id
|
||||
RETURNING renewed.user_id
|
||||
)
|
||||
SELECT input.row_index, CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1 FROM present_assignment present
|
||||
WHERE present.row_index = input.row_index
|
||||
) THEN 'assignment_not_found'
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1 FROM assignment_state locked WHERE locked.row_index = input.row_index
|
||||
) THEN 'assignment_lock_unavailable'
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1 FROM authorization_state authorized
|
||||
WHERE authorized.row_index = input.row_index
|
||||
) THEN 'activity_cell_not_authoritative'
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1 FROM lease_state state WHERE state.row_index = input.row_index
|
||||
) THEN 'control_activity_not_found'
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM lease_state state
|
||||
WHERE state.row_index = input.row_index
|
||||
AND (state.activity_kind <> 'control' OR state.cell_id <> input.cell_id)
|
||||
) THEN 'control_activity_moved'
|
||||
-- Read from renewed_lease, which has one row per input row. The
|
||||
-- assignment update collapses to one row per host, so it cannot answer
|
||||
-- for a host that brought two leases to the same batch.
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM renewed_lease renewed
|
||||
WHERE renewed.row_index = input.row_index
|
||||
) THEN 'renewed'
|
||||
ELSE 'control_activity_not_found'
|
||||
END AS outcome
|
||||
FROM renewal_input input
|
||||
ORDER BY input.row_index`
|
||||
|
||||
export function controlRenewalBatchParams(
|
||||
rows: readonly ControlRenewalRequest[],
|
||||
now: number
|
||||
): unknown[] {
|
||||
return [
|
||||
rows.map((row) => row.identity.userId),
|
||||
rows.map((row) => row.identity.relayHostId),
|
||||
rows.map((row) => row.activityId),
|
||||
rows.map((row) => row.cellId),
|
||||
rows.map((row) => row.expiresAt),
|
||||
now,
|
||||
now
|
||||
]
|
||||
}
|
||||
|
||||
// Rows come back ordered by row_index, which is the 1-based position in the
|
||||
// statement's parameter arrays.
|
||||
export function readControlRenewalOutcomes(
|
||||
rows: SqlRow[],
|
||||
expected: number
|
||||
): ControlRenewalOutcome[] {
|
||||
if (rows.length !== expected) throw new Error('missing_control_renewal_outcome')
|
||||
return rows.map((row, position) => {
|
||||
if (Number(row.row_index) !== position + 1) {
|
||||
throw new Error('misordered_control_renewal_outcome')
|
||||
}
|
||||
const outcome = row.outcome
|
||||
if (
|
||||
typeof outcome !== 'string' ||
|
||||
!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(outcome as ControlRenewalOutcome)
|
||||
) {
|
||||
throw new Error('invalid_control_renewal_outcome')
|
||||
}
|
||||
// SAFETY: the membership check above is what narrows this string.
|
||||
return outcome as ControlRenewalOutcome
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
// The outage this guards against: the credential cleanup ran every 30s in all 23 cells and both
|
||||
// sweeps over relay_invites had no usable index, so each one seq-scanned the whole table inside the
|
||||
// maintenance transaction. Only a real planner can show the partial indexes take that away, and
|
||||
// only a real server has ctid.
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_credential_sweep_test'
|
||||
|
||||
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const NOW = 100 * DAY_MS
|
||||
|
||||
function scopedUrl(): string {
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
async function onAdmin<T>(operation: (client: pg.Client) => Promise<T>): Promise<T> {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
return await operation(client)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
}
|
||||
|
||||
describePostgres('credential cleanup against PostgreSQL', () => {
|
||||
let database: RelayDatabase
|
||||
let store: RelayCredentialStore
|
||||
const opened: RelayDatabase[] = []
|
||||
|
||||
beforeEach(async () => {
|
||||
await onAdmin(async (client) => {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
})
|
||||
database = await openRelayDatabase({ databaseUrl: scopedUrl(), dataDir: '' })
|
||||
opened.push(database)
|
||||
store = new RelayCredentialStore(database, () => NOW)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await Promise.all(opened.map((open) => open.close().catch(() => undefined)))
|
||||
await onAdmin((client) => client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`))
|
||||
})
|
||||
|
||||
async function seedInvites(
|
||||
count: number,
|
||||
state: string,
|
||||
updatedAt: number,
|
||||
expiresAt = NOW - DAY_MS
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_invites
|
||||
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
|
||||
max_attempts, expires_at, created_at, updated_at)
|
||||
SELECT ?, ?, 'device-' || n, 'token-' || ? || '-' || n, ?, 0, 3, ?, ?, ?
|
||||
FROM generate_series(1, ?) AS n`,
|
||||
[identity.userId, identity.relayHostId, state, state, expiresAt, updatedAt, updatedAt, count]
|
||||
)
|
||||
}
|
||||
|
||||
// Not through RelayDatabase: it routes anything that is not a SELECT to the row-count path, and
|
||||
// EXPLAIN on an UPDATE is neither.
|
||||
async function plan(sql: string, params: unknown[]): Promise<string> {
|
||||
const client = new pg.Client({ connectionString: scopedUrl() })
|
||||
await client.connect()
|
||||
try {
|
||||
let index = 0
|
||||
const result = await client.query(
|
||||
`EXPLAIN ${sql.replace(/\?/g, () => `$${(index += 1)}`)}`,
|
||||
params
|
||||
)
|
||||
return result.rows.map((row) => String(row['QUERY PLAN'])).join('\n')
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
}
|
||||
|
||||
it('plans both invite sweeps as index scans instead of scanning the whole table', async () => {
|
||||
// Production's shape: terminal invites outnumber live ones by orders of magnitude, which is
|
||||
// what makes the partial predicates worth having.
|
||||
await seedInvites(20_000, 'consumed', NOW)
|
||||
for (const state of ['available', 'reserved', 'cooldown']) {
|
||||
await seedInvites(200, state, NOW, NOW + DAY_MS)
|
||||
}
|
||||
await database.query(
|
||||
`UPDATE relay_invites SET reservation_expires_at = ? WHERE state = 'reserved'`,
|
||||
[NOW + 1]
|
||||
)
|
||||
// Only ANALYZE makes the planner's row estimates real; without it a cold table looks tiny and
|
||||
// a seq scan wins on any index.
|
||||
await database.query(`ANALYZE relay_invites`)
|
||||
|
||||
const expiry = await plan(
|
||||
`UPDATE relay_invites SET state = 'expired'
|
||||
WHERE expires_at <= ? AND state IN ('available', 'reserved', 'cooldown')`,
|
||||
[NOW]
|
||||
)
|
||||
const reservation = await plan(
|
||||
`UPDATE relay_invites SET state = 'cooldown'
|
||||
WHERE state = 'reserved' AND reservation_expires_at <= ? AND expires_at > ?`,
|
||||
[NOW, NOW]
|
||||
)
|
||||
|
||||
// Which of the two partial indexes serves the reservation pass is the planner's call: both
|
||||
// predicates hold only live invites, so either one reads a handful of rows. The invariant is
|
||||
// that neither pass reads the whole table any more.
|
||||
for (const sweep of [expiry, reservation]) {
|
||||
expect(sweep).not.toContain('Seq Scan on relay_invites')
|
||||
expect(sweep).toMatch(/using relay_invites_sweep_(expiry|reservation)/)
|
||||
}
|
||||
expect(expiry).toContain('relay_invites_sweep_expiry')
|
||||
})
|
||||
|
||||
it('plans the basis sweep off the composite index rather than the 1.5 GB heap', async () => {
|
||||
// The shape that made this the most expensive statement in the sweep: 20,000 settled bases to
|
||||
// 50 live ones. A partial index on active = 1 looks like the answer to that ratio and is not:
|
||||
// a basis is inserted active and flipped to 0, so it accumulates the same dead entries, and
|
||||
// the planner picks the composite index anyway. See the schema comment beside it.
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
|
||||
FROM generate_series(1, 20000) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
SELECT 'live-' || n, ?, ?, 'device-1', 1, 'invite', ?, 1, ?
|
||||
FROM generate_series(1, 50) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
|
||||
)
|
||||
await database.query(`ANALYZE relay_connection_bases`)
|
||||
|
||||
const sweep = await plan(
|
||||
`UPDATE relay_connection_bases SET active = 0 WHERE active = 1 AND deadline <= ?`,
|
||||
[NOW]
|
||||
)
|
||||
|
||||
expect(sweep).not.toContain('Seq Scan on relay_connection_bases')
|
||||
expect(sweep).toContain('using relay_connection_bases_active_deadline')
|
||||
})
|
||||
|
||||
it('plans the drained basis reaper off the composite index, not the heap', async () => {
|
||||
// Why the composite index stays for now: it is the only one covering active = 0, and the case
|
||||
// that needs it is the steady state, where every row is inside retention and the reaper must
|
||||
// learn there is nothing to do. While the backlog drains the planner rightly prefers a bounded
|
||||
// sequential scan, because it finds its 5,000 rows and stops; measured at 200k rows, the
|
||||
// drained batch costs 5 buffers with this index and 1,274 without it.
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
|
||||
FROM generate_series(1, 20000) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW - 60_000, NOW - 60_000]
|
||||
)
|
||||
await database.query(`ANALYZE relay_connection_bases`)
|
||||
|
||||
const reaper = await plan(
|
||||
`DELETE FROM relay_connection_bases WHERE ctid IN (
|
||||
SELECT ctid FROM relay_connection_bases WHERE active = ? AND deadline <= ? LIMIT 5000
|
||||
)`,
|
||||
[0, NOW - DAY_MS]
|
||||
)
|
||||
|
||||
expect(reaper).toContain('relay_connection_bases_active_deadline')
|
||||
expect(reaper).not.toContain('Seq Scan on relay_connection_bases')
|
||||
})
|
||||
|
||||
it('plans the pending-authorization and rate-window sweeps as index scans', async () => {
|
||||
await database.query(
|
||||
`INSERT INTO relay_direct_authorizations
|
||||
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
deadline, consumed_at)
|
||||
SELECT 'auth-' || n, ?, ?, 'device-1', 1, ?, ?
|
||||
FROM generate_series(1, 20000) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW - 1, NOW - 1]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
|
||||
SELECT 'scope-' || n, 'invite-mint', ?, 1 FROM generate_series(1, 20000) AS n`,
|
||||
[NOW]
|
||||
)
|
||||
await database.query(`ANALYZE relay_direct_authorizations`)
|
||||
await database.query(`ANALYZE relay_rate_windows`)
|
||||
|
||||
const pending = await plan(
|
||||
`UPDATE relay_direct_authorizations SET consumed_at = ?
|
||||
WHERE consumed_at IS NULL AND deadline <= ?`,
|
||||
[NOW, NOW]
|
||||
)
|
||||
const windows = await plan(`DELETE FROM relay_rate_windows WHERE window_started_at < ?`, [
|
||||
NOW - DAY_MS
|
||||
])
|
||||
|
||||
expect(pending).toContain('relay_direct_authorizations_pending_deadline')
|
||||
expect(pending).not.toContain('Seq Scan on relay_direct_authorizations')
|
||||
expect(windows).toContain('relay_rate_windows_started')
|
||||
expect(windows).not.toContain('Seq Scan on relay_rate_windows')
|
||||
})
|
||||
|
||||
it('reaps settled bases and consumed authorizations through ctid', async () => {
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
|
||||
FROM generate_series(1, 5002) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
VALUES ('live', ?, ?, 'device-1', 1, 'invite', ?, 1, ?)`,
|
||||
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_direct_authorizations
|
||||
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
deadline, consumed_at)
|
||||
SELECT 'consumed-' || n, ?, ?, 'device-1', 1, ?, ?
|
||||
FROM generate_series(1, 5002) AS n`,
|
||||
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_direct_authorizations
|
||||
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
deadline, consumed_at)
|
||||
VALUES ('pending', ?, ?, 'device-1', 1, ?, NULL)`,
|
||||
[identity.userId, identity.relayHostId, NOW + 30_000]
|
||||
)
|
||||
|
||||
await store.cleanup()
|
||||
expect(
|
||||
await database.query(`SELECT count(*) AS total FROM relay_connection_bases`)
|
||||
).toEqual([{ total: '3' }])
|
||||
expect(
|
||||
await database.query(`SELECT count(*) AS total FROM relay_direct_authorizations`)
|
||||
).toEqual([{ total: '3' }])
|
||||
|
||||
await store.cleanup()
|
||||
// Only the rows a reader could still accept are left.
|
||||
expect(
|
||||
await database.query(`SELECT basis_conn_id FROM relay_connection_bases`)
|
||||
).toEqual([{ basis_conn_id: 'live' }])
|
||||
expect(
|
||||
await database.query(`SELECT direct_auth_id FROM relay_direct_authorizations`)
|
||||
).toEqual([{ direct_auth_id: 'pending' }])
|
||||
})
|
||||
|
||||
it('reaps terminal invites past retention through ctid, one bounded batch per cycle', async () => {
|
||||
await seedInvites(5_002, 'consumed', NOW - 30 * DAY_MS)
|
||||
await seedInvites(3, 'invalidated', NOW - 6 * DAY_MS)
|
||||
await seedInvites(2, 'available', NOW - 400 * DAY_MS, NOW + DAY_MS)
|
||||
|
||||
await store.cleanup()
|
||||
expect(await database.query(`SELECT count(*) AS total FROM relay_invites`)).toEqual([
|
||||
{ total: '7' }
|
||||
])
|
||||
|
||||
await store.cleanup()
|
||||
// The two live invites and the three inside retention survive; the batch remainder is gone.
|
||||
expect(
|
||||
await database.query(`SELECT state, count(*) AS total FROM relay_invites GROUP BY state ORDER BY state`)
|
||||
).toEqual([
|
||||
{ state: 'available', total: '2' },
|
||||
{ state: 'invalidated', total: '3' }
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,297 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
|
||||
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const NOW = 100 * DAY_MS
|
||||
|
||||
async function insertInvite(
|
||||
database: RelayDatabase,
|
||||
invite: { token: string; state: string; updatedAt: number; expiresAt?: number }
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_invites
|
||||
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
|
||||
max_attempts, expires_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
`device-${invite.token}`,
|
||||
invite.token,
|
||||
invite.state,
|
||||
0,
|
||||
3,
|
||||
invite.expiresAt ?? NOW + DAY_MS,
|
||||
invite.updatedAt,
|
||||
invite.updatedAt
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
async function remainingTokens(database: RelayDatabase): Promise<string[]> {
|
||||
const rows = await database.query(`SELECT token_hash FROM relay_invites ORDER BY token_hash`)
|
||||
return rows.map((row) => String(row.token_hash))
|
||||
}
|
||||
|
||||
async function insertBasis(
|
||||
database: RelayDatabase,
|
||||
basis: { id: string; active: number; deadline: number }
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
basis.id,
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
'device-1',
|
||||
1,
|
||||
'invite',
|
||||
basis.deadline,
|
||||
basis.active,
|
||||
NOW
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
async function insertDirectAuthorization(
|
||||
database: RelayDatabase,
|
||||
auth: { id: string; deadline: number; consumedAt: number | null }
|
||||
): Promise<void> {
|
||||
await database.query(
|
||||
`INSERT INTO relay_direct_authorizations
|
||||
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
deadline, consumed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[auth.id, identity.userId, identity.relayHostId, 'device-1', 1, auth.deadline, auth.consumedAt]
|
||||
)
|
||||
}
|
||||
|
||||
async function remainingIds(database: RelayDatabase, table: string, column: string): Promise<string[]> {
|
||||
const rows = await database.query(`SELECT ${column} FROM ${table} ORDER BY ${column}`)
|
||||
return rows.map((row) => String(row[column]))
|
||||
}
|
||||
|
||||
describe('credential cleanup invite reaper', () => {
|
||||
it('deletes terminal invites past retention and keeps everything else', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
const stale = NOW - 8 * DAY_MS
|
||||
const recent = NOW - 6 * DAY_MS
|
||||
for (const state of ['expired', 'consumed', 'invalidated']) {
|
||||
await insertInvite(database, { token: `stale-${state}`, state, updatedAt: stale })
|
||||
await insertInvite(database, { token: `recent-${state}`, state, updatedAt: recent })
|
||||
}
|
||||
|
||||
await store.cleanup()
|
||||
|
||||
expect(await remainingTokens(database)).toEqual([
|
||||
'recent-consumed',
|
||||
'recent-expired',
|
||||
'recent-invalidated'
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('never deletes an invite that a reader could still consume, however old', async () => {
|
||||
// Retention is measured on updated_at, and a long-lived available invite has an old one. The
|
||||
// state filter is what keeps the reaper from deleting a credential still in use.
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
const ancient = NOW - 400 * DAY_MS
|
||||
for (const state of ['available', 'reserved', 'cooldown']) {
|
||||
await insertInvite(database, {
|
||||
token: `live-${state}`,
|
||||
state,
|
||||
updatedAt: ancient,
|
||||
expiresAt: NOW + DAY_MS
|
||||
})
|
||||
}
|
||||
|
||||
await store.cleanup()
|
||||
|
||||
expect(await remainingTokens(database)).toEqual(['live-available', 'live-cooldown', 'live-reserved'])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('bounds one cycle to a single batch and drains the rest on later cycles', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
const stale = NOW - 30 * DAY_MS
|
||||
for (let index = 0; index < 5_002; index += 1) {
|
||||
await insertInvite(database, {
|
||||
token: `consumed-${String(index).padStart(5, '0')}`,
|
||||
state: 'consumed',
|
||||
updatedAt: stale
|
||||
})
|
||||
}
|
||||
|
||||
await store.cleanup()
|
||||
expect(await remainingTokens(database)).toHaveLength(2)
|
||||
|
||||
await store.cleanup()
|
||||
expect(await remainingTokens(database)).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('still expires credentials the sweep owns, and only those past their deadline', async () => {
|
||||
// The reaper runs after the sweep in the same call, so this pins that adding it did not
|
||||
// displace any of the five state transitions the sweep is there for.
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
await insertInvite(database, {
|
||||
token: 'lapsed',
|
||||
state: 'available',
|
||||
updatedAt: NOW,
|
||||
expiresAt: NOW - 1
|
||||
})
|
||||
await insertInvite(database, {
|
||||
token: 'current',
|
||||
state: 'available',
|
||||
updatedAt: NOW,
|
||||
expiresAt: NOW + DAY_MS
|
||||
})
|
||||
await database.query(
|
||||
`UPDATE relay_invites SET state = ?, reservation_expires_at = ? WHERE token_hash = ?`,
|
||||
['reserved', NOW - 1, 'current']
|
||||
)
|
||||
await database.query(
|
||||
`INSERT INTO relay_connection_bases
|
||||
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
|
||||
credential_kind, deadline, active, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
'basis-lapsed', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW - 1, 1, NOW,
|
||||
'basis-live', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW + 1, 1, NOW
|
||||
]
|
||||
)
|
||||
await store.recordDirectAuthorization({
|
||||
...identity,
|
||||
relayDeviceId: 'device-1',
|
||||
directAuthId: 'direct-lapsed',
|
||||
owningControlGeneration: 1,
|
||||
deadline: NOW - 1
|
||||
})
|
||||
await store.recordDirectAuthorization({
|
||||
...identity,
|
||||
relayDeviceId: 'device-1',
|
||||
directAuthId: 'direct-live',
|
||||
owningControlGeneration: 1,
|
||||
deadline: NOW + 1
|
||||
})
|
||||
await database.query(
|
||||
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
|
||||
VALUES (?, ?, ?, ?), (?, ?, ?, ?)`,
|
||||
['scope', 'invite-mint', NOW - 2 * DAY_MS, 1, 'scope', 'invite-mint', NOW - 1, 1]
|
||||
)
|
||||
|
||||
await store.cleanup()
|
||||
|
||||
expect(
|
||||
await database.query(`SELECT token_hash, state FROM relay_invites ORDER BY token_hash`)
|
||||
).toEqual([
|
||||
{ token_hash: 'current', state: 'cooldown' },
|
||||
{ token_hash: 'lapsed', state: 'expired' }
|
||||
])
|
||||
expect(
|
||||
await database.query(`SELECT basis_conn_id, active FROM relay_connection_bases ORDER BY basis_conn_id`)
|
||||
).toEqual([
|
||||
{ basis_conn_id: 'basis-lapsed', active: 0 },
|
||||
{ basis_conn_id: 'basis-live', active: 1 }
|
||||
])
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT direct_auth_id FROM relay_direct_authorizations
|
||||
WHERE consumed_at IS NULL ORDER BY direct_auth_id`
|
||||
)
|
||||
).toEqual([{ direct_auth_id: 'direct-live' }])
|
||||
expect(await database.query(`SELECT window_started_at FROM relay_rate_windows`)).toEqual([
|
||||
{ window_started_at: NOW - 1 }
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('reaps connection bases whose deadline passed over a day ago, and nothing else', async () => {
|
||||
// Retention is measured on deadline, and both readers of a basis require deadline >= now, so a
|
||||
// deadline a day in the past is already unusable however the active flag reads. The active = 0
|
||||
// clause is what keeps the batch an index range, not what makes the row safe to delete.
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
await insertBasis(database, { id: 'stale-inactive', active: 0, deadline: NOW - 2 * DAY_MS })
|
||||
await insertBasis(database, { id: 'recent-inactive', active: 0, deadline: NOW - 60_000 })
|
||||
// A long-lived splice: still active hours after the 30s deadline it was created with. The
|
||||
// sweep deactivates it this cycle and the reaper takes it in the same call, which is safe
|
||||
// precisely because no reader would have accepted it since its deadline passed.
|
||||
await insertBasis(database, { id: 'stale-active', active: 1, deadline: NOW - 400 * DAY_MS })
|
||||
await insertBasis(database, { id: 'live-active', active: 1, deadline: NOW + DAY_MS })
|
||||
|
||||
await store.cleanup()
|
||||
|
||||
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([
|
||||
'live-active',
|
||||
'recent-inactive'
|
||||
])
|
||||
// The one row a reader can still use is untouched, active flag included.
|
||||
expect(
|
||||
await database.query(
|
||||
`SELECT active FROM relay_connection_bases WHERE basis_conn_id = 'live-active'`
|
||||
)
|
||||
).toEqual([{ active: 1 }])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('reaps consumed direct authorizations past retention and never a pending one', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
await insertDirectAuthorization(database, {
|
||||
id: 'stale-consumed',
|
||||
deadline: NOW - 2 * DAY_MS,
|
||||
consumedAt: NOW - 2 * DAY_MS
|
||||
})
|
||||
await insertDirectAuthorization(database, {
|
||||
id: 'recent-consumed',
|
||||
deadline: NOW - 60_000,
|
||||
consumedAt: NOW - 60_000
|
||||
})
|
||||
await insertDirectAuthorization(database, {
|
||||
id: 'pending-ancient',
|
||||
deadline: NOW + DAY_MS,
|
||||
consumedAt: null
|
||||
})
|
||||
|
||||
await store.cleanup()
|
||||
|
||||
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([
|
||||
'pending-ancient',
|
||||
'recent-consumed'
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('bounds each table to one batch per cycle', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const store = new RelayCredentialStore(database, () => NOW)
|
||||
for (let index = 0; index < 5_001; index += 1) {
|
||||
const id = String(index).padStart(5, '0')
|
||||
await insertBasis(database, { id: `basis-${id}`, active: 0, deadline: NOW - 2 * DAY_MS })
|
||||
await insertDirectAuthorization(database, {
|
||||
id: `auth-${id}`,
|
||||
deadline: NOW - 2 * DAY_MS,
|
||||
consumedAt: NOW - 2 * DAY_MS
|
||||
})
|
||||
}
|
||||
|
||||
await store.cleanup()
|
||||
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toHaveLength(1)
|
||||
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toHaveLength(1)
|
||||
|
||||
await store.cleanup()
|
||||
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([])
|
||||
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
})
|
||||
@@ -12,6 +12,17 @@ const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
|
||||
// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing
|
||||
// working for clients whose clocks trail the cell by up to this margin.
|
||||
const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000
|
||||
// Terminal invites are read by nothing: every reader re-checks expiry and state at read time, so
|
||||
// the row only serves the audit trail, which relay_audit_events already keeps. A week is long
|
||||
// enough to answer a support question about a pairing that failed.
|
||||
const TERMINAL_INVITE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000
|
||||
// Why: both readers of a connection basis and of a direct authorization require it still
|
||||
// active/unconsumed AND inside its deadline, and every deadline is set at most 30s past insert, so
|
||||
// a settled row can never authorize anything again. A day is margin for forensics, not for reads.
|
||||
const INACTIVE_AUTHORIZATION_RETENTION_MS = 24 * 60 * 60 * 1000
|
||||
// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over
|
||||
// however many cycles it takes.
|
||||
const REAP_BATCH_ROWS = 5000
|
||||
|
||||
export type RelayIdentity = { userId: string; relayHostId: string }
|
||||
export type CredentialReservation = RelayIdentity & {
|
||||
@@ -643,6 +654,44 @@ export class RelayCredentialStore {
|
||||
[now - 24 * 60 * 60 * 1000]
|
||||
)
|
||||
})
|
||||
await this.reapSettledCredentials(now)
|
||||
}
|
||||
|
||||
// Outside the sweep transaction on purpose: each delete is idempotent and independent of the
|
||||
// state transitions above, so batching them in would only hold their row locks for longer.
|
||||
private async reapSettledCredentials(now: number): Promise<void> {
|
||||
await this.reapBatch(
|
||||
'relay_invites',
|
||||
'state IN (?, ?, ?) AND updated_at <= ?',
|
||||
['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]
|
||||
)
|
||||
// deadline, not created_at: it is the second column of relay_connection_bases_active_deadline,
|
||||
// so once the backlog is drained this batch learns there is nothing left to do from the index
|
||||
// instead of the 1.5 GB heap. Both readers reject a passed deadline, so a day past one is
|
||||
// unusable whatever the active flag says.
|
||||
await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [
|
||||
0,
|
||||
now - INACTIVE_AUTHORIZATION_RETENTION_MS
|
||||
])
|
||||
// consumed_at, not deadline: consumption is what settles this row, and it can happen well
|
||||
// before the deadline, so measuring from it retains the row for the full window either way.
|
||||
await this.reapBatch(
|
||||
'relay_direct_authorizations',
|
||||
'consumed_at IS NOT NULL AND consumed_at <= ?',
|
||||
[now - INACTIVE_AUTHORIZATION_RETENTION_MS]
|
||||
)
|
||||
}
|
||||
|
||||
// ctid/rowid, not the primary key: the physical address lets the delete re-find exactly the batch
|
||||
// the subquery located instead of re-matching the predicate per row.
|
||||
private async reapBatch(table: string, predicate: string, params: unknown[]): Promise<void> {
|
||||
const address = this.database.dialect === 'sqlite' ? 'rowid' : 'ctid'
|
||||
await this.database.query(
|
||||
`DELETE FROM ${table} WHERE ${address} IN (
|
||||
SELECT ${address} FROM ${table} WHERE ${predicate} LIMIT ${REAP_BATCH_ROWS}
|
||||
)`,
|
||||
params
|
||||
)
|
||||
}
|
||||
|
||||
private async installStatusWith(
|
||||
|
||||
@@ -134,7 +134,7 @@ describe('PostgreSQL relay deadlines', () => {
|
||||
statements.every(
|
||||
(statement) =>
|
||||
statement === POSTGRES_STATEMENT_STATS_MIGRATION.trim() ||
|
||||
/^(?:CREATE|ALTER TABLE)\b/i.test(body(statement))
|
||||
/^(?:CREATE|ALTER TABLE|DROP INDEX)\b/i.test(body(statement))
|
||||
)
|
||||
).toBe(true)
|
||||
// The backfill is DML, so it stays on the deadline-bearing serving pool.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { isRelayDatabaseTransientError } from './database.js'
|
||||
|
||||
export type DatabaseStartupRetryPolicy = {
|
||||
attempts: number
|
||||
windowMs: number
|
||||
baseDelayMs: number
|
||||
maxDelayMs: number
|
||||
jitterMs: number
|
||||
// Which failures this particular startup step may repeat. Not every caller can
|
||||
// repeat everything the request path calls transient: what the retry re-runs
|
||||
// decides that, so the call site owns it.
|
||||
isRetryable?: (error: unknown) => boolean
|
||||
}
|
||||
|
||||
export type DatabaseStartupRetryObserver = {
|
||||
onRetry?: (event: { attempt: number; delayMs: number; error: unknown }) => void
|
||||
onRecovered?: (event: { attempts: number }) => void
|
||||
onGaveUp?: (event: { attempts: number; error: unknown; retryable: boolean }) => void
|
||||
}
|
||||
|
||||
function retryDelayMs(policy: DatabaseStartupRetryPolicy, attempt: number): number {
|
||||
const backoffMs = Math.min(policy.baseDelayMs * 2 ** (attempt - 1), policy.maxDelayMs)
|
||||
return backoffMs + Math.floor(Math.random() * (policy.jitterMs + 1))
|
||||
}
|
||||
|
||||
// Startup work that a cold dependency - a proxy sidecar that just started, a
|
||||
// database still accepting the fleet back - can fail once and serve a moment
|
||||
// later. The wall-clock window, not the attempt count, is the real bound.
|
||||
export async function retryTransientDatabaseStartup<T>(
|
||||
operation: () => Promise<T>,
|
||||
policy: DatabaseStartupRetryPolicy,
|
||||
observer: DatabaseStartupRetryObserver = {}
|
||||
): Promise<T> {
|
||||
const retryDeadline = Date.now() + policy.windowMs
|
||||
for (let attempt = 1; ; attempt += 1) {
|
||||
try {
|
||||
const result = await operation()
|
||||
if (attempt > 1) observer.onRecovered?.({ attempts: attempt })
|
||||
return result
|
||||
} catch (error) {
|
||||
const remainingMs = retryDeadline - Date.now()
|
||||
const retryable = (policy.isRetryable ?? isRelayDatabaseTransientError)(error)
|
||||
if (attempt === policy.attempts || remainingMs <= 0 || !retryable) {
|
||||
observer.onGaveUp?.({ attempts: attempt, error, retryable })
|
||||
throw error
|
||||
}
|
||||
const delayMs = Math.min(retryDelayMs(policy, attempt), remainingMs)
|
||||
observer.onRetry?.({ attempt, delayMs, error })
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,20 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { isRelayDatabaseTransientError } from './database.js'
|
||||
import { PostgresPoolPressure } from './postgres-pool-pressure.js'
|
||||
|
||||
// The only way to mark an error as an acquire failure is to fail a real
|
||||
// acquire, so the gated cases go through the pressure wrapper the pool uses.
|
||||
async function failedAcquire(message: string): Promise<unknown> {
|
||||
const pool = {
|
||||
totalCount: 0,
|
||||
idleCount: 0,
|
||||
waitingCount: 0,
|
||||
connect: vi.fn(async () => {
|
||||
throw new Error(message)
|
||||
})
|
||||
}
|
||||
return await new PostgresPoolPressure(pool as never).connect().catch((error: unknown) => error)
|
||||
}
|
||||
|
||||
describe('relay database transient errors', () => {
|
||||
it.each(['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'])(
|
||||
@@ -15,4 +30,39 @@ describe('relay database transient errors', () => {
|
||||
).toBe(true)
|
||||
expect(isRelayDatabaseTransientError(new TypeError('broken invariant'))).toBe(false)
|
||||
})
|
||||
|
||||
it('classifies a pool connect timeout that node-postgres reports with no code', () => {
|
||||
// pg-pool raises this only from its own connect path, so no statement ran.
|
||||
expect(
|
||||
isRelayDatabaseTransientError(
|
||||
new Error('Connection terminated due to connection timeout')
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('classifies an early-ended socket only when it ended during the acquire', async () => {
|
||||
expect(
|
||||
isRelayDatabaseTransientError(await failedAcquire('Connection terminated unexpectedly'))
|
||||
).toBe(true)
|
||||
// The same message mid-statement leaves the commit outcome unknown, so it
|
||||
// must stay a hard failure rather than invite a retry.
|
||||
expect(
|
||||
isRelayDatabaseTransientError(new Error('Connection terminated unexpectedly'))
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it.each([null, undefined, 'a thrown string'])(
|
||||
'survives %s reaching it instead of an error object',
|
||||
(thrown) => {
|
||||
expect(isRelayDatabaseTransientError(thrown)).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps a failed acquire that is not transient out of the retry path', async () => {
|
||||
expect(
|
||||
isRelayDatabaseTransientError(
|
||||
await failedAcquire('password authentication failed for user "relay"')
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -6,6 +6,7 @@ import pg from 'pg'
|
||||
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||||
import {
|
||||
emptyPostgresPoolPressureCounts,
|
||||
isPostgresPoolConnectFailure,
|
||||
PostgresPoolPressure,
|
||||
type PostgresPoolPressureCounts
|
||||
} from './postgres-pool-pressure.js'
|
||||
@@ -99,6 +100,18 @@ CREATE TABLE IF NOT EXISTS relay_invites (
|
||||
CREATE INDEX IF NOT EXISTS relay_invites_device
|
||||
ON relay_invites(user_id, relay_host_id, relay_device_id);
|
||||
|
||||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||||
-- Why: the credential sweep matches (state, expires_at) every cycle while invites in a terminal
|
||||
-- state accumulate for the life of the database. Unindexed it seq-scans the whole table inside the
|
||||
-- maintenance transaction. Partial, so the index holds only the states the sweep can act on.
|
||||
CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry
|
||||
ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown');
|
||||
|
||||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||||
-- Why: the second sweep pass matches (state, reservation_expires_at) over the same table.
|
||||
CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation
|
||||
ON relay_invites(reservation_expires_at) WHERE state = 'reserved';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_devices (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
@@ -159,6 +172,11 @@ CREATE TABLE IF NOT EXISTS relay_connection_bases (
|
||||
-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle
|
||||
-- and holds the maintenance transaction open long enough to time out
|
||||
-- assignment lock waits.
|
||||
-- Why not a partial index on active = 1: a basis is inserted active and flipped to 0, so each
|
||||
-- deactivation leaves a dead entry in that index too. Measured on production-shaped history it
|
||||
-- carries the same dead entries as this one, the planner picks this one in every state, and it
|
||||
-- costs ~65 bytes of WAL per insert. Bloat here is cured by reaping and vacuum, not by a narrower
|
||||
-- index.
|
||||
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
|
||||
ON relay_connection_bases(active, deadline);
|
||||
|
||||
@@ -172,6 +190,12 @@ CREATE TABLE IF NOT EXISTS relay_direct_authorizations (
|
||||
consumed_at BIGINT
|
||||
);
|
||||
|
||||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||||
-- Why: the sweep expires pending authorizations by (consumed_at IS NULL, deadline), and consumed
|
||||
-- rows are never deleted. Partial, so the index stays the size of the pending set.
|
||||
CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline
|
||||
ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_confirm_results (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
@@ -523,8 +547,9 @@ CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases (
|
||||
updated_at BIGINT NOT NULL,
|
||||
PRIMARY KEY (user_id, relay_host_id, activity_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS relay_assignment_activity_expiry
|
||||
ON relay_assignment_activity_leases(expires_at);
|
||||
-- expires_at is deliberately unindexed: every control renewal writes it (~471/s), so an index on
|
||||
-- it makes each renewal a non-HOT update that rewrites index entries. Its only reader is the 30s
|
||||
-- expiry sweep, which seq-scans 14.8k rows / 7MB in a few milliseconds.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_control_connection_reservations (
|
||||
reservation_id TEXT PRIMARY KEY,
|
||||
@@ -558,6 +583,12 @@ CREATE TABLE IF NOT EXISTS relay_rate_windows (
|
||||
PRIMARY KEY (scope_key, window_kind, window_started_at)
|
||||
);
|
||||
|
||||
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
|
||||
-- Why: window_started_at is the PRIMARY KEY's last column, so the sweep's 24h retention delete
|
||||
-- cannot use it and seq-scans instead.
|
||||
CREATE INDEX IF NOT EXISTS relay_rate_windows_started
|
||||
ON relay_rate_windows(window_started_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_migration_leases (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
@@ -644,7 +675,24 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||||
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
|
||||
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
|
||||
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
|
||||
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
|
||||
// Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because
|
||||
// this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write,
|
||||
// and all 28 directors reach it at once; a lock timeout here must not restart the instance, which
|
||||
// would only re-queue the same DDL behind the same writers. Once it wins, the pre-check answers
|
||||
// absent and no later boot sends it at all.
|
||||
`-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s
|
||||
DROP INDEX IF EXISTS relay_assignment_activity_expiry`,
|
||||
// The drop is what makes HOT legal; this is what makes it possible. A renewal can only reuse the
|
||||
// row's own page when that page has room for a second version, and at the default fillfactor of
|
||||
// 100 a freshly filled page has none - measured at 0.5% HOT with the index gone and the default,
|
||||
// against 100% at 70. Takes SHARE UPDATE EXCLUSIVE, which blocks vacuum and DDL but no reader or
|
||||
// writer, and only for the catalog write. Applies to pages as they refill, so the table converges
|
||||
// over its own renewal cycle rather than at boot.
|
||||
// Deferrable for the same reason, though SHARE UPDATE EXCLUSIVE blocks only vacuum and DDL: it
|
||||
// buys nothing until the drop lands, so a boot that deferred the drop should defer this too.
|
||||
`-- schema-deferrable: buys nothing until the drop above lands
|
||||
ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`
|
||||
]
|
||||
|
||||
// The exact statement list a Postgres boot applies, in order, so the lock-target census can read
|
||||
@@ -924,13 +972,15 @@ function retryablePostgresTransactionError(error: unknown): boolean {
|
||||
}
|
||||
|
||||
export function isRelayDatabaseTransientError(error: unknown): boolean {
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
// Runs inside the query catch, where a thrown null or undefined would turn a
|
||||
// database failure into a TypeError that buries it.
|
||||
const code = String((error as { code?: unknown } | null)?.code)
|
||||
if (['40P01', '40001', '55P03', '57014', '53300', '57P03', '08001', '08006'].includes(code)) {
|
||||
return true
|
||||
}
|
||||
return String((error as { message?: unknown }).message).includes(
|
||||
'timeout exceeded when trying to connect'
|
||||
)
|
||||
// A pool that cannot hand out a client reports no SQLSTATE at all, so the
|
||||
// acquire boundary owns that vocabulary.
|
||||
return isPostgresPoolConnectFailure(error)
|
||||
}
|
||||
|
||||
async function waitForPostgresRetry(random: () => number = Math.random): Promise<void> {
|
||||
@@ -965,6 +1015,9 @@ class PostgresDatabase implements RelayDatabase {
|
||||
error,
|
||||
phase,
|
||||
sql,
|
||||
// Passed in rather than re-derived: the log has to say what the routes
|
||||
// actually did, and one classifier cannot drift from itself.
|
||||
transient: isRelayDatabaseTransientError(error),
|
||||
elapsedMs: performance.now() - startedAt,
|
||||
pool: this.pool
|
||||
})
|
||||
@@ -1155,13 +1208,15 @@ async function backfillRelayCellRegions(database: RelayDatabase): Promise<void>
|
||||
)
|
||||
}
|
||||
|
||||
export async function openRelayDatabase(input: {
|
||||
export type RelayDatabaseOpenInput = {
|
||||
databaseUrl?: string
|
||||
dataDir: string
|
||||
poolMax?: number
|
||||
applicationName?: string
|
||||
statementTimeoutMs?: number
|
||||
}): Promise<RelayDatabase> {
|
||||
}
|
||||
|
||||
export async function openRelayDatabase(input: RelayDatabaseOpenInput): Promise<RelayDatabase> {
|
||||
let database: RelayDatabase
|
||||
if (input.databaseUrl) {
|
||||
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
|
||||
|
||||
@@ -729,3 +729,73 @@ describe('control lease jitter', () => {
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('paced drain and the phones of a host not yet told', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
const laterHostId = 'qrstuvwxyz012345'
|
||||
const laterIdentity = { ...identity, sub: 'user-2', relayHostId: laterHostId }
|
||||
|
||||
async function twoHostCell(): Promise<{
|
||||
h: ReturnType<typeof harness>
|
||||
told: FakeSocket
|
||||
untold: FakeSocket
|
||||
}> {
|
||||
const h = harness()
|
||||
const told = await activeHost(h)
|
||||
const untold = new FakeSocket()
|
||||
await h.activate(untold as unknown as WebSocket, laterIdentity, null, 1, false, 1, '1.4.197')
|
||||
// Both hosts now dial in, so the credential mocks have to answer for either.
|
||||
h.store.resolveResume.mockImplementation(async (hostId: string) => ({
|
||||
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub
|
||||
}))
|
||||
h.store.reserveCredential.mockImplementation(async (hostId: string) => ({
|
||||
...reservation,
|
||||
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub,
|
||||
relayHostId: hostId
|
||||
}))
|
||||
return { h, told, untold }
|
||||
}
|
||||
|
||||
async function dial(h: ReturnType<typeof harness>, hostId: string): Promise<FakeSocket> {
|
||||
const client = new FakeSocket()
|
||||
await h.registry.acceptClient(client as unknown as WebSocket, hostId, 'credential')
|
||||
return client
|
||||
}
|
||||
|
||||
it('serves a host whose drain has not been sent and refuses one whose has', async () => {
|
||||
const { h, told, untold } = await twoHostCell()
|
||||
h.registry.drain(0, { paceWindowMs: 40_000 })
|
||||
|
||||
const refused = await dial(h, identity.relayHostId)
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
expect(told.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
|
||||
|
||||
const served = await dial(h, laterHostId)
|
||||
expect(served.close).not.toHaveBeenCalled()
|
||||
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
|
||||
})
|
||||
|
||||
it('refuses that host\'s phones as soon as its own drain is sent', async () => {
|
||||
const { h, untold } = await twoHostCell()
|
||||
h.registry.drain(0, { paceWindowMs: 40_000 })
|
||||
await vi.advanceTimersByTimeAsync(40_000)
|
||||
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
|
||||
|
||||
const refused = await dial(h, laterHostId)
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
})
|
||||
|
||||
it('keeps an unpaced drain refusing every phone at once', async () => {
|
||||
const { h } = await twoHostCell()
|
||||
h.registry.drain(0)
|
||||
for (const hostId of [identity.relayHostId, laterHostId]) {
|
||||
const refused = await dial(h, hostId)
|
||||
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -12,6 +12,12 @@ import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayCredentialStore } from './credential-store.js'
|
||||
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
|
||||
import {
|
||||
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
|
||||
type ControlRenewalOutcome,
|
||||
type ControlRenewalRequest
|
||||
} from './control-renewal-statement.js'
|
||||
import { HostSessionRegistry, type HostSession } from './host-session-registry.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayRuntimeObserver } from './relay-observability.js'
|
||||
@@ -23,6 +29,12 @@ import {
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
|
||||
|
||||
// A due renewal leaves the heartbeat as a batch enqueue, so the store only sees
|
||||
// the tick once the batch window closes.
|
||||
async function closeRenewalWindow(): Promise<void> {
|
||||
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
|
||||
}
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly OPEN = 1
|
||||
readonly CLOSING = 2
|
||||
@@ -109,6 +121,7 @@ function createRegistry(
|
||||
activate: ActivateSession
|
||||
acquireActivity: ReturnType<typeof vi.fn>
|
||||
renewControlActivity: ReturnType<typeof vi.fn>
|
||||
renewControlActivities: ReturnType<typeof vi.fn>
|
||||
releaseActivity: ReturnType<typeof vi.fn>
|
||||
observer: {
|
||||
recordAuth: ReturnType<typeof vi.fn>
|
||||
@@ -119,12 +132,38 @@ function createRegistry(
|
||||
const acquireActivity = vi.fn().mockResolvedValue(undefined)
|
||||
const renewControlActivity = vi.fn().mockResolvedValue(undefined)
|
||||
const releaseActivity = vi.fn().mockResolvedValue(true)
|
||||
// Mirrors the store's own batch semantics over the single-renewal mock: a known
|
||||
// outcome becomes that row's verdict, and any other failure reaches the caller
|
||||
// as the driver's error. Keeps every per-call expectation below aimed at the
|
||||
// renewal a session actually asked for.
|
||||
const renewControlActivities = vi.fn(
|
||||
async (rows: readonly ControlRenewalRequest[]): Promise<ControlRenewalOutcome[]> =>
|
||||
await Promise.all(
|
||||
rows.map(async (row): Promise<ControlRenewalOutcome> => {
|
||||
try {
|
||||
await renewControlActivity(row.identity, {
|
||||
activityId: row.activityId,
|
||||
cellId: row.cellId,
|
||||
expiresAt: row.expiresAt
|
||||
})
|
||||
return 'renewed'
|
||||
} catch (error) {
|
||||
const message = String((error as { message?: unknown }).message)
|
||||
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) {
|
||||
throw error
|
||||
}
|
||||
return message as ControlRenewalOutcome
|
||||
}
|
||||
})
|
||||
)
|
||||
)
|
||||
const assignments = {
|
||||
activateControl,
|
||||
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
|
||||
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
|
||||
acquireActivity,
|
||||
renewControlActivity,
|
||||
renewControlActivities,
|
||||
releaseActivity
|
||||
} as unknown as RelayAssignmentStore
|
||||
const observer = {
|
||||
@@ -176,6 +215,7 @@ function createRegistry(
|
||||
activate,
|
||||
acquireActivity,
|
||||
renewControlActivity,
|
||||
renewControlActivities,
|
||||
releaseActivity,
|
||||
observer
|
||||
}
|
||||
@@ -336,15 +376,15 @@ describe('host session cleanup races', () => {
|
||||
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
|
||||
|
||||
// POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both
|
||||
// reach drain(), so a second teardown can be scheduled for the same session.
|
||||
// reach drain(), so a retry re-sends to every session. It must re-arm the pending
|
||||
// teardown rather than stack a second one: across a paced cell that is 800 orphaned
|
||||
// timers per retry, each one holding the loop open for the rest of the window.
|
||||
registry.drain(0)
|
||||
const scheduled = vi.getTimerCount()
|
||||
registry.drain(0)
|
||||
// Pin the premise: if drain ever gains an idempotency guard, the retry schedules no
|
||||
// second teardown and the assertion below stops defending the write-once snapshot
|
||||
// while still passing. Compare against the count before the retry rather than an
|
||||
// absolute, since the session's heartbeat interval is also pending.
|
||||
expect(vi.getTimerCount()).toBe(scheduled + 1)
|
||||
// Compare against the count before the retry rather than an absolute, since the
|
||||
// session's heartbeat interval is also pending.
|
||||
expect(vi.getTimerCount()).toBe(scheduled)
|
||||
vi.advanceTimersByTime(1)
|
||||
|
||||
// Asserting registry state, not the log line: FakeSocket closes synchronously, so
|
||||
@@ -689,7 +729,8 @@ describe('host session cleanup races', () => {
|
||||
expect(original).not.toBeNull()
|
||||
|
||||
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
|
||||
vi.advanceTimersByTime(15_000)
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledOnce()
|
||||
expect(renewControlActivity).toHaveBeenCalledWith(
|
||||
@@ -715,6 +756,7 @@ describe('host session cleanup races', () => {
|
||||
})
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
await closeRenewalWindow()
|
||||
const replacement = new FakeSocket()
|
||||
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
|
||||
reject(new Error('activity_cell_not_authoritative'))
|
||||
@@ -737,6 +779,7 @@ describe('host session cleanup races', () => {
|
||||
})
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
await closeRenewalWindow()
|
||||
h.registry.drainHost({
|
||||
attemptId: 'attempt',
|
||||
userId: identity.sub,
|
||||
@@ -762,6 +805,7 @@ describe('host session cleanup races', () => {
|
||||
for (let interval = 0; interval < 4; interval++) {
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
|
||||
await closeRenewalWindow()
|
||||
}
|
||||
|
||||
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
|
||||
@@ -791,8 +835,10 @@ describe('host session cleanup races', () => {
|
||||
try {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
expect(renewControlActivity).toHaveBeenCalledOnce()
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
@@ -812,6 +858,7 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
stalled.resolve(undefined)
|
||||
@@ -831,13 +878,16 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
|
||||
await closeRenewalWindow()
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
stalled.resolve(undefined)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(3)
|
||||
registry.drain(0)
|
||||
@@ -855,6 +905,7 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(acquireActivity).toHaveBeenCalledWith(
|
||||
{ userId: identity.sub, relayHostId: identity.relayHostId },
|
||||
@@ -880,6 +931,7 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(acquireActivity).not.toHaveBeenCalled()
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
|
||||
@@ -899,6 +951,7 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
@@ -918,6 +971,7 @@ describe('host session cleanup races', () => {
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
|
||||
expect(socket.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.DRAINING,
|
||||
@@ -939,6 +993,7 @@ describe('host session cleanup races', () => {
|
||||
for (let interval = 0; interval < 3; interval++) {
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
|
||||
await closeRenewalWindow()
|
||||
}
|
||||
|
||||
expect(renewControlActivity).toHaveBeenCalledTimes(2)
|
||||
@@ -966,6 +1021,7 @@ describe('control renewal cadence across a rebind', () => {
|
||||
const beat = async (target: FakeSocket): Promise<void> => {
|
||||
await vi.advanceTimersByTimeAsync(ping)
|
||||
target.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
|
||||
await closeRenewalWindow()
|
||||
}
|
||||
|
||||
// Age the session so its attempt counter is well above zero.
|
||||
@@ -993,6 +1049,57 @@ describe('control renewal cadence across a rebind', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('control renewals shared by one batch', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('renews two due hosts in one call and leaves a stale one alone', async () => {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValueOnce('control:production-gce-c3:1')
|
||||
.mockResolvedValueOnce('control:production-gce-c3:1')
|
||||
const { registry, activate, renewControlActivities } = createRegistry(activateControl)
|
||||
const other = { ...identity, sub: 'user-2', relayHostId: 'ponmlkjihgfedcba' }
|
||||
const staleSocket = new FakeSocket()
|
||||
const liveSocket = new FakeSocket()
|
||||
await activate(staleSocket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
await activate(liveSocket as unknown as WebSocket, other, null, 1, false, 1)
|
||||
const stale = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
const live = registry.get({ userId: other.sub, relayHostId: other.relayHostId })!
|
||||
|
||||
// Both come due inside the same window, and one socket goes away while the
|
||||
// statement is still in PostgreSQL.
|
||||
let release!: () => void
|
||||
renewControlActivities.mockImplementationOnce(
|
||||
async (rows: readonly ControlRenewalRequest[]) => {
|
||||
staleSocket.close()
|
||||
await new Promise<void>((resolve) => (release = resolve))
|
||||
return rows.map((): ControlRenewalOutcome => 'renewed')
|
||||
}
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
const staleDueAt = stale.activityRenewalDueAt
|
||||
await closeRenewalWindow()
|
||||
release()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(renewControlActivities).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
renewControlActivities.mock.calls[0]![0].map(
|
||||
(row: ControlRenewalRequest) => row.identity.relayHostId
|
||||
)
|
||||
).toEqual([identity.relayHostId, other.relayHostId])
|
||||
expect(live.activityRenewalCompletedAttempt).toBe(1)
|
||||
expect(stale.activityRenewalCompletedAttempt).toBe(0)
|
||||
expect(stale.activityRenewalDueAt).toBe(staleDueAt)
|
||||
registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('control lease recovery after the session is gone', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
@@ -1019,6 +1126,7 @@ describe('control lease recovery after the session is gone', () => {
|
||||
new Promise<void>((_resolve, reject) => (failRenewal = reject))
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
await closeRenewalWindow()
|
||||
expect(renewControlActivity).toHaveBeenCalledOnce()
|
||||
|
||||
const newer = new FakeSocket()
|
||||
@@ -1696,3 +1804,124 @@ describe('host data attach owner lookup', () => {
|
||||
expect(h.owner.activeConnIds.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('paced drain', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
async function connectHosts(count: number): Promise<{
|
||||
registry: HostSessionRegistry
|
||||
sockets: FakeSocket[]
|
||||
}> {
|
||||
const activateControl = vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValue('control:production-gce-c3:1')
|
||||
const { registry, activate } = createRegistry(activateControl)
|
||||
const sockets: FakeSocket[] = []
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const socket = new FakeSocket()
|
||||
sockets.push(socket)
|
||||
await activate(
|
||||
socket as unknown as WebSocket,
|
||||
{ ...identity, sub: `user-${index}` },
|
||||
null,
|
||||
1,
|
||||
false,
|
||||
1
|
||||
)
|
||||
socket.send.mockClear()
|
||||
}
|
||||
return { registry, sockets }
|
||||
}
|
||||
|
||||
function drainsSent(sockets: FakeSocket[]): number {
|
||||
return sockets.filter((socket) =>
|
||||
socket.send.mock.calls.some(([payload]) => String(payload).includes('"type":"drain"'))
|
||||
).length
|
||||
}
|
||||
|
||||
it('sends every drain at once when no window is given', async () => {
|
||||
const { registry, sockets } = await connectHosts(4)
|
||||
registry.drain(0)
|
||||
expect(drainsSent(sockets)).toBe(4)
|
||||
})
|
||||
|
||||
// Windows here stay under the 75s control-silence watchdog, which would otherwise close
|
||||
// a test socket that never heartbeats before its paced send is due.
|
||||
it('spreads the sends evenly across the window', async () => {
|
||||
const { registry, sockets } = await connectHosts(5)
|
||||
registry.drain(0, { paceWindowMs: 40_000 })
|
||||
// The first host is sent synchronously; the last lands on the window's closing edge.
|
||||
expect(drainsSent(sockets)).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(drainsSent(sockets)).toBe(2)
|
||||
await vi.advanceTimersByTimeAsync(20_000)
|
||||
expect(drainsSent(sockets)).toBe(4)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(drainsSent(sockets)).toBe(5)
|
||||
})
|
||||
|
||||
it('fences admission for every session before the first paced send lands', async () => {
|
||||
const { registry, sockets } = await connectHosts(3)
|
||||
registry.drain(0, { paceWindowMs: 40_000 })
|
||||
expect(registry.isDraining()).toBe(true)
|
||||
// A host whose drain has not been sent yet must already be non-authoritative.
|
||||
const socket = new FakeSocket()
|
||||
registry.acceptControl(socket as unknown as WebSocket, { ...identity, sub: 'user-late' })
|
||||
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
expect(drainsSent(sockets)).toBe(1)
|
||||
})
|
||||
|
||||
it('gives each host its own grace after its own send, not after the call', async () => {
|
||||
const { registry, sockets } = await connectHosts(2)
|
||||
registry.drain(10_000, { paceWindowMs: 40_000 })
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
|
||||
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
|
||||
// Its own send at 40s plus its own 10s grace, not 10s from the drain call.
|
||||
await vi.advanceTimersByTimeAsync(39_999)
|
||||
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
|
||||
await vi.advanceTimersByTimeAsync(10_001)
|
||||
expect(sockets[1]!.readyState).toBe(sockets[1]!.CLOSED)
|
||||
})
|
||||
|
||||
it('leaves no timer behind once an emergency drain cuts a window short', async () => {
|
||||
const { registry } = await connectHosts(4)
|
||||
registry.drain(0, { paceWindowMs: 40_000 })
|
||||
registry.drain(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
// Every session is closed, so anything still pending is an orphan of the cut window.
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps the first teardown snapshot when a regional drain fires before the fleet one', async () => {
|
||||
const { registry, sockets } = await connectHosts(1)
|
||||
const session = registry.get({ userId: 'user-0', relayHostId: identity.relayHostId })!
|
||||
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
|
||||
registry.drainHost({
|
||||
attemptId: 'attempt',
|
||||
userId: 'user-0',
|
||||
relayHostId: identity.relayHostId,
|
||||
sourceAssignmentEpoch: 1,
|
||||
graceMs: 0
|
||||
})
|
||||
registry.drain(10)
|
||||
await vi.advanceTimersByTimeAsync(11)
|
||||
expect(session.closingCounts).toEqual({ splices: 1, pending: 0 })
|
||||
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
|
||||
})
|
||||
|
||||
it('lets an emergency drain supersede the sends still queued by a paced one', async () => {
|
||||
const { registry, sockets } = await connectHosts(4)
|
||||
registry.drain(0, { paceWindowMs: 40_000 })
|
||||
expect(drainsSent(sockets)).toBe(1)
|
||||
registry.drain(0)
|
||||
expect(drainsSent(sockets)).toBe(4)
|
||||
const sendsAfterEmergency = sockets.map((socket) => socket.send.mock.calls.length)
|
||||
await vi.advanceTimersByTimeAsync(40_000)
|
||||
expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -26,6 +26,7 @@ import type WebSocket from 'ws'
|
||||
import type { RawData } from 'ws'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { ControlRenewalBatch } from './control-renewal-batch.js'
|
||||
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
@@ -179,6 +180,10 @@ export class HostSessionRegistry {
|
||||
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
|
||||
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
|
||||
private draining = false
|
||||
private readonly drainTimers = new Set<ReturnType<typeof setTimeout>>()
|
||||
// Hosts whose drain has been sent. Paced sends land minutes apart, so "this cell is
|
||||
// draining" is not the same question as "this host has been told to leave".
|
||||
private readonly drainSentHosts = new Set<string>()
|
||||
|
||||
private readonly idleWork = new Map<string, number>()
|
||||
private readonly idleAttempts = new Map<
|
||||
@@ -299,6 +304,15 @@ export class HostSessionRegistry {
|
||||
private readonly cellIncarnation?: string
|
||||
) {}
|
||||
|
||||
// Renewals leave the heartbeat as an enqueue: one statement per cell per
|
||||
// window replaces one write transaction per host, which is what keeps the
|
||||
// shared PostgreSQL instance out of buffer-header contention.
|
||||
private readonly controlRenewals = new ControlRenewalBatch(
|
||||
async (rows) => await this.assignments.renewControlActivities(rows),
|
||||
() => this.logIdentity(),
|
||||
(flush) => this.observer.recordControlRenewalFlush?.(flush)
|
||||
)
|
||||
|
||||
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
|
||||
private controlLeaseExpiresAt(): number {
|
||||
const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
|
||||
@@ -330,7 +344,10 @@ export class HostSessionRegistry {
|
||||
credential: string,
|
||||
capacityReservation?: PendingHostDataReservation
|
||||
): Promise<void> {
|
||||
if (this.draining) {
|
||||
// Not `this.draining`: a paced drain tells hosts minutes apart, and the director keeps
|
||||
// pointing phones here until their own host has moved. Refusing them for the whole
|
||||
// window would turn a 2 min drain into a 2 min outage for hosts not yet told.
|
||||
if (this.drainSentHosts.has(hostId)) {
|
||||
capacityReservation?.release()
|
||||
this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
|
||||
return
|
||||
@@ -450,7 +467,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
// Admission may have crossed a drain or control replacement while persisting activity.
|
||||
if (
|
||||
this.draining ||
|
||||
this.drainSentHosts.has(hostId) ||
|
||||
this.sessions.get(sessionKey) !== session ||
|
||||
session.state !== 'active' ||
|
||||
session.socket !== admittingSocket ||
|
||||
@@ -592,7 +609,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
// Already admitted attachments may finish a regional drain, but never a retired generation.
|
||||
if (
|
||||
this.draining ||
|
||||
this.drainSentHosts.has(identity.relayHostId) ||
|
||||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
|
||||
this.get(identity)?.state === 'closed' ||
|
||||
!session.activeConnIds.has(connId) ||
|
||||
@@ -846,17 +863,49 @@ export class HostSessionRegistry {
|
||||
return { controls, splices, pendingSplices }
|
||||
}
|
||||
|
||||
drain(graceMs: number): void {
|
||||
drain(graceMs: number, options: { paceWindowMs?: number } = {}): void {
|
||||
this.draining = true
|
||||
for (const session of this.sessions.values()) {
|
||||
if (session.state === 'closed') continue
|
||||
session.authorityRevision += 1
|
||||
session.state = 'drain-only'
|
||||
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
|
||||
setTimeout(() => this.closeDrainedSession(session), graceMs)
|
||||
// A later drain (an emergency one, or shutdown) owns every session again, so nothing
|
||||
// queued by an earlier paced drain may still fire: it would re-send and, worse, keep
|
||||
// the event loop alive for the rest of a window the operator just cut short.
|
||||
for (const timer of this.drainTimers) clearTimeout(timer)
|
||||
this.drainTimers.clear()
|
||||
const paceWindowMs = Math.max(0, Math.trunc(options.paceWindowMs ?? 0))
|
||||
const targets = [...this.sessions.values()].filter((session) => session.state !== 'closed')
|
||||
// The desktop re-dials the director as soon as it reads `drain`, whatever graceMs says,
|
||||
// so spreading the send is the only thing that spreads the reconnect load.
|
||||
const step = paceWindowMs > 0 && targets.length > 1 ? paceWindowMs / (targets.length - 1) : 0
|
||||
for (const [index, session] of targets.entries()) {
|
||||
const delay = Math.round(step * index)
|
||||
if (delay === 0) {
|
||||
this.sendDrain(session, graceMs)
|
||||
continue
|
||||
}
|
||||
this.scheduleDrainTimer(delay, () => this.sendDrain(session, graceMs))
|
||||
}
|
||||
}
|
||||
|
||||
// A session is only fenced when it is told, not when the drain starts: until its send
|
||||
// lands it is an ordinary live host, and its phones have to keep being able to reach it.
|
||||
private sendDrain(session: HostSession, graceMs: number): void {
|
||||
if (session.state === 'closed') return
|
||||
session.authorityRevision += 1
|
||||
session.state = 'drain-only'
|
||||
this.drainSentHosts.add(session.relayHostId)
|
||||
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
|
||||
this.scheduleDrainTimer(graceMs, () => this.closeDrainedSession(session))
|
||||
}
|
||||
|
||||
// Unref'd so a drain in flight never holds the process open past its own work.
|
||||
private scheduleDrainTimer(delayMs: number, run: () => void): void {
|
||||
const timer: ReturnType<typeof setTimeout> = setTimeout(() => {
|
||||
this.drainTimers.delete(timer)
|
||||
run()
|
||||
}, delayMs)
|
||||
timer.unref?.()
|
||||
this.drainTimers.add(timer)
|
||||
}
|
||||
|
||||
drainHost(input: {
|
||||
attemptId: string
|
||||
userId: string
|
||||
@@ -1345,15 +1394,13 @@ export class HostSessionRegistry {
|
||||
session.controlActivityId === controlActivityId &&
|
||||
session.authorityRevision === authorityRevision &&
|
||||
attempt > session.activityRenewalCompletedAttempt
|
||||
void this.assignments
|
||||
.renewControlActivity(
|
||||
{ userId: session.identity.sub, relayHostId: session.relayHostId },
|
||||
{
|
||||
activityId: controlActivityId,
|
||||
cellId: this.config.cellId,
|
||||
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
|
||||
}
|
||||
)
|
||||
void this.controlRenewals
|
||||
.enqueue({
|
||||
identity: { userId: session.identity.sub, relayHostId: session.relayHostId },
|
||||
activityId: controlActivityId,
|
||||
cellId: this.config.cellId,
|
||||
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
|
||||
})
|
||||
.then(() => {
|
||||
if (!current()) return
|
||||
session.activityRenewalCompletedAttempt = attempt
|
||||
@@ -1419,6 +1466,13 @@ export class HostSessionRegistry {
|
||||
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
|
||||
return
|
||||
}
|
||||
if (error instanceof Error && error.message === 'assignment_lock_unavailable') {
|
||||
// A per-host transaction held the row, so the batch passed over it
|
||||
// rather than making every other host in the flush wait. The next
|
||||
// tick is 15s away against a 105s lease, and the flush line already
|
||||
// reports the count, so this needs no line of its own.
|
||||
return
|
||||
}
|
||||
console.warn('[orca-relay] control activity renewal failed')
|
||||
})
|
||||
// Terminal handler: a throw inside the async catch above (e.g. a
|
||||
|
||||
@@ -4,114 +4,296 @@ import type { RelayDatabase, SqlRow } from './database.js'
|
||||
|
||||
export const IDLE_REHOME_PAGE_SIZE = 100
|
||||
|
||||
export async function selectIdleRegionalRehomes(input: {
|
||||
// How many decision rows one poll is allowed to look at. The poll runs about
|
||||
// fifty times a minute across the directors, so its cost has to be set by this
|
||||
// number and not by the size of the fleet or the width of the cohort.
|
||||
export const IDLE_REHOME_DECISION_WINDOW = 500
|
||||
|
||||
// Where the last window ended. A keyset beats OFFSET: `OFFSET n` still has to
|
||||
// produce and throw away n rows, and n grew by a page on every poll that
|
||||
// dispatched, so the scan got more expensive the longer the rollout ran.
|
||||
export type IdleRehomeHostCursor = { userId: string; relayHostId: string } | null
|
||||
|
||||
export type IdleRegionalRehomeCandidate = IdleRegionalRehomeRequest & { sourceCellUrl: string }
|
||||
|
||||
export type IdleRegionalRehomeSelection = {
|
||||
candidates: IdleRegionalRehomeCandidate[]
|
||||
cursor: IdleRehomeHostCursor
|
||||
}
|
||||
|
||||
type SourceCell = {
|
||||
cellId: string
|
||||
region: string
|
||||
cellIncarnation: string
|
||||
startedAt: number
|
||||
cellUrl: string
|
||||
}
|
||||
|
||||
type TargetCell = { cellId: string; capacityRequests: number; reservedRequests: number }
|
||||
|
||||
type SelectionInput = {
|
||||
database: RelayDatabase
|
||||
now: number
|
||||
heartbeatTtlMs: number
|
||||
cohortPercent: number
|
||||
offset: number
|
||||
connectionHeadroom: Map<string, boolean>
|
||||
preferenceMaxAgeMs: number
|
||||
hostCooldownMs: number
|
||||
cursor: IdleRehomeHostCursor
|
||||
connectionHeadroom: ReadonlyMap<string, boolean>
|
||||
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
|
||||
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
|
||||
const [runtimes, safetyRows] = await Promise.all([
|
||||
input.database.query('SELECT * FROM relay_cell_runtime'),
|
||||
input.database.query('SELECT * FROM relay_cell_rehome_safety')
|
||||
])
|
||||
const cleanCells = runtimes
|
||||
.filter((runtime) =>
|
||||
input.cellIsClean(
|
||||
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
|
||||
runtime,
|
||||
input.now
|
||||
)
|
||||
)
|
||||
.map((runtime) => String(runtime.cell_id))
|
||||
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
|
||||
if (!cleanCells.length || !targetCells.length) return []
|
||||
}
|
||||
|
||||
const CELL_INVENTORY_QUERY = `SELECT cell.cell_id, cell.cell_url, cell.enabled,
|
||||
cell.capacity_requests, cell.reserved_requests, region.region,
|
||||
admission.admission_state, capability.cell_incarnation AS capability_incarnation,
|
||||
capability.regional_rehome_protocol
|
||||
FROM relay_cells cell
|
||||
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
|
||||
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
|
||||
LEFT JOIN relay_cell_capabilities capability ON capability.cell_id = cell.cell_id`
|
||||
|
||||
export async function selectIdleRegionalRehomes(
|
||||
input: SelectionInput
|
||||
): Promise<IdleRegionalRehomeSelection> {
|
||||
const cells = await readCellInventory(input)
|
||||
if (!cells.sources.size || !cells.targetsByRegion.size) return { candidates: [], cursor: null }
|
||||
const sourceRegions = [...new Set([...cells.sources.values()].map((cell) => cell.region))]
|
||||
const targetRegions = [...cells.targetsByRegion.keys()]
|
||||
const decisionFilter = `outcome = 'conclusive' AND policy_version = 1
|
||||
AND preferred_region IN (${placeholders(targetRegions.length)})
|
||||
AND incumbent_region IN (${placeholders(sourceRegions.length)})
|
||||
AND preferred_region <> incumbent_region
|
||||
AND expires_at > ? AND observed_at >= ? AND cohort_bucket < ?`
|
||||
const decisionParams = [
|
||||
...targetRegions,
|
||||
...sourceRegions,
|
||||
input.now,
|
||||
input.now - input.preferenceMaxAgeMs,
|
||||
input.cohortPercent
|
||||
]
|
||||
const after = input.cursor ? [input.cursor.userId, input.cursor.relayHostId] : []
|
||||
const afterFilter = input.cursor ? 'AND (user_id, relay_host_id) > (?, ?)' : ''
|
||||
|
||||
// The window is taken first and on its own so the poll knows where it stopped
|
||||
// reading, not just where it stopped emitting. Every gate below this point can
|
||||
// reject a host, and a cursor that only advanced past emitted rows would park
|
||||
// on a rejected host forever.
|
||||
const window = await input.database.query(
|
||||
`SELECT user_id, relay_host_id FROM relay_region_decisions
|
||||
WHERE ${decisionFilter} ${afterFilter}
|
||||
ORDER BY user_id, relay_host_id LIMIT ?`,
|
||||
[...decisionParams, ...after, IDLE_REHOME_DECISION_WINDOW]
|
||||
)
|
||||
if (!window.length) return { candidates: [], cursor: null }
|
||||
const windowEnd = window[window.length - 1]!
|
||||
const windowWasFull = window.length === IDLE_REHOME_DECISION_WINDOW
|
||||
|
||||
const sourceList = [...cells.sources.values()]
|
||||
const rows = await input.database.query(
|
||||
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
|
||||
a.assignment_epoch, host.generation, r.cell_incarnation,
|
||||
s.cell_url, target.cell_id AS target_cell_id
|
||||
FROM relay_region_rehome_control policy
|
||||
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
|
||||
// The verification names the window's keys rather than repeating its LIMIT:
|
||||
// the two reads take separate snapshots, and a decision that turned eligible
|
||||
// between them would otherwise shift the second LIMIT and push the last host
|
||||
// out of it while the cursor still advanced past it.
|
||||
`SELECT d.user_id, d.relay_host_id, d.preferred_region, a.cell_id AS source_cell_id,
|
||||
a.assignment_epoch, host.generation
|
||||
FROM (SELECT user_id, relay_host_id, preferred_region, incumbent_region, assignment_epoch
|
||||
FROM relay_region_decisions
|
||||
WHERE ${decisionFilter}
|
||||
AND (user_id, relay_host_id) IN (${Array.from({ length: window.length }, () => '(?,?)').join(',')})
|
||||
-- The LIMIT cannot truncate a key set this size; it is here because without
|
||||
-- it Postgres flattens the subquery, estimates one row out of the join, and
|
||||
-- drives the whole plan from a sequential scan of the capability table.
|
||||
ORDER BY user_id, relay_host_id LIMIT ?) d
|
||||
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
|
||||
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
|
||||
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
|
||||
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
|
||||
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
|
||||
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
|
||||
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
|
||||
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
|
||||
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
|
||||
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
|
||||
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
|
||||
AND a.assignment_epoch = d.assignment_epoch
|
||||
JOIN (${inlineRows(SOURCE_CELL_COLUMNS, sourceList.length)}) source
|
||||
ON source.cell_id = a.cell_id AND source.region = d.incumbent_region
|
||||
JOIN relay_control_capabilities host ON host.user_id = d.user_id
|
||||
AND host.relay_host_id = d.relay_host_id AND host.cell_id = a.cell_id
|
||||
AND host.assignment_epoch = a.assignment_epoch
|
||||
AND host.cell_incarnation = source.cell_incarnation AND host.idle_regional_rehome = 1
|
||||
JOIN relay_assignment_activity_leases lease ON lease.user_id = d.user_id
|
||||
AND lease.relay_host_id = d.relay_host_id AND lease.activity_id = host.activity_id
|
||||
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
|
||||
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
|
||||
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
|
||||
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
|
||||
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
|
||||
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
|
||||
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
|
||||
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
|
||||
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
|
||||
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
|
||||
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
|
||||
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
|
||||
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
|
||||
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
|
||||
-- Reserve the moving host's source activity plus its assignment on the target.
|
||||
AND target.reserved_requests + 1 + (
|
||||
SELECT COALESCE(SUM(activity.request_units), 0)
|
||||
FROM relay_assignment_activity_leases activity
|
||||
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
|
||||
AND activity.cell_id = a.cell_id
|
||||
) <= target.capacity_requests
|
||||
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
|
||||
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
|
||||
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
|
||||
AND lease.expires_at > ? AND lease.updated_at >= source.started_at
|
||||
WHERE NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
|
||||
WHERE migration.user_id = d.user_id AND migration.relay_host_id = d.relay_host_id
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
|
||||
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
|
||||
AND attempt.created_at > ? - policy.host_cooldown_ms)
|
||||
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
|
||||
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
|
||||
target.cell_id
|
||||
LIMIT ? OFFSET ?`,
|
||||
WHERE attempt.user_id = d.user_id AND attempt.relay_host_id = d.relay_host_id
|
||||
AND attempt.created_at > ?)
|
||||
ORDER BY d.user_id, d.relay_host_id, host.generation DESC
|
||||
-- Counted in hosts, because a host with one eligible target has to be able
|
||||
-- to fill a page on its own. A host with many leaves part of this page
|
||||
-- unread, and the cursor stops where the page stopped, so it is re-read
|
||||
-- next poll rather than skipped.
|
||||
LIMIT ?`,
|
||||
[
|
||||
...decisionParams,
|
||||
...window.flatMap((row) => [row.user_id, row.relay_host_id]),
|
||||
IDLE_REHOME_DECISION_WINDOW,
|
||||
...sourceList.flatMap((cell) => [cell.cellId, cell.region, cell.cellIncarnation, cell.startedAt]),
|
||||
input.now,
|
||||
input.now,
|
||||
input.now,
|
||||
input.cohortPercent,
|
||||
input.now,
|
||||
input.now - input.heartbeatTtlMs,
|
||||
input.now - input.heartbeatTtlMs,
|
||||
...cleanCells,
|
||||
...targetCells,
|
||||
input.now,
|
||||
IDLE_REHOME_PAGE_SIZE,
|
||||
input.offset
|
||||
input.now - input.hostCooldownMs,
|
||||
IDLE_REHOME_PAGE_SIZE
|
||||
]
|
||||
)
|
||||
return rows.map((row) => {
|
||||
const request = {
|
||||
v: 1 as const,
|
||||
userId: String(row.user_id),
|
||||
relayHostId: String(row.relay_host_id),
|
||||
sourceCellId: String(row.source_cell_id),
|
||||
sourceCellIncarnation: String(row.cell_incarnation),
|
||||
sourceAssignmentEpoch: Number(row.assignment_epoch),
|
||||
sourceGeneration: Number(row.generation),
|
||||
targetCellId: String(row.target_cell_id)
|
||||
const units = rows.length ? await sourceRequestUnits(input.database, rows) : new Map<string, number>()
|
||||
|
||||
const candidates: IdleRegionalRehomeCandidate[] = []
|
||||
let stoppedAt: IdleRehomeHostCursor = null
|
||||
for (const row of rows) {
|
||||
// Whole hosts only: the lower-priority targets are a host's fallbacks when
|
||||
// the first one defers, and splitting them across pages loses them.
|
||||
if (candidates.length >= IDLE_REHOME_PAGE_SIZE) {
|
||||
return { candidates, cursor: stoppedAt }
|
||||
}
|
||||
// UUIDv5 keeps retries on every director bound to the same source authority and target.
|
||||
const digest = createHash('sha1')
|
||||
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
|
||||
.update(JSON.stringify(request))
|
||||
.digest()
|
||||
digest[6] = (digest[6]! & 0x0f) | 0x50
|
||||
digest[8] = (digest[8]! & 0x3f) | 0x80
|
||||
const hex = digest.subarray(0, 16).toString('hex')
|
||||
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
|
||||
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
|
||||
})
|
||||
const source = cells.sources.get(String(row.source_cell_id))!
|
||||
const sourceUnits = units.get(hostKey(row)) ?? 0
|
||||
for (const target of cells.targetsByRegion.get(String(row.preferred_region)) ?? []) {
|
||||
if (target.reservedRequests + 1 + sourceUnits > target.capacityRequests) continue
|
||||
candidates.push(idleRehomeCandidate(row, source, target.cellId))
|
||||
}
|
||||
stoppedAt = { userId: String(row.user_id), relayHostId: String(row.relay_host_id) }
|
||||
}
|
||||
// A full verification page may have been cut short of the window's end, so only
|
||||
// a page that ran the window out may wrap to the head of the keyspace.
|
||||
if (rows.length === IDLE_REHOME_PAGE_SIZE) return { candidates, cursor: stoppedAt }
|
||||
return {
|
||||
candidates,
|
||||
cursor: windowWasFull
|
||||
? { userId: String(windowEnd.user_id), relayHostId: String(windowEnd.relay_host_id) }
|
||||
: null
|
||||
}
|
||||
}
|
||||
|
||||
// Every cell predicate the candidate join used to re-evaluate per (host, cell)
|
||||
// pair. There are tens of cells and tens of thousands of hosts, so this is
|
||||
// resolved once per poll against the four small inventory tables.
|
||||
async function readCellInventory(
|
||||
input: SelectionInput
|
||||
): Promise<{ sources: Map<string, SourceCell>; targetsByRegion: Map<string, TargetCell[]> }> {
|
||||
const { database, now } = input
|
||||
const [runtimeRows, safetyRows, inventory] = await Promise.all([
|
||||
database.query('SELECT * FROM relay_cell_runtime'),
|
||||
database.query('SELECT * FROM relay_cell_rehome_safety'),
|
||||
database.query(CELL_INVENTORY_QUERY)
|
||||
])
|
||||
const runtimes = new Map(runtimeRows.map((row) => [String(row.cell_id), row]))
|
||||
const safety = new Map(safetyRows.map((row) => [String(row.cell_id), row]))
|
||||
const sources = new Map<string, SourceCell>()
|
||||
const targetsByRegion = new Map<string, TargetCell[]>()
|
||||
const load = new Map<string, number>()
|
||||
for (const cell of inventory) {
|
||||
const cellId = String(cell.cell_id)
|
||||
const runtime = runtimes.get(cellId)
|
||||
if (!runtime || !input.cellIsClean(safety.get(cellId), runtime, now)) continue
|
||||
if (
|
||||
Number(cell.enabled) !== 1 ||
|
||||
cell.admission_state !== 'general' ||
|
||||
cell.region == null ||
|
||||
Number(runtime.ready) !== 1 ||
|
||||
Number(runtime.last_heartbeat_at) <= now - input.heartbeatTtlMs ||
|
||||
cell.capability_incarnation == null ||
|
||||
String(cell.capability_incarnation) !== String(runtime.cell_incarnation) ||
|
||||
Number(cell.regional_rehome_protocol) < 3
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const region = String(cell.region)
|
||||
sources.set(cellId, {
|
||||
cellId,
|
||||
region,
|
||||
cellIncarnation: String(runtime.cell_incarnation),
|
||||
startedAt: Number(runtime.started_at),
|
||||
cellUrl: String(cell.cell_url)
|
||||
})
|
||||
if (input.connectionHeadroom.get(cellId) === false) continue
|
||||
const capacityRequests = Number(cell.capacity_requests)
|
||||
const reservedRequests = Number(cell.reserved_requests)
|
||||
const targets = targetsByRegion.get(region) ?? []
|
||||
targets.push({ cellId, capacityRequests, reservedRequests })
|
||||
targetsByRegion.set(region, targets)
|
||||
load.set(cellId, (reservedRequests + Number(runtime.observed_requests)) / capacityRequests)
|
||||
}
|
||||
for (const targets of targetsByRegion.values()) {
|
||||
targets.sort(
|
||||
(left, right) =>
|
||||
load.get(left.cellId)! - load.get(right.cellId)! || (left.cellId < right.cellId ? -1 : 1)
|
||||
)
|
||||
}
|
||||
return { sources, targetsByRegion }
|
||||
}
|
||||
|
||||
// One grouped read for the page instead of a correlated aggregate per (host, cell) pair.
|
||||
async function sourceRequestUnits(
|
||||
database: RelayDatabase,
|
||||
rows: SqlRow[]
|
||||
): Promise<Map<string, number>> {
|
||||
const seen = new Set<string>()
|
||||
const params: unknown[] = []
|
||||
for (const row of rows) {
|
||||
if (seen.has(hostKey(row))) continue
|
||||
seen.add(hostKey(row))
|
||||
params.push(row.user_id, row.relay_host_id, row.source_cell_id)
|
||||
}
|
||||
const sums = await database.query(
|
||||
`SELECT user_id, relay_host_id, COALESCE(SUM(request_units), 0) AS request_units
|
||||
FROM relay_assignment_activity_leases
|
||||
WHERE (user_id, relay_host_id, cell_id) IN (${Array.from({ length: seen.size }, () => '(?,?,?)').join(',')})
|
||||
GROUP BY user_id, relay_host_id`,
|
||||
params
|
||||
)
|
||||
return new Map(sums.map((row) => [hostKey(row), Number(row.request_units)]))
|
||||
}
|
||||
|
||||
const SOURCE_CELL_COLUMNS = [
|
||||
['cell_id', 'TEXT'],
|
||||
['region', 'TEXT'],
|
||||
['cell_incarnation', 'TEXT'],
|
||||
['started_at', 'BIGINT']
|
||||
] as const
|
||||
|
||||
function placeholders(count: number): string {
|
||||
return Array.from({ length: count }, () => '?').join(',')
|
||||
}
|
||||
|
||||
// A derived table the planner can hash, in the one syntax both Postgres and the
|
||||
// SQLite test engine accept (`VALUES ... AS t(col)` and LATERAL are not common to
|
||||
// both). Only the first branch is cast; both engines take the union's types from it.
|
||||
function inlineRows(columns: ReadonlyArray<readonly [string, string]>, rows: number): string {
|
||||
const first = columns.map(([name, type]) => `CAST(? AS ${type}) AS ${name}`)
|
||||
const rest = Array.from({ length: rows - 1 }, () => `UNION ALL SELECT ${placeholders(columns.length)}`)
|
||||
return `SELECT ${first.join(', ')} ${rest.join(' ')}`
|
||||
}
|
||||
|
||||
function hostKey(row: SqlRow): string {
|
||||
return `${String(row.user_id)} | ||||