Merge origin/main into brennanb2025/fix-modal-recheck-action

This commit is contained in:
Brennan Benson
2026-09-18 02:58:13 -04:00
1617 changed files with 86154 additions and 3497 deletions
+7
View File
@@ -41,3 +41,10 @@
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
# Mobile web bundle source. Every text byte here is hashed into an asset digest and
# from there into buildId, so a CRLF checkout produces a different bundle id for the
# same commit (91af2897 vs 9d78435e). The PNG is -text because it must not be touched.
/src/mobile-web/index.html text eol=lf
/src/mobile-web/src/*.ts text eol=lf
/src/mobile-web/src/*.css text eol=lf
/src/mobile-web/src/*.png -text
@@ -57,6 +57,9 @@ jobs:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
# ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
# 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
DRAIN_PACE_WINDOW_MS: '120000'
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
@@ -84,12 +87,6 @@ jobs:
"SKIP_RELAY_MONITOR_GATE ${TARGET_IMAGE_DIGEST}"
[[ "${GATE_OVERRIDE_REASON}" =~ ^[[:print:]]{12,500}$ ]]
fi
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
fi
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
echo "so recover each remaining cell with its own fresh monitor" >&2
@@ -121,6 +118,26 @@ jobs:
- uses: hashicorp/setup-terraform@v3
with: { terraform_wrapper: false }
# One approved-cell table, in the wave validator the dispatch gate already uses, so
# a cell's class and its wave's selector delta cannot drift apart between the two.
- name: Resolve this cell's admission class and wave selector delta
run: |
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
--cell-id "${TARGET_CELL_ID}")"
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
fi
{
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
} >> "${GITHUB_ENV}"
- name: Require fresh aggregate monitor evidence reference
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
@@ -214,11 +231,13 @@ jobs:
--no-monitor-state \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" --retry-freshness
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
else
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" --retry-freshness
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
fi
- name: Require durable rehome disabled and exact selector
@@ -248,6 +267,11 @@ jobs:
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c17|c18)
EXPECTED_HARD_CAP=600
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
@@ -263,10 +287,6 @@ jobs:
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
if test "${EXPECTED_REGION}" = us-central1; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
@@ -288,6 +308,14 @@ jobs:
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
fi
# The startup template emits rehome trust lines only for a declared source cell, so
# require membership exactly when either side of this roll expects those lines.
if test "${EXPECTED_REGION}" = us-central1 && {
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
}; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
@@ -332,20 +360,66 @@ jobs:
}
CURRENT_RUNTIME="$(admin_post current-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
# A rollback that failed between template apply and admission restore
# leaves the cell already on the rollback image; resume from that
# state instead of demanding the pre-rollback predecessor.
# Two different failures leave the cell on the rollback image, and the image
# alone cannot tell them apart. A rollback that failed between its template
# apply and its admission restore restarted the cell, so that cell is not
# draining and resumes. A wave that stopped after its drain and before its
# template apply never restarted anything, so its cell is still draining and
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
if test "${DEPLOY_MODE}" = rollback \
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
ROLLBACK_RESUME=true
if test "${LIVE_DRAINING}" = true; then
ROLLBACK_STAGE=stranded
else
ROLLBACK_STAGE=resume
fi
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
else
ROLLBACK_RESUME=false
ROLLBACK_STAGE=roll
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
fi
if test "${ROLLBACK_STAGE}" = resume; then
ROLLBACK_RESUME=true
else
ROLLBACK_RESUME=false
fi
# A stranded cell's template still carries the image the cell is serving, so that
# is the predecessor its plan is reviewed against. A template already moved on to
# the target is refused here rather than rolled backwards under a stale review.
if test "${ROLLBACK_STAGE}" = stranded; then
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
else
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
fi
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require the cell pristine in the
# class it is declared to serve in.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
else
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
fi
# Draining sheds connections, and a migration-only cell holds none, so the flag
# carries no precondition there. It also outlives a failed wave, because the drain
# that set it is followed by no restart, which is the state a failed canary leaves.
if test "${DEPLOY_MODE}" = rollback \
|| test "${ENTRY_ADMISSION}" = migration-only; then
PRECHECK_DRAINING=either
else
PRECHECK_DRAINING=forbidden
fi
# A resumed rollback already restarted, so its cell has to come back not draining;
# that is what separates it from a wave that stopped before its template apply.
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
PREDECESSOR_DRAINING_OK=true
else
PREDECESSOR_DRAINING_OK=false
fi
RESTORED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
@@ -366,8 +440,19 @@ jobs:
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
# A migration-only cell is already isolated and is handed back isolated, so both
# halves of its wave see exactly the membership it entered with.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
fi
{
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
# The drain wait and the plan review both read the image this cell actually
# serves, which is the rollback image on a stranded cell and not the current one.
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
# The failsafe consumes these; deriving them here keeps them
# defined for a failure in any later step.
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
@@ -386,8 +471,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
(.region == $region or
($region == "us-central1" and $protocol == 0 and .region == null)) and
@@ -403,8 +487,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'[
if .role != "cell" then "role" else empty end,
if .cellId != $cell then "cellId" else empty end,
@@ -440,16 +523,6 @@ jobs:
fi
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require a pristine general cell.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
PRECHECK_DRAINING=either
else
PRECHECK_ADMISSION=general
PRECHECK_DRAINING=forbidden
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -475,18 +548,25 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Isolating a migration-only cell must be a read-only no-op; a change here would
# mean the live class is not the one this wave planned around.
if test "${ENTRY_ADMISSION}" = migration-only; then
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
fi
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
# The wait has to outlast the pacing window as well as the leases it waits on.
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat either --admission migration-only --draining required \
--activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
--timeout-ms 900000
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
--timeout-ms 1020000
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
@@ -499,6 +579,7 @@ jobs:
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
@@ -544,6 +625,7 @@ jobs:
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--rollback-image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
@@ -571,20 +653,37 @@ jobs:
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}"
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
# A stranded cell already runs the reviewed template, so the apply above replaces
# no instance and the drain flag, which only a restart clears, would survive the
# whole wave. Roll the MIG explicitly on exactly the policy a template change uses.
# Every field is passed: gcloud persists these into the MIG's update policy, and it
# defaults the method to substitute on a group with no stateful config, so omitting
# one drifts the policy off the reviewed one and fails every later targeted plan.
if test "${ROLLBACK_STAGE}" = stranded \
&& test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then
gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--replacement-method recreate --max-surge 0 --max-unavailable 1
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
@@ -659,28 +758,36 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
--cell-incarnation "${TARGET_INCARNATION}"
- name: Restore only the verified selected cell to general admission
- name: Restore only the verified selected cell to its entry admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
# Activating a migration-only cell would promote it to a serving cell for good, so
# restore it with the idempotent isolate that reports the authoritative generation.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORE_MODE=isolate
else
RESTORE_MODE=activate
fi
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
echo "${ACTIVATE_RESULT}"
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
<<< "${ACTIVATE_RESULT}")"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
echo "${RESTORE_RESULT}"
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
<<< "${RESTORE_RESULT}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission general --draining forbidden --activity allowed \
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
--draining forbidden --activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
@@ -181,14 +181,20 @@ jobs:
if: ${{ inputs.mode == 'batch-apply' }}
env:
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
run: |
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
--target-digest "${{ inputs.target-image-digest }}" \
--rollback-digest "${{ inputs.rollback-image-digest }}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}"
--cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" \
--rollback-digest "${ROLLBACK_DIGEST}" \
--selector-generation "${SELECTOR_GENERATION}" \
--rehome-generation "${REHOME_GENERATION}"
- name: Reject previously consumed aggregate safety evidence
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
+7
View File
@@ -96,6 +96,13 @@ jobs:
- name: Typecheck
run: pnpm typecheck
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
# drifted. This fails when a test file that checks today stops checking, when a test leaves
# the program, and on @ts-nocheck; the baseline may only shrink.
- name: Typecheck tests (ratchet)
run: pnpm run check:tests-typecheck
- name: Test
run: pnpm test
+6
View File
@@ -780,6 +780,12 @@ jobs:
- name: Project web client from renderer build
run: pnpm run build:web-from-renderer
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
# step instead of calling build:release, and electron-builder's beforePack guard hard-fails
# without out/mobile-web.
- name: Build mobile web bundle
run: pnpm run build:mobile-web
- name: Build native components
run: pnpm run build:native
@@ -738,6 +738,42 @@ describe('relay incident live preflight', () => {
expect(seen[0]!.generation).toBe(5)
})
it('offsets by the wave delta the cell class declares', async () => {
const generationFor = async (args: string[]) => {
const seen: AdmissionSelector[] = []
await expect(runIncidentLivePreflight(args, {
now: () => now,
collect: async (expected) => {
seen.push(expected)
const next = canonicalSample(expected.generation)
next.expectedSelector = expected
return next
}
})).resolves.toBeUndefined()
return seen[0]!.generation
}
// A migration-only cell's wave isolates and restores nothing, so no predecessor moved it.
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '0'])
)).toBe(1)
expect(await generationFor(
overrideArgs(['--wave-index', '2', '--selector-wave-delta', '2'])
)).toBe(5)
})
it('rejects a selector wave delta no cell class produces', async () => {
for (const delta of ['1', '3', '4', '', '-0', '02']) {
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', delta]),
{ now: () => now }
)).rejects.toThrow('usage:')
}
await expect(runIncidentLivePreflight(
overrideArgs(['--selector-wave-delta', '0', '--selector-wave-delta', '0']),
{ now: () => now }
)).rejects.toThrow('usage:')
})
it('pins the strictest migration policy', async () => {
// An inactive migration target is tolerable only under recover-forward,
// and an override cannot elect that policy, so this must still fail.
@@ -31,6 +31,8 @@ const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX_PATTERN = /^[0-3]$/
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/
export function livePreflightGcloud(
gcloud: ReturnType<typeof createGcloudClient>,
@@ -93,13 +95,16 @@ function describeFailure(failure: IncidentFailure): string {
}
const PREFLIGHT_USAGE =
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>] [--retry-freshness]' +
'usage: --state-file <verified-monitor-state> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>] [--retry-freshness]' +
' | --no-monitor-state --expected-selector-generation <n>' +
' --selector-membership-file <json> [--wave-index <0-3>]'
' --selector-membership-file <json> [--wave-index <0-3>]' +
' [--selector-wave-delta <0|2>]'
const VALUE_OPTIONS = new Set([
'--state-file',
'--wave-index',
'--selector-wave-delta',
'--expected-selector-generation',
'--selector-membership-file'
])
@@ -241,6 +246,8 @@ export async function runIncidentLivePreflight(
const parsed = parsePreflightArgs(argv)
const waveIndex = parsed.options.get('--wave-index') ?? '0'
if (!WAVE_INDEX_PATTERN.test(waveIndex)) throw new Error(PREFLIGHT_USAGE)
const selectorWaveDelta = parsed.options.get('--selector-wave-delta') ?? '2'
if (!SELECTOR_WAVE_DELTA_PATTERN.test(selectorWaveDelta)) throw new Error(PREFLIGHT_USAGE)
const now = dependencies.now ?? Date.now
const plan = parsed.flags.has('--no-monitor-state')
? await overridePreflightPlan(parsed.options, now())
@@ -252,14 +259,16 @@ export async function runIncidentLivePreflight(
dependencies.environment
)
// Each predecessor same-cap apply wave reversibly isolates and restores its
// cell, advancing the selector generation by exactly 2 with membership
// unchanged (rollback is single-cell, so it never reaches a later wave), so
// the live selector comparison must expect the wave-adjusted generation.
// cell with membership unchanged (rollback is single-cell, so it never reaches
// a later wave), so the live selector comparison must expect the wave-adjusted
// generation. A general cell advances it by 2; a migration-only cell is already
// isolated and stays that way, so its wave advances it by 0. A wave is never
// mixed, so one delta covers every predecessor.
const collectOptions = {
environment: plan.environment,
expectedSelector: {
...plan.expectedSelector,
generation: plan.expectedSelector.generation + 2 * Number(waveIndex)
generation: plan.expectedSelector.generation + Number(selectorWaveDelta) * Number(waveIndex)
},
...(dependencies.now ? { now: dependencies.now } : {})
}
+12 -4
View File
@@ -58,6 +58,8 @@ const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
const REGION_CATALOG_CACHE_MS = 30_000
// A drain that outlives the roll step it belongs to is an outage, not a pacing win.
const DRAIN_PACE_WINDOW_MAX_MS = 5 * 60 * 1_000
type AdmissionRejectionLogEntry = {
route: 'assign' | 'resolve'
@@ -72,7 +74,7 @@ export function createRelayApp(
operations: {
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
drain: (graceMs: number, options?: { paceWindowMs?: number }) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
@@ -488,12 +490,18 @@ export function createRelayApp(
return context.json({ error: 'invalid_token' }, 401)
}
const body = z
.object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) })
.object({
v: z.literal(1),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
// Spreads the drain sends, and so the re-dials, over this window.
paceWindowMs: z.number().int().nonnegative().max(DRAIN_PACE_WINDOW_MAX_MS).optional()
})
.strict()
.safeParse(await context.req.json().catch(() => null))
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
operations.drain(body.data.graceMs)
return context.json({ ok: true })
const paceWindowMs = body.data.paceWindowMs ?? 0
operations.drain(body.data.graceMs, { paceWindowMs })
return context.json({ ok: true, paceWindowMs })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
+159 -119
View File
@@ -45,6 +45,15 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
CONTROL_RENEWAL_BATCH_SQL,
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
controlRenewalBatchParams,
orderedControlRenewalRows,
readControlRenewalOutcomes,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
@@ -101,21 +110,8 @@ type RelayAssignmentStoreOptions = {
recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
}
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
| 'database_error'
export type { ControlRenewalOutcome, ControlRenewalRequest }
const CONTROL_RENEWAL_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved'
])
export type RelayAssignment = AssignmentIdentity & {
cellId: string
cellUrl: string
@@ -3478,132 +3474,149 @@ export class RelayAssignmentStore {
})
}
// Kept as the single-row contract for callers and tests: resolves on a
// renewal and throws the outcome (or the driver's own error) otherwise.
async renewControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number }
): Promise<void> {
validateActivityId(input.activityId)
const now = this.now()
const maximumExpiresAt =
now +
ASSIGNMENT_LIMITS.activityLeaseMs +
RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
if (
!Number.isSafeInteger(input.expiresAt) ||
input.expiresAt <= now ||
input.expiresAt > maximumExpiresAt
) {
if (!controlRenewalExpiryIsValid(input.expiresAt, now)) {
throw new Error('invalid_activity_expiry')
}
const startedAt = performance.now()
let outcome: ControlRenewalOutcome = 'database_error'
try {
outcome =
this.database.dialect === 'postgres'
? await this.renewPostgresControlActivity(identity, input, now)
: await this.renewTransactionalControlActivity(identity, input, now)
outcome = await this.renewOneControlActivity({ identity, ...input }, now)
if (outcome !== 'renewed') throw new Error(outcome)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (CONTROL_RENEWAL_OUTCOMES.has(message as ControlRenewalOutcome)) {
outcome = message as ControlRenewalOutcome
}
outcome = controlRenewalOutcomeOfError(error)
throw error
} finally {
this.recordControlRenewal?.(performance.now() - startedAt, outcome)
}
}
private async renewPostgresControlActivity(
identity: AssignmentIdentity,
input: { activityId: string; cellId: string; expiresAt: number },
// Renews every due control lease on a cell in one write transaction, returning
// one outcome per input row in input order. Never throws for a multi-row batch:
// a caller routes its own session on its own outcome.
async renewControlActivities(
rows: readonly ControlRenewalRequest[]
): Promise<ControlRenewalOutcome[]> {
const now = this.now()
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
const accepted: Array<ControlRenewalRequest & { index: number }> = []
for (const [index, row] of rows.entries()) {
const rejection = controlRenewalRejection(row, now)
if (rejection) outcomes[index] = rejection
else accepted.push({ ...row, index })
}
const startedAt = performance.now()
try {
if (accepted.length === 0) return outcomes
let results: ControlRenewalOutcome[]
try {
results = await this.executeControlRenewals(accepted, now)
} catch (error) {
if (rows.length === 1) {
outcomes[accepted[0]!.index] = controlRenewalOutcomeOfError(error)
throw error
}
results = accepted.map(() => 'database_error')
}
for (const [position, row] of accepted.entries()) outcomes[row.index] = results[position]!
return outcomes
} finally {
// Every path, so a rethrown lone renewal and an all-invalid batch are
// counted the same as a batch that reached PostgreSQL.
const durationMs = performance.now() - startedAt
for (const outcome of outcomes) this.recordControlRenewal?.(durationMs, outcome)
}
}
private async executeControlRenewals(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
if (rows.length === 1) return [await this.renewOneControlActivity(rows[0]!, now)]
if (this.database.dialect !== 'postgres') {
// Correctness over throughput: the SQLite writer is serialized anyway, and
// this is the dialect the unit suites run on.
return await this.renewControlActivitiesInSeries(rows, now)
}
const ordered = orderedControlRenewalRows(rows.map((row, index) => ({ ...row, index })))
const outcomes = new Array<ControlRenewalOutcome>(rows.length)
try {
const parsed = readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams(ordered, now)
),
ordered.length
)
for (const [position, row] of ordered.entries()) outcomes[row.index] = parsed[position]!
return outcomes
} catch (error) {
// One statement means one contended assignment row can fail the whole
// batch, so a failure degrades to the per-host statements this replaced
// rather than costing every other host on the cell its renewal.
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_batch_failed',
rows: ordered.length,
message: String((error as { message?: unknown }).message)
})
)
await Promise.all(
ordered.map(async (row) => {
try {
outcomes[row.index] = await this.renewOneControlActivity(row, now)
} catch {
outcomes[row.index] = 'database_error'
}
})
)
return outcomes
}
}
private async renewControlActivitiesInSeries(
rows: readonly ControlRenewalRequest[],
now: number
): Promise<ControlRenewalOutcome[]> {
const outcomes: ControlRenewalOutcome[] = []
for (const row of rows) {
try {
outcomes.push(await this.renewOneControlActivity(row, now))
} catch {
outcomes.push('database_error')
}
}
return outcomes
}
// Returns the outcome; a driver or pool failure reaches the caller unchanged.
private async renewOneControlActivity(
row: ControlRenewalRequest,
now: number
): Promise<ControlRenewalOutcome> {
const row = (
await this.database.query(
`WITH assignment_state AS MATERIALIZED (
SELECT cell_id, assignment_epoch
FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?
FOR UPDATE
), migration_state AS MATERIALIZED (
SELECT migration.assignment_epoch
FROM relay_assignment_migrations migration
JOIN assignment_state assignment
ON migration.target_cell_id = assignment.cell_id
AND migration.assignment_epoch = assignment.assignment_epoch
WHERE migration.user_id = ? AND migration.relay_host_id = ?
AND migration.source_cell_id = ?
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT 1 AS authorized
FROM assignment_state assignment
WHERE assignment.cell_id = ? OR EXISTS (SELECT 1 FROM migration_state)
), lease_state AS MATERIALIZED (
SELECT lease.activity_kind, lease.cell_id
FROM relay_assignment_activity_leases lease
CROSS JOIN authorization_state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, ?),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
WHERE lease.user_id = ? AND lease.relay_host_id = ? AND lease.activity_id = ?
AND state.activity_kind = 'control' AND state.cell_id = ?
RETURNING 1
), renewed_assignment AS (
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, ?),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
AND EXISTS (SELECT 1 FROM renewed_lease)
RETURNING 1
)
SELECT CASE
WHEN NOT EXISTS (SELECT 1 FROM assignment_state)
THEN 'assignment_not_found'
WHEN NOT EXISTS (SELECT 1 FROM authorization_state)
THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (SELECT 1 FROM lease_state)
THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state
WHERE activity_kind <> 'control' OR cell_id <> ?
) THEN 'control_activity_moved'
WHEN EXISTS (SELECT 1 FROM renewed_assignment) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome`,
[
identity.userId,
identity.relayHostId,
identity.userId,
identity.relayHostId,
input.cellId,
input.cellId,
identity.userId,
identity.relayHostId,
input.activityId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.activityId,
input.cellId,
input.expiresAt,
now,
identity.userId,
identity.relayHostId,
input.cellId
]
)
)[0]
if (!row) throw new Error('missing_control_renewal_outcome')
const outcome = text(row, 'outcome') as ControlRenewalOutcome
if (!CONTROL_RENEWAL_OUTCOMES.has(outcome)) throw new Error('invalid_control_renewal_outcome')
return outcome
if (this.database.dialect === 'postgres') {
return readControlRenewalOutcomes(
await this.database.query(
CONTROL_RENEWAL_BATCH_SQL,
controlRenewalBatchParams([row], now)
),
1
)[0]!
}
try {
return await this.renewTransactionalControlActivity(row.identity, row, now)
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) throw error
return message as ControlRenewalOutcome
}
}
private async renewTransactionalControlActivity(
@@ -7900,6 +7913,33 @@ function validateActivityId(activityId: string): void {
if (!activityId || activityId.length > 256) throw new Error('invalid_activity_id')
}
function controlRenewalExpiryIsValid(expiresAt: number, now: number): boolean {
const maximumExpiresAt =
now + ASSIGNMENT_LIMITS.activityLeaseMs + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
return Number.isSafeInteger(expiresAt) && expiresAt > now && expiresAt <= maximumExpiresAt
}
// A renewal that threw still owes the metric an outcome: the message carries one
// when the statement decided it, and anything else is the driver failing.
function controlRenewalOutcomeOfError(error: unknown): ControlRenewalOutcome {
const message = String((error as { message?: unknown }).message)
return CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)
? (message as ControlRenewalOutcome)
: 'database_error'
}
function controlRenewalRejection(
row: ControlRenewalRequest,
now: number
): ControlRenewalOutcome | null {
try {
validateActivityId(row.activityId)
} catch {
return 'invalid_activity_id'
}
return controlRenewalExpiryIsValid(row.expiresAt, now) ? null : 'invalid_activity_expiry'
}
function activityKind(row: SqlRow): AssignmentActivityKind {
const value = text(row, 'activity_kind')
if (!(value in ACTIVITY_REQUEST_UNITS)) throw new Error('invalid_activity_kind')
@@ -3,6 +3,7 @@ import { ASSIGNMENT_LIMITS, RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
@@ -133,6 +134,10 @@ describePostgres('expired control lease after a database outage', () => {
internals.heartbeat(session)
}
// A due renewal leaves the heartbeat as a batch enqueue, so a poll has to
// outlast the batch window before it can call the renewal missing.
const renewalPoll = { timeout: CONTROL_RENEWAL_BATCH_INTERVAL_MS + 4_000 }
const leaseRows = async (relayHostId: string) =>
await database.query(
`SELECT activity_id, cell_id FROM relay_assignment_activity_leases
@@ -149,7 +154,8 @@ describePostgres('expired control lease after a database outage', () => {
.poll(
async () =>
socket.close.mock.calls.length > 0 ||
(await leaseRows(relayHostId)).length === expectedRows
(await leaseRows(relayHostId)).length === expectedRows,
renewalPoll
)
.toBe(true)
}
@@ -212,7 +218,7 @@ describePostgres('expired control lease after a database outage', () => {
).rejects.toThrow('control_activity_moved')
heartbeat(registry, session)
await expect.poll(() => socket.close.mock.calls.length).toBe(1)
await expect.poll(() => socket.close.mock.calls.length, renewalPoll).toBe(1)
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
expect(await leaseRows(identity.relayHostId)).toEqual([
@@ -0,0 +1,291 @@
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type SqlRow
} from './database.js'
const now = 1_900_000_000_000
const expiresAt = now + 105_000
function renewal(userId: string, relayHostId: string, expiry = expiresAt) {
return {
identity: { userId, relayHostId },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: expiry
}
}
// A PostgreSQL-dialect database that answers the renewal statement without a
// server, so the statement count and its parameter arrays are observable.
class RenewalStatementProbe implements RelayDatabase {
readonly dialect = 'postgres' as const
readonly statements: Array<{ sql: string; params: unknown[] }> = []
failuresRemaining = 0
constructor(private readonly outcomeFor: (userId: string) => ControlRenewalOutcome) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push({ sql, params })
if (this.failuresRemaining > 0) {
this.failuresRemaining -= 1
throw new Error('canceling statement due to statement timeout')
}
const userIds = params[0] as string[]
return userIds.map((userId, index) => ({
row_index: String(index + 1),
outcome: this.outcomeFor(userId)
}))
}
async queryLocked(): Promise<SqlRow[]> {
throw new Error('unexpected_locked_query')
}
async transaction<T>(): Promise<T> {
// Renewals must never open one: that is the write transaction per host this
// batch exists to remove.
throw new Error('unexpected_transaction')
}
async close(): Promise<void> {}
}
describe('batched control renewals on PostgreSQL', () => {
it('spends one statement on every host that came due', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000003')
])
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
expect(probe.statements).toHaveLength(1)
expect(probe.statements[0]!.sql).toBe(CONTROL_RENEWAL_BATCH_SQL)
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
})
it('locks assignment rows in primary-key order and still answers in input order', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-c', 'host000000000003'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000001'),
renewal('user-a', 'host000000000001')
])
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
// statement's ORDER BY repeats it: no batch can queue against another in a
// different sequence.
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
expect(probe.statements[0]!.params[1]).toEqual([
'host000000000001',
'host000000000002',
'host000000000001',
'host000000000003'
])
expect(outcomes).toEqual([
'renewed',
'renewed',
'control_activity_moved',
'renewed'
])
})
it('keeps a malformed request out of the statement and fails only that row', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002', now + ASSIGNMENT_LIMITS.activityLeaseMs * 10),
{ ...renewal('user-a', 'host000000000003'), activityId: '' },
renewal('user-a', 'host000000000004')
])
expect(outcomes).toEqual([
'renewed',
'invalid_activity_expiry',
'invalid_activity_id',
'renewed'
])
expect(probe.statements[0]!.params[1]).toEqual(['host000000000001', 'host000000000004'])
})
it('degrades to one statement per host when the batch statement fails', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['renewed', 'renewed'])
expect(probe.statements).toHaveLength(3)
expect(probe.statements[1]!.params[1]).toEqual(['host000000000001'])
expect(probe.statements[2]!.params[1]).toEqual(['host000000000002'])
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_batch_failed',
rows: 2
})
} finally {
warn.mockRestore()
}
})
it('reports a host that fails its own fallback statement without touching the rest', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 2
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes.filter((outcome) => outcome === 'renewed')).toHaveLength(1)
expect(outcomes.filter((outcome) => outcome === 'database_error')).toHaveLength(1)
} finally {
warn.mockRestore()
}
})
it('reports a contended assignment row apart from a missing one', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_lock_unavailable' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
// Retryable: SKIP LOCKED passed over the row rather than queueing the whole
// flush behind whoever held it.
expect(outcomes).toEqual(['renewed', 'assignment_lock_unavailable'])
})
it('counts a lone renewal that threw before rethrowing it', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
// still owes an outcome for the attempt.
await expect(
store.renewControlActivities([renewal('user-a', 'host000000000001')])
).rejects.toThrow('statement timeout')
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
})
it('counts a batch in which every row was rejected before the statement', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
const outcomes = await store.renewControlActivities([
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
renewal('user-a', 'host000000000002', now - 1)
])
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
expect(probe.statements).toHaveLength(0)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'invalid_activity_id',
'invalid_activity_expiry'
])
})
it('counts one renewal metric per row against the flush latency', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
)
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'renewed',
'assignment_not_found'
])
})
})
describe('batched control renewals on SQLite', () => {
it('renews every host through the transactional path', async () => {
let clock = now
const database = await openInMemoryRelayDatabase()
try {
const store = new RelayAssignmentStore(database, () => clock)
await store.reconcileCells([
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
])
const hosts = ['host000000000001', 'host000000000002']
const requests = []
for (const relayHostId of hosts) {
const identity = { userId: 'user-a', relayHostId }
const assignment = await store.assign(identity)
await store.activateControl(identity, {
cellId: assignment.cellId,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
requests.push({
identity,
activityId: `control:${assignment.cellId}:1`,
cellId: assignment.cellId,
expiresAt: clock + 105_000
})
}
// A host with no assignment at all must not cost the others their renewal.
requests.push({
identity: { userId: 'user-a', relayHostId: 'host000000000009' },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: clock + 105_000
})
clock += 1_000
const outcomes = await store.renewControlActivities(requests)
expect(outcomes).toEqual(['renewed', 'renewed', 'assignment_not_found'])
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? ORDER BY relay_host_id ASC`,
['user-a']
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
now + 105_000,
now + 105_000
])
} finally {
await database.close()
}
})
})
@@ -0,0 +1,231 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
CONTROL_RENEWAL_BATCH_INTERVAL_MS,
CONTROL_RENEWAL_BATCH_MAX_ROWS,
ControlRenewalBatch,
type ControlRenewalFlush
} from './control-renewal-batch.js'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// Settles into the outcome the caller saw, attached at enqueue so a rejection is
// never momentarily unhandled.
function outcomeOf(renewal: Promise<void>): Promise<string> {
return renewal.then(
() => 'renewed',
(error: unknown) => String((error as { message?: unknown }).message)
)
}
function request(
host: string,
expiresAt = 1_000,
activityId = 'control:cell-a:1'
): ControlRenewalRequest {
return {
identity: { userId: 'user-a', relayHostId: host },
activityId,
cellId: 'cell-a',
expiresAt
}
}
describe('control renewal batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('spends one call on every renewal that came due in the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const settled = [
batch.enqueue(request('host0000000000a1')),
batch.enqueue(request('host0000000000a2')),
batch.enqueue(request('host0000000000a3'))
]
expect(renew).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all(settled)).resolves.toEqual([undefined, undefined, undefined])
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0].map((row) => row.identity.relayHostId)).toEqual([
'host0000000000a1',
'host0000000000a2',
'host0000000000a3'
])
})
it('routes each outcome back to the caller that asked for it', async () => {
const outcomes: ControlRenewalOutcome[] = [
'renewed',
'assignment_not_found',
'control_activity_moved'
]
const batch = new ControlRenewalBatch(async () => outcomes)
const first = outcomeOf(batch.enqueue(request('host0000000000b1')))
const second = outcomeOf(batch.enqueue(request('host0000000000b2')))
const third = outcomeOf(batch.enqueue(request('host0000000000b3')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second, third])).resolves.toEqual([
'renewed',
'assignment_not_found',
'control_activity_moved'
])
})
it('flushes on reaching the row ceiling instead of waiting out the window', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
for (let row = 0; row < CONTROL_RENEWAL_BATCH_MAX_ROWS - 1; row++) {
void batch.enqueue(request(`host${String(row).padStart(12, '0')}`))
}
expect(renew).not.toHaveBeenCalled()
void batch.enqueue(request('host0000000000zz'))
await vi.advanceTimersByTimeAsync(0)
expect(renew).toHaveBeenCalledOnce()
expect(renew.mock.calls[0]![0]).toHaveLength(CONTROL_RENEWAL_BATCH_MAX_ROWS)
// The window timer must not fire a second, empty statement.
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledOnce()
})
it('does not hold a new window behind a statement still in PostgreSQL', async () => {
let release!: (outcomes: ControlRenewalOutcome[]) => void
const renew = vi
.fn<(rows: readonly ControlRenewalRequest[]) => Promise<ControlRenewalOutcome[]>>()
.mockImplementationOnce(
async () => await new Promise<ControlRenewalOutcome[]>((resolve) => (release = resolve))
)
.mockResolvedValue(['renewed'])
const batch = new ControlRenewalBatch(renew)
const stalled = batch.enqueue(request('host0000000000c1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
const next = batch.enqueue(request('host0000000000c2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
await expect(next).resolves.toBeUndefined()
release(['renewed'])
await expect(stalled).resolves.toBeUndefined()
})
it('reports the driver failure to every caller in the flush', async () => {
const batch = new ControlRenewalBatch(async () => {
throw new Error('pool timeout')
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const first = outcomeOf(batch.enqueue(request('host0000000000d1')))
const second = outcomeOf(batch.enqueue(request('host0000000000d2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(Promise.all([first, second])).resolves.toEqual([
'pool timeout',
'pool timeout'
])
} finally {
warn.mockRestore()
}
})
it('supersedes a second attempt for one lease and answers both callers', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const earlier = batch.enqueue(request('host0000000000e1', 1_000))
const later = batch.enqueue(request('host0000000000e1', 2_000))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew.mock.calls[0]![0]).toEqual([
expect.objectContaining({ expiresAt: 2_000 })
])
await expect(earlier).resolves.toBeUndefined()
await expect(later).resolves.toBeUndefined()
})
it('holds a second activity for one host back to the next flush', async () => {
const renew = vi.fn(async (rows: readonly ControlRenewalRequest[]) =>
rows.map((): ControlRenewalOutcome => 'renewed')
)
const batch = new ControlRenewalBatch(renew)
const first = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:1'))
const second = batch.enqueue(request('host0000000000h1', 1_000, 'control:cell-a:2'))
const other = batch.enqueue(request('host0000000000h2'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
// One statement updates a host's assignment row once, so the host appears in
// one flush only; the newer generation leads the next one.
expect(renew.mock.calls[0]![0].map((row) => row.activityId)).toEqual([
'control:cell-a:1',
'control:cell-a:1'
])
await expect(Promise.all([first, other])).resolves.toEqual([undefined, undefined])
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(renew).toHaveBeenCalledTimes(2)
expect(renew.mock.calls[1]![0].map((row) => row.activityId)).toEqual(['control:cell-a:2'])
await expect(second).resolves.toBeUndefined()
})
it('stays quiet for a fast flush that renewed everything', async () => {
const flushes: ControlRenewalFlush[] = []
const batch = new ControlRenewalBatch(
async () => ['renewed'],
() => ({ cellId: 'cell-a' }),
(flush) => flushes.push(flush)
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void batch.enqueue(request('host0000000000f1'))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
expect(warn).not.toHaveBeenCalled()
expect(flushes).toEqual([
{ rows: 1, durationMs: expect.any(Number), outcomes: { renewed: 1 } }
])
} finally {
warn.mockRestore()
}
})
it('logs one line with the outcome counts when a flush did not renew everything', async () => {
const batch = new ControlRenewalBatch(
async () => ['renewed', 'control_activity_not_found'],
() => ({ cellId: 'cell-a' })
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
void outcomeOf(batch.enqueue(request('host0000000000g1')))
const missing = outcomeOf(batch.enqueue(request('host0000000000g2')))
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
await expect(missing).resolves.toBe('control_activity_not_found')
expect(warn).toHaveBeenCalledOnce()
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_flush',
cellId: 'cell-a',
rows: 2,
outcomes: { renewed: 1, control_activity_not_found: 1 }
})
} finally {
warn.mockRestore()
}
})
})
@@ -0,0 +1,148 @@
import { performance } from 'node:perf_hooks'
import type {
ControlRenewalOutcome,
ControlRenewalRequest
} from './control-renewal-statement.js'
// One flush per second turns the fleet's control-lease write rate into a
// function of the cell count rather than the host count: a cell's ~10 due
// renewals per second become one write transaction instead of ten. Well inside
// the 105s lease runway, so a host that misses a window is never at risk.
export const CONTROL_RENEWAL_BATCH_INTERVAL_MS = 1_000
// Ceiling on the parameter arrays. Row locks live until the statement commits,
// so this is what bounds how long one flush holds them: measured at 11.5ms for
// 200 rows against a 20,000-row table, and 9.4ms with a host wedged in a
// per-host transaction.
export const CONTROL_RENEWAL_BATCH_MAX_ROWS = 200
// A flush slower than this is the only latency worth a line; the metrics event
// carries the distribution.
const CONTROL_RENEWAL_SLOW_FLUSH_MS = 250
export type ControlRenewalFlush = {
rows: number
durationMs: number
outcomes: Record<string, number>
}
type PendingWaiter = { resolve: () => void; reject: (error: unknown) => void }
type PendingRenewal = { request: ControlRenewalRequest; waiters: PendingWaiter[] }
type QueuedRenewal = { request: ControlRenewalRequest; waiter: PendingWaiter }
// Per host, not per activity: one statement updates a host's assignment row
// once, so two activities for the same host must not share a flush.
function pendingKey(request: ControlRenewalRequest): string {
return [request.identity.userId, request.identity.relayHostId].join('\u0000')
}
// Collects the control-lease renewals a cell owes and spends one statement on
// them. Each caller still gets the single-renewal contract: the promise resolves
// on `renewed` and rejects with the outcome as its message otherwise, so callers
// keep their per-session error routing unchanged.
export class ControlRenewalBatch {
private pending = new Map<string, PendingRenewal>()
// Renewals a host cannot contribute to the flush being built; they open the
// next one.
private deferred: QueuedRenewal[] = []
private timer: ReturnType<typeof setTimeout> | null = null
constructor(
private readonly renew: (
rows: readonly ControlRenewalRequest[]
) => Promise<ControlRenewalOutcome[]>,
private readonly logFields: () => Record<string, unknown> = () => ({}),
private readonly observe?: (flush: ControlRenewalFlush) => void
) {}
enqueue(request: ControlRenewalRequest): Promise<void> {
return new Promise<void>((resolve, reject) => {
this.admit({ request, waiter: { resolve, reject } })
})
}
private admit(queued: QueuedRenewal): void {
const key = pendingKey(queued.request)
const existing = this.pending.get(key)
if (existing && existing.request.activityId !== queued.request.activityId) {
this.deferred.push(queued)
this.scheduleFlush()
return
}
if (existing) {
// A second attempt at the same lease inside one window supersedes the
// first expiry; both callers still hear the outcome they waited for.
existing.request = {
...queued.request,
expiresAt: Math.max(existing.request.expiresAt, queued.request.expiresAt)
}
existing.waiters.push(queued.waiter)
return
}
this.pending.set(key, { request: queued.request, waiters: [queued.waiter] })
if (this.pending.size >= CONTROL_RENEWAL_BATCH_MAX_ROWS) {
void this.flush()
return
}
this.scheduleFlush()
}
private scheduleFlush(): void {
this.timer ??= setTimeout(() => {
this.timer = null
void this.flush()
}, CONTROL_RENEWAL_BATCH_INTERVAL_MS)
this.timer.unref?.()
}
// Flushes run concurrently on purpose: a statement stalled in PostgreSQL must
// not hold back the renewals that came due while it was waiting.
async flush(): Promise<void> {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
const batch = [...this.pending.values()]
this.pending = new Map()
// Re-admitted against the empty map, so a host deferred out of this flush
// leads the next one.
const deferred = this.deferred
this.deferred = []
for (const queued of deferred) this.admit(queued)
if (batch.length === 0) return
const startedAt = performance.now()
let outcomes: ControlRenewalOutcome[]
try {
outcomes = await this.renew(batch.map((entry) => entry.request))
} catch (error) {
for (const entry of batch) for (const waiter of entry.waiters) waiter.reject(error)
this.report(batch.length, performance.now() - startedAt, { flush_failed: batch.length })
return
}
const counts: Record<string, number> = {}
for (const [index, entry] of batch.entries()) {
const outcome = outcomes[index] ?? 'database_error'
counts[outcome] = (counts[outcome] ?? 0) + 1
for (const waiter of entry.waiters) {
if (outcome === 'renewed') waiter.resolve()
else waiter.reject(new Error(outcome))
}
}
this.report(batch.length, performance.now() - startedAt, counts)
}
private report(rows: number, durationMs: number, outcomes: Record<string, number>): void {
this.observe?.({ rows, durationMs, outcomes })
const renewed = outcomes.renewed ?? 0
if (durationMs <= CONTROL_RENEWAL_SLOW_FLUSH_MS && renewed === rows) return
console.warn(
JSON.stringify({
event: 'orca_relay_control_renewal_flush',
...this.logFields(),
rows,
durationMs: Math.round(durationMs),
outcomes
})
)
}
}
@@ -1,8 +1,11 @@
import { performance } from 'node:perf_hooks'
import { ASSIGNMENT_LIMITS, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { CONTROL_RENEWAL_BATCH_SQL } from './control-renewal-statement.js'
import {
openRelayDatabase,
POSTGRES_LOCK_TIMEOUT_MS,
type RelayDatabase,
type RelayLockOptions,
type SqlRow
@@ -22,7 +25,9 @@ const targetCell = {
capacityRequests: 100
}
const userId = 'control-renewal-postgres-user'
const identities = Array.from({ length: 6 }, (_, index) => ({
// Indexes 0-5 belong to the single-renewal cases below, which mutate their
// host's migration and lease state; the batch cases own 6-13.
const identities = Array.from({ length: 14 }, (_, index) => ({
userId,
relayHostId: `controlrenewal${index + 1}`
}))
@@ -72,7 +77,7 @@ class StallFirstRenewalQueryDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (this.stallNext && sql.includes('WITH assignment_state AS MATERIALIZED')) {
if (this.stallNext && sql === CONTROL_RENEWAL_BATCH_SQL) {
this.stallNext = false
this.stalled.resolve()
await this.continue.promise
@@ -103,7 +108,7 @@ class RenewalQueryProbeDatabase implements RelayDatabase {
constructor(private readonly database: RelayDatabase) {}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
if (sql.includes('WITH assignment_state AS MATERIALIZED')) this.renewalQueries++
if (sql === CONTROL_RENEWAL_BATCH_SQL) this.renewalQueries++
return await this.database.query(sql, params)
}
@@ -332,6 +337,164 @@ describePostgres('PostgreSQL control renewal', () => {
).rejects.toThrow('invalid_activity_expiry')
})
it('renews every due host in one autocommitted statement', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
const batch = identities.slice(6, 10)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
batch.map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed', 'renewed'])
expect(probe.renewalQueries).toBe(1)
expect(probe.transactions).toBe(0)
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_id = ? ORDER BY relay_host_id ASC`,
[userId, controlId(sourceCell.id)]
)
expect(
leases
.filter((lease) =>
batch.some((identity) => identity.relayHostId === lease.relay_host_id)
)
.map((lease) => Number(lease.expires_at))
).toEqual([expiresAt, expiresAt, expiresAt, expiresAt])
})
it('reports each host its own verdict inside one batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const live = identities[10]!
const absent = { userId, relayHostId: 'controlrenewalgone' }
const outcomes = await store.renewControlActivities([
{ identity: absent, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{ identity: live, activityId: controlId(sourceCell.id), cellId: sourceCell.id, expiresAt },
{
identity: live,
activityId: controlId(targetCell.id),
cellId: targetCell.id,
expiresAt
}
])
expect(outcomes).toEqual([
'assignment_not_found',
'renewed',
'activity_cell_not_authoritative'
])
})
it('passes over a host whose assignment row is held and renews the rest', async () => {
const store = new RelayAssignmentStore(database, () => now)
now += 30_000
const expiresAt = now + 105_000
const held = identities[11]!
const free = identities[12]!
const locked = signal()
const release = signal()
// Holds the row the way every per-host transactional path does.
const holder = database.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[held.userId, held.relayHostId]
)
locked.resolve()
await release.promise
})
await locked.promise
const startedAt = performance.now()
const outcomes = await store.renewControlActivities(
[held, free].map((identity) => ({
identity,
activityId: controlId(sourceCell.id),
cellId: sourceCell.id,
expiresAt
}))
)
const elapsedMs = performance.now() - startedAt
release.resolve()
await holder
expect(outcomes).toEqual(['assignment_lock_unavailable', 'renewed'])
// It skipped rather than queued: a blocking FOR UPDATE would have spent the
// pool's whole lock_timeout here and failed the free host too.
expect(elapsedMs).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS)
const lease = (
await database.query(
`SELECT expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
[free.userId, free.relayHostId, controlId(sourceCell.id)]
)
)[0]
expect(Number(lease!.expires_at)).toBe(expiresAt)
})
it('renews both of one host\u2019s control leases in a single batch', async () => {
const store = new RelayAssignmentStore(database, () => now)
const identity = identities[13]!
// Two live control leases on one host. Written directly because
// activateControl retires the prior generation, and what is under test is the
// statement's row-wise behaviour, not how the second lease came to exist.
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`control:${sourceCell.id}:2`,
sourceCell.id,
now,
now
]
)
now += 30_000
const expiresAt = now + 105_000
const outcomes = await store.renewControlActivities(
[1, 2].map((generation) => ({
identity,
activityId: `control:${sourceCell.id}:${generation}`,
cellId: sourceCell.id,
expiresAt: expiresAt - generation
}))
)
expect(outcomes).toEqual(['renewed', 'renewed'])
const leases = await database.query(
`SELECT activity_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC`,
[identity.userId, identity.relayHostId]
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
expiresAt - 1,
expiresAt - 2
])
// The assignment row is written once, carrying the later of the two.
const row = (
await database.query(
`SELECT lease_expires_at, last_activity_at FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
expect(Number(row!.lease_expires_at)).toBe(expiresAt - 1)
expect(Number(row!.last_activity_at)).toBe(now)
})
it('uses one autocommitted PostgreSQL statement for a steady renewal', async () => {
const probe = new RenewalQueryProbeDatabase(database)
const store = new RelayAssignmentStore(probe, () => now)
@@ -0,0 +1,230 @@
import type { AssignmentIdentity } from './assignment-identity-queue.js'
import type { SqlRow } from './database.js'
export type ControlRenewalOutcome =
| 'renewed'
| 'assignment_not_found'
| 'activity_cell_not_authoritative'
| 'control_activity_not_found'
| 'control_activity_moved'
// The host's assignment row was already locked by one of the per-host
// transactional paths. Retryable, and never a reason to close a control: the
// next tick is 15s away and the lease has 105s on it.
| 'assignment_lock_unavailable'
// Decided per row before the statement runs, so one malformed request cannot
// cost the rest of the batch its renewal.
| 'invalid_activity_id'
| 'invalid_activity_expiry'
| 'database_error'
// Outcomes the statement itself can report. `database_error` is raised by the
// driver, and `invalid_activity_expiry` is decided per row before the statement
// is built, so neither can come back as a row.
export const CONTROL_RENEWAL_STATEMENT_OUTCOMES = new Set<ControlRenewalOutcome>([
'renewed',
'assignment_not_found',
'activity_cell_not_authoritative',
'control_activity_not_found',
'control_activity_moved',
'assignment_lock_unavailable'
])
export type ControlRenewalRequest = {
identity: AssignmentIdentity
activityId: string
cellId: string
expiresAt: number
}
// LOCK ORDER - (user_id, relay_host_id), the primary key of relay_assignments,
// applied here and repeated as the statement's ORDER BY so it holds whether the
// planner walks the primary-key index or sorts under the LockRows node.
//
// The batch never waits for an assignment row: SKIP LOCKED reports a contended
// host separately instead. That is what bounds how long a flush holds its locks
// to its own execution time, because row locks live until the statement commits,
// and it is why one host wedged in a per-host transaction cannot stall the
// renewals of every other host sharing the flush.
//
// With no wait on the assignment pass, the deadlock question reduces to the two
// later passes. Every writer in this store locks a host's assignment row before
// that host's lease rows (`assignmentRow` then `lockAssignmentActivities`), and
// a host whose assignment row is held was skipped, so the batch never reaches
// that host's lease: the lease pass cannot wait either.
// `markMigrationTargetRegistered` is the one writer that locks a migration row
// without the assignment row first. It takes no further locks, so it can delay a
// mid-migration row by up to the pool's lock_timeout but cannot close a cycle.
export function orderedControlRenewalRows<Row extends { identity: AssignmentIdentity }>(
rows: readonly Row[]
): Row[] {
return [...rows].sort(
(left, right) =>
left.identity.userId.localeCompare(right.identity.userId) ||
left.identity.relayHostId.localeCompare(right.identity.relayHostId)
)
}
// One statement renewing every due control lease on this cell, row-wise over the
// unnested parameter arrays. Logic per row is what the single-row predecessor
// did: lock the assignment, admit the caller's cell either as the current cell or
// as the source of an active forward migration, lock that host's control lease,
// push both expiries forward, and report one outcome. The one addition is
// `present_assignment`, an unlocked probe that separates a host with no
// assignment row at all from one whose row SKIP LOCKED passed over - the first
// closes the control, the second retries.
export const CONTROL_RENEWAL_BATCH_SQL = `WITH renewal_input AS MATERIALIZED (
SELECT
renewal.ordinality AS row_index,
renewal.user_id,
renewal.relay_host_id,
renewal.activity_id,
renewal.cell_id,
renewal.expires_at
FROM unnest(?::text[], ?::text[], ?::text[], ?::text[], ?::bigint[])
WITH ORDINALITY AS renewal(
user_id, relay_host_id, activity_id, cell_id, expires_at, ordinality
)
), present_assignment AS MATERIALIZED (
SELECT input.row_index
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
), assignment_state AS MATERIALIZED (
SELECT input.row_index, assignment.cell_id, assignment.assignment_epoch
FROM renewal_input input
JOIN relay_assignments assignment
ON assignment.user_id = input.user_id
AND assignment.relay_host_id = input.relay_host_id
ORDER BY assignment.user_id, assignment.relay_host_id
FOR UPDATE OF assignment SKIP LOCKED
), migration_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
JOIN relay_assignment_migrations migration
ON migration.user_id = input.user_id
AND migration.relay_host_id = input.relay_host_id
AND migration.source_cell_id = input.cell_id
AND migration.target_cell_id = locked.cell_id
AND migration.assignment_epoch = locked.assignment_epoch
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
ORDER BY migration.user_id, migration.relay_host_id
FOR UPDATE OF migration
), authorization_state AS MATERIALIZED (
SELECT locked.row_index
FROM assignment_state locked
JOIN renewal_input input ON input.row_index = locked.row_index
WHERE locked.cell_id = input.cell_id
OR EXISTS (
SELECT 1 FROM migration_state moving
WHERE moving.row_index = locked.row_index
)
), lease_state AS MATERIALIZED (
SELECT authorized.row_index, lease.activity_kind, lease.cell_id
FROM authorization_state authorized
JOIN renewal_input input ON input.row_index = authorized.row_index
JOIN relay_assignment_activity_leases lease
ON lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
ORDER BY lease.user_id, lease.relay_host_id, lease.activity_id
FOR UPDATE OF lease
), renewed_lease AS (
UPDATE relay_assignment_activity_leases lease
SET expires_at = GREATEST(lease.expires_at, input.expires_at),
updated_at = GREATEST(lease.updated_at, ?)
FROM lease_state state
JOIN renewal_input input ON input.row_index = state.row_index
WHERE lease.user_id = input.user_id
AND lease.relay_host_id = input.relay_host_id
AND lease.activity_id = input.activity_id
AND state.activity_kind = 'control' AND state.cell_id = input.cell_id
RETURNING state.row_index
), renewed_assignment AS (
-- Grouped per host: an UPDATE whose FROM offers a target row more than
-- once applies one source row and returns one, so two leases on one
-- host would leave the assignment carrying the wrong expiry. The
-- aggregate hands it exactly one row, carrying the later expiry.
UPDATE relay_assignments assignment
SET lease_expires_at = GREATEST(assignment.lease_expires_at, renewed.expires_at),
last_activity_at = GREATEST(assignment.last_activity_at, ?)
FROM (
SELECT input.user_id, input.relay_host_id, MAX(input.expires_at) AS expires_at
FROM renewed_lease renewed
JOIN renewal_input input ON input.row_index = renewed.row_index
GROUP BY input.user_id, input.relay_host_id
) renewed
WHERE assignment.user_id = renewed.user_id
AND assignment.relay_host_id = renewed.relay_host_id
RETURNING renewed.user_id
)
SELECT input.row_index, CASE
WHEN NOT EXISTS (
SELECT 1 FROM present_assignment present
WHERE present.row_index = input.row_index
) THEN 'assignment_not_found'
WHEN NOT EXISTS (
SELECT 1 FROM assignment_state locked WHERE locked.row_index = input.row_index
) THEN 'assignment_lock_unavailable'
WHEN NOT EXISTS (
SELECT 1 FROM authorization_state authorized
WHERE authorized.row_index = input.row_index
) THEN 'activity_cell_not_authoritative'
WHEN NOT EXISTS (
SELECT 1 FROM lease_state state WHERE state.row_index = input.row_index
) THEN 'control_activity_not_found'
WHEN EXISTS (
SELECT 1 FROM lease_state state
WHERE state.row_index = input.row_index
AND (state.activity_kind <> 'control' OR state.cell_id <> input.cell_id)
) THEN 'control_activity_moved'
-- Read from renewed_lease, which has one row per input row. The
-- assignment update collapses to one row per host, so it cannot answer
-- for a host that brought two leases to the same batch.
WHEN EXISTS (
SELECT 1 FROM renewed_lease renewed
WHERE renewed.row_index = input.row_index
) THEN 'renewed'
ELSE 'control_activity_not_found'
END AS outcome
FROM renewal_input input
ORDER BY input.row_index`
export function controlRenewalBatchParams(
rows: readonly ControlRenewalRequest[],
now: number
): unknown[] {
return [
rows.map((row) => row.identity.userId),
rows.map((row) => row.identity.relayHostId),
rows.map((row) => row.activityId),
rows.map((row) => row.cellId),
rows.map((row) => row.expiresAt),
now,
now
]
}
// Rows come back ordered by row_index, which is the 1-based position in the
// statement's parameter arrays.
export function readControlRenewalOutcomes(
rows: SqlRow[],
expected: number
): ControlRenewalOutcome[] {
if (rows.length !== expected) throw new Error('missing_control_renewal_outcome')
return rows.map((row, position) => {
if (Number(row.row_index) !== position + 1) {
throw new Error('misordered_control_renewal_outcome')
}
const outcome = row.outcome
if (
typeof outcome !== 'string' ||
!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(outcome as ControlRenewalOutcome)
) {
throw new Error('invalid_control_renewal_outcome')
}
// SAFETY: the membership check above is what narrows this string.
return outcome as ControlRenewalOutcome
})
}
@@ -0,0 +1,283 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
// The outage this guards against: the credential cleanup ran every 30s in all 23 cells and both
// sweeps over relay_invites had no usable index, so each one seq-scanned the whole table inside the
// maintenance transaction. Only a real planner can show the partial indexes take that away, and
// only a real server has ctid.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_credential_sweep_test'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
function scopedUrl(): string {
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
return url.toString()
}
async function onAdmin<T>(operation: (client: pg.Client) => Promise<T>): Promise<T> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
return await operation(client)
} finally {
await client.end()
}
}
describePostgres('credential cleanup against PostgreSQL', () => {
let database: RelayDatabase
let store: RelayCredentialStore
const opened: RelayDatabase[] = []
beforeEach(async () => {
await onAdmin(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
})
database = await openRelayDatabase({ databaseUrl: scopedUrl(), dataDir: '' })
opened.push(database)
store = new RelayCredentialStore(database, () => NOW)
})
afterAll(async () => {
await Promise.all(opened.map((open) => open.close().catch(() => undefined)))
await onAdmin((client) => client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`))
})
async function seedInvites(
count: number,
state: string,
updatedAt: number,
expiresAt = NOW - DAY_MS
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
SELECT ?, ?, 'device-' || n, 'token-' || ? || '-' || n, ?, 0, 3, ?, ?, ?
FROM generate_series(1, ?) AS n`,
[identity.userId, identity.relayHostId, state, state, expiresAt, updatedAt, updatedAt, count]
)
}
// Not through RelayDatabase: it routes anything that is not a SELECT to the row-count path, and
// EXPLAIN on an UPDATE is neither.
async function plan(sql: string, params: unknown[]): Promise<string> {
const client = new pg.Client({ connectionString: scopedUrl() })
await client.connect()
try {
let index = 0
const result = await client.query(
`EXPLAIN ${sql.replace(/\?/g, () => `$${(index += 1)}`)}`,
params
)
return result.rows.map((row) => String(row['QUERY PLAN'])).join('\n')
} finally {
await client.end()
}
}
it('plans both invite sweeps as index scans instead of scanning the whole table', async () => {
// Production's shape: terminal invites outnumber live ones by orders of magnitude, which is
// what makes the partial predicates worth having.
await seedInvites(20_000, 'consumed', NOW)
for (const state of ['available', 'reserved', 'cooldown']) {
await seedInvites(200, state, NOW, NOW + DAY_MS)
}
await database.query(
`UPDATE relay_invites SET reservation_expires_at = ? WHERE state = 'reserved'`,
[NOW + 1]
)
// Only ANALYZE makes the planner's row estimates real; without it a cold table looks tiny and
// a seq scan wins on any index.
await database.query(`ANALYZE relay_invites`)
const expiry = await plan(
`UPDATE relay_invites SET state = 'expired'
WHERE expires_at <= ? AND state IN ('available', 'reserved', 'cooldown')`,
[NOW]
)
const reservation = await plan(
`UPDATE relay_invites SET state = 'cooldown'
WHERE state = 'reserved' AND reservation_expires_at <= ? AND expires_at > ?`,
[NOW, NOW]
)
// Which of the two partial indexes serves the reservation pass is the planner's call: both
// predicates hold only live invites, so either one reads a handful of rows. The invariant is
// that neither pass reads the whole table any more.
for (const sweep of [expiry, reservation]) {
expect(sweep).not.toContain('Seq Scan on relay_invites')
expect(sweep).toMatch(/using relay_invites_sweep_(expiry|reservation)/)
}
expect(expiry).toContain('relay_invites_sweep_expiry')
})
it('plans the basis sweep off the composite index rather than the 1.5 GB heap', async () => {
// The shape that made this the most expensive statement in the sweep: 20,000 settled bases to
// 50 live ones. A partial index on active = 1 looks like the answer to that ratio and is not:
// a basis is inserted active and flipped to 0, so it accumulates the same dead entries, and
// the planner picks the composite index anyway. See the schema comment beside it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'live-' || n, ?, ?, 'device-1', 1, 'invite', ?, 1, ?
FROM generate_series(1, 50) AS n`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(`ANALYZE relay_connection_bases`)
const sweep = await plan(
`UPDATE relay_connection_bases SET active = 0 WHERE active = 1 AND deadline <= ?`,
[NOW]
)
expect(sweep).not.toContain('Seq Scan on relay_connection_bases')
expect(sweep).toContain('using relay_connection_bases_active_deadline')
})
it('plans the drained basis reaper off the composite index, not the heap', async () => {
// Why the composite index stays for now: it is the only one covering active = 0, and the case
// that needs it is the steady state, where every row is inside retention and the reaper must
// learn there is nothing to do. While the backlog drains the planner rightly prefers a bounded
// sequential scan, because it finds its 5,000 rows and stops; measured at 200k rows, the
// drained batch costs 5 buffers with this index and 1,274 without it.
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 60_000, NOW - 60_000]
)
await database.query(`ANALYZE relay_connection_bases`)
const reaper = await plan(
`DELETE FROM relay_connection_bases WHERE ctid IN (
SELECT ctid FROM relay_connection_bases WHERE active = ? AND deadline <= ? LIMIT 5000
)`,
[0, NOW - DAY_MS]
)
expect(reaper).toContain('relay_connection_bases_active_deadline')
expect(reaper).not.toContain('Seq Scan on relay_connection_bases')
})
it('plans the pending-authorization and rate-window sweeps as index scans', async () => {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'auth-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 20000) AS n`,
[identity.userId, identity.relayHostId, NOW - 1, NOW - 1]
)
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
SELECT 'scope-' || n, 'invite-mint', ?, 1 FROM generate_series(1, 20000) AS n`,
[NOW]
)
await database.query(`ANALYZE relay_direct_authorizations`)
await database.query(`ANALYZE relay_rate_windows`)
const pending = await plan(
`UPDATE relay_direct_authorizations SET consumed_at = ?
WHERE consumed_at IS NULL AND deadline <= ?`,
[NOW, NOW]
)
const windows = await plan(`DELETE FROM relay_rate_windows WHERE window_started_at < ?`, [
NOW - DAY_MS
])
expect(pending).toContain('relay_direct_authorizations_pending_deadline')
expect(pending).not.toContain('Seq Scan on relay_direct_authorizations')
expect(windows).toContain('relay_rate_windows_started')
expect(windows).not.toContain('Seq Scan on relay_rate_windows')
})
it('reaps settled bases and consumed authorizations through ctid', async () => {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
SELECT 'settled-' || n, ?, ?, 'device-1', 1, 'invite', ?, 0, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES ('live', ?, ?, 'device-1', 1, 'invite', ?, 1, ?)`,
[identity.userId, identity.relayHostId, NOW + 30_000, NOW]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
SELECT 'consumed-' || n, ?, ?, 'device-1', 1, ?, ?
FROM generate_series(1, 5002) AS n`,
[identity.userId, identity.relayHostId, NOW - 2 * DAY_MS, NOW - 2 * DAY_MS]
)
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES ('pending', ?, ?, 'device-1', 1, ?, NULL)`,
[identity.userId, identity.relayHostId, NOW + 30_000]
)
await store.cleanup()
expect(
await database.query(`SELECT count(*) AS total FROM relay_connection_bases`)
).toEqual([{ total: '3' }])
expect(
await database.query(`SELECT count(*) AS total FROM relay_direct_authorizations`)
).toEqual([{ total: '3' }])
await store.cleanup()
// Only the rows a reader could still accept are left.
expect(
await database.query(`SELECT basis_conn_id FROM relay_connection_bases`)
).toEqual([{ basis_conn_id: 'live' }])
expect(
await database.query(`SELECT direct_auth_id FROM relay_direct_authorizations`)
).toEqual([{ direct_auth_id: 'pending' }])
})
it('reaps terminal invites past retention through ctid, one bounded batch per cycle', async () => {
await seedInvites(5_002, 'consumed', NOW - 30 * DAY_MS)
await seedInvites(3, 'invalidated', NOW - 6 * DAY_MS)
await seedInvites(2, 'available', NOW - 400 * DAY_MS, NOW + DAY_MS)
await store.cleanup()
expect(await database.query(`SELECT count(*) AS total FROM relay_invites`)).toEqual([
{ total: '7' }
])
await store.cleanup()
// The two live invites and the three inside retention survive; the batch remainder is gone.
expect(
await database.query(`SELECT state, count(*) AS total FROM relay_invites GROUP BY state ORDER BY state`)
).toEqual([
{ state: 'available', total: '2' },
{ state: 'invalidated', total: '3' }
])
})
})
@@ -0,0 +1,297 @@
import { describe, expect, it } from 'vitest'
import { RelayCredentialStore, type RelayIdentity } from './credential-store.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const identity: RelayIdentity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const DAY_MS = 24 * 60 * 60 * 1000
const NOW = 100 * DAY_MS
async function insertInvite(
database: RelayDatabase,
invite: { token: string; state: string; updatedAt: number; expiresAt?: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_invites
(user_id, relay_host_id, relay_device_id, token_hash, state, attempt_count,
max_attempts, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
identity.userId,
identity.relayHostId,
`device-${invite.token}`,
invite.token,
invite.state,
0,
3,
invite.expiresAt ?? NOW + DAY_MS,
invite.updatedAt,
invite.updatedAt
]
)
}
async function remainingTokens(database: RelayDatabase): Promise<string[]> {
const rows = await database.query(`SELECT token_hash FROM relay_invites ORDER BY token_hash`)
return rows.map((row) => String(row.token_hash))
}
async function insertBasis(
database: RelayDatabase,
basis: { id: string; active: number; deadline: number }
): Promise<void> {
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
basis.id,
identity.userId,
identity.relayHostId,
'device-1',
1,
'invite',
basis.deadline,
basis.active,
NOW
]
)
}
async function insertDirectAuthorization(
database: RelayDatabase,
auth: { id: string; deadline: number; consumedAt: number | null }
): Promise<void> {
await database.query(
`INSERT INTO relay_direct_authorizations
(direct_auth_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
deadline, consumed_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[auth.id, identity.userId, identity.relayHostId, 'device-1', 1, auth.deadline, auth.consumedAt]
)
}
async function remainingIds(database: RelayDatabase, table: string, column: string): Promise<string[]> {
const rows = await database.query(`SELECT ${column} FROM ${table} ORDER BY ${column}`)
return rows.map((row) => String(row[column]))
}
describe('credential cleanup invite reaper', () => {
it('deletes terminal invites past retention and keeps everything else', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 8 * DAY_MS
const recent = NOW - 6 * DAY_MS
for (const state of ['expired', 'consumed', 'invalidated']) {
await insertInvite(database, { token: `stale-${state}`, state, updatedAt: stale })
await insertInvite(database, { token: `recent-${state}`, state, updatedAt: recent })
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual([
'recent-consumed',
'recent-expired',
'recent-invalidated'
])
await database.close()
})
it('never deletes an invite that a reader could still consume, however old', async () => {
// Retention is measured on updated_at, and a long-lived available invite has an old one. The
// state filter is what keeps the reaper from deleting a credential still in use.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const ancient = NOW - 400 * DAY_MS
for (const state of ['available', 'reserved', 'cooldown']) {
await insertInvite(database, {
token: `live-${state}`,
state,
updatedAt: ancient,
expiresAt: NOW + DAY_MS
})
}
await store.cleanup()
expect(await remainingTokens(database)).toEqual(['live-available', 'live-cooldown', 'live-reserved'])
await database.close()
})
it('bounds one cycle to a single batch and drains the rest on later cycles', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
const stale = NOW - 30 * DAY_MS
for (let index = 0; index < 5_002; index += 1) {
await insertInvite(database, {
token: `consumed-${String(index).padStart(5, '0')}`,
state: 'consumed',
updatedAt: stale
})
}
await store.cleanup()
expect(await remainingTokens(database)).toHaveLength(2)
await store.cleanup()
expect(await remainingTokens(database)).toEqual([])
await database.close()
})
it('still expires credentials the sweep owns, and only those past their deadline', async () => {
// The reaper runs after the sweep in the same call, so this pins that adding it did not
// displace any of the five state transitions the sweep is there for.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertInvite(database, {
token: 'lapsed',
state: 'available',
updatedAt: NOW,
expiresAt: NOW - 1
})
await insertInvite(database, {
token: 'current',
state: 'available',
updatedAt: NOW,
expiresAt: NOW + DAY_MS
})
await database.query(
`UPDATE relay_invites SET state = ?, reservation_expires_at = ? WHERE token_hash = ?`,
['reserved', NOW - 1, 'current']
)
await database.query(
`INSERT INTO relay_connection_bases
(basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation,
credential_kind, deadline, active, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
'basis-lapsed', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW - 1, 1, NOW,
'basis-live', identity.userId, identity.relayHostId, 'device-1', 1, 'invite', NOW + 1, 1, NOW
]
)
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-lapsed',
owningControlGeneration: 1,
deadline: NOW - 1
})
await store.recordDirectAuthorization({
...identity,
relayDeviceId: 'device-1',
directAuthId: 'direct-live',
owningControlGeneration: 1,
deadline: NOW + 1
})
await database.query(
`INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count)
VALUES (?, ?, ?, ?), (?, ?, ?, ?)`,
['scope', 'invite-mint', NOW - 2 * DAY_MS, 1, 'scope', 'invite-mint', NOW - 1, 1]
)
await store.cleanup()
expect(
await database.query(`SELECT token_hash, state FROM relay_invites ORDER BY token_hash`)
).toEqual([
{ token_hash: 'current', state: 'cooldown' },
{ token_hash: 'lapsed', state: 'expired' }
])
expect(
await database.query(`SELECT basis_conn_id, active FROM relay_connection_bases ORDER BY basis_conn_id`)
).toEqual([
{ basis_conn_id: 'basis-lapsed', active: 0 },
{ basis_conn_id: 'basis-live', active: 1 }
])
expect(
await database.query(
`SELECT direct_auth_id FROM relay_direct_authorizations
WHERE consumed_at IS NULL ORDER BY direct_auth_id`
)
).toEqual([{ direct_auth_id: 'direct-live' }])
expect(await database.query(`SELECT window_started_at FROM relay_rate_windows`)).toEqual([
{ window_started_at: NOW - 1 }
])
await database.close()
})
it('reaps connection bases whose deadline passed over a day ago, and nothing else', async () => {
// Retention is measured on deadline, and both readers of a basis require deadline >= now, so a
// deadline a day in the past is already unusable however the active flag reads. The active = 0
// clause is what keeps the batch an index range, not what makes the row safe to delete.
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertBasis(database, { id: 'stale-inactive', active: 0, deadline: NOW - 2 * DAY_MS })
await insertBasis(database, { id: 'recent-inactive', active: 0, deadline: NOW - 60_000 })
// A long-lived splice: still active hours after the 30s deadline it was created with. The
// sweep deactivates it this cycle and the reaper takes it in the same call, which is safe
// precisely because no reader would have accepted it since its deadline passed.
await insertBasis(database, { id: 'stale-active', active: 1, deadline: NOW - 400 * DAY_MS })
await insertBasis(database, { id: 'live-active', active: 1, deadline: NOW + DAY_MS })
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([
'live-active',
'recent-inactive'
])
// The one row a reader can still use is untouched, active flag included.
expect(
await database.query(
`SELECT active FROM relay_connection_bases WHERE basis_conn_id = 'live-active'`
)
).toEqual([{ active: 1 }])
await database.close()
})
it('reaps consumed direct authorizations past retention and never a pending one', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
await insertDirectAuthorization(database, {
id: 'stale-consumed',
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
await insertDirectAuthorization(database, {
id: 'recent-consumed',
deadline: NOW - 60_000,
consumedAt: NOW - 60_000
})
await insertDirectAuthorization(database, {
id: 'pending-ancient',
deadline: NOW + DAY_MS,
consumedAt: null
})
await store.cleanup()
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([
'pending-ancient',
'recent-consumed'
])
await database.close()
})
it('bounds each table to one batch per cycle', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
for (let index = 0; index < 5_001; index += 1) {
const id = String(index).padStart(5, '0')
await insertBasis(database, { id: `basis-${id}`, active: 0, deadline: NOW - 2 * DAY_MS })
await insertDirectAuthorization(database, {
id: `auth-${id}`,
deadline: NOW - 2 * DAY_MS,
consumedAt: NOW - 2 * DAY_MS
})
}
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toHaveLength(1)
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toHaveLength(1)
await store.cleanup()
expect(await remainingIds(database, 'relay_connection_bases', 'basis_conn_id')).toEqual([])
expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])
await database.close()
})
})
+49
View File
@@ -12,6 +12,17 @@ const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
// tolerance at exactly inviteTtlMs; issuing under the ceiling keeps pairing
// working for clients whose clocks trail the cell by up to this margin.
const INVITE_ISSUE_SKEW_MARGIN_MS = 30 * 1000
// Terminal invites are read by nothing: every reader re-checks expiry and state at read time, so
// the row only serves the audit trail, which relay_audit_events already keeps. A week is long
// enough to answer a support question about a pairing that failed.
const TERMINAL_INVITE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000
// Why: both readers of a connection basis and of a direct authorization require it still
// active/unconsumed AND inside its deadline, and every deadline is set at most 30s past insert, so
// a settled row can never authorize anything again. A day is margin for forensics, not for reads.
const INACTIVE_AUTHORIZATION_RETENTION_MS = 24 * 60 * 60 * 1000
// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over
// however many cycles it takes.
const REAP_BATCH_ROWS = 5000
export type RelayIdentity = { userId: string; relayHostId: string }
export type CredentialReservation = RelayIdentity & {
@@ -643,6 +654,44 @@ export class RelayCredentialStore {
[now - 24 * 60 * 60 * 1000]
)
})
await this.reapSettledCredentials(now)
}
// Outside the sweep transaction on purpose: each delete is idempotent and independent of the
// state transitions above, so batching them in would only hold their row locks for longer.
private async reapSettledCredentials(now: number): Promise<void> {
await this.reapBatch(
'relay_invites',
'state IN (?, ?, ?) AND updated_at <= ?',
['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]
)
// deadline, not created_at: it is the second column of relay_connection_bases_active_deadline,
// so once the backlog is drained this batch learns there is nothing left to do from the index
// instead of the 1.5 GB heap. Both readers reject a passed deadline, so a day past one is
// unusable whatever the active flag says.
await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [
0,
now - INACTIVE_AUTHORIZATION_RETENTION_MS
])
// consumed_at, not deadline: consumption is what settles this row, and it can happen well
// before the deadline, so measuring from it retains the row for the full window either way.
await this.reapBatch(
'relay_direct_authorizations',
'consumed_at IS NOT NULL AND consumed_at <= ?',
[now - INACTIVE_AUTHORIZATION_RETENTION_MS]
)
}
// ctid/rowid, not the primary key: the physical address lets the delete re-find exactly the batch
// the subquery located instead of re-matching the predicate per row.
private async reapBatch(table: string, predicate: string, params: unknown[]): Promise<void> {
const address = this.database.dialect === 'sqlite' ? 'rowid' : 'ctid'
await this.database.query(
`DELETE FROM ${table} WHERE ${address} IN (
SELECT ${address} FROM ${table} WHERE ${predicate} LIMIT ${REAP_BATCH_ROWS}
)`,
params
)
}
private async installStatusWith(
@@ -134,7 +134,7 @@ describe('PostgreSQL relay deadlines', () => {
statements.every(
(statement) =>
statement === POSTGRES_STATEMENT_STATS_MIGRATION.trim() ||
/^(?:CREATE|ALTER TABLE)\b/i.test(body(statement))
/^(?:CREATE|ALTER TABLE|DROP INDEX)\b/i.test(body(statement))
)
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
+50 -3
View File
@@ -100,6 +100,18 @@ CREATE TABLE IF NOT EXISTS relay_invites (
CREATE INDEX IF NOT EXISTS relay_invites_device
ON relay_invites(user_id, relay_host_id, relay_device_id);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the credential sweep matches (state, expires_at) every cycle while invites in a terminal
-- state accumulate for the life of the database. Unindexed it seq-scans the whole table inside the
-- maintenance transaction. Partial, so the index holds only the states the sweep can act on.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry
ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown');
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the second sweep pass matches (state, reservation_expires_at) over the same table.
CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation
ON relay_invites(reservation_expires_at) WHERE state = 'reserved';
CREATE TABLE IF NOT EXISTS relay_devices (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -160,6 +172,11 @@ CREATE TABLE IF NOT EXISTS relay_connection_bases (
-- accumulate unboundedly. Unindexed it seq-scans millions of rows every cycle
-- and holds the maintenance transaction open long enough to time out
-- assignment lock waits.
-- Why not a partial index on active = 1: a basis is inserted active and flipped to 0, so each
-- deactivation leaves a dead entry in that index too. Measured on production-shaped history it
-- carries the same dead entries as this one, the planner picks this one in every state, and it
-- costs ~65 bytes of WAL per insert. Bloat here is cured by reaping and vacuum, not by a narrower
-- index.
CREATE INDEX IF NOT EXISTS relay_connection_bases_active_deadline
ON relay_connection_bases(active, deadline);
@@ -173,6 +190,12 @@ CREATE TABLE IF NOT EXISTS relay_direct_authorizations (
consumed_at BIGINT
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: the sweep expires pending authorizations by (consumed_at IS NULL, deadline), and consumed
-- rows are never deleted. Partial, so the index stays the size of the pending set.
CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline
ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL;
CREATE TABLE IF NOT EXISTS relay_confirm_results (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -524,8 +547,9 @@ CREATE TABLE IF NOT EXISTS relay_assignment_activity_leases (
updated_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE INDEX IF NOT EXISTS relay_assignment_activity_expiry
ON relay_assignment_activity_leases(expires_at);
-- expires_at is deliberately unindexed: every control renewal writes it (~471/s), so an index on
-- it makes each renewal a non-HOT update that rewrites index entries. Its only reader is the 30s
-- expiry sweep, which seq-scans 14.8k rows / 7MB in a few milliseconds.
CREATE TABLE IF NOT EXISTS relay_control_connection_reservations (
reservation_id TEXT PRIMARY KEY,
@@ -559,6 +583,12 @@ CREATE TABLE IF NOT EXISTS relay_rate_windows (
PRIMARY KEY (scope_key, window_kind, window_started_at)
);
-- schema-deferrable: created out of band, so a boot that cannot take the lock must retry
-- Why: window_started_at is the PRIMARY KEY's last column, so the sweep's 24h retention delete
-- cannot use it and seq-scans instead.
CREATE INDEX IF NOT EXISTS relay_rate_windows_started
ON relay_rate_windows(window_started_at);
CREATE TABLE IF NOT EXISTS relay_migration_leases (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
@@ -645,7 +675,24 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`,
// Dropped, not created: see the comment on relay_assignment_activity_leases. Deferrable because
// this is the one boot where it has to take ACCESS EXCLUSIVE on a table under continuous write,
// and all 28 directors reach it at once; a lock timeout here must not restart the instance, which
// would only re-queue the same DDL behind the same writers. Once it wins, the pre-check answers
// absent and no later boot sends it at all.
`-- schema-deferrable: one boot has to win ACCESS EXCLUSIVE on a table written ~475/s
DROP INDEX IF EXISTS relay_assignment_activity_expiry`,
// The drop is what makes HOT legal; this is what makes it possible. A renewal can only reuse the
// row's own page when that page has room for a second version, and at the default fillfactor of
// 100 a freshly filled page has none - measured at 0.5% HOT with the index gone and the default,
// against 100% at 70. Takes SHARE UPDATE EXCLUSIVE, which blocks vacuum and DDL but no reader or
// writer, and only for the catalog write. Applies to pages as they refill, so the table converges
// over its own renewal cycle rather than at boot.
// Deferrable for the same reason, though SHARE UPDATE EXCLUSIVE blocks only vacuum and DDL: it
// buys nothing until the drop lands, so a boot that deferred the drop should defer this too.
`-- schema-deferrable: buys nothing until the drop above lands
ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)`
]
// The exact statement list a Postgres boot applies, in order, so the lock-target census can read
@@ -729,3 +729,73 @@ describe('control lease jitter', () => {
vi.advanceTimersByTime(0)
})
})
describe('paced drain and the phones of a host not yet told', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const laterHostId = 'qrstuvwxyz012345'
const laterIdentity = { ...identity, sub: 'user-2', relayHostId: laterHostId }
async function twoHostCell(): Promise<{
h: ReturnType<typeof harness>
told: FakeSocket
untold: FakeSocket
}> {
const h = harness()
const told = await activeHost(h)
const untold = new FakeSocket()
await h.activate(untold as unknown as WebSocket, laterIdentity, null, 1, false, 1, '1.4.197')
// Both hosts now dial in, so the credential mocks have to answer for either.
h.store.resolveResume.mockImplementation(async (hostId: string) => ({
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub
}))
h.store.reserveCredential.mockImplementation(async (hostId: string) => ({
...reservation,
userId: hostId === laterHostId ? laterIdentity.sub : identity.sub,
relayHostId: hostId
}))
return { h, told, untold }
}
async function dial(h: ReturnType<typeof harness>, hostId: string): Promise<FakeSocket> {
const client = new FakeSocket()
await h.registry.acceptClient(client as unknown as WebSocket, hostId, 'credential')
return client
}
it('serves a host whose drain has not been sent and refuses one whose has', async () => {
const { h, told, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
const refused = await dial(h, identity.relayHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(told.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
const served = await dial(h, laterHostId)
expect(served.close).not.toHaveBeenCalled()
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('conn-open'))
})
it('refuses that host\'s phones as soon as its own drain is sent', async () => {
const { h, untold } = await twoHostCell()
h.registry.drain(0, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(40_000)
expect(untold.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
const refused = await dial(h, laterHostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
})
it('keeps an unpaced drain refusing every phone at once', async () => {
const { h } = await twoHostCell()
h.registry.drain(0)
for (const hostId of [identity.relayHostId, laterHostId]) {
const refused = await dial(h, hostId)
expect(refused.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
}
})
})
@@ -12,6 +12,12 @@ import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import {
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
type ControlRenewalOutcome,
type ControlRenewalRequest
} from './control-renewal-statement.js'
import { HostSessionRegistry, type HostSession } from './host-session-registry.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
@@ -23,6 +29,12 @@ import {
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
// A due renewal leaves the heartbeat as a batch enqueue, so the store only sees
// the tick once the batch window closes.
async function closeRenewalWindow(): Promise<void> {
await vi.advanceTimersByTimeAsync(CONTROL_RENEWAL_BATCH_INTERVAL_MS)
}
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSING = 2
@@ -109,6 +121,7 @@ function createRegistry(
activate: ActivateSession
acquireActivity: ReturnType<typeof vi.fn>
renewControlActivity: ReturnType<typeof vi.fn>
renewControlActivities: ReturnType<typeof vi.fn>
releaseActivity: ReturnType<typeof vi.fn>
observer: {
recordAuth: ReturnType<typeof vi.fn>
@@ -119,12 +132,38 @@ function createRegistry(
const acquireActivity = vi.fn().mockResolvedValue(undefined)
const renewControlActivity = vi.fn().mockResolvedValue(undefined)
const releaseActivity = vi.fn().mockResolvedValue(true)
// Mirrors the store's own batch semantics over the single-renewal mock: a known
// outcome becomes that row's verdict, and any other failure reaches the caller
// as the driver's error. Keeps every per-call expectation below aimed at the
// renewal a session actually asked for.
const renewControlActivities = vi.fn(
async (rows: readonly ControlRenewalRequest[]): Promise<ControlRenewalOutcome[]> =>
await Promise.all(
rows.map(async (row): Promise<ControlRenewalOutcome> => {
try {
await renewControlActivity(row.identity, {
activityId: row.activityId,
cellId: row.cellId,
expiresAt: row.expiresAt
})
return 'renewed'
} catch (error) {
const message = String((error as { message?: unknown }).message)
if (!CONTROL_RENEWAL_STATEMENT_OUTCOMES.has(message as ControlRenewalOutcome)) {
throw error
}
return message as ControlRenewalOutcome
}
})
)
)
const assignments = {
activateControl,
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity
} as unknown as RelayAssignmentStore
const observer = {
@@ -176,6 +215,7 @@ function createRegistry(
activate,
acquireActivity,
renewControlActivity,
renewControlActivities,
releaseActivity,
observer
}
@@ -336,15 +376,15 @@ describe('host session cleanup races', () => {
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
// POST /v1/admin/drain has no idempotency guard, and SIGTERM then SIGINT both
// reach drain(), so a second teardown can be scheduled for the same session.
// reach drain(), so a retry re-sends to every session. It must re-arm the pending
// teardown rather than stack a second one: across a paced cell that is 800 orphaned
// timers per retry, each one holding the loop open for the rest of the window.
registry.drain(0)
const scheduled = vi.getTimerCount()
registry.drain(0)
// Pin the premise: if drain ever gains an idempotency guard, the retry schedules no
// second teardown and the assertion below stops defending the write-once snapshot
// while still passing. Compare against the count before the retry rather than an
// absolute, since the session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled + 1)
// Compare against the count before the retry rather than an absolute, since the
// session's heartbeat interval is also pending.
expect(vi.getTimerCount()).toBe(scheduled)
vi.advanceTimersByTime(1)
// Asserting registry state, not the log line: FakeSocket closes synchronously, so
@@ -689,7 +729,8 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
expect(renewControlActivity).toHaveBeenCalledWith(
@@ -715,6 +756,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
@@ -737,6 +779,7 @@ describe('host session cleanup races', () => {
})
)
await vi.advanceTimersByTimeAsync(15_000)
await closeRenewalWindow()
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
@@ -762,6 +805,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 4; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
@@ -791,8 +835,10 @@ describe('host session cleanup races', () => {
try {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
} finally {
warn.mockRestore()
@@ -812,6 +858,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
@@ -831,13 +878,16 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
stalled.resolve(undefined)
await vi.advanceTimersByTimeAsync(0)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledTimes(3)
registry.drain(0)
@@ -855,6 +905,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
@@ -880,6 +931,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(acquireActivity).not.toHaveBeenCalled()
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
@@ -899,6 +951,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -918,6 +971,7 @@ describe('host session cleanup races', () => {
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(socket.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.DRAINING,
@@ -939,6 +993,7 @@ describe('host session cleanup races', () => {
for (let interval = 0; interval < 3; interval++) {
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
expect(renewControlActivity).toHaveBeenCalledTimes(2)
@@ -966,6 +1021,7 @@ describe('control renewal cadence across a rebind', () => {
const beat = async (target: FakeSocket): Promise<void> => {
await vi.advanceTimersByTimeAsync(ping)
target.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
await closeRenewalWindow()
}
// Age the session so its attempt counter is well above zero.
@@ -993,6 +1049,57 @@ describe('control renewal cadence across a rebind', () => {
})
})
describe('control renewals shared by one batch', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('renews two due hosts in one call and leaves a stale one alone', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValueOnce('control:production-gce-c3:1')
.mockResolvedValueOnce('control:production-gce-c3:1')
const { registry, activate, renewControlActivities } = createRegistry(activateControl)
const other = { ...identity, sub: 'user-2', relayHostId: 'ponmlkjihgfedcba' }
const staleSocket = new FakeSocket()
const liveSocket = new FakeSocket()
await activate(staleSocket as unknown as WebSocket, identity, null, 1, false, 1)
await activate(liveSocket as unknown as WebSocket, other, null, 1, false, 1)
const stale = registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const live = registry.get({ userId: other.sub, relayHostId: other.relayHostId })!
// Both come due inside the same window, and one socket goes away while the
// statement is still in PostgreSQL.
let release!: () => void
renewControlActivities.mockImplementationOnce(
async (rows: readonly ControlRenewalRequest[]) => {
staleSocket.close()
await new Promise<void>((resolve) => (release = resolve))
return rows.map((): ControlRenewalOutcome => 'renewed')
}
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const staleDueAt = stale.activityRenewalDueAt
await closeRenewalWindow()
release()
await vi.advanceTimersByTimeAsync(0)
expect(renewControlActivities).toHaveBeenCalledOnce()
expect(
renewControlActivities.mock.calls[0]![0].map(
(row: ControlRenewalRequest) => row.identity.relayHostId
)
).toEqual([identity.relayHostId, other.relayHostId])
expect(live.activityRenewalCompletedAttempt).toBe(1)
expect(stale.activityRenewalCompletedAttempt).toBe(0)
expect(stale.activityRenewalDueAt).toBe(staleDueAt)
registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('control lease recovery after the session is gone', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -1019,6 +1126,7 @@ describe('control lease recovery after the session is gone', () => {
new Promise<void>((_resolve, reject) => (failRenewal = reject))
)
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
await closeRenewalWindow()
expect(renewControlActivity).toHaveBeenCalledOnce()
const newer = new FakeSocket()
@@ -1696,3 +1804,124 @@ describe('host data attach owner lookup', () => {
expect(h.owner.activeConnIds.size).toBe(0)
})
})
describe('paced drain', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
async function connectHosts(count: number): Promise<{
registry: HostSessionRegistry
sockets: FakeSocket[]
}> {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
const { registry, activate } = createRegistry(activateControl)
const sockets: FakeSocket[] = []
for (let index = 0; index < count; index += 1) {
const socket = new FakeSocket()
sockets.push(socket)
await activate(
socket as unknown as WebSocket,
{ ...identity, sub: `user-${index}` },
null,
1,
false,
1
)
socket.send.mockClear()
}
return { registry, sockets }
}
function drainsSent(sockets: FakeSocket[]): number {
return sockets.filter((socket) =>
socket.send.mock.calls.some(([payload]) => String(payload).includes('"type":"drain"'))
).length
}
it('sends every drain at once when no window is given', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
})
// Windows here stay under the 75s control-silence watchdog, which would otherwise close
// a test socket that never heartbeats before its paced send is due.
it('spreads the sends evenly across the window', async () => {
const { registry, sockets } = await connectHosts(5)
registry.drain(0, { paceWindowMs: 40_000 })
// The first host is sent synchronously; the last lands on the window's closing edge.
expect(drainsSent(sockets)).toBe(1)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(2)
await vi.advanceTimersByTimeAsync(20_000)
expect(drainsSent(sockets)).toBe(4)
await vi.advanceTimersByTimeAsync(10_000)
expect(drainsSent(sockets)).toBe(5)
})
it('fences admission for every session before the first paced send lands', async () => {
const { registry, sockets } = await connectHosts(3)
registry.drain(0, { paceWindowMs: 40_000 })
expect(registry.isDraining()).toBe(true)
// A host whose drain has not been sent yet must already be non-authoritative.
const socket = new FakeSocket()
registry.acceptControl(socket as unknown as WebSocket, { ...identity, sub: 'user-late' })
expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
expect(drainsSent(sockets)).toBe(1)
})
it('gives each host its own grace after its own send, not after the call', async () => {
const { registry, sockets } = await connectHosts(2)
registry.drain(10_000, { paceWindowMs: 40_000 })
await vi.advanceTimersByTimeAsync(10_000)
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
// Its own send at 40s plus its own 10s grace, not 10s from the drain call.
await vi.advanceTimersByTimeAsync(39_999)
expect(sockets[1]!.readyState).toBe(sockets[1]!.OPEN)
await vi.advanceTimersByTimeAsync(10_001)
expect(sockets[1]!.readyState).toBe(sockets[1]!.CLOSED)
})
it('leaves no timer behind once an emergency drain cuts a window short', async () => {
const { registry } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
// Every session is closed, so anything still pending is an orphan of the cut window.
expect(vi.getTimerCount()).toBe(0)
})
it('keeps the first teardown snapshot when a regional drain fires before the fleet one', async () => {
const { registry, sockets } = await connectHosts(1)
const session = registry.get({ userId: 'user-0', relayHostId: identity.relayHostId })!
session.activeSplices.set('conn-a', () => session.activeSplices.delete('conn-a'))
registry.drainHost({
attemptId: 'attempt',
userId: 'user-0',
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 0
})
registry.drain(10)
await vi.advanceTimersByTimeAsync(11)
expect(session.closingCounts).toEqual({ splices: 1, pending: 0 })
expect(sockets[0]!.readyState).toBe(sockets[0]!.CLOSED)
})
it('lets an emergency drain supersede the sends still queued by a paced one', async () => {
const { registry, sockets } = await connectHosts(4)
registry.drain(0, { paceWindowMs: 40_000 })
expect(drainsSent(sockets)).toBe(1)
registry.drain(0)
expect(drainsSent(sockets)).toBe(4)
const sendsAfterEmergency = sockets.map((socket) => socket.send.mock.calls.length)
await vi.advanceTimersByTimeAsync(40_000)
expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency)
})
})
+73 -19
View File
@@ -26,6 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import { ControlRenewalBatch } from './control-renewal-batch.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
@@ -179,6 +180,10 @@ export class HostSessionRegistry {
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly drainTimers = new Set<ReturnType<typeof setTimeout>>()
// Hosts whose drain has been sent. Paced sends land minutes apart, so "this cell is
// draining" is not the same question as "this host has been told to leave".
private readonly drainSentHosts = new Set<string>()
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
@@ -299,6 +304,15 @@ export class HostSessionRegistry {
private readonly cellIncarnation?: string
) {}
// Renewals leave the heartbeat as an enqueue: one statement per cell per
// window replaces one write transaction per host, which is what keeps the
// shared PostgreSQL instance out of buffer-header contention.
private readonly controlRenewals = new ControlRenewalBatch(
async (rows) => await this.assignments.renewControlActivities(rows),
() => this.logIdentity(),
(flush) => this.observer.recordControlRenewalFlush?.(flush)
)
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
private controlLeaseExpiresAt(): number {
const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
@@ -330,7 +344,10 @@ export class HostSessionRegistry {
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
// Not `this.draining`: a paced drain tells hosts minutes apart, and the director keeps
// pointing phones here until their own host has moved. Refusing them for the whole
// window would turn a 2 min drain into a 2 min outage for hosts not yet told.
if (this.drainSentHosts.has(hostId)) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
return
@@ -450,7 +467,7 @@ export class HostSessionRegistry {
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.drainSentHosts.has(hostId) ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
@@ -592,7 +609,7 @@ export class HostSessionRegistry {
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.drainSentHosts.has(identity.relayHostId) ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
@@ -846,17 +863,49 @@ export class HostSessionRegistry {
return { controls, splices, pendingSplices }
}
drain(graceMs: number): void {
drain(graceMs: number, options: { paceWindowMs?: number } = {}): void {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
// A later drain (an emergency one, or shutdown) owns every session again, so nothing
// queued by an earlier paced drain may still fire: it would re-send and, worse, keep
// the event loop alive for the rest of a window the operator just cut short.
for (const timer of this.drainTimers) clearTimeout(timer)
this.drainTimers.clear()
const paceWindowMs = Math.max(0, Math.trunc(options.paceWindowMs ?? 0))
const targets = [...this.sessions.values()].filter((session) => session.state !== 'closed')
// The desktop re-dials the director as soon as it reads `drain`, whatever graceMs says,
// so spreading the send is the only thing that spreads the reconnect load.
const step = paceWindowMs > 0 && targets.length > 1 ? paceWindowMs / (targets.length - 1) : 0
for (const [index, session] of targets.entries()) {
const delay = Math.round(step * index)
if (delay === 0) {
this.sendDrain(session, graceMs)
continue
}
this.scheduleDrainTimer(delay, () => this.sendDrain(session, graceMs))
}
}
// A session is only fenced when it is told, not when the drain starts: until its send
// lands it is an ordinary live host, and its phones have to keep being able to reach it.
private sendDrain(session: HostSession, graceMs: number): void {
if (session.state === 'closed') return
session.authorityRevision += 1
session.state = 'drain-only'
this.drainSentHosts.add(session.relayHostId)
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
this.scheduleDrainTimer(graceMs, () => this.closeDrainedSession(session))
}
// Unref'd so a drain in flight never holds the process open past its own work.
private scheduleDrainTimer(delayMs: number, run: () => void): void {
const timer: ReturnType<typeof setTimeout> = setTimeout(() => {
this.drainTimers.delete(timer)
run()
}, delayMs)
timer.unref?.()
this.drainTimers.add(timer)
}
drainHost(input: {
attemptId: string
userId: string
@@ -1345,15 +1394,13 @@ export class HostSessionRegistry {
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
{
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
}
)
void this.controlRenewals
.enqueue({
identity: { userId: session.identity.sub, relayHostId: session.relayHostId },
activityId: controlActivityId,
cellId: this.config.cellId,
expiresAt: startedAt + CONTROL_ACTIVITY_LEASE_MS
})
.then(() => {
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
@@ -1419,6 +1466,13 @@ export class HostSessionRegistry {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control activity moved')
return
}
if (error instanceof Error && error.message === 'assignment_lock_unavailable') {
// A per-host transaction held the row, so the batch passed over it
// rather than making every other host in the flush wait. The next
// tick is 15s away against a 105s lease, and the flush line already
// reports the count, so this needs no line of its own.
return
}
console.warn('[orca-relay] control activity renewal failed')
})
// Terminal handler: a throw inside the async catch above (e.g. a
+15 -10
View File
@@ -46,14 +46,19 @@ const {
ready,
cellIncarnation
} = createRelayServer(config, database)
const cleanupTimer = setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
30_000
)
// Same owner as the assignment sweep: the cleanup only expires credentials that every reader
// already re-checks at read time, so running it in all 23 cells multiplied one table scan by 23
// without changing any answer.
const cleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(
() =>
void runRelayBackgroundOperation(
() => store.cleanup(),
'[orca-relay] credential cleanup failed'
),
jitteredSweepIntervalMs(30_000)
)
: null
const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role)
? setInterval(() => {
void runAssignmentCleanup(assignments)
@@ -82,7 +87,7 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cleanupTimer.unref()
cleanupTimer?.unref()
assignmentCleanupTimer?.unref()
inventorySnapshotTimer?.unref()
migrationInventoryTimer?.unref()
@@ -128,7 +133,7 @@ server.listen(config.port, () => {
})
const shutdown = (): void => {
clearInterval(cleanupTimer)
if (cleanupTimer) clearInterval(cleanupTimer)
if (assignmentCleanupTimer) clearInterval(assignmentCleanupTimer)
if (inventorySnapshotTimer) clearInterval(inventorySnapshotTimer)
if (migrationInventoryTimer) clearInterval(migrationInventoryTimer)
@@ -697,6 +697,78 @@ async function postPath(
})
}
describe('cell drain endpoint pacing', () => {
function appWithDrain(): {
app: ReturnType<typeof createRelayApp>
drain: ReturnType<typeof vi.fn>
} {
const drain = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
return { app, drain }
}
it('drains everything at once when the caller asks for no pacing', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', { v: 1, graceMs: 0 })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 0 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 0 })
})
it('passes the requested window through and echoes what it accepted', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ ok: true, paceWindowMs: 120_000 })
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 120_000 })
})
it('refuses a window that is negative, fractional, or past the cap', async () => {
for (const paceWindowMs of [-1, 1.5, 300_001]) {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs
})
expect(response.status).toBe(400)
expect(drain).not.toHaveBeenCalled()
}
})
it('accepts the cap itself', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'deploy-token', {
v: 1,
graceMs: 0,
paceWindowMs: 300_000
})
expect(response.status).toBe(200)
expect(drain).toHaveBeenCalledWith(0, { paceWindowMs: 300_000 })
})
it('still rejects an unauthenticated pacing request', async () => {
const { app, drain } = appWithDrain()
const response = await postPath(app, '/v1/admin/drain', 'wrong-token', {
v: 1,
graceMs: 0,
paceWindowMs: 120_000
})
expect(response.status).toBe(401)
expect(drain).not.toHaveBeenCalled()
})
})
function config(overrides: Partial<RelayConfig> = {}): RelayConfig {
return {
port: 8080,
@@ -1,6 +1,7 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { ControlRenewalFlush } from './control-renewal-batch.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
import type { RelayReadinessGraceEvent, RelayReadinessObservation } from './relay-readiness.js'
@@ -53,6 +54,7 @@ export interface RelayRuntimeObserver {
recordReconnect(): void
recordSql(durationMs: number, success: boolean): void
recordControlRenewal?(durationMs: number, outcome: ControlRenewalOutcome): void
recordControlRenewalFlush?(flush: ControlRenewalFlush): void
recordControlActivityRecovery?(success: boolean): void
recordAssignmentAdmission?(outcome: AssignmentAdmissionOutcome): void
recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void
@@ -119,6 +121,8 @@ type RelayMetricDeltas = {
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlRenewalFlushLatenciesMs: number[]
controlRenewalFlushRowsMax: number
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
@@ -164,6 +168,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlRenewalFlushLatenciesMs: [],
controlRenewalFlushRowsMax: 0,
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
@@ -274,6 +280,14 @@ export class RelayObservability implements RelayRuntimeObserver {
(this.deltas.controlRenewalsByOutcome[outcome] ?? 0) + 1
}
recordControlRenewalFlush(flush: ControlRenewalFlush): void {
this.deltas.controlRenewalFlushLatenciesMs.push(flush.durationMs)
this.deltas.controlRenewalFlushRowsMax = Math.max(
this.deltas.controlRenewalFlushRowsMax,
flush.rows
)
}
recordControlActivityRecovery(success: boolean): void {
if (success) this.deltas.controlActivityRecoveries++
else this.deltas.controlActivityRecoveryFailures++
@@ -379,6 +393,7 @@ export class RelayObservability implements RelayRuntimeObserver {
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const controlRenewalFlush = latencySummary(deltas.controlRenewalFlushLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -447,9 +462,16 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
// Meaning changed when renewals began batching: for a batched row this is
// the flush's duration, not that row's own statement latency. The
// per-flush fields below are the ones to read for statement cost.
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
controlRenewalFlushesDelta: deltas.controlRenewalFlushLatenciesMs.length,
controlRenewalFlushRowsMax: deltas.controlRenewalFlushRowsMax,
controlRenewalFlushLatencyMsP95: controlRenewalFlush.p95,
controlRenewalFlushLatencyMsMax: controlRenewalFlush.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
@@ -1,5 +1,5 @@
import pg from 'pg'
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import {
applyPostgresSchema,
catalogObjectPresence,
@@ -190,6 +190,79 @@ describePostgres('relay boot-time schema against PostgreSQL', () => {
}
})
it('defers the activity-lease migrations and still boots while their table is locked', async () => {
// The migration boot, reproduced: the index is there, the table is locked by someone else, and
// all the drop can do is time out. It has to leave the statement for the next boot rather than
// fail, or 28 directors crash-loop through a stall on a table written ~475/s.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
opened.push(cold)
// Put the database back in its pre-migration shape, which is what makes the drop lock-taking.
await pool.query(
`CREATE INDEX relay_assignment_activity_expiry
ON ${schema}.relay_assignment_activity_leases(expires_at)`
)
await pool.query(`ALTER TABLE ${schema}.relay_assignment_activity_leases RESET (fillfactor)`)
const warned: string[] = []
const warn = vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const holder = new pg.Client({ connectionString: url })
await holder.connect()
await holder.query('BEGIN')
await holder.query(
`LOCK TABLE ${schema}.relay_assignment_activity_leases IN ACCESS EXCLUSIVE MODE`
)
let summary: Awaited<ReturnType<typeof applyPostgresSchema>>
try {
summary = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
} finally {
await holder.query('ROLLBACK')
await holder.end()
warn.mockRestore()
}
// Both statements deferred, and the boot still applied everything else.
expect(summary.deferred).toBe(2)
expect(summary.ran).toBeGreaterThan(0)
const deferred = warned
.map((line) => JSON.parse(line) as { event?: string; name?: string })
.filter((event) => event.event === 'orca_relay_postgres_schema_object_deferred')
expect(deferred.map((event) => event.name)).toEqual([
'relay_assignment_activity_expiry',
'fillfactor=70'
])
// Nothing was applied, so the next boot has the same work to do, not half of it.
const stillThere = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(stillThere.rowCount).toBe(1)
// And the next boot, with the lock gone, finishes the job.
const retry = await applyPostgresSchema(
relayPostgresSchemaStatements(),
(statement) => pool.query(statement),
{ catalogQuery: async (sql, params) => (await pool.query(sql, params)).rows }
)
expect(retry.deferred).toBe(0)
const gone = await pool.query(
`SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2`,
[schema, 'relay_assignment_activity_expiry']
)
expect(gone.rowCount).toBe(0)
const options = await pool.query(
`SELECT reloptions FROM pg_class WHERE oid = to_regclass($1)`,
[`${schema}.relay_assignment_activity_leases`]
)
expect(options.rows[0]?.reloptions).toEqual(['fillfactor=70'])
await pool.end()
})
it('fails that same boot with 55P03 when the pre-check is not wired in', async () => {
// Keeps the test above from passing vacuously: the lock really does block relay's DDL.
const cold = await openRelayDatabase({ databaseUrl: url, dataDir: '' })
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest'
import {
requireSchemaLockTarget,
schemaDeferrable,
schemaLockTarget,
sqlWithoutComments,
takesRelationLock,
@@ -16,9 +17,22 @@ import { relayPostgresSchemaStatements } from './database.js'
// CREATE INDEX CONCURRENTLY first, then add it to SCHEMA and update this list.
const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_device', skipWhen: 'present' },
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_current_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_devices', name: 'relay_devices_grace_hash', skipWhen: 'present' },
{ kind: 'index', table: 'relay_connection_bases', name: 'relay_connection_bases_active_deadline', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_assignment_region_preferences',
@@ -72,12 +86,6 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
name: 'relay_cell_drain_attempt_states_cell',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_assignment_activity_leases',
name: 'relay_assignment_activity_expiry',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_control_connection_reservations',
@@ -90,6 +98,7 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
name: 'relay_control_connection_reservation_assignment',
skipWhen: 'present'
},
{ kind: 'index', table: 'relay_rate_windows', name: 'relay_rate_windows_started', skipWhen: 'present' },
{ kind: 'index', table: 'relay_assignment_migrations', name: 'relay_assignment_migrations_active', skipWhen: 'present' },
{
kind: 'index',
@@ -116,7 +125,14 @@ const GOLDEN_LOCK_TAKING: SchemaLockTarget[] = [
},
{ kind: 'column', table: 'relay_region_rehome_control', name: 'host_cooldown_ms', skipWhen: 'present' },
{ kind: 'column', table: 'relay_control_capabilities', name: 'idle_regional_rehome', skipWhen: 'present' },
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' }
{ kind: 'column', table: 'relay_region_rehome_attempts', name: 'source_generation', skipWhen: 'present' },
{ kind: 'index-by-name', name: 'relay_assignment_activity_expiry', skipWhen: 'absent' },
{
kind: 'reloption',
table: 'relay_assignment_activity_leases',
name: 'fillfactor=70',
skipWhen: 'present'
}
]
const INDEX_OR_ADD_COLUMN = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|ALTER\s+TABLE\s+[^\s]+\s+ADD\s+COLUMN)/i
@@ -149,8 +165,12 @@ describe('relay boot-time lock targets', () => {
for (const statement of relayPostgresSchemaStatements()) {
const target = schemaLockTarget(statement)
if (!target) continue
expect(target.name).toMatch(/^[a-z_][a-z0-9_]*$/)
expect(target.table).toMatch(/^[a-z_][a-z0-9_]*$/)
// A reloption is the one target whose name is a pair rather than an identifier, because
// pg_class stores reloptions as `name=value` text and the value is half the question.
const shape = target.kind === 'reloption' ? /^[a-z_][a-z0-9_]*=[A-Za-z0-9_.]+$/ : /^[a-z_][a-z0-9_]*$/
expect(target.name).toMatch(shape)
// A DROP INDEX names no table, so there is none to check.
if (target.kind !== 'index-by-name') expect(target.table).toMatch(/^[a-z_][a-z0-9_]*$/)
}
})
@@ -194,7 +214,76 @@ describe('relay boot-time lock targets', () => {
it('leaves every statement classifiable once its leading comments are stripped', () => {
for (const statement of relayPostgresSchemaStatements()) {
expect(sqlWithoutComments(statement)).toMatch(/^(?:CREATE|ALTER|DO)\s/i)
expect(sqlWithoutComments(statement)).toMatch(/^(?:CREATE|ALTER|DROP|DO)\s/i)
}
})
it('pre-checks the activity-expiry drop by name, and skips it once the index is gone', () => {
// A DROP INDEX takes ACCESS EXCLUSIVE on the index's table for as long as the index is there,
// so it is in the census like any other lock-taking statement. Its target resolves by name
// alone, because the statement names no table and needs none.
const drops = relayPostgresSchemaStatements().filter((statement) =>
/^DROP\s/i.test(sqlWithoutComments(statement))
)
expect(drops.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
for (const statement of drops) {
expect(takesRelationLock(statement)).toBe(true)
expect(schemaLockTarget(statement)).toEqual({
kind: 'index-by-name',
name: 'relay_assignment_activity_expiry',
skipWhen: 'absent'
})
}
})
it('marks the out-of-band sweep indexes and the activity-lease migrations deferrable, and nothing else', () => {
// The statements a lock timeout must not turn into a crash loop, and the only ones: every
// other statement still fails the boot loudly, which is what keeps the marker meaningful.
const deferrable = relayPostgresSchemaStatements().filter(schemaDeferrable)
expect(deferrable.map((statement) => sqlWithoutComments(statement).replace(/\s+/g, ' '))).toEqual([
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_expiry ON relay_invites(expires_at) WHERE state IN ('available', 'reserved', 'cooldown')",
"CREATE INDEX IF NOT EXISTS relay_invites_sweep_reservation ON relay_invites(reservation_expires_at) WHERE state = 'reserved'",
'CREATE INDEX IF NOT EXISTS relay_direct_authorizations_pending_deadline ON relay_direct_authorizations(deadline) WHERE consumed_at IS NULL',
'CREATE INDEX IF NOT EXISTS relay_rate_windows_started ON relay_rate_windows(window_started_at)',
'DROP INDEX IF EXISTS relay_assignment_activity_expiry',
'ALTER TABLE relay_assignment_activity_leases SET (fillfactor = 70)'
])
})
it('derives a target for a partial index, WHERE clause and all', () => {
// The pre-check reads the index name and table from the head of the statement, so a trailing
// WHERE is invisible to it. Asserted because the sweep indexes depend on that: a parser that
// gave a partial index no target would send it unchecked on every boot.
const partial = relayPostgresSchemaStatements().filter((statement) =>
/^CREATE\s+INDEX\b[\s\S]*\bWHERE\b/i.test(sqlWithoutComments(statement))
)
expect(partial.map(schemaLockTarget)).toEqual([
{ kind: 'index', table: 'relay_invites', name: 'relay_invites_sweep_expiry', skipWhen: 'present' },
{
kind: 'index',
table: 'relay_invites',
name: 'relay_invites_sweep_reservation',
skipWhen: 'present'
},
{
kind: 'index',
table: 'relay_direct_authorizations',
name: 'relay_direct_authorizations_pending_deadline',
skipWhen: 'present'
}
])
})
it('no longer creates an index on the column every control renewal writes', () => {
// The regression this drop exists to prevent: re-adding it would make ~471 renewals/s non-HOT
// again. A CREATE anywhere in the schema naming that index fails here.
const creates = relayPostgresSchemaStatements().filter((statement) =>
/relay_assignment_activity_expiry/i.test(sqlWithoutComments(statement))
)
expect(creates.map(sqlWithoutComments)).toEqual([
'DROP INDEX IF EXISTS relay_assignment_activity_expiry'
])
})
})
+1 -1
View File
@@ -136,7 +136,7 @@ export function createRelayServer(
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drain: (graceMs, options) => sessions.drain(graceMs, options ?? {}),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
@@ -52,4 +52,23 @@ describe('sweep schedule jitter', () => {
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
it('jitters the credential cleanup tick', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const cleanup = /'\[orca-relay\] credential cleanup failed'\s*\),\s*([^\n]*?)\n/.exec(source)
expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
})
// A census, not a list of the timers that happen to be gated today: an ungated sweep runs in
// every cell as well as the director, which multiplies one table scan by the fleet size.
it('gates every periodic sweep in index.ts on the maintenance role', () => {
const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
const timers = source.match(/setInterval\(/g) ?? []
const gated =
source.match(/roleOwnsAssignmentMaintenance\(config\.role\)\s*\?\s*setInterval\(/g) ?? []
expect(timers.length).toBeGreaterThan(0)
expect(gated.length).toBe(timers.length)
})
})
@@ -35,6 +35,9 @@ function cellOrigin(cellId) {
// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
// Matches the cell's own cap on /v1/admin/drain.
const MAX_PACE_WINDOW_MS = 5 * 60 * 1_000
export function parseProductionCapacityCellArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
@@ -64,11 +67,22 @@ export function parseProductionCapacityCellArguments(argv) {
) {
throw new Error('production capacity target origin is not exact')
}
const paceWindowMs = values['pace-window-ms'] === undefined
? 0
: Number(values['pace-window-ms'])
if (
!Number.isSafeInteger(paceWindowMs) ||
paceWindowMs < 0 ||
paceWindowMs > MAX_PACE_WINDOW_MS
) {
throw new Error('--pace-window-ms must be an integer between 0 and 300000')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellOrigin: expectedCellOrigin,
cellId,
mode: values.mode
mode: values.mode,
paceWindowMs
}
}
@@ -82,24 +96,39 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const postAt = async (origin, path, body) =>
await responseJson(
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
),
path
const postRaw = async (origin, path, body) =>
await fetchAdminOnceMore(
fetchImpl,
`${origin}${path}`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body)
},
{ wait: overrides.wait }
)
const postAt = async (origin, path, body) =>
await responseJson(await postRaw(origin, path, body), path)
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
if (config.mode === 'drain') {
const paceWindowMs = config.paceWindowMs ?? 0
if (paceWindowMs > 0) {
const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', {
v: 1,
graceMs: 0,
paceWindowMs
})
if (paced.ok) {
await paced.json().catch(() => ({}))
return { changed: false, drained: true, paceWindowMs }
}
// A cell still on an image without paced drain rejects the unknown field outright.
// An unpaced drain is the behaviour that cell already has, so fall back to it.
if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`)
await paced.json().catch(() => ({}))
}
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
return { changed: false, drained: true }
return { changed: false, drained: true, paceWindowMs: 0 }
}
const before = await inspectAdmissionSelector(post)
const state = selectorCellState(before.selector, config.cellId)
@@ -90,7 +90,8 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: 'https://c7.relay.onorca.dev',
cellId: 'production-gce-c7',
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -112,8 +113,12 @@ describe('production Relay capacity cell admission', () => {
]), /not approved/)
})
it('admits the same-cap Asia cells only under the same-cap allowlist', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => {
for (const cellId of [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
// Migration-only canaries: the US-only capacity rollout never touches them either.
'production-gce-c17', 'production-gce-c18'
]) {
const hostname = cellId.slice('production-gce-'.length)
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -125,10 +130,11 @@ describe('production Relay capacity cell admission', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname}.relay.onorca.dev`,
cellId,
mode: 'isolate'
mode: 'isolate',
paceWindowMs: 0
})
}
for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
for (const cellId of ['production-gce-c12', 'production-gce-c30']) {
const hostname = cellId.slice('production-gce-'.length)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -168,13 +174,77 @@ describe('production Relay capacity cell admission', () => {
{ ...config, mode: 'drain' },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0 }
}])
})
it('paces the drain send when the roll asks for a window', async () => {
const fake = canaryFetch()
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0, paceWindowMs: 120_000 }
}])
})
it('drains unpaced when the cell image rejects the pacing field', async () => {
const bodies = []
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async (url, init) => {
assert.equal(new URL(url).pathname, '/v1/admin/drain')
const body = JSON.parse(init.body)
bodies.push(body)
if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400)
return response({ v: 1, draining: true })
}
}
)
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
assert.deepEqual(bodies, [
{ v: 1, graceMs: 0, paceWindowMs: 120_000 },
{ v: 1, graceMs: 0 }
])
})
it('fails a paced drain that the cell rejects for any other reason', async () => {
await assert.rejects(
prepareProductionCapacityCell(
{ ...config, mode: 'drain', paceWindowMs: 120_000 },
{
token: 'token',
wait: async () => {},
fetch: async () => response({ error: 'invalid_token' }, 401)
}
),
/returned 401/
)
})
it('refuses a pacing window that is not a bounded integer', () => {
const argv = (value) => [
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c26.relay.onorca.dev',
'--cell-id', 'production-gce-c26',
'--mode', 'drain',
'--pace-window-ms', value
]
for (const value of ['-1', '300001', '1.5', 'soon']) {
assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/)
}
assert.equal(parseProductionCapacityCellArguments(argv('300000')).paceWindowMs, 300_000)
})
it('restores only the selected cell to general admission', async () => {
const fake = canaryFetch()
await prepareProductionCapacityCell(
@@ -228,7 +298,7 @@ describe('production Relay capacity cell admission', () => {
}
)
assert.equal(calls, 2)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
})
it('fails when both drain attempts return a transient 503', async () => {
@@ -2,14 +2,29 @@ import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without
// displacing anybody. It enters and must leave migration-only, never general.
export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18']
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
'production-gce-c27', 'production-gce-c28', 'production-gce-c29',
...SAME_CAP_MIGRATION_ONLY_CELLS
]
// A general cell's wave isolates and restores it, advancing the selector twice; a
// migration-only cell's isolate and restore are both no-ops, so its wave advances nothing.
export function selectorWaveDelta(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 0 : 2
}
export function entryAdmission(cellId) {
return SAME_CAP_MIGRATION_ONLY_CELLS.includes(cellId) ? 'migration-only' : 'general'
}
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
@@ -23,6 +38,11 @@ function cells(value) {
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
// Every later cell offsets from one per-wave selector delta, and the two classes
// have different ones, so a mixed wave has no single offset any cell could use.
if (new Set(parsed.map(selectorWaveDelta)).size > 1) {
throw new Error('same-cap wave cells must be all general or all migration-only')
}
return parsed
}
@@ -101,7 +121,7 @@ export function canaryAuthority(input) {
cellId: wave.cells[0],
targetDigest: wave.targetDigest,
rollbackDigest: wave.rollbackDigest,
selectorGeneration: selectorGeneration + 2,
selectorGeneration: selectorGeneration + selectorWaveDelta(wave.cells[0]),
rehomeGeneration,
// Audit trail, not authority: a batch reusing this canary is authorized by
// its own confirmation, so verification below neither requires nor forbids it.
@@ -114,6 +134,8 @@ export function canaryAuthority(input) {
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
// A mixed wave is already rejected, so the batch's first cell names the whole batch's class.
const batchAdmission = entryAdmission(cells(expected.cellIds ?? '')[0])
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
@@ -127,6 +149,14 @@ export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// A migration-only cell carries no hosts and a different cap, so rolling it proves nothing
// about a general batch, and its wave advances a different selector delta.
if (entryAdmission(authority.cellId) !== batchAdmission) {
throw new Error(
`canary authority cell ${authority.cellId} is ${entryAdmission(authority.cellId)}, ` +
`but this batch is ${batchAdmission}`
)
}
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
@@ -182,10 +212,20 @@ export function main(argv = process.argv.slice(2)) {
}))}\n`)
return
}
if (command === 'cell-class') {
const cellId = input['cell-id']
if (!SAME_CAP_CELLS.includes(cellId)) throw new Error('same-cap wave cells are invalid')
process.stdout.write(`${JSON.stringify({
entryAdmission: entryAdmission(cellId),
selectorWaveDelta: selectorWaveDelta(cellId)
})}\n`)
return
}
if (command === 'verify-canary') {
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
commitSha: input['commit-sha'],
runId: input['run-id'],
cellIds: input['cell-ids'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
selectorGeneration: input['selector-generation'],
@@ -1,14 +1,19 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { execFileSync, spawnSync } from 'node:child_process'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { test } from 'node:test'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
canaryAuthority,
entryAdmission,
main,
validateSameCapWave,
verifyCanaryAuthority
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
const targetDigest = `sha256:${'a'.repeat(64)}`
const rollbackDigest = `sha256:${'b'.repeat(64)}`
@@ -52,6 +57,82 @@ test('requires one canary or a bounded reviewed batch', () => {
}), /cells/)
})
test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)
assert.equal(entryAdmission(cellId), 'migration-only', cellId)
assert.deepEqual(validateSameCapWave({
mode: 'canary-apply',
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`
}).cells, [cellId])
}
const cellIds = 'production-gce-c17,production-gce-c18'
assert.deepEqual(validateSameCapWave({
mode: 'batch-apply',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
canaryRunId: '42'
}).cells, ['production-gce-c17', 'production-gce-c18'])
// A mixed wave has no single selector delta for its later cells to offset from.
const mixed = 'production-gce-c7,production-gce-c17'
assert.throws(() => validateSameCapWave({
mode: 'batch-apply',
cellIds: mixed,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed}`,
canaryRunId: '42'
}), /all general or all migration-only/)
})
test('seals a migration-only canary at the generation its wave leaves behind', () => {
const seal = (cellId) => canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
// Isolate and restore are both no-ops on a migration-only cell, so nothing advances.
assert.equal(seal('production-gce-c17').selectorGeneration, 11)
assert.equal(seal('production-gce-c7').selectorGeneration, 13)
// That canary still authorizes a later batch of its own class; it is evidence about the image.
assert.equal(verifyCanaryAuthority(seal('production-gce-c17'), {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c17,production-gce-c18',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
rehomeGeneration: '4'
}).cellId, 'production-gce-c17')
})
test('reports each approved cell\'s class and selector delta', () => {
const printed = []
const write = process.stdout.write.bind(process.stdout)
process.stdout.write = (chunk) => printed.push(String(chunk))
try {
main(['cell-class', '--cell-id', 'production-gce-c17'])
main(['cell-class', '--cell-id', 'production-gce-c7'])
} finally {
process.stdout.write = write
}
assert.deepEqual(printed.map((line) => JSON.parse(line)), [
{ entryAdmission: 'migration-only', selectorWaveDelta: 0 },
{ entryAdmission: 'general', selectorWaveDelta: 2 }
])
assert.throws(() => main(['cell-class', '--cell-id', 'production-gce-c12']), /cells are invalid/)
})
test('binds rollback confirmation to the exact digest and ordered cells', () => {
assert.throws(() => validateSameCapWave({
mode: 'rollback',
@@ -94,6 +175,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.equal(verifyCanaryAuthority(authority, {
commitSha: 'c'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
@@ -102,6 +184,7 @@ test('seals and verifies canary authority for later batches', () => {
assert.throws(() => verifyCanaryAuthority(authority, {
commitSha: 'd'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
@@ -116,8 +199,8 @@ test('reuses a canary across selector advances only within the same control epoc
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9',
targetDigest, rollbackDigest, selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
@@ -191,6 +274,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
commitSha,
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
@@ -313,6 +397,7 @@ test('seals the override into the canary authority as audit trail only', () => {
const expected = {
commitSha: 'f'.repeat(40),
runId: '42',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
selectorGeneration: '21',
@@ -337,3 +422,124 @@ test('seals the override into the canary authority as audit trail only', () => {
'production-gce-c7'
)
})
function sealedCanary(cellId) {
return canaryAuthority({
cellIds: cellId,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
}
// Why: a migration-only cell holds zero hosts at a different cap and its wave advances no
// selector, so rolling one is no evidence for a general batch, and the reverse is no evidence
// either. Nothing but the sealed cell id says which class a canary actually proved.
test('refuses a canary sealed on a cell of the other admission class', () => {
const expected = {
commitSha: 'c'.repeat(40),
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '99',
rehomeGeneration: '4'
}
const general = 'production-gce-c8,production-gce-c9'
const migrationOnly = SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: general
}),
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: migrationOnly
}),
/canary authority cell production-gce-c7 is general, but this batch is migration-only/
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c7'), {
...expected, cellIds: general
}).cellId,
'production-gce-c7'
)
assert.equal(
verifyCanaryAuthority(sealedCanary('production-gce-c17'), {
...expected, cellIds: migrationOnly
}).cellId,
'production-gce-c17'
)
// A caller that names no batch at all gets no verdict, rather than an unchecked class.
assert.throws(
() => verifyCanaryAuthority(sealedCanary('production-gce-c7'), expected),
/same-cap wave cells are invalid/
)
})
// The dispatch workflow is the only caller, so the class check only binds anything if that
// step actually hands the batch over; run the step's own shell exactly as written.
function verifyCanaryStepScript() {
const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
const first = ' node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \\\n'
const start = dispatch.indexOf(first)
assert.notEqual(start, -1, 'the dispatch workflow has no verify-canary step')
const last = ' --rehome-generation "${REHOME_GENERATION}"\n'
const end = dispatch.indexOf(last, start)
assert.notEqual(end, -1, 'the verify-canary step does not end at the rehome generation')
return dispatch.slice(start, end + last.length).replace(/^ {10}/gm, '')
}
async function runVerifyCanaryStep(authority, cellIds) {
const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-verify-'))
try {
await mkdir(join(temporary, 'relay-same-cap-canary'), { recursive: true })
await writeFile(
join(temporary, 'relay-same-cap-canary', 'authority.json'),
JSON.stringify(authority)
)
return spawnSync('bash', ['-euo', 'pipefail', '-c', verifyCanaryStepScript()], {
cwd: new URL('../..', import.meta.url),
env: {
...process.env,
RUNNER_TEMP: temporary,
GITHUB_SHA: authority.commitSha,
CANARY_RUN_ID: authority.runId,
CELL_IDS: cellIds,
TARGET_DIGEST: targetDigest,
ROLLBACK_DIGEST: rollbackDigest,
SELECTOR_GENERATION: '99',
REHOME_GENERATION: '4'
},
encoding: 'utf8'
})
} finally {
await rm(temporary, { recursive: true, force: true })
}
}
test('the batch gate hands its own cells to the canary check', async () => {
const accepted = await runVerifyCanaryStep(
sealedCanary('production-gce-c7'),
'production-gce-c8,production-gce-c9'
)
assert.equal(accepted.status, 0, accepted.stderr)
const crossed = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
'production-gce-c8,production-gce-c9'
)
assert.equal(crossed.status, 1, crossed.stdout)
assert.match(
crossed.stderr,
/canary authority cell production-gce-c17 is migration-only, but this batch is general/
)
const migrationOnly = await runVerifyCanaryStep(
sealedCanary('production-gce-c17'),
SAME_CAP_MIGRATION_ONLY_CELLS.join(',')
)
assert.equal(migrationOnly.status, 0, migrationOnly.stderr)
})
@@ -67,11 +67,11 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
)
assert.match(
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(
job,
@@ -87,7 +87,7 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_RESTORE\}"/)
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
assert.match(job, /FAILSAFE_GENERATION/)
@@ -98,14 +98,17 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
// Wave 0 must retry freshness-only failures too: one Cloud Monitoring publish
// lag at the sample instant is not health evidence, and single-shot wave 0
// failed a whole batch on a series that was fresh again a minute later.
assert.match(job, /dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" --retry-freshness/)
assert.match(
job,
/dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
assert.doesNotMatch(job, /RETRY_ARGS/)
// Break-glass: the override skips the aggregate 15-minute monitor evidence and
// nothing else. The live per-wave recheck still runs on the override path, off
// the dispatch inputs the rehome inspect below verifies against the director.
assert.match(
job,
/if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,120}?--wave-index "\$\{WAVE_INDEX\}" --retry-freshness/
/if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,160}?--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
// The override is re-validated here, not trusted from the caller, and it is
// bound to the digest this wave installs.
@@ -3,7 +3,12 @@ import { spawnSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { describe, it } from 'node:test'
import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs'
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
import {
SAME_CAP_CELLS,
SAME_CAP_MIGRATION_ONLY_CELLS,
entryAdmission,
selectorWaveDelta
} from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs'
@@ -15,6 +20,7 @@ const production = readFileSync(
)
const REHOME_SOURCE_CELLS = rehomeSourceCells()
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com'
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
@@ -33,16 +39,28 @@ function rehomeSourceCells() {
// The job cross-checks its pinned pool against the committed map; model the same read.
function tfvarsDatabasePoolMax(cellId) {
const start = production.indexOf(`"${cellId}" = {`)
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
const block = production.slice(start, production.indexOf('\n }', start))
return /database_pool_max\s*=\s*(\d+)/.exec(block)?.[1] ?? '10'
return tfvarsCellBlock(cellId).match(/database_pool_max\s*=\s*(\d+)/)?.[1] ?? '10'
}
function startupScript({ cap, image, trusted, pool }) {
function tfvarsHardCap(cellId) {
const cap = /connection_hard_cap\s*=\s*(\d+)/.exec(tfvarsCellBlock(cellId))?.[1]
assert.notEqual(cap, undefined, `${cellId} has no connection_hard_cap`)
return cap
}
function tfvarsCellBlock(cellId) {
const start = production.indexOf(`"${cellId}" = {`)
assert.notEqual(start, -1, `${cellId} is missing from production.tfvars`)
return production.slice(start, production.indexOf('\n }', start))
}
function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) {
return [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacityIdentity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]),
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -59,7 +77,11 @@ function startupScript({ cap, image, trusted, pool }) {
}
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
function rollPlan({ cellId, cap, protocol, pool }) {
function rollPlan({
cellId, cap, protocol, pool,
beforeCapacityIdentity = CAPACITY_IDENTITY,
afterCapacityIdentity = CAPACITY_IDENTITY
}) {
return {
configuration: {
root_module: {
@@ -90,7 +112,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
image: ROLLBACK_IMAGE,
trusted: protocol >= 1,
// The live template predates the reviewed pool raise, as every asia cell's does.
pool: pool === undefined ? undefined : '10'
pool: pool === undefined ? undefined : '10',
capacityIdentity: beforeCapacityIdentity
})
},
after: {
@@ -98,7 +121,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
cap,
image: TARGET_IMAGE,
trusted: protocol >= 1,
pool
pool,
capacityIdentity: afterCapacityIdentity
}),
self_link: null
},
@@ -145,6 +169,95 @@ function cellShape(cellId) {
return { cap: Number(cap), pool: pool.slice('pool='.length) || undefined }
}
// The class block runs before checkout-independent work and decides the whole wave shape.
function resolveCellClass(cellId) {
return spawnSync('bash', [
'-euo',
'pipefail',
'-c',
`${jobBlock(
' CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \\',
' SELECTOR_WAVE_DELTA="$(jq -er \'.selectorWaveDelta\' <<< "${CELL_CLASS}")"'
)}\necho "\${ENTRY_ADMISSION} \${SELECTOR_WAVE_DELTA}"`
], { cwd: new URL('../..', import.meta.url), env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
}
function drainingBlock() {
return `${jobBlock(
' # Rollback is the documented recovery from a failed canary, which',
' PREDECESSOR_DRAINING_OK=false\n fi'
)}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"`
}
// The three fields gcloud would otherwise default, as the MIG resource declares them.
function migUpdatePolicy() {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const policy = terraform.split(' update_policy {')[1]?.split('\n }')[0] ?? ''
const method = /replacement_method\s+= "([A-Z]+)"/.exec(policy)?.[1]
assert.notEqual(method, undefined, 'the MIG declares no replacement method')
// Both fixed bounds come from the topology locals the MIG resource points at.
const surgeLocal = /max_surge_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
const unavailableLocal =
/max_unavailable_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
assert.notEqual(surgeLocal, undefined, 'the MIG pins no surge local')
assert.notEqual(unavailableLocal, undefined, 'the MIG pins no unavailable local')
const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? ''
const local = (name) => {
const value = new RegExp(`${name}\\s+= (\\d+)`).exec(topology)?.[1]
assert.notEqual(value, undefined, `the topology locals pin no ${name}`)
return value
}
return {
replacementMethod: method.toLowerCase(),
maxSurge: local(surgeLocal),
maxUnavailable: local(unavailableLocal)
}
}
// The stage decides the predecessor, the plan's reviewed rollback image, and whether the
// MIG is rolled explicitly, so run the real block rather than restating its rule.
function stageBlock() {
return `${jobBlock(
' # Two different failures leave the cell on the rollback image, and the image',
' PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"\n fi'
)}\necho "\${ROLLBACK_STAGE} \${ROLLBACK_RESUME} \${PREDECESSOR_IMAGE_DIGEST}` +
` \${PREDECESSOR_REHOME_PROTOCOL} \${PLAN_ROLLBACK_IMAGE}"`
}
function generationBlock() {
return `${jobBlock(
' if test "${DEPLOY_MODE}" = verify; then',
' fi'
)}\necho "\${EFFECTIVE_SELECTOR_GENERATION}"`
}
// The job derives both memberships in one block; run that block alone for each class.
function membership(env) {
const script = `${jobBlock(
' RESTORED_MIGRATION_CELLS="$(jq -rn \\',
' fi'
)}\njq -cn --arg a "\${ISOLATED_MIGRATION_CELLS}" --arg b "\${ISOLATED_GENERAL_CELLS}" \\
--arg c "\${RESTORED_MIGRATION_CELLS}" --arg d "\${RESTORED_GENERAL_CELLS}" \\
'{isolatedMigration:$a,isolatedGeneral:$b,restoredMigration:$c,restoredGeneral:$d}'`
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
env: { ...process.env, ...env },
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
return JSON.parse(resolved.stdout)
}
function jobBlock(firstLine, lastLine) {
const start = workflow.indexOf(`${firstLine}\n`)
assert.notEqual(start, -1, `the job has no ${firstLine.trim()}`)
const end = workflow.indexOf(`\n${lastLine}\n`, start)
assert.notEqual(end, -1, `that block has no ${lastLine.trim()}`)
return workflow.slice(start, end + lastLine.length + 1).replace(/^ {10}/gm, '')
}
describe('same-cap roll scripts accept every same-cap cell', () => {
it('parses every wave cell through the same-cap canary allowlist', () => {
for (const cellId of SAME_CAP_CELLS) {
@@ -159,7 +272,8 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
cellId,
mode
mode,
paceWindowMs: 0
})
}
}
@@ -171,12 +285,13 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.match(
resolved.stdout.trim(),
/^(us-central1 1000 pool=|asia-east2 3000 pool=16)$/,
/^(us-central1 1000 pool=|us-central1 600 pool=|asia-east2 3000 pool=16)$/,
cellId
)
assert.equal(tfvarsDatabasePoolMax(cellId), cellShape(cellId).pool ?? '10', cellId)
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
}
assert.equal(resolveCellShape('production-gce-c17').status, 1)
assert.equal(resolveCellShape('production-gce-c12').status, 1)
assert.equal(resolveCellShape('production-gce-c30').status, 1)
})
@@ -188,10 +303,19 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const end = lines.findIndex((line) => !line.endsWith('\\'))
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--approved-cells same-cap/)
assert.match(call, /--mode (isolate|drain|activate)/)
// The restore call picks its mode from the cell's entry admission class.
assert.match(call, /--mode (isolate|drain|activate|"\$\{RESTORE_MODE\}")/)
}
})
it('paces the drain it sends to the selected cell', () => {
const drain = workflow.split('--mode drain')[1] ?? ''
assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
// The transition wait has to outlast the pacing window on top of the leases it waits on.
assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
})
it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
@@ -213,9 +337,15 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
// the rest before it plans, and the next test covers them at protocol 0.
assert.deepEqual(
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
SAME_CAP_MIGRATION_ONLY_CELLS
)
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const { cap, pool } = cellShape(cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -223,6 +353,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: String(protocol),
@@ -261,15 +392,16 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
const plan = rollPlan({ cellId, cap: 600, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
@@ -278,6 +410,304 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
)
})
it('resolves the class and selector delta the wave validator declares', () => {
for (const cellId of SAME_CAP_CELLS) {
const resolved = resolveCellClass(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.equal(
resolved.stdout.trim(),
`${entryAdmission(cellId)} ${selectorWaveDelta(cellId)}`,
cellId
)
}
assert.equal(resolveCellClass('production-gce-c12').status, 1)
})
it('offsets a later wave by this cell class\'s own selector delta', () => {
for (const [waveIndex, delta] of [['0', 2], ['3', 2], ['0', 0], ['3', 0]]) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', generationBlock()], {
env: {
...process.env,
DEPLOY_MODE: 'apply',
EXPECTED_SELECTOR_GENERATION: '40',
WAVE_INDEX: waveIndex,
SELECTOR_WAVE_DELTA: String(delta)
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, resolved.stderr)
assert.equal(resolved.stdout.trim(), String(40 + delta * Number(waveIndex)))
}
})
it('hands a migration-only cell back the exact membership it entered with', () => {
const entry = {
EXPECTED_MIGRATION_ONLY_CELLS: 'production-gce-c17,production-gce-c18',
EXPECTED_GENERAL_CELLS: 'production-gce-c7,production-gce-c8'
}
const isolated = membership({
...entry,
TARGET_CELL_ID: 'production-gce-c17',
ENTRY_ADMISSION: 'migration-only'
})
assert.deepEqual(isolated, {
isolatedMigration: 'production-gce-c17,production-gce-c18',
isolatedGeneral: 'production-gce-c7,production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
})
// A general cell still leaves migration-only and returns to general.
assert.deepEqual(
membership({
...entry,
TARGET_CELL_ID: 'production-gce-c7',
ENTRY_ADMISSION: 'general'
}),
{
isolatedMigration: 'production-gce-c17,production-gce-c18,production-gce-c7',
isolatedGeneral: 'production-gce-c8',
restoredMigration: 'production-gce-c17,production-gce-c18',
restoredGeneral: 'production-gce-c7,production-gce-c8'
}
)
})
it('never activates a migration-only cell and proves its isolate changed nothing', () => {
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /if test "\$\{ENTRY_ADMISSION\}" = migration-only; then\n\s+RESTORE_MODE=isolate/)
assert.match(restore, /--admission "\$\{ENTRY_ADMISSION\}"/)
// The pre-mutation check must demand the class the cell is declared to serve in.
assert.match(workflow, /PRECHECK_ADMISSION="\$\{ENTRY_ADMISSION\}"/)
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /migration-only; then\n\s+jq -e '\.changed == false'/)
})
it('requires rehome source membership exactly when a roll carries trust lines', () => {
const step = workflow
.split('name: Resolve immutable same-cap cell configuration')[1]
.split('\n - name:')[0]
const guard = step.indexOf('jq -e --arg cell "${TARGET_CELL_ID}" \'index($cell) != null\'')
assert.notEqual(guard, -1)
// The guard reads both protocols, so it has to sit after they are resolved.
assert.ok(step.indexOf('DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"') < guard)
assert.match(
step.slice(0, guard),
/test "\$\{DESIRED_REHOME_PROTOCOL\}" != 0 \|\| test "\$\{CURRENT_REHOME_PROTOCOL\}" != 0\n\s+\}; then\s+$/
)
})
it('rolls a template stale enough to predate the pinned capacity identity', () => {
// Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no
// capacity identity line, so the roll adds one. Run 35290908836 failed closed here.
const cellId = 'production-gce-c17'
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 600,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
capacityServiceAccount: CAPACITY_IDENTITY,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null })
assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 })
// The line may only be gained. A roll may not rewrite it,
assert.throws(
() => validateCapacityPlan(stale, {
...config,
capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com'
}),
/reviewed image and capacity/
)
// nor drop it from a template that already carries one.
assert.throws(
() => validateCapacityPlan(
rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }),
config
),
/reviewed image and capacity/
)
// A same-cap roll cannot run without the identity pinned at all.
assert.throws(
() => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }),
/invalid service account/
)
})
it('pins the capacity identity on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
for (const invocation of invocations) {
const lines = invocation.split('\n')
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
assert.match(
lines.slice(0, end + 1).join(' '),
/--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/
)
}
// Both steps must read it from the same repository variable the job already requires.
assert.equal(
workflow.split(
'CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}'
).length,
4
)
})
it('decides the predecessor draining rule from the real block, for both classes', () => {
// A zero-host cell sheds nothing, and a failed canary's own drain leaves the flag set
// with no restart behind it; run 35292335415 stopped on exactly that residue.
const cases = [
// mode, entry class, resume, expected [precheck admission, precheck draining, jq ok]
['apply', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['apply', 'general', 'false', ['general', 'forbidden', 'false']],
['verify', 'migration-only', 'false', ['migration-only', 'either', 'true']],
['verify', 'general', 'false', ['general', 'forbidden', 'false']],
// Every rollback path keeps exactly the behaviour it had.
['rollback', 'general', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'general', 'true', ['general-or-migration-only', 'either', 'false']],
['rollback', 'migration-only', 'false', ['general-or-migration-only', 'either', 'true']],
['rollback', 'migration-only', 'true', ['general-or-migration-only', 'either', 'false']]
]
for (const [mode, entry, resume, expected] of cases) {
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', drainingBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
ENTRY_ADMISSION: entry,
ROLLBACK_RESUME: resume
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${entry}/${resume}: ${resolved.stderr}`)
assert.deepEqual(
resolved.stdout.trim().split(' '),
expected,
`${mode}/${entry}/${resume}`
)
}
})
it('reads one draining decision in both predecessor checks', () => {
const step = workflow
.split('name: Verify exact current generation, digest, cap, and rollback point')[1]
.split('\n - name:')[0]
// The jq assertion and its diagnostic must not be able to disagree.
assert.equal(step.split('--argjson drainingOk "${PREDECESSOR_DRAINING_OK}"').length, 3)
assert.doesNotMatch(step, /drainingOk "\$\(test/)
// The fresh VM is still required not to be draining, on every path.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /--admission migration-only --draining forbidden/)
const restore = workflow
.split('name: Restore only the verified selected cell to its entry admission')[1]
.split('\n - id:')[0]
assert.match(restore, /--draining forbidden --activity allowed/)
})
it('classifies every rollback stage from the real block', () => {
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
const target = `sha256:${'7'.repeat(64)}`
const rollback = `sha256:${'0'.repeat(64)}`
const stage = (mode, live, draining) => {
// Exactly how the job assigns them: rollback swaps desired and current.
const desired = mode === 'rollback' ? rollback : target
const current = mode === 'rollback' ? target : rollback
const resolved = spawnSync('bash', ['-euo', 'pipefail', '-c', stageBlock()], {
env: {
...process.env,
DEPLOY_MODE: mode,
CURRENT_RUNTIME: JSON.stringify({ imageDigest: live, draining }),
DESIRED_IMAGE_DIGEST: desired,
CURRENT_IMAGE_DIGEST: current,
DESIRED_IMAGE: `${repository}@${desired}`,
IMAGE_REPOSITORY: repository,
DESIRED_REHOME_PROTOCOL: '1',
CURRENT_REHOME_PROTOCOL: '0'
},
encoding: 'utf8'
})
assert.equal(resolved.status, 0, `${mode}/${live}/${draining}: ${resolved.stderr}`)
return resolved.stdout.trim().split(' ')
}
const roll = (current, protocol) =>
['roll', 'false', current, protocol, `${repository}@${current}`]
// Only the last row differs from main: it used to read `resume` and wedge, because the
// resume path refuses a draining cell and never restarts one.
assert.deepEqual(stage('apply', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('apply', rollback, true), roll(rollback, '0'))
assert.deepEqual(stage('apply', target, false), roll(rollback, '0'))
assert.deepEqual(stage('verify', rollback, false), roll(rollback, '0'))
assert.deepEqual(stage('rollback', target, false), roll(target, '0'))
assert.deepEqual(stage('rollback', target, true), roll(target, '0'))
assert.deepEqual(
stage('rollback', rollback, false),
['resume', 'true', rollback, '1', `${repository}@${target}`]
)
assert.deepEqual(
stage('rollback', rollback, true),
['stranded', 'false', rollback, '1', `${repository}@${rollback}`]
)
// A runtime that reports no drain flag at all must never read as stranded.
const [missing] = stage('rollback', rollback, null)
assert.equal(missing, 'resume')
})
it('rolls the MIG itself when a stranded plan changes nothing', () => {
const apply = workflow
.split('name: Apply only the selected same-cap template and MIG')[1]
.split('\n - id:')[0]
// The plan is reviewed against the image the cell serves, not an assumed predecessor.
assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/)
assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/)
assert.match(
apply,
/test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0/
)
// gcloud persists all three fields into the MIG's update policy and defaults the
// method to substitute here, so every one has to match what Terraform declares or the
// recovery drifts the policy and the next targeted plan is refused as an unreviewed
// MIG change. Read the declared values rather than restating them.
assert.match(apply, /rolling-action replace "\$\{MIG_NAME\}"/)
const declared = migUpdatePolicy()
assert.deepEqual(declared, {
replacementMethod: 'recreate',
maxSurge: '0',
maxUnavailable: '1'
})
assert.match(
apply,
new RegExp(
`--replacement-method ${declared.replacementMethod}` +
` --max-surge ${declared.maxSurge} --max-unavailable ${declared.maxUnavailable}`
)
)
// Nothing else may reach the group, and the roll has to be waited on.
assert.equal(apply.split('rolling-action').length, 2)
assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3)
})
it('waits on the image a stranded cell actually serves', () => {
const isolate = workflow
.split('name: Reversibly isolate and drain only the selected cell')[1]
.split('\n - id:')[0]
assert.match(isolate, /--expected-image-digests "\$\{PREDECESSOR_IMAGE_DIGEST\}"/)
// A stranded cell has to come back on a new process, which is what clears the drain.
const after = workflow
.split('name: Verify new incarnation, exact image, protocol, and durable safety')[1]
.split('\n - name:')[0]
assert.match(after, /test "\$\{TARGET_INCARNATION\}" != "\$\{SOURCE_INCARNATION\}"/)
assert.match(after, /if test "\$\{ROLLBACK_RESUME\}" = true; then/)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
@@ -47,7 +47,10 @@ export function parseCapacityPlanArguments(argv) {
return value
}
if (!values.image) throw new Error('missing --image')
if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) {
if (
['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
!values['capacity-service-account']
) {
throw new Error('missing --capacity-service-account')
}
if (
@@ -239,7 +242,10 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
]
]
if (config.mode === 'bootstrap-cell') {
// A same-cap cell whose template predates this line gains it on its next roll, so the
// before/after comparison ignores it; pinning the exact identity here is what reviews it,
// and what stops a roll dropping or rewriting the line it lets through.
if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
expected.push([
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
@@ -454,18 +460,22 @@ function cellPlan(plan, changes, config) {
const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
// Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
// Only a template stale enough to predate the line may move it, and only by gaining it;
// requireDesiredStartupScript holds the after value to the exact reviewed identity.
const stripCapacityIdentity =
['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
if (
typeof beforeScript !== 'string' ||
(sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
normalizedStartupScript(
beforeScript,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap,
stripPool
) !== normalizedStartupScript(
script,
config.mode === 'bootstrap-cell',
stripCapacityIdentity,
config.mode === 'same-cap-cell',
sameCap,
stripPool
@@ -501,7 +511,7 @@ export function validateCapacityPlan(plan, config) {
throw new Error('capacity Terraform plans may change only a cell')
}
if (
config.mode === 'bootstrap-cell' &&
['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
!SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
) {
throw new Error('capacity Terraform plan has an invalid service account')
@@ -428,10 +428,14 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, cap = 1_000, trust = false }) => [
const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
@@ -468,6 +472,7 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
@@ -653,10 +658,12 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
@@ -693,6 +700,7 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
@@ -737,6 +745,133 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
}
})
test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, capacity }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(capacity === null
? []
: [` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const plan = (beforeCapacity, afterCapacity) => ({
resource_changes: [
{
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startup({
selectedImage: rollbackImage,
capacity: beforeCapacity
})
},
after: {
metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
})
const config = {
cellId: 'production-gce-c17',
hardCap: 600,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
// A template old enough to predate the line gains it, which is the only move allowed.
assert.deepEqual(
validateCapacityPlan(plan(null, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
assert.deepEqual(
validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
{ mode: 'same-cap-cell', changes: 2 }
)
for (const [before, after] of [
[capacityIdentity, null],
[null, null],
[capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
[null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
]) {
assert.throws(
() => validateCapacityPlan(plan(before, after), config),
/reviewed image and capacity/,
`${before} -> ${after}`
)
}
// Without the pin there is nothing reviewing the line the comparison now ignores.
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: undefined
}),
/invalid service account/
)
assert.throws(
() => validateCapacityPlan(plan(null, capacityIdentity), {
...config,
capacityServiceAccount: 'not-an-email'
}),
/invalid service account/
)
})
test('the capacity identity argument is required by same-cap-cell mode', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const base = [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c17',
'--hard-cap', '600',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '0'
]
assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
assert.throws(
() => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
/--capacity-service-account is invalid/
)
assert.equal(
parseCapacityPlanArguments([
...base,
'--capacity-service-account',
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
]).capacityServiceAccount,
'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
)
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
@@ -747,6 +882,7 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
@@ -788,10 +924,12 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, pool }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
...(pool === undefined
? []
: [` printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -837,6 +975,7 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
mode: 'same-cap-cell',
image,
rollbackImage,
capacityServiceAccount: capacityIdentity,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '1'
@@ -891,6 +1030,7 @@ test('the database pool argument is accepted by same-cap-cell mode alone', () =>
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
'--regional-rehome-protocol', '1',
+61 -1
View File
@@ -374,7 +374,8 @@ remain general at 1,000/60. The workflow lock, single-use evidence marker, exact
targeted Terraform plan, and per-cell heartbeat/admission oracle are unchanged.
`Deploy Relay Production Same-Cap` rolls only the reviewed US 1,000/60 and Asia 3,000/60 serving
sets without changing a cell's connection shape. Use `canary-apply` for exactly one cell. A successful canary
sets and the two migration-only US 600/60 cells, C17 and C18, without changing a cell's connection
shape. Use `canary-apply` for exactly one cell. A successful canary
seals its commit, target and rollback digests, selector generation, and durable rehome generation;
`batch-apply` accepts only that same authority and rolls two to four cells sequentially. Each cell is
isolated, drained to two restart-safe samples, replaced from a targeted saved plan, and restored only
@@ -384,6 +385,65 @@ director; the workflow never receives or mints a director or stamped-cell runtim
keeps only the selected cell migration-only, while the exact rollback digest remains dispatchable via
the same workflow's `rollback` mode.
A roll holds the whole startup script identical before and after except the image, so a
template stale enough to predate a pinned line fails closed rather than absorbing the drift.
The one exception is the capacity identity: a cell that predates it gains it on its next roll,
and the plan validator pins the exact reviewed identity instead of comparing that line, so a
roll can never drop or rewrite it. Any other stale line still fails closed and needs a
convergence apply first.
A drained cell is refused before a roll, because draining means something is already
shedding its connections. A migration-only cell has none to shed, so the flag decides nothing
there and is accepted on entry; the replacement VM is still required not to be draining, and
the incarnation check still proves it was replaced. That also unwedges the state a failed
canary leaves behind, where the wave's own drain set the flag and no restart followed.
C17 and C18 hold no hosts and are not general, so rolling one displaces nobody: they are the
zero-displacement canary for a new image. Their wave enters and leaves migration-only, so its
isolate and its restore are both no-ops and the selector generation does not move; a general
cell's wave still advances it by two. One wave may not mix the two classes, because every cell
after the first offsets from a single per-wave delta. Neither cell is a declared regional-rehome
source, so its template carries no rehome trust lines and it may roll only at rehome protocol `0`;
the job refuses a trusted protocol for it before it plans anything.
### Recovering a wave that died after its drain
A cell's drain flag is a one-way latch on the running process. Only a restart clears it, and
the failsafe that isolates a failed cell does not restart anything. So a wave that stopped
any time after its drain step leaves the cell migration-only and draining, and it stays that
way until the cell is rolled.
Read the failed run before dispatching anything. If its log has a
`"event":"relay_production_capacity_canary","mode":"drain"` line for the cell, the cell is
drained. Then read the cell's live runtime image from
`POST https://<hostname>.relay.onorca.dev/v1/admin/runtime-status`.
1. **Do not re-dispatch `apply`.** It requires the cell general and not draining, and a
drained cell is neither. It will fail closed at the predecessor check.
2. **Dispatch `rollback`,** with the same `target-image-digest` and `rollback-image-digest`
the failed wave used, the live selector generation, and the live tri-state membership
with the failed cell listed under migration-only. The confirmation is
`ROLL_BACK_RELAY_SAME_CAP <rollback-digest> <cell-id>`.
3. The job classifies the cell itself and needs no extra input:
- serving the **rollback** image and draining, it is `stranded`. The wave stopped before
or during its template apply. The job re-isolates, re-drains, applies the reviewed
template, and rolls the MIG explicitly if that template was already in place. The cell
comes back on a new instance, so the drain clears, and it is restored to its entry class.
- serving the **target** image, it is `roll`, the ordinary rollback. The template applied
and the instance was replaced.
- serving the **rollback** image and not draining, it is `resume`: a rollback that failed
after its own template apply. Nothing is applied and nothing restarts.
4. Rollback takes exactly one cell per dispatch. Recover the cells one at a time.
5. If the run died inside `wait-until stable`, the MIG is still rolling on its own. Wait for
it to settle and re-read the runtime before dispatching, or the stage will be read off a
state that is about to change.
6. A `stranded` dispatch that fails at plan review means the template already carries the
target image while the old instance is still up. Wait for the MIG to finish replacing it,
then dispatch again; it will classify as `roll`.
A mutating dispatch still needs a fresh aggregate monitor dry-run unless the break-glass
override below is used.
### Gate override (break-glass)
Every mutating same-cap wave normally consumes a fresh 15-minute aggregate monitor dry-run.
@@ -60,6 +60,10 @@ locals {
control_renewal_latency_ms_p50 = { field = "controlRenewalLatencyMsP50", description = "Control renewal latency p50 in the interval." }
control_renewal_latency_ms_p95 = { field = "controlRenewalLatencyMsP95", description = "Control renewal latency p95 in the interval." }
control_renewal_latency_ms_max = { field = "controlRenewalLatencyMsMax", description = "Maximum control renewal latency in the interval." }
control_renewal_flushes = { field = "controlRenewalFlushesDelta", description = "Batched control-renewal statements issued in the interval, one per cell per flush window." }
control_renewal_flush_rows_max = { field = "controlRenewalFlushRowsMax", description = "Largest number of hosts renewed by a single statement in the interval; the row ceiling is what bounds how long one flush holds its row locks." }
control_renewal_flush_ms_p95 = { field = "controlRenewalFlushLatencyMsP95", description = "Batched control-renewal statement duration p95 in the interval. Row locks live until the statement commits, so this is the lock hold." }
control_renewal_flush_ms_max = { field = "controlRenewalFlushLatencyMsMax", description = "Maximum batched control-renewal statement duration in the interval." }
control_renewals = { field = "controlRenewalsDelta", description = "Control renewal attempts in the interval." }
control_renewal_successes = { field = "controlRenewalSuccessesDelta", description = "Successful control renewals in the interval." }
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
@@ -84,7 +84,7 @@ describe('applyPostgresSchema classification', () => {
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(3)
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('treats an already-applied constraint as skipped rather than an error', async () => {
@@ -94,7 +94,110 @@ describe('applyPostgresSchema classification', () => {
throw postgresError('42710')
})
const summary = await applyPostgresSchema(['ALTER TABLE t ADD CONSTRAINT c CHECK (x > 0)'], query)
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('treats an index another director already dropped as skipped rather than an error', async () => {
// Every director boots at once on a deploy and all of them send the same DROP INDEX IF EXISTS.
// Only one can win; the losers must not fail their boot over a drop that already happened.
const query = vi.fn(async () => {
throw postgresError('42704')
})
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(query).toHaveBeenCalledTimes(1)
})
it('still propagates 42704 from a statement that is not a DROP IF EXISTS', async () => {
// Keeps the case above narrow: an undefined object anywhere else is a real boot failure.
const query = vi.fn(async () => {
throw postgresError('42704')
})
await expect(
applyPostgresSchema(['ALTER TABLE t ADD COLUMN IF NOT EXISTS c BIGINT'], query)
).rejects.toThrow(/42704/)
})
it('leaves a deferrable statement unapplied on a lock timeout instead of failing the boot', async () => {
// The crash loop this prevents: 28 directors reach the same DROP INDEX at once on a table
// under continuous write, all of them time out, and every one restarts to re-queue the same
// DDL behind the same writers.
const warned: string[] = []
vi.spyOn(console, 'warn').mockImplementation((line: string) => {
warned.push(line)
})
const query = vi.fn(async () => {
throw postgresError('55P03')
})
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'],
query
)
expect(summary).toEqual({ ran: 0, skipped: 0, deferred: 1 })
expect(query).toHaveBeenCalledTimes(1)
const event = JSON.parse(warned[warned.length - 1] ?? '{}')
expect(event.event).toBe('orca_relay_postgres_schema_object_deferred')
expect(event.code).toBe('55P03')
expect(event.name).toBe('i')
})
it('runs the statements after a deferral, rather than abandoning the boot at that point', async () => {
// A deferral is not a failure, so nothing behind it may be skipped: the schema still has
// tables to create, and a boot that stopped here would come up against a partial schema.
const sent: string[] = []
const query = vi.fn(async (statement: string) => {
sent.push(statement)
if (statement.includes('DROP INDEX')) throw postgresError('55P03')
return undefined
})
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const summary = await applyPostgresSchema(
['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i', 'CREATE TABLE IF NOT EXISTS t (id TEXT)'],
query
)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 1 })
expect(sent).toHaveLength(2)
})
it('still fails the boot on a lock timeout for a statement that is not marked deferrable', async () => {
// Keeps the marker meaningful. An unmarked statement retains the old contract: fail once and
// loudly, because retrying parks every writer behind the same queue again.
vi.spyOn(console, 'error').mockImplementation(() => undefined)
const query = vi.fn(async () => {
throw postgresError('55P03')
})
await expect(applyPostgresSchema(['DROP INDEX IF EXISTS i'], query)).rejects.toMatchObject({
code: '55P03'
})
})
it('defers only on a lock timeout, not on any other error from a deferrable statement', async () => {
// A deferrable statement is not a statement whose failures stop mattering. A permission error
// is still a boot failure.
const query = vi.fn(async () => {
throw postgresError('42501')
})
await expect(
applyPostgresSchema(['-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'], query)
).rejects.toThrow(/42501/)
})
it('asks the catalog for a dropped index by name and skips the DROP once it is gone', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery, asked } = catalogAnswers([])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
// One parameter, the index name: the statement names no table, and the query references no $2.
expect(asked).toEqual([[expect.stringContaining("relkind = 'i'"), 'i']])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('sends the DROP while the index is still there, which is the boot that has to win', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const { catalogQuery } = catalogAnswers([{}])
const summary = await applyPostgresSchema(['DROP INDEX IF EXISTS i'], query, { catalogQuery })
expect(query).toHaveBeenCalledTimes(1)
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('propagates an unrelated error without retrying', async () => {
@@ -168,7 +271,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
expect(asked).toEqual([
[expect.stringContaining('pg_catalog.pg_index'), 'relay_connection_bases', 'relay_bases_active']
])
expect(summary).toEqual({ ran: 1, skipped: 1 })
expect(summary).toEqual({ ran: 1, skipped: 1, deferred: 0 })
})
it('skips an index the catalog reports as invalid rather than rebuilding it', async () => {
@@ -206,7 +309,8 @@ describe('applyPostgresSchema catalog pre-check', () => {
expect(JSON.parse(logged[logged.length - 1] ?? '{}')).toEqual({
event: 'orca_push_postgres_schema_applied',
ran: 1,
skipped: 1
skipped: 1,
deferred: 0
})
})
@@ -219,7 +323,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
[expect.stringContaining('pg_catalog.pg_attribute'), 'relay_control_capabilities', 'idle']
])
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('never probes the catalog for a statement that takes no relation lock', async () => {
@@ -247,7 +351,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
]
])
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('skips a DROP CONSTRAINT IF EXISTS when the constraint is already gone', async () => {
@@ -265,7 +369,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
{ catalogQuery }
)
expect(query).not.toHaveBeenCalled()
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
expect(logged).toContainEqual({
event: 'orca_relay_postgres_schema_object_absent',
kind: 'constraint',
@@ -281,7 +385,7 @@ describe('applyPostgresSchema catalog pre-check', () => {
const statement = 'ALTER TABLE t DROP CONSTRAINT IF EXISTS region_check'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends an ADD CONSTRAINT the catalog does not name yet', async () => {
@@ -290,14 +394,14 @@ describe('applyPostgresSchema catalog pre-check', () => {
const statement = 'ALTER TABLE t ADD CONSTRAINT region_valid CHECK (r IN (1))'
const summary = await applyPostgresSchema([statement], query, { catalogQuery })
expect(query.mock.calls.map(([sql]) => sql)).toEqual([statement])
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('sends every statement when no catalog query is supplied', async () => {
const query = vi.fn(async (_statement: string) => undefined)
const summary = await applyPostgresSchema([COMMENTED_TABLE, COMMENTED_INDEX], query)
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 2, skipped: 0 })
expect(summary).toEqual({ ran: 2, skipped: 0, deferred: 0 })
})
})
@@ -316,7 +420,7 @@ describe('applyPostgresSchema concurrent creates', () => {
})
expect(query).toHaveBeenCalledTimes(1)
expect(asked).toHaveLength(2)
expect(summary).toEqual({ ran: 0, skipped: 1 })
expect(summary).toEqual({ ran: 0, skipped: 1, deferred: 0 })
})
it('still retries when the catalog says the object is not there after all', async () => {
@@ -332,7 +436,7 @@ describe('applyPostgresSchema concurrent creates', () => {
wait: async () => undefined
})
expect(query).toHaveBeenCalledTimes(2)
expect(summary).toEqual({ ran: 1, skipped: 0 })
expect(summary).toEqual({ ran: 1, skipped: 0, deferred: 0 })
})
it('retries a CREATE TABLE collision without a catalog re-ask, having no target to ask about', async () => {
@@ -25,7 +25,7 @@ export type SchemaStartupOptions = {
wait?: (delayMs: number) => Promise<void>
}
export type SchemaApplySummary = { ran: number; skipped: number }
export type SchemaApplySummary = { ran: number; skipped: number; deferred: number }
function retryDelayMs(attempt: number, random: () => number): number {
const ceiling = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
@@ -39,6 +39,17 @@ function wait(delayMs: number): Promise<void> {
const CREATE_TABLE_IF_NOT_EXISTS = /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
const CREATE_INDEX_IF_NOT_EXISTS = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
const ALTER_TABLE_ADD_CONSTRAINT = /^ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
const DROP_INDEX_IF_EXISTS = /^DROP\s+INDEX\s+(?:CONCURRENTLY\s+)?IF\s+EXISTS\b/i
// Marked in the schema text, beside the SQL it applies to, and read from the raw statement because
// classification strips comments. Says: this boot may leave the statement unapplied rather than
// fail. Only sound for a statement that is idempotent AND that nothing this boot goes on to do
// depends on, because the database is then simply as it was and the next boot re-sends it.
const DEFERRABLE = /^\s*--[^\n]*\bschema-deferrable\b/
export function schemaDeferrable(statement: string): boolean {
return DEFERRABLE.test(statement)
}
// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
@@ -70,6 +81,14 @@ function constraintAlreadyApplied(error: unknown, sql: string): boolean {
)
}
// `IF EXISTS` resolves the name, then locks; between those two steps another director's drop can
// commit and the loser raises 42704 instead of the notice it would have got a moment later. Every
// director boots at once on a deploy, so without this the losers fail their boot over a drop that
// already happened.
function dropAlreadyApplied(error: unknown, sql: string): boolean {
return DROP_INDEX_IF_EXISTS.test(sql) && (error as { code?: unknown } | null)?.code === '42704'
}
function retryableSchemaError(error: unknown, sql: string): boolean {
const value = (error as { code?: unknown; constraint?: unknown } | null) ?? {}
return RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, sql)
@@ -90,7 +109,7 @@ async function nothingToDo(
JSON.stringify({
event: `${eventPrefix}_object_${target.skipWhen}`,
kind: target.kind,
table: target.table,
table: target.kind === 'index-by-name' ? undefined : target.table,
name: target.name,
indisvalid: presence.indisvalid
})
@@ -108,7 +127,7 @@ export async function applyPostgresSchema(
const random = options.random ?? Math.random
const pause = options.wait ?? wait
const deadlineAt = now() + (options.retryDeadlineMs ?? DEFAULT_RETRY_DEADLINE_MS)
const summary: SchemaApplySummary = { ran: 0, skipped: 0 }
const summary: SchemaApplySummary = { ran: 0, skipped: 0, deferred: 0 }
for (const statement of statements) {
// Throws when an index or column statement's target cannot be read, rather than sending it
@@ -126,7 +145,7 @@ export async function applyPostgresSchema(
summary.ran += 1
break
} catch (error) {
if (constraintAlreadyApplied(error, sql)) {
if (constraintAlreadyApplied(error, sql) || dropAlreadyApplied(error, sql)) {
summary.skipped += 1
break
}
@@ -135,6 +154,24 @@ export async function applyPostgresSchema(
// this boot lost the queue. Relation locks are granted in queue order, so each retry parks
// every writer behind it again for another timeout. Fail once, loudly.
if (code === LOCK_NOT_AVAILABLE && !options.retryLockTimeout) {
// A deferrable statement yields the queue instead of crash-looping the instance. Every
// director boots at once on a migration, so a table under continuous write can hand the
// whole fleet a lock timeout on the one statement that has to win once; failing the boot
// for it restarts the instance, which re-queues the same DDL behind the same writers.
if (schemaDeferrable(statement)) {
console.warn(
JSON.stringify({
event: `${eventPrefix}_object_deferred`,
code,
kind: target?.kind,
name: target?.name,
statement: sql.split('\n')[0],
detail: 'could not take its lock; left unapplied for the next boot to retry'
})
)
summary.deferred += 1
break
}
console.error(
JSON.stringify({
event: `${eventPrefix}_lock_timeout`,
@@ -26,10 +26,26 @@ WHERE attrelid = to_regclass($1) AND attname = $2 AND attnum > 0 AND NOT attisdr
const CONSTRAINT_PRESENT = `SELECT 1 FROM pg_catalog.pg_constraint
WHERE conrelid = to_regclass($1) AND conname = $2`
// By name through the search_path, with no table condition, because a DROP INDEX has no table to
// condition on and does not need one: a name that resolves to no visible index is nothing to drop.
// `relkind = 'i'` keeps a same-named table or view from answering for an index. Partitioned indexes
// are 'I', which this deliberately does not match - relay has none, and dropping one is not a
// boot-time operation.
const INDEX_BY_NAME_PRESENT = `SELECT 1 FROM pg_catalog.pg_class c
WHERE c.relname = $1 AND c.relkind = 'i' AND pg_catalog.pg_table_is_visible(c.oid)`
// reloptions is a text[] of `name=value` pairs, absent entirely while the option is at its
// default. Comparing the whole pair is what makes a changed value re-run: `@>` on a different
// value answers no, and the statement runs and overwrites it.
const RELOPTION_PRESENT = `SELECT 1 FROM pg_catalog.pg_class
WHERE oid = to_regclass($1) AND reloptions @> ARRAY[$2]`
const PRESENCE_SQL = {
index: INDEX_PRESENT,
column: COLUMN_PRESENT,
constraint: CONSTRAINT_PRESENT
constraint: CONSTRAINT_PRESENT,
reloption: RELOPTION_PRESENT,
'index-by-name': INDEX_BY_NAME_PRESENT
} as const
export type SchemaCatalogPresence = { present: boolean; indisvalid: unknown }
@@ -42,7 +58,10 @@ export async function catalogObjectPresence(
target: SchemaLockTarget
): Promise<SchemaCatalogPresence> {
const sql = PRESENCE_SQL[target.kind]
const rows = await query(sql, [target.table, target.name])
// The name-only lookup binds one parameter; every other shape binds the table first. Passing a
// parameter the SQL never references is a bind error, not a harmless extra.
const params = target.kind === 'index-by-name' ? [target.name] : [target.table, target.name]
const rows = await query(sql, params)
const row = rows[0]
return row ? { present: true, indisvalid: row.indisvalid } : { present: false, indisvalid: undefined }
}
@@ -1,5 +1,6 @@
export {
applyPostgresSchema,
schemaDeferrable,
type SchemaApplySummary,
type SchemaStartupOptions
} from './apply-postgres-schema.js'
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
import { schemaDeferrable } from './apply-postgres-schema.js'
import {
requireSchemaLockTarget,
schemaLockTarget,
@@ -417,3 +418,105 @@ describe('dollar-quoted bodies', () => {
)
})
})
describe('schemaLockTarget storage parameters', () => {
it('reads a storage parameter as a name=value target the catalog can be asked about', () => {
expect(schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')).toEqual({
kind: 'reloption',
table: 't',
name: 'fillfactor=70',
skipWhen: 'present'
})
})
it('folds the option name but keeps the value as written, the way pg_class stores the pair', () => {
expect(schemaLockTarget('ALTER TABLE t SET (FillFactor=70)')?.name).toBe('fillfactor=70')
})
it('makes a changed value a different target, so it re-runs instead of skipping', () => {
// The failure this prevents: matching on the option name alone would read `fillfactor=100` as
// already satisfying `fillfactor = 70` and skip the statement for the life of the database.
const seventy = schemaLockTarget('ALTER TABLE t SET (fillfactor = 70)')
const eighty = schemaLockTarget('ALTER TABLE t SET (fillfactor = 80)')
expect(seventy?.name).not.toBe(eighty?.name)
})
it('refuses a multi-option SET rather than skipping on only the first option', () => {
// Same reason a multi-action ALTER TABLE is refused: skipping on one option would silently
// drop the others for good.
expect(() =>
requireSchemaLockTarget('ALTER TABLE t SET (fillfactor = 70, autovacuum_enabled = false)')
).toThrow(/unparsed_schema_lock_target/)
})
it('fails the boot on a SET whose shape it cannot read, rather than sending it unchecked', () => {
// A storage parameter takes a relation lock, so no target means the lock is taken on every
// boot. RESET has no value to compare and is not supported.
expect(() => requireSchemaLockTarget('ALTER TABLE t RESET (fillfactor)')).not.toThrow()
expect(() => requireSchemaLockTarget('ALTER TABLE t SET (fillfactor)')).toThrow(
/unparsed_schema_lock_target/
)
})
it('takes a relation lock, so the census requires it to carry a target', () => {
expect(takesRelationLock('ALTER TABLE t SET (fillfactor = 70)')).toBe(true)
})
})
describe('schemaLockTarget dropped indexes', () => {
it('resolves a dropped index by name, with no table to name', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS i')).toEqual({
kind: 'index-by-name',
name: 'i',
skipWhen: 'absent'
})
})
it('reads CONCURRENTLY as a modifier rather than the index name', () => {
expect(schemaLockTarget('DROP INDEX CONCURRENTLY IF EXISTS i')?.name).toBe('i')
})
it('folds an unquoted name and keeps a quoted one, the way relname stores it', () => {
expect(schemaLockTarget('DROP INDEX IF EXISTS MyIndex')?.name).toBe('myindex')
expect(schemaLockTarget('DROP INDEX IF EXISTS "MyIndex"')?.name).toBe('MyIndex')
})
it('takes a relation lock, because the index is there on the boot that has to drop it', () => {
expect(takesRelationLock('DROP INDEX IF EXISTS i')).toBe(true)
})
it('requires IF EXISTS, so a bare DROP fails the boot instead of running unchecked', () => {
// Same contract as DROP CONSTRAINT: a bare DROP on a missing index is an error the server is
// supposed to raise, and a pre-check that skipped it would swallow that.
expect(() => requireSchemaLockTarget('DROP INDEX i')).toThrow(/unparsed_schema_lock_target/)
})
it('refuses a multi-index DROP rather than pre-checking only the first name', () => {
// Skipping on one name would leave the other index in place for the life of the database.
expect(() => requireSchemaLockTarget('DROP INDEX IF EXISTS a, b')).toThrow(
/unparsed_schema_lock_target/
)
})
it('derives the target through a leading deferrable marker', () => {
// The real shape in relay's schema: the marker is a comment, so classification must see past
// it or the statement would reach the server with no pre-check at all.
const statement = '-- schema-deferrable: reason\nDROP INDEX IF EXISTS i'
expect(sqlWithoutComments(statement)).toBe('DROP INDEX IF EXISTS i')
expect(schemaLockTarget(statement)?.name).toBe('i')
})
})
describe('schemaDeferrable', () => {
it('reads the marker only from a leading comment, never from the SQL body', () => {
// A name or a string containing the word must not make a statement deferrable.
expect(schemaDeferrable('-- schema-deferrable: reason\nDROP INDEX IF EXISTS i')).toBe(true)
expect(schemaDeferrable('DROP INDEX IF EXISTS schema_deferrable')).toBe(false)
expect(schemaDeferrable("CREATE TABLE t (c TEXT DEFAULT 'schema-deferrable')")).toBe(false)
})
it('treats an unmarked statement as fatal on a lock timeout, which is the default', () => {
expect(schemaDeferrable('DROP INDEX IF EXISTS i')).toBe(false)
expect(schemaDeferrable('ALTER TABLE t SET (fillfactor = 70)')).toBe(false)
})
})
@@ -2,14 +2,20 @@
// already exists before the statement joins the lock queue. `table` is kept exactly as the
// statement wrote it, schema qualification and quoting included, because it is fed to
// `to_regclass`; `name` is the bare identifier the catalog stores in `relname`/`attname`.
export type SchemaLockTarget = {
kind: 'index' | 'column' | 'constraint'
table: string
name: string
// The catalog answer that means this statement has nothing left to do. Creating statements skip
// on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is done.
skipWhen: 'present' | 'absent'
}
export type SchemaLockTarget =
| {
kind: 'index' | 'column' | 'constraint' | 'reloption'
table: string
name: string
// The catalog answer that means this statement has nothing left to do. Creating statements
// skip on present; `DROP CONSTRAINT IF EXISTS` is the inverse, because nothing to drop is
// done.
skipWhen: 'present' | 'absent'
}
// A `DROP INDEX` names no table, and needs none: an index name that resolves to nothing is
// nothing to drop, whatever table it used to belong to. Resolution is by name through the
// search_path, which is how the DROP itself would resolve it.
| { kind: 'index-by-name'; name: string; skipWhen: 'absent' }
// Keywords that sit in an identifier position when the optional clause before them is absent.
// Without this, `CREATE UNIQUE INDEX CONCURRENTLY ON t(c)` reads CONCURRENTLY as the index name and
@@ -119,10 +125,27 @@ const DROP_CONSTRAINT = new RegExp(
'i'
)
// `IF EXISTS` is required for the same reason it is on DROP CONSTRAINT: a bare `DROP INDEX` on a
// missing index is an error the server is supposed to raise. Without a target the statement throws
// at boot instead, which tells the author to write `IF EXISTS`.
const DROP_INDEX = new RegExp(`^DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?IF\\s+EXISTS\\s+${QUALIFIED}\\s*$`, 'i')
// One option per statement, and a literal value: the catalog stores reloptions as `name=value`
// text, so the pre-check compares the written pair against that array verbatim. A list of options
// is refused by `hasTopLevelComma` before it reaches here, the same as a multi-action ALTER TABLE.
const SET_RELOPTION = new RegExp(
`^ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?${QUALIFIED}\\s+` +
`SET\\s+\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*([A-Za-z0-9_.]+)\\s*\\)\\s*$`,
'i'
)
// Every statement shape that takes a relation lock before Postgres evaluates its existence test.
// `CREATE TABLE IF NOT EXISTS` is absent on purpose: it resolves a name against the schema and
// takes no lock on an existing table.
const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|ALTER\s+TABLE)\b/i
// `DROP INDEX` is here because it takes ACCESS EXCLUSIVE on the index's table whenever the index is
// actually there, which is every boot until the first one wins. That it takes no lock once the
// index is gone is what the pre-check turns into the steady state, not a reason to omit it.
const TAKES_RELATION_LOCK = /^(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|ALTER\s+TABLE)\b/i
export function takesRelationLock(statement: string): boolean {
return TAKES_RELATION_LOCK.test(sqlWithoutComments(statement))
@@ -177,7 +200,9 @@ const MUST_PARSE = [
/^CREATE\s+(?:UNIQUE\s+)?INDEX\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+COLUMN\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bADD\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bDROP\s+CONSTRAINT\b/i
/^ALTER\s+TABLE\b[\s\S]*\bDROP\s+CONSTRAINT\b/i,
/^ALTER\s+TABLE\b[\s\S]*\bSET\s+\(/i,
/^DROP\s+INDEX\b/i
]
// Derived from the statement itself so a renamed index cannot drift away from its pre-check.
@@ -209,6 +234,22 @@ export function schemaLockTarget(statement: string): SchemaLockTarget | undefine
skipWhen: 'absent'
}
}
const droppedIndex = DROP_INDEX.exec(sql)
if (droppedIndex?.[1]) {
return { kind: 'index-by-name', name: catalogName(droppedIndex[1]), skipWhen: 'absent' }
}
const option = SET_RELOPTION.exec(sql)
if (option?.[1] && option[2] && option[3]) {
// Option names are always folded, but the value is stored as written, so only the name goes
// through catalogName. `fillfactor=70` and `fillfactor=80` are different targets, which is
// what makes a changed value re-run rather than skip.
return {
kind: 'reloption',
table: option[1],
name: `${catalogName(option[2])}=${option[3]}`,
skipWhen: 'present'
}
}
return undefined
}
+11 -1
View File
@@ -14,6 +14,10 @@ const {
} = require('./packaged-runtime-node-modules.cjs')
const { verifyLinuxGlibcFloor } = require('./scripts/verify-linux-glibc-floor.cjs')
const { writeMacBuildCompatibility } = require('./scripts/mac-build-compatibility.cjs')
const {
MOBILE_WEB_BUNDLE_DIR,
assertMobileWebBundleBuilt
} = require('./scripts/verify-packaged-mobile-web-bundle.cjs')
const { verifyPackagedPluginResources } = require('./scripts/verify-packaged-plugin-resources.cjs')
const {
verifyPackagedWindowsNodePty
@@ -177,6 +181,9 @@ module.exports = {
// Why: these repo-only inputs are either bundled into out/ or copied via
// extraResources. Shipping them in app.asar bloats the desktop bundle.
'!src{,/**/*}',
// Redundant under !src above, kept explicit: the built bundle ships from out/mobile-web via the
// out rules exactly as out/web does, and the source tree must never be mistaken for it.
'!src/mobile-web{,/**/*}',
'!config{,/**/*}',
'!docs{,/**/*}',
'!mobile{,/**/*}',
@@ -289,8 +296,11 @@ module.exports = {
verifyStaticAppImagePackage(file, arch)
}
},
beforePack: (context) => {
// electron-builder calls this with the context alone. The second parameter is the bundle root,
// so a test can point the guard at a scratch bundle instead of needing the repo's out/ built.
beforePack: (context, mobileWebBundleDir = MOBILE_WEB_BUNDLE_DIR) => {
assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch)
assertMobileWebBundleBuilt(mobileWebBundleDir)
},
afterPack: async (context) => {
const resourcesDir =
+1
View File
@@ -16,6 +16,7 @@
"src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts",
"src/main/agent-hooks/managed-agent-hook-controls.ts",
"src/main/claude-accounts/keychain.ts",
"src/mobile-web/src/bootstrap.ts",
"src/renderer/src/main.tsx",
"src/renderer/src/popout.tsx",
"src/renderer/src/web/main.tsx",
+9
View File
@@ -56,6 +56,15 @@
"anti-slop/no-module-mocking": "off"
}
},
// mock-descendant-sweep.ts (daemon and relay) is a test-only side-effect shim: its whole body
// is one vi.mock that keeps mock PTY PIDs away from the host process table, and it exists so
// 60 suites do not each inline the same hoisted factory. It is never imported by product code.
{
"files": ["**/mock-descendant-sweep.ts"],
"rules": {
"anti-slop/no-module-mocking": "off"
}
},
// The exemptions below are file-scoped rather than inline `oxlint-disable` comments
// because the root lint scan does not load this plugin, so an inline directive naming
// an anti-slop rule always reads back as an unused directive there.
+230
View File
@@ -0,0 +1,230 @@
import { createHash } from 'node:crypto'
import { realpathSync } from 'node:fs'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import * as esbuild from 'esbuild'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const sourceDir = join(projectDir, 'src', 'mobile-web')
const defaultOutDir = join(projectDir, 'out', 'mobile-web')
export const MOBILE_WEB_BUNDLE_SCHEMA_VERSION = 1
export const MOBILE_WEB_BUNDLE_ENTRYPOINT = 'index.html'
const CONTENT_TYPE_BY_EXTENSION = {
css: 'text/css; charset=utf-8',
html: 'text/html; charset=utf-8',
js: 'text/javascript; charset=utf-8',
png: 'image/png'
}
/**
* Canonical serialization the buildId hashes. Key order is fixed and the list is sorted by path,
* so the id is a pure function of content. Must stay byte-identical to the contract module's
* serializer in src/shared/mobile-web-bundle/.
*/
export function serializeMobileWebBundleAssets(assets) {
return JSON.stringify(
[...assets]
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
.map(({ path, sha256, byteLength, contentType }) => ({
path,
sha256,
byteLength,
contentType
}))
)
}
export function computeMobileWebBundleBuildId(assets) {
return createHash('sha256').update(serializeMobileWebBundleAssets(assets), 'utf8').digest('hex')
}
function sha256Hex(bytes) {
return createHash('sha256').update(bytes).digest('hex')
}
function contentTypeForExtension(extension) {
const contentType = CONTENT_TYPE_BY_EXTENSION[extension]
if (!contentType) {
throw new Error(`[build-mobile-web-bundle] no content type registered for .${extension}`)
}
return contentType
}
function readIntegerConstant(source, name) {
const match = new RegExp(`export const ${name} = (\\d+)`).exec(source)
if (!match) {
throw new Error(`[build-mobile-web-bundle] ${name} not found in src/shared/protocol-version.ts`)
}
return Number.parseInt(match[1], 10)
}
/**
* Parsed rather than imported because protocol-version.ts is TypeScript and this script runs on
* bare node during packaging, before any build output exists.
*/
async function readProtocolWindow() {
const source = await readFile(join(projectDir, 'src', 'shared', 'protocol-version.ts'), 'utf8')
return {
runtimeProtocolVersion: readIntegerConstant(source, 'RUNTIME_PROTOCOL_VERSION'),
// The bundle is a client: the floor it cares about is the oldest host protocol it can talk to.
minCompatibleRuntimeProtocolVersion: readIntegerConstant(
source,
'MIN_COMPATIBLE_RUNTIME_SERVER_VERSION'
)
}
}
async function readDesktopVersion() {
const packageJson = JSON.parse(await readFile(join(projectDir, 'package.json'), 'utf8'))
if (typeof packageJson.version !== 'string' || packageJson.version.length === 0) {
throw new Error('[build-mobile-web-bundle] root package.json has no version')
}
return packageJson.version
}
async function transformEntries(protocolWindow, desktopVersion) {
const result = await esbuild.build({
absWorkingDir: sourceDir,
entryPoints: [join(sourceDir, 'src', 'bootstrap.ts'), join(sourceDir, 'src', 'bootstrap.css')],
bundle: true,
minify: true,
// Virtual: write is false, so outdir only names the emitted files esbuild hands back.
outdir: 'dist',
write: false,
format: 'iife',
target: ['es2022'],
charset: 'utf8',
legalComments: 'none',
// Why no sourcemap and no metafile: both embed absolute paths, which would break reproducibility.
sourcemap: false,
logLevel: 'silent',
define: {
ORCA_MOBILE_WEB_DESKTOP_VERSION: JSON.stringify(desktopVersion),
ORCA_MOBILE_WEB_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.runtimeProtocolVersion
),
ORCA_MOBILE_WEB_MIN_COMPATIBLE_RUNTIME_PROTOCOL_VERSION: JSON.stringify(
protocolWindow.minCompatibleRuntimeProtocolVersion
)
}
})
const byExtension = new Map()
for (const file of result.outputFiles) {
const extension = file.path.endsWith('.css') ? 'css' : 'js'
byExtension.set(extension, Buffer.from(file.contents))
}
const script = byExtension.get('js')
const stylesheet = byExtension.get('css')
if (!script || !stylesheet) {
throw new Error('[build-mobile-web-bundle] esbuild did not emit both a script and a stylesheet')
}
return { script, stylesheet }
}
function hashedAsset(bytes, extension) {
const sha256 = sha256Hex(bytes)
return {
bytes,
path: `assets/${sha256}.${extension}`,
sha256,
byteLength: bytes.byteLength,
contentType: contentTypeForExtension(extension)
}
}
export async function buildMobileWebBundle({ outDir = defaultOutDir } = {}) {
const [desktopVersion, protocolWindow] = await Promise.all([
readDesktopVersion(),
readProtocolWindow()
])
const { script, stylesheet } = await transformEntries(protocolWindow, desktopVersion)
const mark = await readFile(join(sourceDir, 'src', 'orca-mark.png'))
const hashed = [
hashedAsset(script, 'js'),
hashedAsset(stylesheet, 'css'),
hashedAsset(mark, 'png')
]
const [scriptAsset, stylesheetAsset, markAsset] = hashed
const template = await readFile(join(sourceDir, MOBILE_WEB_BUNDLE_ENTRYPOINT), 'utf8')
const substitutions = {
__ORCA_BOOTSTRAP_JS__: scriptAsset.path,
__ORCA_BOOTSTRAP_CSS__: stylesheetAsset.path,
__ORCA_MARK_PNG__: markAsset.path
}
let html = template
for (const [token, value] of Object.entries(substitutions)) {
if (!html.includes(token)) {
throw new Error(`[build-mobile-web-bundle] ${MOBILE_WEB_BUNDLE_ENTRYPOINT} lacks ${token}`)
}
html = html.replaceAll(token, value)
}
const indexBytes = Buffer.from(html, 'utf8')
const indexAsset = {
bytes: indexBytes,
path: MOBILE_WEB_BUNDLE_ENTRYPOINT,
sha256: sha256Hex(indexBytes),
byteLength: indexBytes.byteLength,
contentType: contentTypeForExtension('html')
}
const written = [indexAsset, ...hashed]
const assets = written
.map(({ path, sha256, byteLength, contentType }) => ({ path, sha256, byteLength, contentType }))
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
const manifest = {
schemaVersion: MOBILE_WEB_BUNDLE_SCHEMA_VERSION,
buildId: computeMobileWebBundleBuildId(assets),
desktopVersion,
minCompatibleRuntimeProtocolVersion: protocolWindow.minCompatibleRuntimeProtocolVersion,
runtimeProtocolVersion: protocolWindow.runtimeProtocolVersion,
entrypoint: MOBILE_WEB_BUNDLE_ENTRYPOINT,
totalBytes: assets.reduce((total, asset) => total + asset.byteLength, 0),
assets
}
// Why a full clear: a stale asset left from an earlier build would ship unreferenced inside asar.
await rm(outDir, { recursive: true, force: true })
await mkdir(join(outDir, 'assets'), { recursive: true })
for (const asset of written) {
await writeFile(join(outDir, asset.path), asset.bytes)
}
await writeFile(join(outDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
return { manifest, outDir }
}
/**
* Whether this module was run as the entry script. Two ways to get this wrong, both of which end
* with the builder exiting 0 having written nothing: `file://${path}` never matches on Windows,
* where import.meta.url is `file:///C:/...`; and Node resolves symlinks in import.meta.url but not
* in argv[1], so `node /tmp/...` against a /private/tmp realpath compares two different strings.
* Both seams are injectable so win32 and a missing path can be exercised from a posix runner.
*/
export function isDirectInvocation(
moduleUrl,
scriptPath,
{ toFileUrl = pathToFileURL, realpath = realpathSync } = {}
) {
if (!scriptPath) {
return false
}
let resolved = scriptPath
try {
resolved = realpath(scriptPath)
} catch {
// A path that cannot be resolved cannot be this module; fall through to the literal compare.
}
return moduleUrl === toFileUrl(resolved).href
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
const { manifest, outDir } = await buildMobileWebBundle()
console.log(
`[build-mobile-web-bundle] OK — ${String(manifest.assets.length)} asset(s), ` +
`${String(manifest.totalBytes)} bytes, buildId ${manifest.buildId} -> ${outDir}`
)
}
@@ -0,0 +1,261 @@
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
buildMobileWebBundle,
computeMobileWebBundleBuildId,
isDirectInvocation,
serializeMobileWebBundleAssets
} from './build-mobile-web-bundle.mjs'
import {
MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS,
MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES,
assertNoCarriageReturnsInSource
} from './verify-mobile-web-bundle.mjs'
async function buildIntoScratch() {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-build-'))
const bundleDir = join(scratch, 'mobile-web')
const { manifest } = await buildMobileWebBundle({ outDir: bundleDir })
return { scratch, bundleDir, manifest }
}
describe('buildMobileWebBundle', () => {
it('emits a content-addressed bundle whose only stable name is the entrypoint', async () => {
const { scratch, bundleDir, manifest } = await buildIntoScratch()
try {
const root = await readdir(bundleDir)
expect(root.sort()).toEqual(['assets', 'index.html', 'manifest.json'])
for (const name of await readdir(join(bundleDir, 'assets'))) {
const [digest, extension] = name.split('.')
expect(digest).toMatch(/^[0-9a-f]{64}$/)
const bytes = await readFile(join(bundleDir, 'assets', name))
expect(createHash('sha256').update(bytes).digest('hex')).toBe(digest)
expect(extension).toMatch(/^(js|css|png)$/)
}
const html = await readFile(join(bundleDir, 'index.html'), 'utf8')
for (const asset of manifest.assets) {
if (asset.path !== 'index.html') {
expect(html).toContain(asset.path)
}
}
expect(html).not.toContain('__ORCA_')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('carries every manifest field the Phase A contract names', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(Object.keys(manifest)).toEqual([
'schemaVersion',
'buildId',
'desktopVersion',
'minCompatibleRuntimeProtocolVersion',
'runtimeProtocolVersion',
'entrypoint',
'totalBytes',
'assets'
])
expect(manifest.schemaVersion).toBe(1)
expect(manifest.entrypoint).toBe('index.html')
const packageJson = JSON.parse(
await readFile(new URL('../../package.json', import.meta.url), 'utf8')
)
expect(manifest.desktopVersion).toBe(packageJson.version)
const protocolSource = await readFile(
new URL('../../src/shared/protocol-version.ts', import.meta.url),
'utf8'
)
expect(protocolSource).toContain(
`export const RUNTIME_PROTOCOL_VERSION = ${String(manifest.runtimeProtocolVersion)}`
)
expect(protocolSource).toContain(
`export const MIN_COMPATIBLE_RUNTIME_SERVER_VERSION = ${String(manifest.minCompatibleRuntimeProtocolVersion)}`
)
expect(manifest.totalBytes).toBe(
manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('produces the same buildId from two independent builds', async () => {
const first = await buildIntoScratch()
const second = await buildIntoScratch()
try {
expect(second.manifest.buildId).toBe(first.manifest.buildId)
expect(second.manifest).toEqual(first.manifest)
} finally {
await rm(first.scratch, { recursive: true, force: true })
await rm(second.scratch, { recursive: true, force: true })
}
})
it('embeds no absolute path from the machine that built it', async () => {
const { scratch, bundleDir } = await buildIntoScratch()
try {
const names = [
'index.html',
'manifest.json',
...(await readdir(join(bundleDir, 'assets'))).map((name) => join('assets', name))
]
for (const name of names) {
const text = (await readFile(join(bundleDir, name))).toString('latin1')
expect(text).not.toContain(scratch)
expect(text).not.toContain(process.cwd())
}
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('stays inside the Phase A budget', async () => {
const { scratch, manifest } = await buildIntoScratch()
try {
expect(manifest.assets.length).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS)
expect(manifest.totalBytes).toBeLessThanOrEqual(MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
describe('computeMobileWebBundleBuildId', () => {
const assets = [
{ path: 'index.html', sha256: 'a'.repeat(64), byteLength: 3, contentType: 'text/html' },
{ path: 'assets/b.js', sha256: 'b'.repeat(64), byteLength: 5, contentType: 'text/javascript' }
]
it('sorts by path, so input order cannot change the id', () => {
expect(computeMobileWebBundleBuildId(assets.toReversed())).toBe(
computeMobileWebBundleBuildId(assets)
)
})
it('serializes a fixed key order regardless of the input object key order', () => {
const reordered = assets.map(({ contentType, byteLength, sha256, path }) => ({
contentType,
byteLength,
sha256,
path
}))
expect(serializeMobileWebBundleAssets(reordered)).toBe(serializeMobileWebBundleAssets(assets))
})
it('changes when any hashed field changes', () => {
const baseline = computeMobileWebBundleBuildId(assets)
for (const field of ['sha256', 'byteLength', 'contentType', 'path']) {
const mutated = assets.map((asset, index) =>
index === 0 ? { ...asset, [field]: field === 'byteLength' ? 4 : `${asset[field]}x` } : asset
)
expect(computeMobileWebBundleBuildId(mutated)).not.toBe(baseline)
}
})
})
describe('isDirectInvocation', () => {
const thisFile = import.meta.filename
it('matches the path this module was loaded from', () => {
expect(isDirectInvocation(import.meta.url, thisFile)).toBe(true)
})
it('does not match a different script', () => {
expect(isDirectInvocation(import.meta.url, join(thisFile, '..', 'other.mjs'))).toBe(false)
})
it('tolerates an absent argv[1]', () => {
expect(isDirectInvocation(import.meta.url, undefined)).toBe(false)
expect(isDirectInvocation(import.meta.url, '')).toBe(false)
})
// Why an injected converter: a win32 path cannot be exercised through node:url's pathToFileURL
// on a posix runner, and CI is ubuntu.
const toWin32FileUrl = (windowsPath) => new URL(`file:///${windowsPath.replaceAll('\\', '/')}`)
it('matches a Windows entry path, which the file:// template form never does', () => {
const scriptPath = 'C:\\orca\\config\\scripts\\build-mobile-web-bundle.mjs'
const moduleUrl = 'file:///C:/orca/config/scripts/build-mobile-web-bundle.mjs'
const keepAsIs = (path) => path
expect(
isDirectInvocation(moduleUrl, scriptPath, {
toFileUrl: toWin32FileUrl,
realpath: keepAsIs
})
).toBe(true)
// The regression this guards: `file://${argv[1]}` yields file://C:\orca\... on Windows,
// so the builder exited 0 having written nothing and packaging failed downstream.
expect(`file://${scriptPath}`).not.toBe(moduleUrl)
})
it('is not written with the file:// template form', async () => {
const source = await readFile(new URL('./build-mobile-web-bundle.mjs', import.meta.url), 'utf8')
expect(source).not.toMatch(/file:\/\/\$\{process\.argv\[1\]\}/)
expect(source).toContain('pathToFileURL')
})
})
describe('mobile web source line endings', () => {
it('accepts the committed source tree', async () => {
await expect(assertNoCarriageReturnsInSource()).resolves.toBeUndefined()
})
it('rejects a CRLF source file, because CRLF changes every asset hash and the buildId', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-eol-'))
try {
await writeFile(join(scratch, 'bootstrap.ts'), 'const a = 1\r\nconst b = 2\r\n', 'utf8')
await expect(assertNoCarriageReturnsInSource(scratch)).rejects.toThrow(
/CRLF in mobile web source/
)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('pins eol=lf for every committed text source and -text for the binary', () => {
const files = execFileSync('git', ['ls-files', 'src/mobile-web'], { encoding: 'utf8' })
.split('\n')
.filter(Boolean)
expect(files.length).toBeGreaterThanOrEqual(4)
for (const file of files) {
const attributes = execFileSync('git', ['check-attr', 'text', 'eol', '--', file], {
encoding: 'utf8'
})
if (file.endsWith('.png')) {
expect(attributes).toContain('text: unset')
} else {
expect(attributes).toContain('eol: lf')
}
}
})
})
describe('running the builder through a symlink', () => {
// Node resolves symlinks in import.meta.url but not in argv[1]. Before the guard realpath'd the
// entry path, `node /tmp/<link>` compared /tmp against /private/tmp and the builder exited 0
// having written nothing — a green packaging job with no bundle in it.
it('still recognises the entry module', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-link-'))
try {
const builderUrl = new URL('./build-mobile-web-bundle.mjs', import.meta.url).href
const real = join(scratch, 'entry.mjs')
await writeFile(
real,
`import { isDirectInvocation } from ${JSON.stringify(builderUrl)}\n` +
'process.stdout.write(String(isDirectInvocation(import.meta.url, process.argv[1])))\n',
'utf8'
)
const link = join(scratch, 'entry-link.mjs')
await symlink(real, link)
expect(execFileSync(process.execPath, [link], { encoding: 'utf8' })).toBe('true')
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
})
@@ -3,7 +3,8 @@ import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { buildMobileWebBundle } from './build-mobile-web-bundle.mjs'
const REPO_ROOT = join(import.meta.dirname, '..', '..')
const SRC_MAIN_DIR = join(REPO_ROOT, 'src', 'main')
@@ -442,8 +443,23 @@ describe('arch-aware packaging guard', () => {
const OTHER_ARCH_NAME = process.arch === 'arm64' ? 'x64' : 'arm64'
const SHERPA_PLATFORM = process.platform === 'win32' ? 'win' : process.platform
const otherSherpa = `sherpa-onnx-${SHERPA_PLATFORM}-${OTHER_ARCH_NAME}`
// beforePack also hash-verifies the mobile web bundle, which the unit-test job never builds.
// Point it at a real bundle built into a temp dir: these tests are about the native-variant
// guard, and the bundle guard has its own suite.
let scratch
let bundleDir
beforeAll(async () => {
scratch = await mkdtemp(join(tmpdir(), 'orca-electron-builder-guard-'))
bundleDir = join(scratch, 'mobile-web')
await buildMobileWebBundle({ outDir: bundleDir })
})
afterAll(async () => {
await rm(scratch, { recursive: true, force: true })
})
const packHost = (arch) =>
electronBuilderConfig.beforePack({ electronPlatformName: process.platform, arch })
electronBuilderConfig.beforePack({ electronPlatformName: process.platform, arch }, bundleDir)
it('allows packaging the host platform and architecture', () => {
expect(() => packHost(HOST_ARCH)).not.toThrow()
@@ -471,7 +487,7 @@ describe('arch-aware packaging guard', () => {
(resource) => resource.to === join('node_modules', '@vscode', 'windows-process-tree')
)
const packWindows = () =>
electronBuilderConfig.beforePack({ electronPlatformName: 'win32', arch: 1 })
electronBuilderConfig.beforePack({ electronPlatformName: 'win32', arch: 1 }, bundleDir)
if (process.platform === 'win32' || windowsAddon) {
expect(packWindows).not.toThrow()
} else {
@@ -0,0 +1,159 @@
import { readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import { parseDocument } from 'yaml'
const workflowsDir = fileURLToPath(new URL('../../.github/workflows', import.meta.url))
// Every script whose chain reaches build:mobile-web. build:unpack -> build -> build:desktop, and
// build:mac/linux/win each call build:desktop, so all of them produce out/mobile-web. The chain
// itself is not an assumption here: 'the build scripts' below resolves each one for real.
const BUNDLE_PRODUCING_SCRIPTS = [
'build',
'build:desktop',
'build:release',
'build:release:parallel',
'build:unpack',
'build:mobile-web',
'build:mac',
'build:mac:release',
'build:linux',
'build:win'
]
const BUNDLE_PRODUCER = new RegExp(
`pnpm (?:run )?(?:${BUNDLE_PRODUCING_SCRIPTS.join('|')})(?=$|[\\s'"&|;])`,
'm'
)
const packageScripts = JSON.parse(
readFileSync(fileURLToPath(new URL('../../package.json', import.meta.url)), 'utf8')
).scripts
const SCRIPT_INVOCATION = /pnpm (?:run )?([\w:-]+)(?=$|[\s'"&|;])/g
/** Whether `pnpm run <name>` eventually runs build:mobile-web. */
function reachesBundleBuild(name, seen = new Set()) {
if (name === 'build:mobile-web') {
return true
}
if (seen.has(name)) {
return false
}
seen.add(name)
const body = packageScripts[name]
if (typeof body !== 'string') {
return false
}
return [...body.matchAll(SCRIPT_INVOCATION)].some((match) => reachesBundleBuild(match[1], seen))
}
/**
* Whether `pnpm run <name>` eventually runs electron-builder without --prepackaged, i.e. runs
* beforePack. A workflow job that packs through such a script is a packaging job even though the
* literal electron-builder line lives in package.json (daemon-relocation-spike's build:unpack).
*/
function reachesElectronBuilder(name, seen = new Set()) {
if (seen.has(name)) {
return false
}
seen.add(name)
const body = packageScripts[name]
if (typeof body !== 'string') {
return false
}
if (packsWithBeforePack(body)) {
return true
}
return [...body.matchAll(SCRIPT_INVOCATION)].some((match) =>
reachesElectronBuilder(match[1], seen)
)
}
/** Whether text invokes electron-builder in a way that reaches beforePack. */
function packsWithBeforePack(text) {
const invocations = [...text.matchAll(/[^\n]*electron-builder --config[^\n]*/g)].map(
(match) => match[0]
)
// --prepackaged short-circuits doPack before emitBeforePack, so those jobs never run the guard.
return (
invocations.length > 0 &&
!invocations.every((invocation) => invocation.includes('--prepackaged'))
)
}
// Every job that packs an app and therefore runs beforePack. Listed so that a new packaging
// workflow has to be added here deliberately, with its bundle step, rather than slipping in.
const EXPECTED_PACKAGING_JOBS = [
'adhoc-mac-build.yml build-adhoc-mac',
'daemon-relocation-spike.yml spike',
'daily-mac-build.yml build-daily-mac',
'dev-channel-win-build.yml build-win',
'hourly-mac-build.yml build-hourly-mac',
'pr.yml package',
'pr.yml package_windows',
'release-cut.yml build',
'release-mac-build.yml build-mac',
'win-crash-survival-e2e.yml crash-survival',
'win-update-survival-e2e.yml survival',
'windows-signing-rehearsal.yml rehearse'
]
/**
* Raw source text per job, sliced by the parsed job boundaries. Why not yaml.stringify(job):
* re-serializing folds long lines, and the fold in dev-channel-win-build's build-win landed
* between `electron-builder` and `--config`, hiding a whole packaging job from this census.
*/
function packagingJobs() {
const jobs = []
for (const file of readdirSync(workflowsDir).filter((name) => name.endsWith('.yml'))) {
const source = readFileSync(join(workflowsDir, file), 'utf8')
const jobsNode = parseDocument(source).get('jobs', true)
const items = jobsNode?.items ?? []
for (const [index, pair] of items.entries()) {
const end = index + 1 < items.length ? items[index + 1].key.range[0] : jobsNode.range[2]
const text = source.slice(pair.key.range[0], end)
const packsViaScript = [...text.matchAll(SCRIPT_INVOCATION)].some((match) =>
reachesElectronBuilder(match[1])
)
if (!packsWithBeforePack(text) && !packsViaScript) {
continue
}
jobs.push({ label: `${file} ${String(pair.key.value)}`, text })
}
}
return jobs
}
describe('mobile web bundle packaging coverage', () => {
it('finds every packaging job', () => {
// A rename or a restructure that shrank this list would make every assertion below vacuous.
const labels = packagingJobs().map((job) => job.label)
expect(labels.length).toBeGreaterThanOrEqual(EXPECTED_PACKAGING_JOBS.length)
expect(labels.toSorted()).toEqual(EXPECTED_PACKAGING_JOBS.toSorted())
})
it.each(packagingJobs().map((job) => [job.label, job]))(
'produces out/mobile-web before electron-builder packs: %s',
(_label, job) => {
// Job granularity, not step ordering: the failure this exists for is a job that never builds
// the bundle at all, which is what beforePack turns into a hard packaging failure.
expect(job.text).toMatch(BUNDLE_PRODUCER)
}
)
})
describe('the build scripts the census trusts', () => {
// The census only checks that a packaging job invokes one of these. If a chain stopped calling
// build:mobile-web, every job would still look covered while packaging failed at beforePack.
it.each(BUNDLE_PRODUCING_SCRIPTS)('%s runs build:mobile-web', (name) => {
expect(packageScripts[name]).toBeTypeOf('string')
expect(reachesBundleBuild(name)).toBe(true)
})
it('pr.yml package builds the bundle by hand, because it never calls build:release', () => {
const source = readFileSync(join(workflowsDir, 'pr.yml'), 'utf8')
expect(source).toMatch(/- name: Build mobile web bundle\n\s+run: pnpm run build:mobile-web\n/)
})
})
@@ -0,0 +1,119 @@
/**
* The canonical serialization that buildId hashes exists three times, because the two packaging
* scripts run on bare node before any build output exists and so cannot import the TypeScript
* contract. Three copies drift; this is what stops them. A divergence in any one of them would
* reject every honest bundle at packaging, or ship a bundle whose id the phone recomputes
* differently and re-downloads forever.
*/
import { createHash } from 'node:crypto'
import { createRequire } from 'node:module'
import { describe, expect, it } from 'vitest'
import {
computeMobileWebBundleBuildId,
serializeMobileWebBundleAssets as serializeInBuilder
} from './build-mobile-web-bundle.mjs'
import {
computeMobileWebBundleId,
MobileWebBundleManifestSchema,
serializeMobileWebBundleAssets as serializeInContract
} from '../../src/shared/mobile-web-bundle/manifest-contract'
const require = createRequire(import.meta.url)
const { serializeAssets: serializeInGuard } = require('./verify-packaged-mobile-web-bundle.cjs')
const digest = (hex) => `${hex}`.padStart(64, '0')
/**
* Mixed content types, a nested path, and an uppercase segment that sorts before a lowercase one
* only under code-unit order: `localeCompare` would put `assets/aQ.js` first, so any serializer
* that reached for it produces a different string here.
*/
const ASSETS = [
{
path: 'assets/Za.js',
sha256: digest('a1'),
byteLength: 2048,
contentType: 'text/javascript; charset=utf-8'
},
{ path: 'assets/aQ.css', sha256: digest('b2'), byteLength: 512, contentType: 'text/css' },
{
path: 'assets/nested/mark.png',
sha256: digest('c3'),
byteLength: 40_960,
contentType: 'image/png'
},
{
path: 'index.html',
sha256: digest('d4'),
byteLength: 640,
contentType: 'text/html; charset=utf-8'
}
]
const REORDERED = [ASSETS[3], ASSETS[1], ASSETS[0], ASSETS[2]]
const REVERSED = ASSETS.toReversed()
const sha256Hex = (value) => createHash('sha256').update(value, 'utf8').digest('hex')
describe('the three mobile web bundle serializers', () => {
it('produce one string for the builder, the packaging guard, and the shared contract', () => {
const fromContract = serializeInContract(ASSETS)
expect(serializeInBuilder(ASSETS)).toBe(fromContract)
expect(serializeInGuard(ASSETS)).toBe(fromContract)
})
it.each([
['reordered', REORDERED],
['reversed', REVERSED]
])('are order-independent, so %s input serializes identically', (_label, input) => {
const expected = serializeInContract(ASSETS)
expect(serializeInContract(input)).toBe(expected)
expect(serializeInBuilder(input)).toBe(expected)
expect(serializeInGuard(input)).toBe(expected)
})
it('leaves the caller-supplied array untouched, so a build cannot depend on the sort', () => {
const input = [...REORDERED]
serializeInContract(input)
serializeInBuilder(input)
serializeInGuard(input)
expect(input).toEqual(REORDERED)
})
it('emit exactly path, sha256, byteLength, contentType, in that order, and nothing else', () => {
const decorated = ASSETS.map((asset) => ({ ...asset, sourcePath: '/tmp/ignored', extra: 1 }))
expect(serializeInContract(decorated)).toBe(serializeInContract(ASSETS))
expect(serializeInBuilder(decorated)).toBe(serializeInContract(ASSETS))
expect(serializeInGuard(decorated)).toBe(serializeInContract(ASSETS))
expect(JSON.parse(serializeInContract(ASSETS))[0]).toEqual({
path: 'assets/Za.js',
sha256: digest('a1'),
byteLength: 2048,
contentType: 'text/javascript; charset=utf-8'
})
})
it('hash to one buildId, which the manifest schema then accepts', () => {
const buildId = computeMobileWebBundleId(REORDERED)
expect(computeMobileWebBundleBuildId(REORDERED)).toBe(buildId)
expect(sha256Hex(serializeInGuard(REORDERED))).toBe(buildId)
const manifest = {
schemaVersion: 1,
buildId,
desktopVersion: '1.4.200',
minCompatibleRuntimeProtocolVersion: 2,
runtimeProtocolVersion: 2,
entrypoint: 'index.html',
totalBytes: ASSETS.reduce((total, asset) => total + asset.byteLength, 0),
assets: [...ASSETS]
}
expect(MobileWebBundleManifestSchema.parse(manifest).buildId).toBe(buildId)
})
})
+11
View File
@@ -178,6 +178,17 @@ export const PR_E2E_SOURCE_ROUTES = [
file
)
},
{
// Why: layout resolution is the only place a split direction can be invented, and the
// loss is one-way — the guess is published and written back over the real tree.
id: 'terminal-session.split-orientation-resolution',
specs: ['tests/e2e/desktop-published-split-orientation-legacy-leaf.spec.ts'],
matches: (file) =>
isProductSource(file) &&
/^src\/renderer\/src\/runtime\/(?:remote-terminal-layout-resolution\.ts|sync-runtime-graph\/(?:graph-publication|mobile-session-terminal-tabs|mobile-session-surfaces)\.ts|web-session-tabs-sync\/terminal-surfaces\.ts)$/.test(
file
)
},
{
id: 'terminal-session.remote-pane-layout-retry',
specs: ['tests/e2e/paired-remote-pane-layout-retry.spec.ts'],
@@ -40,6 +40,8 @@ const knobByFlag = {
'--metadata-status': 'ORCA_TYPING_BENCH_METADATA_STATUS',
'--metadata-titles': 'ORCA_TYPING_BENCH_METADATA_TITLES',
'--instrumentation': 'ORCA_TYPING_BENCH_INSTRUMENTATION',
'--graph-probe': 'ORCA_TYPING_BENCH_GRAPH_PROBE',
'--cpu-throttle': 'ORCA_TYPING_BENCH_CPU_THROTTLE',
'--label': 'ORCA_TYPING_BENCH_LABEL'
}
+1
View File
@@ -78,6 +78,7 @@ const result = spawnSync(
'tests/e2e/ssh-reconnect-tab-destruction.spec.ts',
'tests/e2e/ssh-restart-tab-accumulation.spec.ts',
'tests/e2e/ssh-skill-installation.spec.ts',
'tests/e2e/ssh-stale-resume-execution-host-scope.spec.ts',
'tests/e2e/ssh-terminal-window-wake-stale-grid-repro.spec.ts',
'--config',
'tests/playwright.config.ts',
@@ -2,9 +2,14 @@ import { spawn } from 'node:child_process'
import { availableParallelism } from 'node:os'
import { fileURLToPath } from 'node:url'
// The three projects overlap heavily in src/shared but have no build dependency on
// These projects overlap heavily in src/shared but have no build dependency on
// each other, so tsc can check them concurrently instead of in a `&&` chain.
const projects = ['tsconfig.node.json', 'tsconfig.tc.cli.json', 'tsconfig.tc.web.json']
const projects = [
'tsconfig.node.json',
'tsconfig.tc.cli.json',
'tsconfig.tc.web.json',
'tsconfig.mobile-web.json'
]
const repoRoot = fileURLToPath(new URL('../..', import.meta.url))
const tsc = fileURLToPath(new URL('../../node_modules/typescript/bin/tsc', import.meta.url))
+117
View File
@@ -0,0 +1,117 @@
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { buildMobileWebBundle, isDirectInvocation } from './build-mobile-web-bundle.mjs'
import { assertMobileWebBundleBuilt } from './verify-packaged-mobile-web-bundle.cjs'
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
const bundleDir = join(projectDir, 'out', 'mobile-web')
const sourceDir = join(projectDir, 'src', 'mobile-web')
// Phase A budget, not the contract ceiling: a bootstrap page past a quarter-megabyte has stopped
// being a bootstrap. Phase C raises these deliberately.
export const MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS = 16
export const MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES = 256 * 1024
class VerificationError extends Error {}
function fail(message) {
throw new VerificationError(message)
}
async function buildIntoScratch() {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-verify-'))
try {
const { manifest } = await buildMobileWebBundle({ outDir: join(scratch, 'mobile-web') })
return manifest
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
async function listSourceFiles(directory) {
const entries = await readdir(directory, { withFileTypes: true })
const files = []
for (const entry of entries) {
const entryPath = join(directory, entry.name)
if (entry.isDirectory()) {
files.push(...(await listSourceFiles(entryPath)))
} else if (entry.isFile()) {
files.push(entryPath)
}
}
return files.sort()
}
/**
* A CRLF checkout changes the bytes of every text source, which changes every asset hash and so
* the buildId. .gitattributes pins eol=lf; this is what notices when that pin stops working.
*/
export async function assertNoCarriageReturnsInSource(directory = sourceDir) {
const offenders = []
for (const file of await listSourceFiles(directory)) {
// Binary assets are pinned -text and may legitimately contain 0x0d.
if (file.endsWith('.png')) {
continue
}
if ((await readFile(file)).includes(0x0d)) {
offenders.push(file.slice(directory.length + 1))
}
}
if (offenders.length > 0) {
fail(
`CRLF in mobile web source, which would change every asset hash and the buildId: ` +
`${offenders.join(', ')}. Check the .gitattributes eol=lf pin for src/mobile-web.`
)
}
}
export async function verifyMobileWebBundle() {
await assertNoCarriageReturnsInSource()
// The packaging guard owns manifest integrity (safe paths, recomputed buildId, totalBytes, hashes,
// no stray files); a manifest edited after the build fails here exactly as it would at beforePack.
const manifest = assertMobileWebBundleBuilt(bundleDir)
if (manifest.assets.length > MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS) {
fail(
`bundle has ${String(manifest.assets.length)} assets, over the Phase A budget of ` +
`${String(MOBILE_WEB_BUNDLE_PHASE_A_MAX_ASSETS)}`
)
}
if (manifest.totalBytes > MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES) {
fail(
`bundle is ${String(manifest.totalBytes)} bytes, over the Phase A budget of ` +
`${String(MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES)}`
)
}
// Two fresh builds into scratch dirs: a timestamp, an absolute path, or an unstable ordering
// anywhere in the pipeline shows up here as a buildId mismatch rather than as a phone cache miss.
const first = await buildIntoScratch()
const second = await buildIntoScratch()
if (first.buildId !== second.buildId) {
fail(`buildId is not reproducible: ${first.buildId} then ${second.buildId}`)
}
if (first.buildId !== manifest.buildId) {
fail(
`${bundleDir} is stale: it carries buildId ${manifest.buildId}, a fresh build produces ${first.buildId}`
)
}
return manifest
}
if (isDirectInvocation(import.meta.url, process.argv[1])) {
try {
const manifest = await verifyMobileWebBundle()
console.log(
`[verify-mobile-web-bundle] OK — ${String(manifest.assets.length)} asset(s), ` +
`${String(manifest.totalBytes)}/${String(MOBILE_WEB_BUNDLE_PHASE_A_MAX_TOTAL_BYTES)} bytes, ` +
`reproducible buildId ${manifest.buildId}`
)
} catch (error) {
console.error(`[verify-mobile-web-bundle] ${error.message}`)
process.exit(1)
}
}
@@ -0,0 +1,193 @@
const { createHash } = require('node:crypto')
const { readFileSync, readdirSync, statSync } = require('node:fs')
const { join, resolve } = require('node:path')
const projectDir = resolve(__dirname, '..', '..')
const MOBILE_WEB_BUNDLE_DIR = join(projectDir, 'out', 'mobile-web')
const REMEDY = 'Run pnpm build:mobile-web (build:desktop and build:release already do).'
const ENTRYPOINT = 'index.html'
const SHA256_PATTERN = /^[0-9a-f]{64}$/
function failure(message) {
return new Error(`[verify-packaged-mobile-web-bundle] ${message}`)
}
function assertSafeRelativePath(path) {
if (typeof path !== 'string' || path.length === 0) {
throw failure('manifest asset has a missing or empty path')
}
const segments = path.split('/')
if (
path.includes('\\') ||
path.startsWith('/') ||
/^[a-zA-Z]:/.test(path) ||
segments.some((segment) => segment === '' || segment === '.' || segment === '..')
) {
throw failure(`manifest asset path is not a safe relative path: ${path}`)
}
}
function assertInteger(value, field) {
if (!Number.isSafeInteger(value) || value < 0) {
throw failure(`manifest field ${field} is not a non-negative integer: ${String(value)}`)
}
}
/**
* Canonical serialization of the asset list. Must stay byte-identical to
* serializeMobileWebBundleAssets in config/scripts/build-mobile-web-bundle.mjs; a divergence here
* would reject every honest bundle, so the two move together.
*/
function serializeAssets(assets) {
return JSON.stringify(
[...assets]
.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
.map(({ path, sha256, byteLength, contentType }) => ({
path,
sha256,
byteLength,
contentType
}))
)
}
function parseManifest(bundleDir) {
const manifestPath = join(bundleDir, 'manifest.json')
let raw
try {
raw = readFileSync(manifestPath, 'utf8')
} catch (error) {
throw failure(
`no bundle manifest at ${manifestPath} (${error.code ?? error.message}). ${REMEDY}`
)
}
let manifest
try {
manifest = JSON.parse(raw)
} catch (error) {
throw failure(`${manifestPath} is not valid JSON: ${error.message}. ${REMEDY}`)
}
if (typeof manifest !== 'object' || manifest === null || Array.isArray(manifest)) {
throw failure(`${manifestPath} is not a JSON object. ${REMEDY}`)
}
if (manifest.schemaVersion !== 1) {
throw failure(`unsupported manifest schemaVersion: ${String(manifest.schemaVersion)}`)
}
if (typeof manifest.buildId !== 'string' || !SHA256_PATTERN.test(manifest.buildId)) {
throw failure(`manifest buildId is not a sha256 digest: ${String(manifest.buildId)}`)
}
if (typeof manifest.desktopVersion !== 'string' || manifest.desktopVersion.length === 0) {
throw failure('manifest desktopVersion is missing')
}
assertInteger(manifest.minCompatibleRuntimeProtocolVersion, 'minCompatibleRuntimeProtocolVersion')
assertInteger(manifest.runtimeProtocolVersion, 'runtimeProtocolVersion')
assertInteger(manifest.totalBytes, 'totalBytes')
if (manifest.entrypoint !== ENTRYPOINT) {
throw failure(`manifest entrypoint must be ${ENTRYPOINT}, got ${String(manifest.entrypoint)}`)
}
if (!Array.isArray(manifest.assets) || manifest.assets.length === 0) {
throw failure('manifest lists no assets')
}
for (const asset of manifest.assets) {
if (typeof asset !== 'object' || asset === null) {
throw failure('manifest asset entry is not an object')
}
assertSafeRelativePath(asset.path)
if (typeof asset.sha256 !== 'string' || !SHA256_PATTERN.test(asset.sha256)) {
throw failure(`manifest asset ${asset.path} has no sha256 digest`)
}
assertInteger(asset.byteLength, `assets[${asset.path}].byteLength`)
if (typeof asset.contentType !== 'string' || asset.contentType.length === 0) {
throw failure(`manifest asset ${asset.path} has no contentType`)
}
}
if (!manifest.assets.some((asset) => asset.path === manifest.entrypoint)) {
throw failure(`manifest entrypoint ${manifest.entrypoint} is not one of its assets`)
}
const declaredTotal = manifest.assets.reduce((total, asset) => total + asset.byteLength, 0)
if (declaredTotal !== manifest.totalBytes) {
throw failure(
`manifest totalBytes is ${String(manifest.totalBytes)}, its assets sum to ${String(declaredTotal)}`
)
}
const recomputed = createHash('sha256')
.update(serializeAssets(manifest.assets), 'utf8')
.digest('hex')
if (recomputed !== manifest.buildId) {
throw failure(
`manifest buildId ${manifest.buildId} does not match its asset list (expected ${recomputed}). ${REMEDY}`
)
}
return manifest
}
/** Every file under the bundle directory, as a manifest-shaped relative path. */
function listBundleFiles(directory, prefix = '') {
const found = []
for (const entry of readdirSync(directory, { withFileTypes: true })) {
const relativePath = prefix === '' ? entry.name : `${prefix}/${entry.name}`
if (entry.isDirectory()) {
found.push(...listBundleFiles(join(directory, entry.name), relativePath))
} else {
found.push(relativePath)
}
}
return found
}
/**
* Nothing in the bundle directory may be unaccounted for. An asset dropped from the manifest but
* left on disk by an interrupted build ships inside asar, unreachable and unverified, and grows
* the installer; content-addressed names mean stale copies never get overwritten.
*/
function assertNoUnlistedFiles(bundleDir, manifest) {
const listed = new Set(['manifest.json', ...manifest.assets.map((asset) => asset.path)])
const strays = listBundleFiles(bundleDir).filter((path) => !listed.has(path))
if (strays.length > 0) {
throw failure(
`${bundleDir} holds ${String(strays.length)} file(s) the manifest does not list: ` +
`${strays.sort().join(', ')}. ${REMEDY}`
)
}
}
/**
* Packaging guard: electron-builder only warns about a missing input, so without this a release
* would ship an app that advertises the bundle capability and then errors on every request. The
* hash check, not the existence check, is what catches a half-written or stale out/.
*/
function assertMobileWebBundleBuilt(bundleDir = MOBILE_WEB_BUNDLE_DIR) {
const manifest = parseManifest(bundleDir)
assertNoUnlistedFiles(bundleDir, manifest)
for (const asset of manifest.assets) {
const assetPath = join(bundleDir, asset.path)
let size
try {
size = statSync(assetPath).size
} catch (error) {
throw failure(
`manifest lists ${asset.path}, which is missing from ${bundleDir} (${error.code ?? error.message}). ${REMEDY}`
)
}
if (size !== asset.byteLength) {
throw failure(
`${asset.path} is ${String(size)} bytes on disk, manifest says ${String(asset.byteLength)}. ${REMEDY}`
)
}
const sha256 = createHash('sha256').update(readFileSync(assetPath)).digest('hex')
if (sha256 !== asset.sha256) {
throw failure(
`${asset.path} hashes to ${sha256} on disk, manifest says ${asset.sha256}. ${REMEDY}`
)
}
}
console.log(
`[verify-packaged-mobile-web-bundle] OK — buildId ${manifest.buildId}, ` +
`${String(manifest.assets.length)} asset(s), ${String(manifest.totalBytes)} bytes`
)
return manifest
}
// serializeAssets is exported for the parity test that pins it against the builder's and the
// contract's serializers; nothing in packaging calls it from outside this module.
module.exports = { MOBILE_WEB_BUNDLE_DIR, assertMobileWebBundleBuilt, serializeAssets }
@@ -0,0 +1,214 @@
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { buildMobileWebBundle } from './build-mobile-web-bundle.mjs'
const require = createRequire(import.meta.url)
const {
MOBILE_WEB_BUNDLE_DIR,
assertMobileWebBundleBuilt
} = require('./verify-packaged-mobile-web-bundle.cjs')
const electronBuilderConfig = require('../electron-builder.config.cjs')
const REPO_ROOT = join(import.meta.dirname, '..', '..')
async function withBundle(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-guard-'))
const bundleDir = join(scratch, 'mobile-web')
try {
const { manifest } = await buildMobileWebBundle({ outDir: bundleDir })
await run({ bundleDir, manifest })
} finally {
await rm(scratch, { recursive: true, force: true })
}
}
async function rewriteManifest(bundleDir, mutate) {
const manifestPath = join(bundleDir, 'manifest.json')
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
mutate(manifest)
await writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf8')
}
describe('assertMobileWebBundleBuilt', () => {
beforeEach(() => {
vi.spyOn(console, 'log').mockImplementation(() => {})
})
afterEach(() => {
vi.restoreAllMocks()
})
it('accepts a freshly built bundle', async () => {
await withBundle(({ bundleDir, manifest }) => {
expect(() => assertMobileWebBundleBuilt(bundleDir)).not.toThrow()
expect(manifest.entrypoint).toBe('index.html')
expect(manifest.assets.length).toBeGreaterThanOrEqual(3)
expect(
new Set(manifest.assets.map((asset) => asset.contentType)).size
).toBeGreaterThanOrEqual(2)
})
})
it('fails on a file the manifest does not list, so no stale asset ships inside asar', async () => {
await withBundle(async ({ bundleDir }) => {
// An asset dropped from the manifest keeps its content-addressed name, so nothing ever
// overwrites it; without this check it packs unreachable and unverified.
await writeFile(join(bundleDir, 'assets', 'stale.js'), '// from an earlier build\n', 'utf8')
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(
/does not list: assets\/stale\.js/
)
})
})
it('accepts exactly the manifest, the entrypoint and the listed assets', async () => {
await withBundle(async ({ bundleDir, manifest }) => {
const onDisk = (await readdir(bundleDir, { recursive: true, withFileTypes: true }))
.filter((entry) => entry.isFile())
.map((entry) => join(entry.parentPath, entry.name).slice(bundleDir.length + 1))
expect(onDisk.toSorted()).toEqual(
['manifest.json', ...manifest.assets.map((asset) => asset.path)].toSorted()
)
})
})
it('fails when the manifest is missing', async () => {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-guard-'))
try {
expect(() => assertMobileWebBundleBuilt(scratch)).toThrow(/no bundle manifest/)
} finally {
await rm(scratch, { recursive: true, force: true })
}
})
it('fails when the manifest is not JSON', async () => {
await withBundle(async ({ bundleDir }) => {
await writeFile(join(bundleDir, 'manifest.json'), 'not json', 'utf8')
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/not valid JSON/)
})
})
it('fails when an asset is tampered with on disk', async () => {
await withBundle(async ({ bundleDir, manifest }) => {
const asset = manifest.assets.find((entry) => entry.path.endsWith('.js'))
const bytes = await readFile(join(bundleDir, asset.path))
// Same length, different content: only the hash check can catch this.
bytes[bytes.length - 1] = bytes.at(-1) === 0x20 ? 0x09 : 0x20
await writeFile(join(bundleDir, asset.path), bytes)
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/hashes to .* on disk/)
})
})
it('fails when an asset is truncated', async () => {
await withBundle(async ({ bundleDir, manifest }) => {
const asset = manifest.assets.find((entry) => entry.path.endsWith('.css'))
await writeFile(join(bundleDir, asset.path), 'truncated', 'utf8')
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/bytes on disk, manifest says/)
})
})
it('fails when a listed asset was never written', async () => {
await withBundle(async ({ bundleDir, manifest }) => {
const asset = manifest.assets.find((entry) => entry.path.endsWith('.png'))
await rm(join(bundleDir, asset.path))
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/which is missing from/)
})
})
it('fails when the manifest buildId no longer matches its asset list', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
manifest.buildId = 'f'.repeat(64)
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/does not match its asset list/)
})
})
it('fails on an unknown schemaVersion', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
manifest.schemaVersion = 2
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(
/unsupported manifest schemaVersion/
)
})
})
it('fails when a required field is dropped', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
delete manifest.desktopVersion
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/desktopVersion is missing/)
})
})
it('fails when totalBytes disagrees with the asset list', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
manifest.totalBytes += 1
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/its assets sum to/)
})
})
it('refuses an asset path that escapes the bundle directory', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
manifest.assets[0].path = '../outside.js'
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/not a safe relative path/)
})
})
it('refuses a manifest whose entrypoint is not one of its assets', async () => {
await withBundle(async ({ bundleDir }) => {
await rewriteManifest(bundleDir, (manifest) => {
manifest.entrypoint = 'index.html'
manifest.assets = manifest.assets.filter((asset) => asset.path !== 'index.html')
})
expect(() => assertMobileWebBundleBuilt(bundleDir)).toThrow(/is not one of its assets/)
})
})
})
describe('electron-builder packaging wiring', () => {
it('excludes the mobile-web source tree from app.asar', () => {
expect(electronBuilderConfig.files).toContain('!src/mobile-web{,/**/*}')
// The source tree lives under src/, which is excluded wholesale; the explicit entry above
// only survives as a marker, so assert the broad rule is still what does the work.
expect(electronBuilderConfig.files).toContain('!src{,/**/*}')
})
it('does not exclude the built bundle, so out/mobile-web ships like out/web', () => {
const excludesBuiltBundle = electronBuilderConfig.files.some(
(entry) => typeof entry === 'string' && entry.startsWith('!out/mobile-web')
)
expect(excludesBuiltBundle).toBe(false)
})
it('runs the bundle guard in beforePack', () => {
expect(String(electronBuilderConfig.beforePack)).toContain('assertMobileWebBundleBuilt')
})
it('defaults the bundle root to out/mobile-web when electron-builder calls it', () => {
expect(MOBILE_WEB_BUNDLE_DIR).toBe(join(REPO_ROOT, 'out', 'mobile-web'))
// electron-builder passes the context alone, so the default is what ships.
expect(electronBuilderConfig.beforePack.length).toBe(1)
})
it('verifies the bundle root it is given, not the repo one', async () => {
// The seam exists so unit tests need no built out/; it would be worthless if the root were
// accepted and then ignored.
await withBundle(async ({ bundleDir }) => {
await rm(join(bundleDir, 'manifest.json'))
expect(() =>
electronBuilderConfig.beforePack(
{ electronPlatformName: process.platform, arch: process.arch === 'arm64' ? 3 : 1 },
bundleDir
)
).toThrow(/no bundle manifest/)
})
})
})
+8
View File
@@ -0,0 +1,8 @@
{
"extends": "@electron-toolkit/tsconfig/tsconfig.web.json",
"include": ["../src/mobile-web/src/**/*"],
"compilerOptions": {
"composite": true,
"types": []
}
}
+4 -4
View File
@@ -1,5 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 61m">
<title>downloads: 61m</title>
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 62m">
<title>downloads: 62m</title>
<linearGradient id="s" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
@@ -15,7 +15,7 @@
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="11">
<text x="37" y="15" fill="#010101" fill-opacity=".3">downloads</text>
<text x="37" y="14">downloads</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">61m</text>
<text x="90" y="14">61m</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">62m</text>
<text x="90" y="14">62m</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 935 B

After

Width:  |  Height:  |  Size: 935 B

@@ -0,0 +1,32 @@
# Release aborted shared auth filesystem waits
When a filesystem operation remains pending, later Codex/Kimi quota polls reuse it and wait with new deadlines. The old waiter uses `Promise.race` for every poll. On the installed Electron runtime, each abandoned race retains its rejection reason until the raw operation settles, despite removing its abort listener. The fix uses the existing `PromiseSettlementWaiters` registry, which attaches one raw-result reaction and removes expired waiters.
The original ownership symbols, last-waiter cancellation finalizer, one-raw-operation behavior, live callers, and future reads of a late result are preserved. Auth opts into deferred abort settlement so an already-queued raw result keeps its `Promise.race` priority; 24 success/failure/abort schedules compare equal before and after. Existing registry consumers keep their immediate-abort behavior. The abort factory type accepts `unknown` so false, zero, strings, and objects retain the existing auth rejection semantics. No admission or timeout limit changes.
## Reproduce
```sh
ORCA_BACKGROUND_LAUNCH=1 node --expose-gc --max-old-space-size=128 docs/audits/auth-filesystem-wait-retention/reproduce.cjs
```
For Electron, run the installed Electron executable with `ELECTRON_RUN_AS_NODE=1` and the same arguments/environment. This launches no window. The proof reconstructs the original sources by reversing `fix.patch`; expected baseline hashes in `source-versions.json` make source drift fail. Both versions run the actual production scheduler/waiter code, with only the raw filesystem operation replaced by one manually settled promise. A 15-second deadline fails stalled proof execution.
| Runtime / source | Plain aborted Errors alive while raw result pending | Amplified payload objects alive | After raw result settles | After owner drops |
| ---------------------------------------- | --------------------------------------------------- | ------------------------------- | ------------------------ | ----------------- |
| Node 26.6.0 / original | 1 of 128 | 1 of 128 | 1 | 0 |
| Node 26.6.0 / fixed | 1 of 128 | 1 of 128 | 1 | 0 |
| Electron 43.7.0, Node 24.21.0 / original | 128 of 128 | 128 of 128 | 1 | 0 |
| Electron 43.7.0, Node 24.21.0 / fixed | 1 of 128 | 1 of 128 | 1 | 0 |
The one remaining reason belongs to the existing cancellation controller's first abort. Dropping the shared operation releases it. AbortController objects and abort listeners are released by both versions. The amplified arm attaches a **synthetic 64 KiB Uint8Array** to each Error, at most 8 MiB per case. Ordinary timeout errors are much smaller; the separate plain-Error arm verifies that artificial bytes are not needed to reproduce retention. Runtime differences are measured, without attributing them to a particular V8 change.
Controls check an already-aborted first caller never starting raw work, raw rejection identity, aborted caller identity, future callers receiving late and already-settled results, a live sibling surviving cancellation, arbitrary abort reasons, and removed listeners. The unit regression additionally checks that repeated expired waits add no raw-result reactions and that their plain Error objects are collectible while a live caller still needs the operation.
Validation: 50 auth/registry tests pass, including 18 new cases; the reverse-patch configuration produces two expected failures and 48 passes. Six existing watcher-consumer suites pass another 39 tests. Node, Web, and CLI typechecks, focused ordinary/type-aware lint, formatting, and the changed-code quality gate pass. `validation.json` records the test paths and scope. To run the prior implementation against the current tests, use `--config docs/audits/auth-filesystem-wait-retention/before.config.mjs` with those six auth/registry test paths.
## Scope and limits
The three production consumers are `codex-auth-presence.ts`, `codex-backend-auth.ts`, and `kimi-fetcher.ts`. Each intentionally retains the shared operation until actual filesystem settlement to avoid stacking native requests when UNC/WSL reads stall. This audit does not reproduce a real filesystem stall, historical Electron binary behavior, or an affected-host workload.
The auth source matches `v1.4.198` and the audited main revision; the existing registry also matches main. The earlier broad accumulator PR #10179, reverted by #10255, added path/waiter/admission limits to this module. This change instead removes abandoned wait reactions and introduces no such limits. Nothing here attributes #19831 or #19768 to this mechanism or claims an incident-scale memory slope.
@@ -0,0 +1,24 @@
import { resolve } from 'node:path'
import { createRequire } from 'node:module'
import { defineConfig, mergeConfig } from 'vitest/config'
import baseConfig from '../../../config/vitest.config.ts'
const loadSources = createRequire(import.meta.url)(
resolve('docs/audits/auth-filesystem-wait-retention/sources.cjs')
)
const { before } = loadSources()
export default mergeConfig(
baseConfig,
defineConfig({
plugins: [
{
name: 'auth-wait-before-fix',
enforce: 'pre',
transform(_code, id) {
const source = before.get(resolve(id.split('?')[0]))
return source === undefined ? undefined : { code: source, map: null }
}
}
]
})
)
@@ -0,0 +1,598 @@
{
"sourceHashes": {
"src/main/rate-limits/auth-filesystem-operation.ts": {
"before": "581d7045220e92602ca83ec335576889dcba5ada4a699e866ea1dba10afbecac",
"after": "bfac9dc25c3f3c36ab590bc6e01be39ec19dea5a872f5c409ed149e65258f9e2"
},
"src/shared/promise-settlement-waiters.ts": {
"before": "b0b60bae6dcca4bb4f335ef7dda250f4350aa5b0306d3595b887cc4af4493060",
"after": "92618db591cf1fc1ad52f257cc9f3314310bf902fbd24e5d18abcf93afb8e722"
}
},
"runtime": {
"node": "24.21.0",
"acorn": "8.18.0",
"ada": "4.0.0",
"amaro": "1.1.11",
"ares": "1.34.8",
"brotli": "1.2.0",
"cldr": "48.0",
"icu": "78.2",
"llhttp": "9.4.3",
"merve": "1.2.2",
"modules": "148",
"napi": "10",
"nbytes": "0.1.4",
"ncrypto": "0.0.1",
"nghttp2": "1.70.0",
"nghttp3": "",
"ngtcp2": "",
"openssl": "0.0.0",
"simdjson": "4.6.7",
"simdutf": "7.7.0",
"sqlite": "3.53.4",
"tz": "2025c",
"undici": "7.29.1",
"unicode": "17.0",
"uv": "1.52.1",
"uvwasi": "0.0.23",
"v8": "15.0.245.31-electron.0",
"zlib": "1.3.2.1-motley",
"zstd": "1.6.0",
"electron": "43.7.0",
"chrome": "150.0.7871.250"
},
"scenario": "128 aborted waits on one already-started unresolved operation; amplified arm has 64 KiB synthetic payload per abort Error; real native filesystem stall not reproduced",
"before": {
"cases": [
{
"amplify": false,
"abortedWaits": 128,
"unresolved": {
"reasons": 128,
"controllers": 0,
"payloads": 0,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 0
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
},
{
"amplify": true,
"abortedWaits": 128,
"unresolved": {
"reasons": 128,
"controllers": 0,
"payloads": 128,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 1
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
}
],
"controls": {
"lateResultDelivered": true,
"settledResultDelivered": true,
"oneRawCall": 1,
"rawRejectionPreserved": true,
"preAbortedRawCalls": 0,
"allAbortListenersRemoved": true,
"arbitraryAbortReasonsPreserved": 4,
"liveSiblingSurvivesAbort": true
},
"settlementOrder": [
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
}
],
"importedHashes": {
"src/shared/wsl-paths.ts": "1d9dcf5a1ff6693c02ff60a6b074eb2ebb7e83bda12aceba7e0b208d410b6d1a"
},
"bundleSha256": "f895e71ca0f2102e7ff36fa133ae2065fa9810769d7ee6059c826f1a72bcc122"
},
"after": {
"cases": [
{
"amplify": false,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 0,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 0
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
},
{
"amplify": true,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 1,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 1
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
}
],
"controls": {
"lateResultDelivered": true,
"settledResultDelivered": true,
"oneRawCall": 1,
"rawRejectionPreserved": true,
"preAbortedRawCalls": 0,
"allAbortListenersRemoved": true,
"arbitraryAbortReasonsPreserved": 4,
"liveSiblingSurvivesAbort": true
},
"settlementOrder": [
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
}
],
"importedHashes": {
"src/shared/wsl-paths.ts": "1d9dcf5a1ff6693c02ff60a6b074eb2ebb7e83bda12aceba7e0b208d410b6d1a"
},
"bundleSha256": "04324b281a776aa1018995a36d11ff7cf56b4dd737761c3d5b0fb1ebc0047652"
}
}
@@ -0,0 +1,98 @@
diff --git a/src/main/rate-limits/auth-filesystem-operation.ts b/src/main/rate-limits/auth-filesystem-operation.ts
index 228234e92c..7d030ab01e 100644
--- a/src/main/rate-limits/auth-filesystem-operation.ts
+++ b/src/main/rate-limits/auth-filesystem-operation.ts
@@ -1,4 +1,5 @@
import { parseWslUncPath } from '../../shared/wsl-paths'
+import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters'
const MAX_CONCURRENT_WSL_AUTH_OPERATIONS = 2
const activeWslOperationDistros = new Set<string>()
@@ -139,10 +140,9 @@ export function createAuthFilesystemOperation<T>(
const waiters = new Set<symbol>()
let settled = false
const result = scheduleAuthFilesystemOperation(authPath, neededController.signal, operation)
- const markSettled = (): void => {
+ const settlementWaiters = new PromiseSettlementWaiters(result, () => {
settled = true
- }
- void result.then(markSettled, markSettled)
+ })
return {
result,
@@ -156,20 +156,18 @@ export function createAuthFilesystemOperation<T>(
const waiter = Symbol('auth-filesystem-waiter')
waiters.add(waiter)
- let onAbort: (() => void) | null = null
- const aborted = new Promise<never>((_resolve, reject) => {
- onAbort = () => reject(getAbortReason(signal))
- signal.addEventListener('abort', onAbort, { once: true })
- })
- return Promise.race([result, aborted]).finally(() => {
- if (onAbort) {
- signal.removeEventListener('abort', onAbort)
- }
- waiters.delete(waiter)
- if (!settled && waiters.size === 0) {
- neededController.abort(getAbortReason(signal))
- }
- })
+ return settlementWaiters
+ .wait({
+ signal,
+ abortInMicrotask: true,
+ createAbortError: () => getAbortReason(signal)
+ })
+ .finally(() => {
+ waiters.delete(waiter)
+ if (!settled && waiters.size === 0) {
+ neededController.abort(getAbortReason(signal))
+ }
+ })
}
}
}
diff --git a/src/shared/promise-settlement-waiters.ts b/src/shared/promise-settlement-waiters.ts
index 98ec24b25c..97230d54dd 100644
--- a/src/shared/promise-settlement-waiters.ts
+++ b/src/shared/promise-settlement-waiters.ts
@@ -13,8 +13,10 @@ type PromiseSettlementWaiter<T> = {
export type PromiseSettlementWaitOptions<T> = {
signal?: AbortSignal
+ /** Preserve Promise.race ordering when raw settlement and abort share a turn. */
+ abortInMicrotask?: boolean
timeoutMs?: number
- createAbortError?: () => Error
+ createAbortError?: () => unknown
createTimeoutError?: () => Error
onFulfilled?: (value: T) => void
onAbandon?: (reason: 'abort' | 'timeout') => void
@@ -50,7 +52,7 @@ export class PromiseSettlementWaiters<T> {
}
return new Promise<T>((resolve, reject) => {
let waiter!: PromiseSettlementWaiter<T>
- const abandon = (reason: 'abort' | 'timeout', error: Error): void => {
+ const abandon = (reason: 'abort' | 'timeout', error: unknown): void => {
if (!this.waiters.delete(waiter)) {
return
}
@@ -58,8 +60,14 @@ export class PromiseSettlementWaiters<T> {
options.onAbandon?.(reason)
reject(error)
}
- const onAbort = (): void =>
- abandon('abort', options.createAbortError?.() ?? createDefaultAbortError())
+ const onAbort = (): void => {
+ const error = options.createAbortError?.() ?? createDefaultAbortError()
+ if (options.abortInMicrotask) {
+ queueMicrotask(() => abandon('abort', error))
+ } else {
+ abandon('abort', error)
+ }
+ }
waiter = {
resolve,
reject,
@@ -0,0 +1,597 @@
{
"sourceHashes": {
"src/main/rate-limits/auth-filesystem-operation.ts": {
"before": "581d7045220e92602ca83ec335576889dcba5ada4a699e866ea1dba10afbecac",
"after": "bfac9dc25c3f3c36ab590bc6e01be39ec19dea5a872f5c409ed149e65258f9e2"
},
"src/shared/promise-settlement-waiters.ts": {
"before": "b0b60bae6dcca4bb4f335ef7dda250f4350aa5b0306d3595b887cc4af4493060",
"after": "92618db591cf1fc1ad52f257cc9f3314310bf902fbd24e5d18abcf93afb8e722"
}
},
"runtime": {
"node": "26.6.0",
"acorn": "8.17.0",
"ada": "4.0.0",
"amaro": "1.1.11",
"ares": "1.34.8",
"brotli": "1.2.0",
"cldr": "48.0",
"icu": "78.3",
"libffi": "3.7.1",
"llhttp": "9.4.3",
"merve": "1.2.2",
"modules": "147",
"napi": "10",
"nbytes": "0.1.4",
"ncrypto": "0.0.1",
"nghttp2": "1.70.0",
"nghttp3": "",
"ngtcp2": "",
"openssl": "3.6.3",
"simdjson": "4.6.6",
"simdutf": "7.7.0",
"sqlite": "3.53.4",
"tz": "2026a",
"undici": "8.9.0",
"unicode": "17.0",
"uv": "1.52.1",
"uvwasi": "0.0.23",
"v8": "14.6.202.34-node.26",
"zlib": "1.2.12",
"zstd": "1.5.7"
},
"scenario": "128 aborted waits on one already-started unresolved operation; amplified arm has 64 KiB synthetic payload per abort Error; real native filesystem stall not reproduced",
"before": {
"cases": [
{
"amplify": false,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 0,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 0
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
},
{
"amplify": true,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 1,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 1
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
}
],
"controls": {
"lateResultDelivered": true,
"settledResultDelivered": true,
"oneRawCall": 1,
"rawRejectionPreserved": true,
"preAbortedRawCalls": 0,
"allAbortListenersRemoved": true,
"arbitraryAbortReasonsPreserved": 4,
"liveSiblingSurvivesAbort": true
},
"settlementOrder": [
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
}
],
"importedHashes": {
"src/shared/wsl-paths.ts": "1d9dcf5a1ff6693c02ff60a6b074eb2ebb7e83bda12aceba7e0b208d410b6d1a"
},
"bundleSha256": "f895e71ca0f2102e7ff36fa133ae2065fa9810769d7ee6059c826f1a72bcc122"
},
"after": {
"cases": [
{
"amplify": false,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 0,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 0
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
},
{
"amplify": true,
"abortedWaits": 128,
"unresolved": {
"reasons": 1,
"controllers": 0,
"payloads": 1,
"rawCalls": 1
},
"settled": {
"reasons": 1,
"controllers": 0,
"payloads": 1
},
"dropped": {
"reasons": 0,
"controllers": 0,
"payloads": 0
}
}
],
"controls": {
"lateResultDelivered": true,
"settledResultDelivered": true,
"oneRawCall": 1,
"rawRejectionPreserved": true,
"preAbortedRawCalls": 0,
"allAbortListenersRemoved": true,
"arbitraryAbortReasonsPreserved": 4,
"liveSiblingSurvivesAbort": true
},
"settlementOrder": [
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": true,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 1,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 2,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 3,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 4,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": false,
"ticks": 5,
"outcome": {
"status": "fulfilled",
"value": "raw success"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 0,
"outcome": {
"status": "rejected",
"reason": "caller aborted"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 1,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 2,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 3,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 4,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
},
{
"startedBefore": false,
"rejectRaw": true,
"ticks": 5,
"outcome": {
"status": "rejected",
"reason": "raw failure"
}
}
],
"importedHashes": {
"src/shared/wsl-paths.ts": "1d9dcf5a1ff6693c02ff60a6b074eb2ebb7e83bda12aceba7e0b208d410b6d1a"
},
"bundleSha256": "04324b281a776aa1018995a36d11ff7cf56b4dd737761c3d5b0fb1ebc0047652"
}
}
@@ -0,0 +1,242 @@
const assert = require('node:assert/strict')
const { createHash } = require('node:crypto')
const { readFileSync, writeFileSync } = require('node:fs')
const { resolve } = require('node:path')
const Module = require('node:module')
const { getEventListeners } = require('node:events')
const { build } = require('esbuild')
const { root, before, after, hashes } = require('./sources.cjs')()
const settlementOrder = require('./settlement-order.cjs')
assert.equal(process.env.ORCA_BACKGROUND_LAUNCH, '1')
assert.equal(typeof global.gc, 'function')
const sourcePath = 'src/main/rate-limits/auth-filesystem-operation.ts'
const entry = resolve(root, sourcePath)
let candidate = false
let createAuthFilesystemOperation
const turn = () => new Promise((resolveTurn) => setImmediate(resolveTurn))
async function collect() {
for (let index = 0; index < 5; index++) {
await turn()
global.gc()
}
await turn()
}
const count = (refs) => refs.reduce((total, ref) => total + Number(ref.deref() !== undefined), 0)
async function abandonedWait(operation, index, amplify) {
const controller = new AbortController()
const reason = new Error(`synthetic expired poll ${index}`)
// Payload amplifies the retained rejection object; normal timeout errors are much smaller.
if (amplify) {
reason.auditPayload = new Uint8Array(64 * 1024)
reason.auditPayload.fill(index & 255)
}
const references = {
reason: new WeakRef(reason),
controller: new WeakRef(controller),
...(amplify ? { payload: new WeakRef(reason.auditPayload) } : {})
}
const waiting = operation.wait(controller.signal)
controller.abort(reason)
await waiting.catch(() => {})
assert.equal(getEventListeners(controller.signal, 'abort').length, 0)
return references
}
function snapshot(refs) {
return {
reasons: count(refs.map((ref) => ref.reason)),
controllers: count(refs.map((ref) => ref.controller)),
payloads: count(refs.flatMap((ref) => (ref.payload ? [ref.payload] : [])))
}
}
async function retention(amplify) {
let settleRaw
let rawCalls = 0
let operation = createAuthFilesystemOperation('/synthetic-local-auth', () => {
rawCalls++
return new Promise((resolveRaw) => {
settleRaw = resolveRaw
})
})
await turn()
assert.equal(rawCalls, 1)
const refs = []
for (let index = 0; index < 128; index++) {
refs.push(await abandonedWait(operation, index, amplify))
}
await collect()
const unresolved = { ...snapshot(refs), rawCalls }
if (candidate) {
assert.equal(unresolved.reasons, 1)
}
settleRaw('finished')
await operation.result
await collect()
const settled = snapshot(refs)
assert.equal(settled.reasons, 1)
operation = null
settleRaw = null
await collect()
const dropped = snapshot(refs)
assert.deepEqual(dropped, { reasons: 0, controllers: 0, payloads: 0 })
return { amplify, abortedWaits: refs.length, unresolved, settled, dropped }
}
async function controls() {
let rawCalls = 0
let finish
const operation = createAuthFilesystemOperation('/synthetic-auth-controls', () => {
rawCalls++
return new Promise((resolveRaw) => {
finish = resolveRaw
})
})
const expired = new AbortController()
const expiredReason = new Error('expired first poll')
const abortedWait = operation.wait(expired.signal)
await turn()
expired.abort(expiredReason)
await assert.rejects(abortedWait, (reason) => reason === expiredReason)
const later = new AbortController()
const lateWait = operation.wait(later.signal)
finish('late raw result')
assert.equal(await lateWait, 'late raw result')
assert.equal(rawCalls, 1)
assert.equal(await operation.wait(later.signal), 'late raw result')
assert.equal(getEventListeners(expired.signal, 'abort').length, 0)
assert.equal(getEventListeners(later.signal, 'abort').length, 0)
let rejectedCalls = 0
const rejectedReason = new Error('raw rejected')
const rejected = createAuthFilesystemOperation('/synthetic-auth-rejected', async () => {
rejectedCalls++
throw rejectedReason
})
await assert.rejects(
rejected.wait(new AbortController().signal),
(reason) => reason === rejectedReason
)
let preAbortedCalls = 0
const preAborted = createAuthFilesystemOperation('/synthetic-auth-pre-aborted', async () => {
preAbortedCalls++
return 'unexpected'
})
const priorAbort = new AbortController()
priorAbort.abort(expiredReason)
await assert.rejects(preAborted.wait(priorAbort.signal), (reason) => reason === expiredReason)
await assert.rejects(preAborted.result, (reason) => reason === expiredReason)
assert.equal(preAbortedCalls, 0)
let finishReasons
const reasonOperation = createAuthFilesystemOperation(
'/synthetic-auth-reasons',
() =>
new Promise((resolveRaw) => {
finishReasons = resolveRaw
})
)
await turn()
const reasons = [false, 0, 'string abort', { code: 'custom' }]
for (const reason of reasons) {
const controller = new AbortController()
const pending = reasonOperation.wait(controller.signal)
controller.abort(reason)
await assert.rejects(pending, (observed) => observed === reason)
assert.equal(getEventListeners(controller.signal, 'abort').length, 0)
}
const activeController = new AbortController()
const cancelledController = new AbortController()
const active = reasonOperation.wait(activeController.signal)
const cancelled = reasonOperation.wait(cancelledController.signal)
cancelledController.abort(expiredReason)
await assert.rejects(cancelled, (reason) => reason === expiredReason)
finishReasons('active result')
assert.equal(await active, 'active result')
assert.equal(getEventListeners(activeController.signal, 'abort').length, 0)
return {
lateResultDelivered: true,
settledResultDelivered: true,
oneRawCall: rawCalls,
rawRejectionPreserved: rejectedCalls === 1,
preAbortedRawCalls: preAbortedCalls,
allAbortListenersRemoved: true,
arbitraryAbortReasonsPreserved: reasons.length,
liveSiblingSurvivesAbort: true
}
}
async function phase(sources, fixed) {
candidate = fixed
const built = await build({
entryPoints: [entry],
bundle: true,
platform: 'node',
format: 'cjs',
write: false,
metafile: true,
logLevel: 'silent',
plugins: [
{
name: 'hash-fenced-proof-source',
setup(api) {
api.onLoad(
{ filter: /(?:auth-filesystem-operation|promise-settlement-waiters)\.ts$/ },
(args) =>
sources.has(args.path)
? {
contents: sources.get(args.path),
loader: 'ts',
resolveDir: resolve(args.path, '..')
}
: undefined
)
}
}
]
})
const bundled = built.outputFiles[0].text
const moduleOwner = new Module(entry, module)
moduleOwner.filename = entry
moduleOwner.paths = module.paths
moduleOwner._compile(bundled, entry)
createAuthFilesystemOperation = moduleOwner.exports.createAuthFilesystemOperation
const importedHashes = Object.fromEntries(
Object.keys(built.metafile.inputs)
.filter((path) => !sources.has(resolve(root, path)))
.map((path) => [
path,
createHash('sha256')
.update(readFileSync(resolve(root, path)))
.digest('hex')
])
)
return {
cases: [await retention(false), await retention(true)],
controls: await controls(),
settlementOrder: await settlementOrder(createAuthFilesystemOperation),
importedHashes,
bundleSha256: createHash('sha256').update(bundled).digest('hex')
}
}
async function run() {
const result = {
sourceHashes: hashes,
runtime: process.versions,
scenario:
'128 aborted waits on one already-started unresolved operation; amplified arm has 64 KiB synthetic payload per abort Error; real native filesystem stall not reproduced',
before: await phase(before, false),
after: await phase(after, true)
}
assert.deepEqual(result.after.settlementOrder, result.before.settlementOrder)
const output = process.argv[2]
? resolve(process.argv[2])
: resolve(__dirname, `${process.versions.electron ? 'electron-' : 'node-'}results.json`)
writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`)
console.log(JSON.stringify(result, null, 2))
}
const deadline = setTimeout(() => {
console.error('Auth wait proof exceeded 15 seconds')
process.exit(1)
}, 15_000)
run()
.catch((error) => {
console.error(error)
process.exitCode = 1
})
.finally(() => clearTimeout(deadline))
@@ -0,0 +1,37 @@
const turn = () => new Promise((resolveTurn) => setImmediate(resolveTurn))
module.exports = async function settlementOrder(create) {
const cases = []
for (const startedBefore of [true, false]) {
for (const rejectRaw of [false, true]) {
for (let ticks = 0; ticks < 6; ticks++) {
let settle
const operation = create(
'synthetic-auth-order',
() =>
new Promise((resolveRaw, failRaw) => {
settle = () => (rejectRaw ? failRaw('raw failure') : resolveRaw('raw success'))
})
)
await turn()
const controller = new AbortController()
const start = () =>
operation.wait(controller.signal).then(
(value) => ({ status: 'fulfilled', value }),
(reason) => ({ status: 'rejected', reason })
)
let waiting = startedBefore ? start() : null
settle()
for (let index = 0; index < ticks; index++) {
await Promise.resolve()
}
if (!startedBefore) {
waiting = start()
}
controller.abort('caller aborted')
cases.push({ startedBefore, rejectRaw, ticks, outcome: await waiting })
}
}
}
return cases
}
@@ -0,0 +1,11 @@
{
"baselineHashes": {
"src/main/rate-limits/auth-filesystem-operation.ts": "581d7045220e92602ca83ec335576889dcba5ada4a699e866ea1dba10afbecac",
"src/shared/promise-settlement-waiters.ts": "b0b60bae6dcca4bb4f335ef7dda250f4350aa5b0306d3595b887cc4af4493060"
},
"checkedMainRevision": "77cd61df396f25ec91ee2d5ddcbd1f55aa94f818",
"sharedTestBaselineSha256": "1e274da93106f31b9b57aa48fd965a4d81c8adf4ebe1bfa20a196aa3e529a73a",
"authSourceIdenticalNamedRefs": ["origin/main", "v1.4.198"],
"registrySourceIdenticalNamedRefs": ["origin/main"],
"historicalRuntimeReproduced": false
}
@@ -0,0 +1,29 @@
const assert = require('node:assert/strict')
const { createHash } = require('node:crypto')
const { readFileSync } = require('node:fs')
const { resolve } = require('node:path')
const { applyPatch, parsePatch, reversePatch } = require('diff')
module.exports = function loadSources() {
const root = resolve(__dirname, '../../..')
const expected = JSON.parse(readFileSync(resolve(__dirname, 'source-versions.json'), 'utf8'))
const parsed = parsePatch(readFileSync(resolve(__dirname, 'fix.patch'), 'utf8'))
const before = new Map()
const after = new Map()
const hashes = {}
assert.equal(parsed.length, 2)
for (const patch of parsed) {
const path = patch.newFileName.replace(/^b\//, '')
assert.ok(Object.hasOwn(expected.baselineHashes, path), `Unexpected patch path: ${path}`)
const absolute = resolve(root, path)
const current = readFileSync(absolute, 'utf8')
const baseline = applyPatch(current, reversePatch(patch))
assert.notEqual(baseline, false, `Source changed; review fix.patch: ${path}`)
const hash = (source) => createHash('sha256').update(source).digest('hex')
assert.equal(hash(baseline), expected.baselineHashes[path], `Baseline drift: ${path}`)
before.set(absolute, baseline)
after.set(absolute, current)
hashes[path] = { before: hash(baseline), after: hash(current) }
}
return { root, before, after, hashes }
}
@@ -0,0 +1,61 @@
{
"backgroundLaunch": true,
"authAndRegistryTests": {
"before": { "passed": 48, "failed": 2, "exitCode": 1 },
"after": { "passed": 50, "failed": 0, "exitCode": 0 },
"newCases": 18,
"expectedBeforeFailures": [
"PromiseSettlementWaiters preserves abort scheduling with abortInMicrotask=true",
"shared auth filesystem wait lifetime releases aborted poll reasons while one native operation remains needed"
],
"paths": [
"src/main/rate-limits/auth-filesystem-operation.test.ts",
"src/main/rate-limits/auth-filesystem-operation-retention.test.ts",
"src/main/rate-limits/codex-auth-presence.test.ts",
"src/main/rate-limits/kimi-fetcher-wsl-home.test.ts",
"src/main/rate-limits/kimi-fetcher.test.ts",
"src/shared/promise-settlement-waiters.test.ts"
]
},
"existingRegistryConsumers": {
"passed": 39,
"failed": 0,
"exitCode": 0,
"paths": [
"src/relay/relay-watcher-setup-wait.test.ts",
"src/relay/relay-filesystem-watch-registry.test.ts",
"src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts",
"src/main/runtime/file-watcher-host.test.ts",
"src/main/ipc/runtime-watcher-pending-assignment.test.ts",
"src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts"
]
},
"typechecks": {
"command": "node config/scripts/run-typecheck-projects-in-parallel.mjs",
"projects": [
"config/tsconfig.node.json",
"config/tsconfig.tc.cli.json",
"config/tsconfig.tc.web.json"
],
"exitCode": 0
},
"focusedOxlint": { "ordinaryExitCode": 0, "typeAwareExitCode": 0 },
"formatCheckExitCode": 0,
"changedCodeQuality": {
"base": "2fccacadbe23",
"changedFiles": 297,
"newFindings": 0,
"exitCode": 0
},
"proof": {
"node": "26.6.0",
"electron": "43.7.0",
"electronNode": "24.21.0",
"bothExitCode": 0,
"orderingCasesPerRuntime": 24,
"beforeAfterOrderingEqual": true,
"rawFilesystemStall": "injected pending promise, not an affected-host capture",
"amplifiedBytesPerCase": 8388608,
"ordinaryErrorBytes": "not measured"
}
}
@@ -0,0 +1,77 @@
# Closed browser dispatcher retains completed results behind pending native work
Before the fix, completed command results stayed in a closed browser dispatcher until its final handler settled. The fix releases settled cache records at close and drops newly settled records while closed. Pending native handlers, page authority, and executor teardown keep their existing lifetime.
## Actual paths and bounds
Source references in this section describe the hash-fenced baseline. `BrowserClientHostCommandDispatcher.dispatch` refuses every command once closed, before authority or duplicate lookup (`browser-client-host-command-dispatcher.ts:7779`). `close` aborts active work and cancels queued work, but retains its pages and cached completed records when the join returns false (`:156179`). `finishHandler` clears those owners only after the last native handler settles (`:268272`). A newly settled cancellation record is also cached while a sibling remains active (`:297302`).
`BrowserClientHostCommandResultCache.clear` drops only its record-to-page index. PageState.records and PageState.sequencesByCommandId also own the records; clearing just that index does not release result graphs. Existing `releasePage` uses exact settled-record eviction to remove both indexes (`browser-client-host-command-result-cache.ts:2755`).
Defaults (`browser-client-host-command-state.ts:713`) are 256 pages, 256 active commands, 8 concurrent handlers, 32 queued/page, 64 cached results/page, 1,024 cached results total, and a 5,000 ms close/retirement join. Automation result schema allows at most 768 KiB of JSON-serialized value (`browser-client-automation-protocol.ts:5,8999,116131`). These are count/serialized-value limits, not a guaranteed heap/RSS size. The audit uses 32 tiny results, one native wait, and one canceled tiny queued input; it does not allocate near those maxima.
Production composition calls dispatcher close via `PairedRuntimeBrowserClientHost.closeHost` (`paired-runtime-browser-client-host.ts:165180`). If close times out, actual `closeBrowserClientHostComposition` defers executor close behind `whenHandlersSettled` (`paired-runtime-browser-client-host-teardown.ts:3756`). Keeping handler/page/native authority alive is intentional. Completed results cannot serve new or duplicate closed requests and need not share that lifetime.
## Ordinary handler time boundaries
- The navigation command checks cancellation before starting, then calls `routeWebContents.navigateGuest` (`browser-client-page-command-execution.ts:2040`). The actual registry delegates to `navigateBrowserRouteGuest`, which awaits native `guest.loadURL` (`browser-route-guest-lifecycle.ts:99123`) without adding a JS deadline or taking the AbortSignal. Native completion/rejection remains its settlement owner.
- Automation checks cancellation before execution, registers the exact guest, and forwards the signal into RPC (`browser-client-page-automation-runtime.ts:4257`; startup `main-process-ready-runtime.ts:6177`). Core handlers such as browser.snapshot destructure runtime and call its method without observing that signal (`runtime/rpc/methods/browser-core.ts:3943`).
- The ordinary agent-browser helper execution has a 90-second default subprocess timeout (`agent-browser-bridge-types.ts:6`; `agent-browser-bridge-raw-process.ts:2036`), with overrides for some operations. The agent-bridge embedded goto wrapper separately has a 30-second navigation timeout. Those deadlines are not a universal bound on all handler phases, and the direct route navigate path above does not use that wrapper.
The condition is a handler that outlives the dispatcher's five-second join. No affected-host occurrence, natural indefinite stall, or incident attribution has been established.
## Bounded actual-source proof
`scenario.test.mjs` uses the actual dispatcher, page executor, automation runtime, browser.snapshot RPC descriptor, native-navigation wrapper, and composition teardown function. Existing page-executor harness supplies renderer/session/route ports. The runtime's browserSnapshot/native loadURL are small controlled ports; no Electron window, OS child, real web request, or network is used. Logger/mock result arrays do not own the produced payloads: the automation output and dispatcher handler are plain functions, and each completed value is observed only through WeakRef after the helper returns.
Both Node 26.6 and Electron 43.7 / Node 24.21 pass all 4 cases before and after the two-file fix. A 15 ms join override keeps the proof bounded; the native port is explicitly resolved in finally and all native custody eventually settles.
| Observation | Before | Fixed |
| ---------------------------------------------------------------------- | -----: | ----: |
| Completed small payload objects alive after timed-out close | 32 | 0 |
| Cached records after close (32 results + create + queued cancellation) | 34 | 0 |
| Canceled queued input still reachable | yes | no |
| Running native handlers after close | 1 | 1 |
| Page/executor and route/session custody retained | yes | yes |
| Close repeated before native settlement | false | false |
| whenClosed pending before native settlement | yes | yes |
| Payload objects alive after explicit native settlement | 0 | 0 |
| First late cancellation cached while sibling remains pending | 1 | 0 |
Open request replay preserves the exact same Promise and runs once. Closed duplicates fail with dispatcher_closed. Normal native resolution and rejection both complete the existing settlement path. Executor close and route/session release happen only after native settlement in both variants.
The initial candidate compatibility run passed 78 tests in 5 files, including its three lifecycle cases and existing dispatcher, page executor, paired-runtime composition, and paired-runtime host tests. The permanent retention suite adds two object-lifetime regressions: releasing completed results while native navigation stays pending, and releasing one late closed record while a sibling handler remains active. Final validation passed 77 tests across 5 production suites, Node typechecking, and all five explicit quality scans over product and artifact code. Reversing the patch produces the two expected lifetime failures while all 16 existing dispatcher tests pass. Commands and outcomes are recorded in `validation.json`.
## Fix scope
`fix.patch` changes only:
- `src/main/browser/browser-client-host-command-dispatcher.ts`: release each page's already-settled cache during close, after cancellation; discard newly settled records instead of caching them when closed.
- `src/main/browser/browser-client-host-command-result-cache.ts`: accept an optional `retain` flag in `record`, defaulting to the existing caching behavior. When false, existing identity-checked eviction drops the settled record before any cache admission.
Active/cancelling records, pages, native promises, abort behavior, join timing, FIFO, generation/authority checks, and the closed-settlement promise remain owned exactly as before. No row/byte cap changes. The host and executor continue waiting for their existing settlement owners.
## retirePage is separate
`selectCommandPage` rejects retiring and retired generations before `findExistingCommand` (`browser-client-host-command-page.ts:3443`). Thus retired duplicate replay is already unavailable, even though the cache remains until forget/replacement. The control confirms that behavior and verifies explicit forget releases the cache and preserves the stale-generation floor. This fix leaves that existing retirement policy unchanged; freeing results on page retirement is a separate possible follow-up, especially while executor cleanup is still pending. Do not assume a live duplicate replay contract where the actual admission path rejects first.
## Reproduction and source fences
```sh
ORCA_BACKGROUND_LAUNCH=1 ORCA_BROWSER_CACHE_VARIANT=before pnpm exec vitest run --config docs/audits/browser-closed-result-retention/vitest.config.mjs docs/audits/browser-closed-result-retention/scenario.test.mjs
ORCA_BACKGROUND_LAUNCH=1 ORCA_BROWSER_CACHE_VARIANT=fixed pnpm exec vitest run --config docs/audits/browser-closed-result-retention/vitest.config.mjs docs/audits/browser-closed-result-retention/scenario.test.mjs
```
For Electron run the installed binary with `ELECTRON_RUN_AS_NODE=1` and `ORCA_BACKGROUND_LAUNCH=1`, passing node_modules/vitest/vitest.mjs and the same arguments. Reports are separate per runtime/variant; set `ORCA_BROWSER_CACHE_OUTPUT` to another file path to preserve captured reports. `sources.cjs` reverses `fix.patch` in memory and checks exact baseline/fixed hashes plus 21 caller/dependency hashes. The config loads those sources at their real production module IDs without changing checkout files. A synthetic CRLF control checks all 24 source/patch reads against canonical LF hashes. Both variants use the same controlled producer and lifecycle ports.
`source-versions.json` records 23 canonical-LF source/caller hashes. All 23 match named main checkpoint 291b4ddd6f1c1af480169885e0fda7f9c78ff053; 21 match v1.4.198. Both fixed source baselines match both named versions. The proof executes current source/dependencies, not a historical application binary.
## Permanent regression validation
```sh
ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/browser/browser-client-host-command-retention.test.ts src/main/browser/browser-client-host-command-dispatcher.test.ts src/main/browser/browser-client-page-command-executor.test.ts src/main/browser/paired-runtime-browser-client-host-composition.test.ts src/main/browser/paired-runtime-browser-client-host.test.ts
ORCA_BACKGROUND_LAUNCH=1 ORCA_BROWSER_CACHE_VARIANT=before pnpm exec vitest run --config docs/audits/browser-closed-result-retention/vitest.config.mjs src/main/browser/browser-client-host-command-retention.test.ts src/main/browser/browser-client-host-command-dispatcher.test.ts
ORCA_BACKGROUND_LAUNCH=1 pnpm tc:node
```
The baseline regression command intentionally fails the two new lifetime assertions. The source and object counts prove a code mechanism, not incident-specific browser use, native stall duration, aggregate app RSS, or attribution to #19831.
@@ -0,0 +1,138 @@
{
"sources": {
"src/main/browser/browser-client-host-command-dispatcher.ts": {
"before": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"after": "0e4074a4cb7379de58d75dd93e3f74090dea18a944bd4f038730cf19f6e68a98"
},
"src/main/browser/browser-client-host-command-result-cache.ts": {
"before": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"after": "f06bd1f31be9ffcbb750f242bacd0c2f16222cde7d011010acbd46ceb70ea83f"
},
"src/main/browser/browser-client-host-command-state.ts": {
"before": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"after": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f"
},
"src/main/browser/browser-client-host-command-page.ts": {
"before": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"after": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb"
},
"src/main/browser/browser-client-host-command-join.ts": {
"before": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"after": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f"
},
"src/main/browser/browser-client-page-command-executor.ts": {
"before": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"after": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01"
},
"src/main/browser/browser-client-page-command-execution.ts": {
"before": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"after": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e"
},
"src/main/browser/browser-client-page-command-executor-test-harness.ts": {
"before": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"after": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24"
},
"src/main/browser/browser-client-page-automation-runtime.ts": {
"before": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"after": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319"
},
"src/main/browser/browser-route-guest-lifecycle.ts": {
"before": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"after": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb"
},
"src/main/browser/browser-route-webcontents-registry.ts": {
"before": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"after": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad"
},
"src/main/browser/paired-runtime-browser-client-host.ts": {
"before": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"after": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038"
},
"src/main/browser/paired-runtime-browser-client-host-composition.ts": {
"before": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"after": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74"
},
"src/main/browser/paired-runtime-browser-client-host-teardown.ts": {
"before": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"after": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59"
},
"src/main/browser/paired-runtime-browser-client-host-runtime.ts": {
"before": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"after": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c"
},
"src/main/runtime/rpc/methods/browser-core.ts": {
"before": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"after": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d"
},
"src/main/runtime/runtime-browser-commands-active-screencasts-by-page-id.ts": {
"before": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"after": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038"
},
"src/main/browser/agent-browser-bridge-types.ts": {
"before": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"after": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f"
},
"src/main/browser/agent-browser-bridge-raw-process.ts": {
"before": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"after": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c"
},
"src/main/browser/agent-browser-bridge-core-commands.ts": {
"before": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"after": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03"
},
"src/main/startup/main-process-ready-runtime.ts": {
"before": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"after": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b"
},
"src/shared/browser-client-host-protocol.ts": {
"before": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"after": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956"
},
"src/shared/browser-client-automation-protocol.ts": {
"before": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"after": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58"
}
},
"runtime": {
"node": "24.21.0",
"electron": "43.7.0",
"v8": "15.0.245.31-electron.0"
},
"variant": "before",
"cases": [
{
"kind": "native-navigation-close",
"completedPayloads": 32,
"heldNativePorts": 1,
"joinTimeoutOverrideMs": 15,
"retainedPayloadsAfterClose": 32,
"cachedResultsAfterClose": 34,
"cancelledQueuedInputRetained": true,
"signalAborted": true,
"executorCustodyPreserved": true,
"routeLeasePreserved": true,
"closedDuplicateRejected": true,
"secondCloseSettled": false,
"retainedAfterNativeSettlement": 0,
"executorClosedAfterNativeSettlement": true
},
{
"kind": "late-sibling-settlement",
"oneHandlerStillOwned": true,
"cachedAfterFirstSettlement": 1,
"closedSettlementStillPending": true
},
{
"kind": "open-replay-and-retire-contract",
"openPromiseIdentityPreserved": true,
"retiredDuplicateRejected": true,
"retireCachePolicyUnchanged": true,
"explicitForgetReleasedCache": true
},
{
"kind": "synthetic-crlf-source-control",
"canonicalHashesMatch": true,
"reads": 24
}
]
}
@@ -0,0 +1,138 @@
{
"sources": {
"src/main/browser/browser-client-host-command-dispatcher.ts": {
"before": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"after": "0e4074a4cb7379de58d75dd93e3f74090dea18a944bd4f038730cf19f6e68a98"
},
"src/main/browser/browser-client-host-command-result-cache.ts": {
"before": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"after": "f06bd1f31be9ffcbb750f242bacd0c2f16222cde7d011010acbd46ceb70ea83f"
},
"src/main/browser/browser-client-host-command-state.ts": {
"before": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"after": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f"
},
"src/main/browser/browser-client-host-command-page.ts": {
"before": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"after": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb"
},
"src/main/browser/browser-client-host-command-join.ts": {
"before": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"after": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f"
},
"src/main/browser/browser-client-page-command-executor.ts": {
"before": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"after": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01"
},
"src/main/browser/browser-client-page-command-execution.ts": {
"before": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"after": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e"
},
"src/main/browser/browser-client-page-command-executor-test-harness.ts": {
"before": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"after": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24"
},
"src/main/browser/browser-client-page-automation-runtime.ts": {
"before": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"after": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319"
},
"src/main/browser/browser-route-guest-lifecycle.ts": {
"before": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"after": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb"
},
"src/main/browser/browser-route-webcontents-registry.ts": {
"before": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"after": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad"
},
"src/main/browser/paired-runtime-browser-client-host.ts": {
"before": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"after": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038"
},
"src/main/browser/paired-runtime-browser-client-host-composition.ts": {
"before": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"after": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74"
},
"src/main/browser/paired-runtime-browser-client-host-teardown.ts": {
"before": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"after": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59"
},
"src/main/browser/paired-runtime-browser-client-host-runtime.ts": {
"before": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"after": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c"
},
"src/main/runtime/rpc/methods/browser-core.ts": {
"before": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"after": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d"
},
"src/main/runtime/runtime-browser-commands-active-screencasts-by-page-id.ts": {
"before": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"after": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038"
},
"src/main/browser/agent-browser-bridge-types.ts": {
"before": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"after": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f"
},
"src/main/browser/agent-browser-bridge-raw-process.ts": {
"before": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"after": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c"
},
"src/main/browser/agent-browser-bridge-core-commands.ts": {
"before": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"after": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03"
},
"src/main/startup/main-process-ready-runtime.ts": {
"before": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"after": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b"
},
"src/shared/browser-client-host-protocol.ts": {
"before": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"after": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956"
},
"src/shared/browser-client-automation-protocol.ts": {
"before": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"after": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58"
}
},
"runtime": {
"node": "26.6.0",
"electron": null,
"v8": "14.6.202.34-node.26"
},
"variant": "before",
"cases": [
{
"kind": "native-navigation-close",
"completedPayloads": 32,
"heldNativePorts": 1,
"joinTimeoutOverrideMs": 15,
"retainedPayloadsAfterClose": 32,
"cachedResultsAfterClose": 34,
"cancelledQueuedInputRetained": true,
"signalAborted": true,
"executorCustodyPreserved": true,
"routeLeasePreserved": true,
"closedDuplicateRejected": true,
"secondCloseSettled": false,
"retainedAfterNativeSettlement": 0,
"executorClosedAfterNativeSettlement": true
},
{
"kind": "late-sibling-settlement",
"oneHandlerStillOwned": true,
"cachedAfterFirstSettlement": 1,
"closedSettlementStillPending": true
},
{
"kind": "open-replay-and-retire-contract",
"openPromiseIdentityPreserved": true,
"retiredDuplicateRejected": true,
"retireCachePolicyUnchanged": true,
"explicitForgetReleasedCache": true
},
{
"kind": "synthetic-crlf-source-control",
"canonicalHashesMatch": true,
"reads": 24
}
]
}
@@ -0,0 +1,16 @@
--- a/src/main/browser/browser-client-host-command-dispatcher.ts
+++ b/src/main/browser/browser-client-host-command-dispatcher.ts
@@ -165,0 +166 @@
+ this.resultCache.releasePage(page)
@@ -300 +301 @@
- this.resultCache.record(page, record)
+ this.resultCache.record(page, record, !this.closed)
--- a/src/main/browser/browser-client-host-command-result-cache.ts
+++ b/src/main/browser/browser-client-host-command-result-cache.ts
@@ -11 +11,5 @@
- record(page: PageState, record: CommandRecord): void {
+ record(page: PageState, record: CommandRecord, retain = true): void {
+ if (!retain) {
+ this.evict(page, record.event.commandSequence, record)
+ return
+ }
@@ -0,0 +1,138 @@
{
"sources": {
"src/main/browser/browser-client-host-command-dispatcher.ts": {
"before": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"after": "0e4074a4cb7379de58d75dd93e3f74090dea18a944bd4f038730cf19f6e68a98"
},
"src/main/browser/browser-client-host-command-result-cache.ts": {
"before": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"after": "f06bd1f31be9ffcbb750f242bacd0c2f16222cde7d011010acbd46ceb70ea83f"
},
"src/main/browser/browser-client-host-command-state.ts": {
"before": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"after": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f"
},
"src/main/browser/browser-client-host-command-page.ts": {
"before": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"after": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb"
},
"src/main/browser/browser-client-host-command-join.ts": {
"before": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"after": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f"
},
"src/main/browser/browser-client-page-command-executor.ts": {
"before": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"after": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01"
},
"src/main/browser/browser-client-page-command-execution.ts": {
"before": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"after": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e"
},
"src/main/browser/browser-client-page-command-executor-test-harness.ts": {
"before": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"after": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24"
},
"src/main/browser/browser-client-page-automation-runtime.ts": {
"before": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"after": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319"
},
"src/main/browser/browser-route-guest-lifecycle.ts": {
"before": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"after": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb"
},
"src/main/browser/browser-route-webcontents-registry.ts": {
"before": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"after": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad"
},
"src/main/browser/paired-runtime-browser-client-host.ts": {
"before": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"after": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038"
},
"src/main/browser/paired-runtime-browser-client-host-composition.ts": {
"before": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"after": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74"
},
"src/main/browser/paired-runtime-browser-client-host-teardown.ts": {
"before": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"after": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59"
},
"src/main/browser/paired-runtime-browser-client-host-runtime.ts": {
"before": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"after": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c"
},
"src/main/runtime/rpc/methods/browser-core.ts": {
"before": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"after": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d"
},
"src/main/runtime/runtime-browser-commands-active-screencasts-by-page-id.ts": {
"before": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"after": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038"
},
"src/main/browser/agent-browser-bridge-types.ts": {
"before": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"after": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f"
},
"src/main/browser/agent-browser-bridge-raw-process.ts": {
"before": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"after": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c"
},
"src/main/browser/agent-browser-bridge-core-commands.ts": {
"before": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"after": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03"
},
"src/main/startup/main-process-ready-runtime.ts": {
"before": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"after": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b"
},
"src/shared/browser-client-host-protocol.ts": {
"before": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"after": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956"
},
"src/shared/browser-client-automation-protocol.ts": {
"before": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"after": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58"
}
},
"runtime": {
"node": "24.21.0",
"electron": "43.7.0",
"v8": "15.0.245.31-electron.0"
},
"variant": "fixed",
"cases": [
{
"kind": "native-navigation-close",
"completedPayloads": 32,
"heldNativePorts": 1,
"joinTimeoutOverrideMs": 15,
"retainedPayloadsAfterClose": 0,
"cachedResultsAfterClose": 0,
"cancelledQueuedInputRetained": false,
"signalAborted": true,
"executorCustodyPreserved": true,
"routeLeasePreserved": true,
"closedDuplicateRejected": true,
"secondCloseSettled": false,
"retainedAfterNativeSettlement": 0,
"executorClosedAfterNativeSettlement": true
},
{
"kind": "late-sibling-settlement",
"oneHandlerStillOwned": true,
"cachedAfterFirstSettlement": 0,
"closedSettlementStillPending": true
},
{
"kind": "open-replay-and-retire-contract",
"openPromiseIdentityPreserved": true,
"retiredDuplicateRejected": true,
"retireCachePolicyUnchanged": true,
"explicitForgetReleasedCache": true
},
{
"kind": "synthetic-crlf-source-control",
"canonicalHashesMatch": true,
"reads": 24
}
]
}
@@ -0,0 +1,138 @@
{
"sources": {
"src/main/browser/browser-client-host-command-dispatcher.ts": {
"before": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"after": "0e4074a4cb7379de58d75dd93e3f74090dea18a944bd4f038730cf19f6e68a98"
},
"src/main/browser/browser-client-host-command-result-cache.ts": {
"before": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"after": "f06bd1f31be9ffcbb750f242bacd0c2f16222cde7d011010acbd46ceb70ea83f"
},
"src/main/browser/browser-client-host-command-state.ts": {
"before": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"after": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f"
},
"src/main/browser/browser-client-host-command-page.ts": {
"before": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"after": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb"
},
"src/main/browser/browser-client-host-command-join.ts": {
"before": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"after": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f"
},
"src/main/browser/browser-client-page-command-executor.ts": {
"before": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"after": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01"
},
"src/main/browser/browser-client-page-command-execution.ts": {
"before": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"after": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e"
},
"src/main/browser/browser-client-page-command-executor-test-harness.ts": {
"before": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"after": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24"
},
"src/main/browser/browser-client-page-automation-runtime.ts": {
"before": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"after": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319"
},
"src/main/browser/browser-route-guest-lifecycle.ts": {
"before": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"after": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb"
},
"src/main/browser/browser-route-webcontents-registry.ts": {
"before": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"after": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad"
},
"src/main/browser/paired-runtime-browser-client-host.ts": {
"before": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"after": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038"
},
"src/main/browser/paired-runtime-browser-client-host-composition.ts": {
"before": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"after": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74"
},
"src/main/browser/paired-runtime-browser-client-host-teardown.ts": {
"before": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"after": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59"
},
"src/main/browser/paired-runtime-browser-client-host-runtime.ts": {
"before": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"after": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c"
},
"src/main/runtime/rpc/methods/browser-core.ts": {
"before": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"after": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d"
},
"src/main/runtime/runtime-browser-commands-active-screencasts-by-page-id.ts": {
"before": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"after": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038"
},
"src/main/browser/agent-browser-bridge-types.ts": {
"before": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"after": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f"
},
"src/main/browser/agent-browser-bridge-raw-process.ts": {
"before": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"after": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c"
},
"src/main/browser/agent-browser-bridge-core-commands.ts": {
"before": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"after": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03"
},
"src/main/startup/main-process-ready-runtime.ts": {
"before": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"after": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b"
},
"src/shared/browser-client-host-protocol.ts": {
"before": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"after": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956"
},
"src/shared/browser-client-automation-protocol.ts": {
"before": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"after": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58"
}
},
"runtime": {
"node": "26.6.0",
"electron": null,
"v8": "14.6.202.34-node.26"
},
"variant": "fixed",
"cases": [
{
"kind": "native-navigation-close",
"completedPayloads": 32,
"heldNativePorts": 1,
"joinTimeoutOverrideMs": 15,
"retainedPayloadsAfterClose": 0,
"cachedResultsAfterClose": 0,
"cancelledQueuedInputRetained": false,
"signalAborted": true,
"executorCustodyPreserved": true,
"routeLeasePreserved": true,
"closedDuplicateRejected": true,
"secondCloseSettled": false,
"retainedAfterNativeSettlement": 0,
"executorClosedAfterNativeSettlement": true
},
{
"kind": "late-sibling-settlement",
"oneHandlerStillOwned": true,
"cachedAfterFirstSettlement": 0,
"closedSettlementStillPending": true
},
{
"kind": "open-replay-and-retire-contract",
"openPromiseIdentityPreserved": true,
"retiredDuplicateRejected": true,
"retireCachePolicyUnchanged": true,
"explicitForgetReleasedCache": true
},
{
"kind": "synthetic-crlf-source-control",
"canonicalHashesMatch": true,
"reads": 24
}
]
}
@@ -0,0 +1,330 @@
import { afterEach, expect, it, vi } from 'vitest'
import { writeFileSync, readFileSync } from 'node:fs'
import { createRequire } from 'node:module'
const { loadSources } = createRequire(import.meta.url)('./sources.cjs')
const sourceInfo = loadSources()
import { BrowserClientHostCommandDispatcher } from '../../../src/main/browser/browser-client-host-command-dispatcher'
import {
createHarness,
createCommand
} from '../../../src/main/browser/browser-client-page-command-executor-test-harness'
import { BrowserClientPageAutomationRuntime } from '../../../src/main/browser/browser-client-page-automation-runtime'
import { navigateBrowserRouteGuest } from '../../../src/main/browser/browser-route-guest-lifecycle'
import { closeBrowserClientHostComposition } from '../../../src/main/browser/paired-runtime-browser-client-host-teardown'
import { BROWSER_CORE_METHODS } from '../../../src/main/runtime/rpc/methods/browser-core'
const fixed = process.env.ORCA_BROWSER_CACHE_VARIANT !== 'before'
const variant = fixed ? 'fixed' : 'before',
reports = []
const authority = {
authorityRuntimeId: 'runtime-a',
authorityEpoch: 'epoch-a',
browserHostClientId: 'client-a',
browserHostGeneration: 3,
pageCommandProtocolVersion: 1
}
function gate() {
let resolve, reject
const promise = new Promise((yes, no) => {
resolve = yes
reject = no
})
return { promise, resolve, reject }
}
async function collect() {
for (let turn = 0; turn < 8; turn++) {
await new Promise(setImmediate)
global.gc()
}
}
function alive(refs) {
return refs.filter((ref) => ref.deref()).length
}
function command(sequence, body, page = 'page-a', generation = 7) {
return createCommand('createPage', {
browserPageId: page,
pageHostGeneration: generation,
commandSequence: sequence,
commandId: `${page}-${generation}-${sequence}`,
command: body
})
}
function cached(dispatcher) {
return [...dispatcher.pages.values()].reduce((sum, page) => sum + page.settledSequences.length, 0)
}
function queueUnstartedPayload(dispatcher) {
const payload = { queued: 'small-command-input' }
return {
ref: new WeakRef(payload),
promise: dispatcher.dispatch(
command(35, { type: 'automation', method: 'browser.snapshot', params: { payload } })
)
}
}
async function appendSnapshot(dispatcher, index) {
const result = await dispatcher.dispatch(
command(index + 2, { type: 'automation', method: 'browser.snapshot', params: {} })
)
expect(result.status).toBe('completed')
return new WeakRef(result.value)
}
afterEach(() => {
vi.restoreAllMocks()
writeFileSync(
process.env.ORCA_BROWSER_CACHE_OUTPUT ??
`docs/audits/browser-closed-result-retention/${variant}-${process.versions.electron ? 'electron' : 'node'}-results.json`,
`${JSON.stringify(
{
sources: sourceInfo.hashes,
runtime: {
node: process.versions.node,
electron: process.versions.electron ?? null,
v8: process.versions.v8
},
variant,
cases: reports
},
null,
2
)}\n`
)
})
it('keeps completed actual automation results behind one pending native navigation after close timeout', async () => {
const h = createHarness(),
native = gate(),
entered = gate()
let signal,
ordinal = 0,
executorClosed = false,
deferredClose
const snapshot = BROWSER_CORE_METHODS.find((method) => method.name === 'browser.snapshot')
const automation = new BrowserClientPageAutomationRuntime({
browserManager: {
getGuestWebContentsId: () => 41,
registerGuest: () => true,
unregisterGuest() {}
},
getAgentBrowserBridge: () => null,
executeRpc: (_method, params, contextSignal) =>
snapshot.handler(params, {
signal: contextSignal,
runtime: {
browserSnapshot: async () => ({ title: `ordinary-result-${ordinal++}`, items: [1, 2, 3] })
}
})
})
h.dependencies.executeAutomation = (input, contextSignal) =>
automation.execute(input, contextSignal)
h.dependencies.retireAutomation = (input) => automation.retire(input)
h.dependencies.routeWebContents.navigateGuest = (claim, url) =>
navigateBrowserRouteGuest(
claim.registration,
url,
{
registration: claim.registration,
navigationGranted: true,
guest: {
loadURL: () => {
entered.resolve()
return native.promise
}
}
},
() => true
)
const dispatcher = new BrowserClientHostCommandDispatcher({
authority,
handler: (event, contextSignal) => {
if (event.command.type === 'navigate') {
signal = contextSignal
}
return h.executor.handle(event, contextSignal)
},
joinTimeoutMs: 15
})
await dispatcher.dispatch(createCommand('createPage'))
const refs = []
for (let index = 0; index < 32; index++) {
refs.push(await appendSnapshot(dispatcher, index))
}
await collect()
expect(alive(refs)).toBe(32)
const pending = dispatcher.dispatch(
command(34, { type: 'navigate', url: 'https://example.invalid/held' })
)
await entered.promise
const queued = queueUnstartedPayload(dispatcher)
h.executor.fenceNavigation()
const closing = closeBrowserClientHostComposition({
host: { close: () => dispatcher.close(), whenHandlersSettled: () => dispatcher.whenClosed() },
executor: {
async close() {
executorClosed = true
await h.executor.close()
}
},
routeSets: { async close() {} },
error: new Error('controlled disconnect'),
deferExecutorClose: (close) => {
deferredClose = close
},
reportCleanupError: (error) => {
throw error
}
})
try {
expect(await closing).toBe(false)
expect(await pending).toMatchObject({
status: 'failed',
errorCode: 'browser_host_command_cancelled'
})
expect(await queued.promise).toMatchObject({
status: 'failed',
errorCode: 'browser_host_command_cancelled'
})
expect(signal.aborted).toBe(true)
expect(executorClosed).toBe(false)
expect(h.executor.hasPage('page-a', 7)).toBe(true)
expect(h.route.release).not.toHaveBeenCalled()
expect(h.routeSession.release).not.toHaveBeenCalled()
expect(() => dispatcher.dispatch(createCommand('createPage'))).toThrow('dispatcher_closed')
expect(await dispatcher.close()).toBe(false)
let settled = false
void dispatcher.whenClosed().then(() => {
settled = true
})
await collect()
expect(settled).toBe(false)
const retained = alive(refs),
cachedResults = cached(dispatcher)
expect(retained).toBe(fixed ? 0 : 32)
expect(cachedResults).toBe(fixed ? 0 : 34)
expect(Boolean(queued.ref.deref())).toBe(!fixed)
expect(dispatcher.runningHandlers).toBe(1)
reports.push({
kind: 'native-navigation-close',
completedPayloads: 32,
heldNativePorts: 1,
joinTimeoutOverrideMs: 15,
retainedPayloadsAfterClose: retained,
cachedResultsAfterClose: cachedResults,
cancelledQueuedInputRetained: Boolean(queued.ref.deref()),
signalAborted: true,
executorCustodyPreserved: true,
routeLeasePreserved: true,
closedDuplicateRejected: true,
secondCloseSettled: false
})
} finally {
native.resolve()
await dispatcher.whenClosed()
await deferredClose
await h.executor.close()
}
await collect()
expect(alive(refs)).toBe(0)
expect(executorClosed).toBe(true)
expect(h.route.release).toHaveBeenCalledOnce()
expect(h.routeSession.release).toHaveBeenCalledOnce()
reports.at(-1).retainedAfterNativeSettlement = alive(refs)
reports.at(-1).executorClosedAfterNativeSettlement = true
})
it('does not retain late completed cancellation records while a sibling native handler remains owned', async () => {
const first = gate(),
second = gate()
const dispatcher = new BrowserClientHostCommandDispatcher({
authority,
joinTimeoutMs: 15,
handler: (event) => (event.browserPageId === 'page-a' ? first.promise : second.promise)
})
const firstResult = dispatcher.dispatch(
command(
1,
{ type: 'createPage', browserProfileId: 'profile-a', executionHostKey: 'execution-host-a' },
'page-a'
)
)
const secondResult = dispatcher.dispatch(
command(
1,
{ type: 'createPage', browserProfileId: 'profile-a', executionHostKey: 'execution-host-a' },
'page-b'
)
)
expect(await dispatcher.close()).toBe(false)
expect(await firstResult).toMatchObject({ errorCode: 'browser_host_command_cancelled' })
expect(await secondResult).toMatchObject({ errorCode: 'browser_host_command_cancelled' })
first.resolve({ status: 'completed', value: { late: 'ignored' } })
await new Promise(setImmediate)
expect(dispatcher.runningHandlers).toBe(1)
expect(cached(dispatcher)).toBe(fixed ? 0 : 1)
expect(dispatcher.pages.get('page-a').records.size).toBe(fixed ? 0 : 1)
let settled = false
void dispatcher.whenClosed().then(() => {
settled = true
})
await new Promise(setImmediate)
expect(settled).toBe(false)
reports.push({
kind: 'late-sibling-settlement',
oneHandlerStillOwned: true,
cachedAfterFirstSettlement: cached(dispatcher),
closedSettlementStillPending: true
})
second.reject(new Error('controlled native failure'))
await dispatcher.whenClosed()
expect(dispatcher.runningHandlers).toBe(0)
expect(dispatcher.pages.size).toBe(0)
})
it('preserves open replay and generation fencing independently of closed cache release', async () => {
let calls = 0
const dispatcher = new BrowserClientHostCommandDispatcher({
authority,
handler: () => {
calls++
return { status: 'completed', value: { ordinary: true } }
}
})
const event = command(1, {
type: 'createPage',
browserProfileId: 'profile-a',
executionHostKey: 'execution-host-a'
})
const original = dispatcher.dispatch(event),
duplicate = dispatcher.dispatch(event)
expect(duplicate).toBe(original)
await original
expect(dispatcher.dispatch(event)).toBe(original)
expect(calls).toBe(1)
expect(await dispatcher.retirePage('page-a', 7)).toBe(true)
expect(() => dispatcher.dispatch(event)).toThrow('generation_stale')
expect(cached(dispatcher)).toBe(1)
expect(dispatcher.forgetPage('page-a', 7)).toBe(true)
expect(cached(dispatcher)).toBe(0)
expect(() => dispatcher.dispatch(event)).toThrow('generation_stale')
expect(await dispatcher.close()).toBe(true)
reports.push({
kind: 'open-replay-and-retire-contract',
openPromiseIdentityPreserved: true,
retiredDuplicateRejected: true,
retireCachePolicyUnchanged: true,
explicitForgetReleasedCache: true
})
})
it('loads identical canonical hashes from synthetic CRLF source and patch reads', () => {
let reads = 0
const crlf = loadSources({
readText: (filename) => {
reads++
return readFileSync(filename, 'utf8').replace(/\r?\n/g, '\r\n')
}
})
expect(crlf.hashes).toEqual(sourceInfo.hashes)
expect([...crlf.before.entries()]).toEqual([...sourceInfo.before.entries()])
expect([...crlf.after.entries()]).toEqual([...sourceInfo.after.entries()])
reports.push({ kind: 'synthetic-crlf-source-control', canonicalHashesMatch: true, reads })
})
@@ -0,0 +1,405 @@
{
"canonicalLF": true,
"sources": [
{
"path": "src/main/browser/browser-client-host-command-dispatcher.ts",
"workingSha256": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"lineCount": 315,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-host-command-result-cache.ts",
"workingSha256": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"lineCount": 51,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-host-command-state.ts",
"workingSha256": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"lineCount": 171,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "a5f6b7a3737f86b2953fc43fa0e3ed5faaa141814d5f94fd192da1bb510f355f",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-host-command-page.ts",
"workingSha256": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"lineCount": 220,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "f5ca7f90511ee0290d2a1d0b3c5c39d3c9841ebcf90a5564bce83fbc9aca06bb",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-host-command-join.ts",
"workingSha256": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"lineCount": 21,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "5bf1a38231100cc416176ef2052bdff7b3fca91b6f6dbefcd222b92be59cbb2f",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-page-command-executor.ts",
"workingSha256": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"lineCount": 319,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "d249de82f89365371fa4f3b457e0da552910819555a5a0bced0c95010ccacb01",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-page-command-execution.ts",
"workingSha256": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"lineCount": 114,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "82b7c87d17b34c267ebfdf3d6ee25058cbd05dbc163ac24c43aefed00f0e3c6e",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-page-command-executor-test-harness.ts",
"workingSha256": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"lineCount": 145,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "b43ec764e4d094070fab51381ceb24487999dca97b4a38131be08fab0c5cbe24",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-client-page-automation-runtime.ts",
"workingSha256": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"lineCount": 141,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "7b1c0c527a87b8c22ea8290a2e1efc7ca659f2b4a5e2bd7c740fb5c22a2a6319",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-route-guest-lifecycle.ts",
"workingSha256": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"lineCount": 172,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "52adbea664bcfc5d774f129d442f4fc36d4a1e1280a073053fc09ae026ca83cb",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/browser-route-webcontents-registry.ts",
"workingSha256": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"lineCount": 325,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "132c6740539aaeb9b1ceb3c108d2aebfb9e8e5c4ce3ff7cd1191e5b45dff8cad",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/paired-runtime-browser-client-host.ts",
"workingSha256": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"lineCount": 193,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "797c0130e30c5f40551f5d392326fff1ac8752e262653b3f855dfd7df9fa0038",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/paired-runtime-browser-client-host-composition.ts",
"workingSha256": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"lineCount": 323,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "f8b55a24035e929664f3c5f61ee8a81ee560250379dcf64b39d35e1b60628b74",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/paired-runtime-browser-client-host-teardown.ts",
"workingSha256": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"lineCount": 68,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "4d9eb53c81ddfa3cb8181735b08c2b90a380bebab1f173ae5d8da9f719671f59",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/paired-runtime-browser-client-host-runtime.ts",
"workingSha256": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"lineCount": 327,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "2d0650d0a5ebe88f1185ab5676762ed93188e7c28a69508137ed7cfa36e3832c",
"matchesWorking": true
}
}
},
{
"path": "src/main/runtime/rpc/methods/browser-core.ts",
"workingSha256": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"lineCount": 293,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "fdc2afeece6689644fbd607a59e484ef9d2ac6db7404e3378f2db10e16984f6d",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "c847be873f4ce19b04796be962985a0234f159da2b38881fb8f6db1a1cbf720b",
"matchesWorking": false
}
}
},
{
"path": "src/main/runtime/runtime-browser-commands-active-screencasts-by-page-id.ts",
"workingSha256": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"lineCount": 209,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "0af16d6fe883f6af79754e064b77a1b9ab8e80953657434eacc88cc1c523e038",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/agent-browser-bridge-types.ts",
"workingSha256": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"lineCount": 65,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "3ea69d15aef862807108835d1cbd536275a105a96bdc86f3be95dfbed55ab45f",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/agent-browser-bridge-raw-process.ts",
"workingSha256": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"lineCount": 108,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "d0e9929c4a4571d8752efa5e4063500efa75fefd228eb6cb1ea6092748befb0c",
"matchesWorking": true
}
}
},
{
"path": "src/main/browser/agent-browser-bridge-core-commands.ts",
"workingSha256": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"lineCount": 169,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "d587a7e7c3bd368c9003802607edd20def98e64442cd6d5914779c633feebd03",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "06d6a61e431680ebd89e9a29b18e2198da3d84df6398b234fa8a255a6fcedf8a",
"matchesWorking": false
}
}
},
{
"path": "src/main/startup/main-process-ready-runtime.ts",
"workingSha256": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"lineCount": 156,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "5bb3615e9dc430e0a9ab61cb3d70e57ecbfa4c5ea82d5414fc82d93e81e9819b",
"matchesWorking": true
}
}
},
{
"path": "src/shared/browser-client-host-protocol.ts",
"workingSha256": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"lineCount": 343,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "712fe3eaed30c2de322466f316c966e526dcb60cf387cc0f6ba48012c70a7956",
"matchesWorking": true
}
}
},
{
"path": "src/shared/browser-client-automation-protocol.ts",
"workingSha256": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"lineCount": 129,
"namedRefs": {
"mainCheckpoint": {
"ref": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"sha256": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"matchesWorking": true
},
"v1.4.198": {
"ref": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"sha256": "f93331c39fd0f8e08518d7a11a461bf5108c1a65be8f213ab2e2eff2ab977e58",
"matchesWorking": true
}
}
}
],
"baselineHashes": {
"src/main/browser/browser-client-host-command-dispatcher.ts": "f8c610a54f7d16c59043250dde5ef399f70e1a974c9492d09d403138e46c1da3",
"src/main/browser/browser-client-host-command-result-cache.ts": "9ec3dad78c83f864f74957cad766cbcf41863e92d81d7e7a471702b0f9ab2f34"
},
"fixedHashes": {
"src/main/browser/browser-client-host-command-dispatcher.ts": "0e4074a4cb7379de58d75dd93e3f74090dea18a944bd4f038730cf19f6e68a98",
"src/main/browser/browser-client-host-command-result-cache.ts": "f06bd1f31be9ffcbb750f242bacd0c2f16222cde7d011010acbd46ceb70ea83f"
},
"auditedHead": "6ab6802df90553d2a26e9a4c7a519ae11d4b5e09"
}
@@ -0,0 +1,42 @@
const assert = require('node:assert/strict')
const { readFileSync } = require('node:fs')
const { createHash } = require('node:crypto')
const { resolve } = require('node:path')
const { applyPatch, parsePatch, reversePatch } = require('diff')
const canonicalLf = (text) => text.replace(/\r\n/g, '\n')
const sha256 = (text) => createHash('sha256').update(text).digest('hex')
function loadSources({ readText = (filename) => readFileSync(filename, 'utf8') } = {}) {
const root = resolve(__dirname, '../../..')
const expected = JSON.parse(readFileSync(resolve(__dirname, 'source-versions.json'), 'utf8'))
const patches = parsePatch(canonicalLf(readText(resolve(__dirname, 'fix.patch'))))
const before = new Map()
const after = new Map()
const hashes = {}
assert.equal(patches.length, 2)
for (const patch of patches) {
const path = patch.newFileName.replace(/^b\//, '')
assert.ok(Object.hasOwn(expected.baselineHashes, path), `Unexpected patch path: ${path}`)
const absolute = resolve(root, path)
const current = canonicalLf(readText(absolute))
const baseline = applyPatch(current, reversePatch(patch))
assert.notEqual(baseline, false, `Patch no longer reverses: ${path}`)
assert.equal(sha256(current), expected.fixedHashes[path], `Fixed source drift: ${path}`)
assert.equal(sha256(baseline), expected.baselineHashes[path], `Baseline source drift: ${path}`)
before.set(absolute, baseline)
after.set(absolute, current)
hashes[path] = { before: sha256(baseline), after: sha256(current) }
}
for (const source of expected.sources) {
if (Object.hasOwn(hashes, source.path)) {
continue
}
const text = canonicalLf(readText(resolve(root, source.path)))
assert.equal(sha256(text), source.workingSha256, `Caller source drift: ${source.path}`)
hashes[source.path] = { before: sha256(text), after: sha256(text) }
}
return { root, before, after, hashes }
}
module.exports = { loadSources }
@@ -0,0 +1,228 @@
{
"tests": {
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/browser/browser-client-host-command-retention.test.ts src/main/browser/browser-client-host-command-dispatcher.test.ts src/main/browser/browser-client-page-command-executor.test.ts src/main/browser/paired-runtime-browser-client-host-composition.test.ts src/main/browser/paired-runtime-browser-client-host.test.ts",
"passed": 77,
"files": 5,
"newRegressionCases": 2,
"exitCode": 0
},
"baselineOverlay": {
"command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_BROWSER_CACHE_VARIANT=before pnpm exec vitest run --config docs/audits/browser-closed-result-retention/vitest.config.mjs src/main/browser/browser-client-host-command-retention.test.ts src/main/browser/browser-client-host-command-dispatcher.test.ts",
"passed": 16,
"expectedFailed": 2,
"failures": [
"32 completed result objects remain reachable while native navigation stays pending.",
"The first late closed input remains reachable while a sibling handler stays pending."
],
"exitCode": 1
},
"typecheck": {
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm tc:node",
"exitCode": 0
},
"publicationQuality": {
"scans": [
{
"label": "code quality",
"command": [
"pnpm",
"exec",
"oxlint",
"--no-ignore",
"--deny-warnings",
"--report-unused-disable-directives-severity",
"warn",
"src/main/browser/browser-client-host-command-dispatcher.ts",
"src/main/browser/browser-client-host-command-result-cache.ts",
"src/main/browser/browser-client-host-command-retention.test.ts",
"docs/audits/browser-closed-result-retention/sources.cjs",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"label": "casting code quality",
"command": [
"pnpm",
"exec",
"oxlint",
"--no-ignore",
"--deny-warnings",
"--config",
"config/oxlint-code-quality-casting.json",
"src/main/browser/browser-client-host-command-dispatcher.ts",
"src/main/browser/browser-client-host-command-result-cache.ts",
"src/main/browser/browser-client-host-command-retention.test.ts",
"docs/audits/browser-closed-result-retention/sources.cjs",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"label": "type-aware code quality",
"command": [
"pnpm",
"exec",
"oxlint",
"--no-ignore",
"--deny-warnings",
"--type-aware",
"--config",
"config/oxlint-code-quality-type-aware.json",
"src/main/browser/browser-client-host-command-dispatcher.ts",
"src/main/browser/browser-client-host-command-result-cache.ts",
"src/main/browser/browser-client-host-command-retention.test.ts",
"docs/audits/browser-closed-result-retention/sources.cjs",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"label": "React Doctor",
"command": [
"pnpm",
"exec",
"oxlint",
"--no-ignore",
"--deny-warnings",
"--config",
"config/oxlint-react-doctor.json",
"src/main/browser/browser-client-host-command-dispatcher.ts",
"src/main/browser/browser-client-host-command-result-cache.ts",
"src/main/browser/browser-client-host-command-retention.test.ts",
"docs/audits/browser-closed-result-retention/sources.cjs",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"label": "design system",
"command": [
"pnpm",
"exec",
"oxlint",
"--no-ignore",
"--deny-warnings",
"--config",
"config/oxlint-design-system.json",
"src/main/browser/browser-client-host-command-dispatcher.ts",
"src/main/browser/browser-client-host-command-result-cache.ts",
"src/main/browser/browser-client-host-command-retention.test.ts",
"docs/audits/browser-closed-result-retention/sources.cjs",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
}
]
},
"changedQuality": {
"command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_CODE_QUALITY_BASE=HEAD pnpm run check:code-quality:changed",
"exitCode": 0,
"note": "The five explicit-file scans include all six TS/CJS/MJS publication paths. The ordinary changed gate does not see ignored new artifacts before staging."
},
"proofs": {
"runs": [
{
"runtime": "node",
"variant": "before",
"command": [
"pnpm",
"exec",
"vitest",
"run",
"--config",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"runtime": "node",
"variant": "fixed",
"command": [
"pnpm",
"exec",
"vitest",
"run",
"--config",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"runtime": "electron",
"variant": "before",
"command": [
"node_modules/electron/dist/Electron.app/Contents/MacOS/Electron",
"node_modules/vitest/vitest.mjs",
"run",
"--config",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
},
{
"runtime": "electron",
"variant": "fixed",
"command": [
"node_modules/electron/dist/Electron.app/Contents/MacOS/Electron",
"node_modules/vitest/vitest.mjs",
"run",
"--config",
"docs/audits/browser-closed-result-retention/vitest.config.mjs",
"docs/audits/browser-closed-result-retention/scenario.test.mjs"
],
"exitCode": 0
}
],
"casesPerVariantPerRuntime": 4,
"variants": ["before", "fixed"],
"node": "26.6.0",
"electron": "43.7.0",
"electronNode": "24.21.0",
"controlledPendingNativePorts": 1,
"smallCompletedResults": 32,
"crlfReadControl": 24,
"environment": {
"ORCA_BACKGROUND_LAUNCH": "1",
"ELECTRON_RUN_AS_NODE": "1 for Electron runs",
"ORCA_BROWSER_CACHE_VARIANT": "before or fixed"
},
"outputOverride": "ORCA_BROWSER_CACHE_OUTPUT"
},
"sourceParity": {
"canonicalLF": true,
"mainCheckpoint": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"historicalVersion": "v1.4.198",
"historicalRef": "e0826956fcfc532f5a1e55b5e081f2e57e553c43",
"namedBaselineTargetMatches": 2,
"mainCheckpointCitedSourceMatches": 23,
"historicalCitedSourceMatches": 21,
"citedSourceCount": 23,
"hashCoverage": "Two product targets plus 21 cited caller/dependency modules, not all transitive imports.",
"historicalApplicationReplay": false
},
"scope": {
"trigger": "A handler outlives the dispatcher close join (default 5 seconds).",
"nativeSettlementAuthorityPreserved": true,
"retirePageCachePolicyChanged": false,
"incidentAttribution": false,
"measuredRSS": false
},
"format": "All 14 publication files except fix.patch checked with oxfmt stdin mode; a second pass produced identical bytes.",
"gitDiffCheckExitCode": 0,
"publicationWhitespace": {
"commandTemplate": "git diff --no-index --check <empty-file> <promoted-path>",
"files": 14,
"expectedExitCode": 1,
"diagnostics": 0,
"note": "The complete content of every publication path is checked, including ignored new artifacts. Exit 1 only means the content differs from an empty file. fix.patch uses zero-context hunks."
}
}
@@ -0,0 +1,30 @@
import { resolve } from 'node:path'
import { createRequire } from 'node:module'
import { defineConfig, mergeConfig } from 'vitest/config'
import base from '../../../config/vitest.config.ts'
const { loadSources } = createRequire(import.meta.url)('./sources.cjs')
const { before, after } = loadSources()
const sources = process.env.ORCA_BROWSER_CACHE_VARIANT === 'before' ? before : after
const config = mergeConfig(
base,
defineConfig({
plugins: [
{
name: 'closed-browser-cache-source-overlay',
enforce: 'pre',
transform(_code, id) {
const source = sources.get(resolve(id.split('?')[0]))
return source === undefined ? undefined : { code: source, map: null }
}
}
]
})
)
config.test.include = [
'docs/audits/browser-closed-result-retention/scenario.test.mjs',
'src/main/browser/browser-client-host-command-retention.test.ts',
'src/main/browser/browser-client-host-command-dispatcher.test.ts'
]
config.test.maxWorkers = 1
export default config
@@ -0,0 +1,40 @@
# Destroyed browser guests retain main-process callbacks
An embedded browser guest's `destroyed` event called `cleanupGuestPolicyAttachment`. That removed its primary page-to-WebContents lookup but left four per-page cleanup callbacks that capture the destroyed WebContents wrapper, plus renderer/workspace/worktree/profile metadata. `unregisterAll` subsequently iterated only the now-empty primary lookup: three callback maps and the renderer/workspace metadata survived that cleanup too.
Renderer reload can destroy guests without each page sending explicit unregister IPC. A later close of a restored, unmounted page does not send that IPC either: `destroyPersistentWebview` returns early when its renderer registry has no guest. Explicit unregister correctly releases these resources; same-page re-registration also replaces its callbacks. The defect affects destroyed owners that do not take either path.
The fix routes destruction through the existing `unregisterGuest` with a guest-retirement reason only when that exact guest still owns the primary page ID. Already bound downloads retain their existing renderer routing until they settle; explicit page close still cancels them. Unregistered guests and popups retain policy-only cleanup. A stale callback cannot unregister a replacement. Normal renderer-process loss keeps its live WebContents and metadata for reload recovery; a fresh guest registration supplies its ownership metadata again. Shared browser sessions and sibling pages are untouched.
## Download lifetime correction
Review found that the initial fix treated guest destruction as logical page closure and canceled bound downloads. An exact-source before/after control confirmed that difference with an EventEmitter guest and controlled DownloadItem. Guest retirement now releases guest-owned callbacks while preserving ongoing page downloads, their destinations and cancel authorization. A retained numeric renderer route drains after the last download settles, provided no replacement guest, other download or newer routing owner needs it.
Nine additional controls cover progress and completion/error delivery, explicit close after guest destruction, multiple downloads, replacement guests/routing, repeated guest destruction and renderer loss. Together with four existing browser suites, 55 tests pass; Node typecheck and ordinary/anti-slop lint pass. These controls do not establish native Chromium download survival after destruction on each operating system. No download capacity or wire format changes.
## Reproduce
With dependencies already installed, run from the repository root:
```sh
ORCA_BACKGROUND_LAUNCH=1 node docs/audits/browser-destroyed-guest-retention/reproduce.mjs
```
The script runs the actual manager and guest callback installers with EventEmitter WebContents fixtures. It removes only the destruction guard in memory for the baseline, then runs the same nine tests on the fixed source. A temporary test observer records actual map sizes before each assertion. It launches no Orca window or browser process, limits each worker to 512 MiB and each run to 60 seconds, uses the shared process launcher, and removes temporary files. Results include source hashes and runtime provenance.
| After 1,000 distinct guest destructions | Before | Fixed |
| -------------------------------------------------------------------------- | -----: | ----: |
| Primary guest lookup | 0 | 0 |
| Each context-menu / grab-shortcut / app-shortcut / wheel cleanup map | 1,000 | 0 |
| Each renderer / workspace / worktree / profile map | 1,000 | 0 |
| Policy cleanup map | 0 | 0 |
| Each context-menu / grab-shortcut / app-shortcut map after `unregisterAll` | 1,000 | 0 |
| Renderer and workspace maps after `unregisterAll` | 1,000 | 0 |
Baseline: six tests pass, three fail. Fixed: all nine pass. Controls cover explicit unregister, same-ID replacement with a captured old callback, popup and pre-registration destruction, renderer-process recovery, fresh guest registration, and two pages sharing one browser session profile. The selected existing browser-manager and offscreen lifecycle suites also passed: 64 tests across seven files including the new suite.
## Version and limits
Targeted source reads of `v1.4.198` confirm the same destroyed-event policy-only cleanup, map ownership, and `unregisterAll` omission. The executable comparison uses current production source; it does not launch the historical app. This is a retaining path present in the version reported by #19831/#19768. It does not establish that either incident followed this destruction sequence, or measure native memory retained by a destroyed WebContents. The 1,000 iterations measure retained callbacks and metadata, not 1,000 surviving Chromium processes or a gigabyte allocation.
Adjacent audit negatives: explicit page close removes the renderer guest registry and main registration; worktree switching deliberately parks guests under the existing hidden-worktree retention policy; offscreen creation is synchronously indexed with shutdown admission and exact-window teardown; client-hosted async page creation checks availability after acquisitions and cleans canceled owners. PDF capture rejects late disconnected-client completion, its stream buffers have a five-minute TTL, and existing screenshot commands have deadlines. No additional native screenshot hang or unbounded native-page acquisition was reproduced. The separate late renderer registration reply can restore small page-ID metadata after close; it is outside this main-process fix.
@@ -0,0 +1,168 @@
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { build } from 'esbuild'
if (process.env.ORCA_BACKGROUND_LAUNCH !== '1') {
throw new Error('Run with ORCA_BACKGROUND_LAUNCH=1.')
}
const root = fileURLToPath(new URL('../../../', import.meta.url))
const productionPath = 'src/main/browser/browser-manager-guest-navigation-policy.ts'
const testPath = 'src/main/browser/browser-manager-destroyed-guest.test.ts'
const fixturePath = 'src/main/browser/browser-manager-destroyed-guest-test-fixture.ts'
const current = await readFile(resolve(root, productionPath), 'utf8')
const fix = ` const browserTabId = this.tabIdByWebContentsId.get(guest.id)
// A destroyed primary guest also owns per-page callbacks that capture its WebContents.
if (browserTabId && this.webContentsIdByTabId.get(browserTabId) === guest.id) {
this.unregisterGuest(browserTabId, 'guest-destroyed')
return
}
`
if (current.split(fix).length !== 2) {
throw new Error('Expected exactly one primary-guest destruction guard; review the transform.')
}
const baseline = current.replace(fix, '')
const test = await readFile(resolve(root, testPath), 'utf8')
const observe = ' const counts = manager.retainedCounts()\n'
if (test.split(observe).length !== 2) {
throw new Error('Expected exactly one retained-count observer; review the transform.')
}
const observedTest = `import { appendFileSync } from 'node:fs'\n${test.replace(
observe,
`${observe} appendFileSync(process.env.ORCA_BROWSER_GUEST_COUNTS_PATH, JSON.stringify({ test: expect.getState().currentTestName, counts }) + '\\n')\n`
)}`
const sha256 = (source) => createHash('sha256').update(source).digest('hex')
const sourceHashes = {
[productionPath]: { before: sha256(baseline), after: sha256(current) },
[testPath]: { current: sha256(test), observed: sha256(observedTest) },
[fixturePath]: { current: sha256(await readFile(resolve(root, fixturePath))) }
}
for (const path of [
'src/main/browser/browser-manager-state.ts',
'src/main/browser/browser-manager-registration.ts',
'src/main/browser/browser-manager-download-lifecycle.ts'
]) {
sourceHashes[path] = { current: sha256(await readFile(resolve(root, path))) }
}
const scratch = await mkdtemp(join(tmpdir(), 'orca-browser-destroyed-guest-'))
const require = createRequire(import.meta.url)
let runnerModuleId
try {
const runnerPath = join(scratch, 'run-process.cjs')
await build({
absWorkingDir: root,
entryPoints: [resolve(root, 'src/shared/child-process/run-process.ts')],
outfile: runnerPath,
bundle: true,
platform: 'node',
format: 'cjs',
logLevel: 'silent'
})
runnerModuleId = require.resolve(runnerPath)
const { runProcess } = require(runnerModuleId)
const configImport = JSON.stringify(pathToFileURL(resolve(root, 'config/vitest.config.ts')).href)
async function run(label, production) {
const config = join(scratch, `${label}.config.mjs`)
const report = join(scratch, `${label}.json`)
const countsPath = join(scratch, `${label}.counts.jsonl`)
const sources = {
[resolve(root, productionPath).replaceAll('\\', '/')]: production,
[resolve(root, testPath).replaceAll('\\', '/')]: observedTest
}
await writeFile(
config,
`import base from ${configImport};
const sources = ${JSON.stringify(sources)};
export default {...base, test: {...base.test, include: [${JSON.stringify(testPath)}], maxWorkers: 1}, plugins: [{
name: 'browser-destroyed-guest-audit', enforce: 'pre',
transform(_code, id) {
const source = sources[id.replaceAll('\\\\', '/').split('?')[0]];
return source === undefined ? null : {code: source, map: null};
}
}]};\n`
)
const result = await runProcess({
program: process.execPath,
args: [
resolve(root, 'node_modules/vitest/vitest.mjs'),
'run',
'--config',
config,
'--reporter=json',
`--outputFile=${report}`
],
cwd: root,
env: {
...process.env,
NODE_OPTIONS: '--max-old-space-size=512',
ORCA_BROWSER_GUEST_COUNTS_PATH: countsPath
},
timeoutMs: 60_000,
maxOutputBytes: 2 * 1024 * 1024
})
let parsed
try {
parsed = JSON.parse(await readFile(report, 'utf8'))
} catch (error) {
throw new Error(`${label} failed: ${result.stderr || result.stdout}`, { cause: error })
}
const counts = (await readFile(countsPath, 'utf8'))
.trim()
.split('\n')
.map((line) => JSON.parse(line))
return {
exitCode: result.code,
passed: parsed.numPassedTests,
failed: parsed.numFailedTests,
counts,
failedCases: parsed.testResults.flatMap((suite) =>
suite.assertionResults
.filter((assertion) => assertion.status === 'failed')
.map((assertion) => assertion.fullName)
)
}
}
const before = await run('before', baseline)
const after = await run('after', current)
const passed =
before.passed === 6 &&
before.failed === 3 &&
after.passed === 9 &&
after.failed === 0 &&
before.counts.length === 9 &&
after.counts.length === 9 &&
before.counts[0].counts.contextMenus === 1000 &&
before.counts[1].counts.contextMenus === 1000 &&
after.counts[0].counts.contextMenus === 0 &&
after.counts[1].counts.contextMenus === 0
console.log(
JSON.stringify(
{
comparison:
'Actual BrowserManager registration, destroyed-event handler, callback maps and unregisterAll; Electron methods use EventEmitter fixtures. Baseline removes only the exact-primary destruction cleanup in a temporary source transform.',
provenance: { node: process.version, platform: process.platform, arch: process.arch },
sourceHashes,
before,
after,
passed
},
null,
2
)
)
if (!passed) {
process.exitCode = 1
}
} finally {
if (runnerModuleId) {
delete require.cache[runnerModuleId]
}
await rm(scratch, { recursive: true, force: true })
}
@@ -0,0 +1,321 @@
{
"comparison": "Actual BrowserManager registration, destroyed-event handler, callback maps and unregisterAll; Electron methods use EventEmitter fixtures. Baseline removes only the exact-primary destruction cleanup in a temporary source transform.",
"provenance": {
"node": "v26.6.0",
"platform": "darwin",
"arch": "arm64"
},
"sourceHashes": {
"src/main/browser/browser-manager-guest-navigation-policy.ts": {
"before": "35aee2b5665a49535897fd0589853248f902061f77b3e142f94e90eabeb7332c",
"after": "741ac6d9f30fcf82993fa5c11f40093ba8a75483866407776c983538453f9b32"
},
"src/main/browser/browser-manager-destroyed-guest.test.ts": {
"current": "25806188a208952a1bebd042ec0dc4552784179ed3b738a5d5336789604ef314",
"observed": "8e59825237dea1b53294361122f0ea681947d4e6e9ea8fa771b5e15e080d88a6"
},
"src/main/browser/browser-manager-destroyed-guest-test-fixture.ts": {
"current": "4e6bf3589a3888860e6cfa8b6c83e8e50b4344650a5aaa85f221930ee8f9c0fd"
},
"src/main/browser/browser-manager-state.ts": {
"current": "1d82e875461984b3bee2d9dc9b477be7518e9394ace77b24e9b513766cc8a210"
},
"src/main/browser/browser-manager-registration.ts": {
"current": "55d66285b1d3ad29a7be596b1ca3536f333f2e3b2580e4d2a41090928ca8edd1"
},
"src/main/browser/browser-manager-download-lifecycle.ts": {
"current": "8c8d0dd488d31f6098f7ea1000ed8d6ffd8b23704a097db974207c9732d77c2a"
}
},
"before": {
"exitCode": 1,
"passed": 6,
"failed": 3,
"counts": [
{
"test": "browser guest destruction ownership > releases registered callbacks and ownership after 1000 distinct guest destructions",
"counts": {
"guests": 0,
"contextMenus": 1000,
"grabShortcuts": 1000,
"appShortcuts": 1000,
"wheelHandlers": 1000,
"renderers": 1000,
"workspaces": 1000,
"worktrees": 1000,
"profiles": 1000,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > leaves no dead-guest callbacks for the window-close unregisterAll path",
"counts": {
"guests": 0,
"contextMenus": 1000,
"grabShortcuts": 1000,
"appShortcuts": 1000,
"wheelHandlers": 0,
"renderers": 1000,
"workspaces": 1000,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > keeps explicit unregister before destruction idempotent",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > does not let a captured old destroyed callback retire a replacement guest",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > cleans a popup without retiring its live primary page",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > cleans policies for a guest destroyed before registration",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > preserves live guest ownership when its renderer process needs reload recovery",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > rebuilds ownership when a restored page registers its fresh guest",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > preserves a sibling page using the same browser session profile",
"counts": {
"guests": 1,
"contextMenus": 2,
"grabShortcuts": 2,
"appShortcuts": 2,
"wheelHandlers": 2,
"renderers": 2,
"workspaces": 2,
"worktrees": 2,
"profiles": 2,
"policies": 1
}
}
],
"failedCases": [
"browser guest destruction ownership releases registered callbacks and ownership after 1000 distinct guest destructions",
"browser guest destruction ownership leaves no dead-guest callbacks for the window-close unregisterAll path",
"browser guest destruction ownership preserves a sibling page using the same browser session profile"
]
},
"after": {
"exitCode": 0,
"passed": 9,
"failed": 0,
"counts": [
{
"test": "browser guest destruction ownership > releases registered callbacks and ownership after 1000 distinct guest destructions",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > leaves no dead-guest callbacks for the window-close unregisterAll path",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > keeps explicit unregister before destruction idempotent",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > does not let a captured old destroyed callback retire a replacement guest",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > cleans a popup without retiring its live primary page",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > cleans policies for a guest destroyed before registration",
"counts": {
"guests": 0,
"contextMenus": 0,
"grabShortcuts": 0,
"appShortcuts": 0,
"wheelHandlers": 0,
"renderers": 0,
"workspaces": 0,
"worktrees": 0,
"profiles": 0,
"policies": 0
}
},
{
"test": "browser guest destruction ownership > preserves live guest ownership when its renderer process needs reload recovery",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > rebuilds ownership when a restored page registers its fresh guest",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
},
{
"test": "browser guest destruction ownership > preserves a sibling page using the same browser session profile",
"counts": {
"guests": 1,
"contextMenus": 1,
"grabShortcuts": 1,
"appShortcuts": 1,
"wheelHandlers": 1,
"renderers": 1,
"workspaces": 1,
"worktrees": 1,
"profiles": 1,
"policies": 1
}
}
],
"failedCases": []
},
"passed": true
}
@@ -0,0 +1,30 @@
# Late browser registration replies restore retired renderer state
`createBrowserPageWebviewGuestSession` awaited `registerGuest` IPC and then wrote the returned guest ID into the renderer's persistent `registeredWebContentsIds` map. An explicit close could remove the webview and map entry before that reply arrived; a delayed success restored the retired entry. An older reply could also overwrite the ID of a replacement guest. Its follow-on callbacks could synchronize an obsolete annotation bridge or mutate recovery state after the listener session was disposed. Separately, recovery validation could issue repair IPC after its initial registration query outlived that owner.
The fix checks the existing recovery disposal state, current listener ref, persistent registry identity, and captured WebContents ID before accepting a reply or running those continuations. It makes no new registry and sends no late unregister IPC. A current hidden guest still accepts successful registration. When a persistent guest remounts, the new session's existing `validateAfterResume` path retries registration if the old reply was ignored. Current unsuccessful replies and current repair retain their prior behavior.
## Reproduce
With dependencies already installed:
```sh
ORCA_BACKGROUND_LAUNCH=1 node docs/audits/browser-registration-reply-retention/reproduce.mjs
```
This runs the actual renderer session, recovery controller, and persistent guest registry against headless DOM fixtures and deferred IPC replies. The baseline reverses only the included production patch in memory. A temporary observer records map/callback counts after all replies settle. The script uses the shared process launcher, 512 MiB workers, a 60-second deadline, and temporary files removed in `finally`. No Orca window, native guest, or remote host is launched.
| After 1,000 explicit guest closes and delayed successful replies | Before | Fixed |
| ---------------------------------------------------------------- | -----: | ----: |
| Live webviews | 0 | 0 |
| Retained registration entries | 1,000 | 0 |
| Late annotation synchronizations | 1,000 | 0 |
| Unregister calls | 1,000 | 1,000 |
The baseline fails ten tests and passes six controls; fixed source passes all 16. Cases cover distinct closed IDs, replacement elements, a changed guest ID on the same element, a remount reusing the same element/ref, disposed and moved refs, registry removal before listener disposal, a throwing identity getter, hidden current guests, successful/inconclusive replies, and late versus current repair. The repair-completion case verifies that an old success cannot clear a newer guest's recovery error. All host-guest suites also pass: 196 tests across 23 files, including recovery, viewport, registry, worktree retention, and paintability. The web typecheck passes.
## Version and limits
Targeted reads of `v1.4.198` confirm the same unconditional registration setter, post-reply callbacks, post-query repair, and close-time map deletion. This establishes a renderer retaining path in the reported version, not that #19831 or #19768 exercised it. Each retained entry is a page ID and numeric guest ID. This proof does not show a surviving native browser process or explain gigabyte-scale memory growth. The independent main-process destroyed-guest callback retention has its own fix and proof.
The registration reply is the only production setter of `registeredWebContentsIds`; explicit close and replacement remove its key. Following callers found no second setter that could recreate this same metadata after removal. The annotation callback uses current page routing, which is why skipping a stale callback is necessary without issuing cleanup against a replacement.
@@ -0,0 +1,126 @@
diff --git a/src/renderer/src/components/browser-pane/host-guest/browser-page-guest-recovery.ts b/src/renderer/src/components/browser-pane/host-guest/browser-page-guest-recovery.ts
index 45f3b354b5..b6e30917d1 100644
--- a/src/renderer/src/components/browser-pane/host-guest/browser-page-guest-recovery.ts
+++ b/src/renderer/src/components/browser-pane/host-guest/browser-page-guest-recovery.ts
@@ -21,6 +21,7 @@ type BrowserPageGuestRecoveryOptions = {
export type BrowserPageGuestRecovery = {
confirmRegistration: () => void
dispose: () => void
+ isDisposed: () => boolean
finish: () => boolean
recoverRenderer: () => void
retryRecovery: () => void
@@ -263,6 +264,7 @@ export function createBrowserPageGuestRecovery(
clearValidationRetry()
clearValidationTimeout()
},
+ isDisposed: () => disposed,
finish,
recoverRenderer,
retryRecovery: () => {
diff --git a/src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts b/src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts
index 4623b817b2..dd5e243ba2 100644
--- a/src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts
+++ b/src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts
@@ -15,7 +15,11 @@ import {
type BrowserPageGuestRecovery
} from './browser-page-guest-recovery'
import { browserPageZoomLevelToPercent, setBrowserPageZoomLevel } from './browser-page-zoom'
-import { registeredWebContentsIds, replacePersistentWebview } from './webview-registry'
+import {
+ registeredWebContentsIds,
+ replacePersistentWebview,
+ webviewRegistry
+} from './webview-registry'
import { browserPageExists } from '../describe-page/browser-page-load-error'
import type {
BrowserPageRecoveryNavigationValidation,
@@ -80,11 +84,21 @@ export function createBrowserPageWebviewGuestSession({
webContentsId: number
promise: Promise<boolean | null>
} | null = null
- const registerGuest = (): Promise<boolean | null> => {
- let webContentsId: number
+ const readWebContentsId = (): number | null => {
try {
- webContentsId = webview.getWebContentsId()
+ return webview.getWebContentsId()
} catch {
+ return null
+ }
+ }
+ const ownsGuest = (webContentsId: number | null): boolean =>
+ webContentsId !== null &&
+ !guestRecovery.isDisposed() &&
+ webviewRef.current === webview &&
+ webviewRegistry.get(browserTabId) === webview &&
+ readWebContentsId() === webContentsId
+ const registerGuest = (webContentsId: number | null): Promise<boolean | null> => {
+ if (webContentsId === null || !ownsGuest(webContentsId)) {
return Promise.resolve(null)
}
if (registrationInFlight?.webContentsId === webContentsId) {
@@ -99,6 +113,9 @@ export function createBrowserPageWebviewGuestSession({
webContentsId
})
.then((registered) => {
+ if (!ownsGuest(webContentsId)) {
+ return null
+ }
if (registered) {
registeredWebContentsIds.set(browserTabId, webContentsId)
return true
@@ -146,22 +163,26 @@ export function createBrowserPageWebviewGuestSession({
return null
}
if (registeredWebContentsIds.get(browserTabId) !== webContentsId) {
- return registerGuest()
+ return registerGuest(webContentsId)
}
const registered = await window.api.browser.isGuestRegistered({
browserPageId: browserTabId,
webContentsId
})
+ if (!ownsGuest(webContentsId)) {
+ return null
+ }
if (registered) {
return true
}
- return window.api.browser.repairGuestRegistration({
+ const repaired = await window.api.browser.repairGuestRegistration({
browserPageId: browserTabId,
workspaceId,
worktreeId,
sessionProfileId,
webContentsId
})
+ return ownsGuest(webContentsId) ? repaired : null
},
replaceGuest: () => replacePersistentWebview(browserTabId),
onReplacementReady: () => setGuestRecoveryGeneration((generation) => generation + 1),
@@ -184,7 +205,11 @@ export function createBrowserPageWebviewGuestSession({
const handleDidAttach = (): void => {
// Why: register at attach since cert failures can precede dom-ready; the dom-ready path stays an idempotent fallback.
- void registerGuest().then((registered) => {
+ const webContentsId = readWebContentsId()
+ void registerGuest(webContentsId).then((registered) => {
+ if (!ownsGuest(webContentsId)) {
+ return
+ }
if (registered === true) {
guestRecovery.confirmRegistration()
}
@@ -207,7 +232,10 @@ export function createBrowserPageWebviewGuestSession({
const queuedAnnotationViewportBridgeSync =
liveWebContentsId === null || registeredWebContentsIds.get(browserTabId) !== liveWebContentsId
if (queuedAnnotationViewportBridgeSync) {
- void registerGuest().then((registered) => {
+ void registerGuest(liveWebContentsId).then((registered) => {
+ if (!ownsGuest(liveWebContentsId)) {
+ return
+ }
const completedRecovery = guestRecovery.finish()
if (registered === true) {
guestRecovery.confirmRegistration()
@@ -0,0 +1,153 @@
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { applyPatch, parsePatch, reversePatch } from 'diff'
import { build } from 'esbuild'
if (process.env.ORCA_BACKGROUND_LAUNCH !== '1') {
throw new Error('Run with ORCA_BACKGROUND_LAUNCH=1.')
}
const root = fileURLToPath(new URL('../../../', import.meta.url))
const patch = await readFile(new URL('./fix.patch', import.meta.url), 'utf8')
const beforeSources = {}
const sourceHashes = {}
const sha256 = (source) => createHash('sha256').update(source).digest('hex')
for (const parsed of parsePatch(patch)) {
const path = parsed.newFileName.replace(/^b\//, '')
const absolute = resolve(root, path)
const current = await readFile(absolute, 'utf8')
const before = applyPatch(current, reversePatch(parsed))
if (before === false) {
throw new Error(`Source changed; review the proof patch: ${path}`)
}
beforeSources[absolute.replaceAll('\\', '/')] = before
sourceHashes[path] = { before: sha256(before), after: sha256(current) }
}
const testPath =
'src/renderer/src/components/browser-pane/host-guest/browser-page-registration-ownership.test.ts'
const test = await readFile(resolve(root, testPath), 'utf8')
const countAssertion = ' expect(webviewRegistry.size).toBe(0)\n'
if (test.split(countAssertion).length !== 2) {
throw new Error('Expected exactly one closed-guest count assertion; review the observer.')
}
const observedTest = `import { writeFileSync } from 'node:fs'\n${test.replace(
countAssertion,
` writeFileSync(process.env.ORCA_BROWSER_REGISTRATION_COUNTS_PATH, JSON.stringify({ liveWebviews: webviewRegistry.size, registeredGuestIds: registeredWebContentsIds.size, lateAnnotationSyncs: sessions.reduce((count, page) => count + page.sync.mock.calls.length, 0), unregisterCalls: unregister.mock.calls.length }))\n${countAssertion}`
)}`
sourceHashes[testPath] = { current: sha256(test), observed: sha256(observedTest) }
const scratch = await mkdtemp(join(tmpdir(), 'orca-browser-registration-reply-'))
const require = createRequire(import.meta.url)
let runnerModuleId
try {
const runnerPath = join(scratch, 'run-process.cjs')
await build({
absWorkingDir: root,
entryPoints: [resolve(root, 'src/shared/child-process/run-process.ts')],
outfile: runnerPath,
bundle: true,
platform: 'node',
format: 'cjs',
logLevel: 'silent'
})
runnerModuleId = require.resolve(runnerPath)
const { runProcess } = require(runnerModuleId)
const configImport = JSON.stringify(pathToFileURL(resolve(root, 'config/vitest.config.ts')).href)
async function run(label, productionSources) {
const config = join(scratch, `${label}.config.mjs`)
const report = join(scratch, `${label}.json`)
const countsPath = join(scratch, `${label}.counts.json`)
const sources = {
...productionSources,
[resolve(root, testPath).replaceAll('\\', '/')]: observedTest
}
await writeFile(
config,
`import base from ${configImport};
const sources = ${JSON.stringify(sources)};
export default {...base, test: {...base.test, include: [${JSON.stringify(testPath)}], maxWorkers: 1}, plugins: [{
name: 'browser-registration-reply-audit', enforce: 'pre',
transform(_code, id) {
const source = sources[id.replaceAll('\\\\', '/').split('?')[0]];
return source === undefined ? null : {code: source, map: null};
}
}]};\n`
)
const result = await runProcess({
program: process.execPath,
args: [
resolve(root, 'node_modules/vitest/vitest.mjs'),
'run',
'--config',
config,
'--reporter=json',
`--outputFile=${report}`
],
cwd: root,
env: {
...process.env,
NODE_OPTIONS: '--max-old-space-size=512',
ORCA_BROWSER_REGISTRATION_COUNTS_PATH: countsPath
},
timeoutMs: 60_000,
maxOutputBytes: 2 * 1024 * 1024
})
let parsed
try {
parsed = JSON.parse(await readFile(report, 'utf8'))
} catch (error) {
throw new Error(`${label} failed: ${result.stderr || result.stdout}`, { cause: error })
}
return {
exitCode: result.code,
passed: parsed.numPassedTests,
failed: parsed.numFailedTests,
after1000ClosedGuests: JSON.parse(await readFile(countsPath, 'utf8')),
failedCases: parsed.testResults.flatMap((suite) =>
suite.assertionResults
.filter((assertion) => assertion.status === 'failed')
.map((assertion) => assertion.fullName)
)
}
}
const before = await run('before', beforeSources)
const after = await run('after', {})
const passed =
before.passed === 6 &&
before.failed === 10 &&
after.passed === 16 &&
after.failed === 0 &&
before.after1000ClosedGuests.liveWebviews === 0 &&
before.after1000ClosedGuests.registeredGuestIds === 1000 &&
after.after1000ClosedGuests.registeredGuestIds === 0 &&
after.after1000ClosedGuests.unregisterCalls === 1000
console.log(
JSON.stringify(
{
comparison:
'Actual renderer guest session, recovery controller and persistent guest registry with deferred IPC replies; baseline reverses only fix.patch in a temporary source transform.',
provenance: { node: process.version, platform: process.platform, arch: process.arch },
sourceHashes,
before,
after,
passed
},
null,
2
)
)
if (!passed) {
process.exitCode = 1
}
} finally {
if (runnerModuleId) {
delete require.cache[runnerModuleId]
}
await rm(scratch, { recursive: true, force: true })
}
@@ -0,0 +1,58 @@
{
"comparison": "Actual renderer guest session, recovery controller and persistent guest registry with deferred IPC replies; baseline reverses only fix.patch in a temporary source transform.",
"provenance": {
"node": "v26.6.0",
"platform": "darwin",
"arch": "arm64"
},
"sourceHashes": {
"src/renderer/src/components/browser-pane/host-guest/browser-page-guest-recovery.ts": {
"before": "044280571771c4e90d07f5f5426878539f2b1729d9a4482c59d8fd236e7d4177",
"after": "2efdea1c4223b2f4114548a7f6ec4b576e1c6bde7819a07dc096d5167c2ba42c"
},
"src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts": {
"before": "2fe17f0fe4f8ef6ca7cff7ca5731d9d725dbf4b5e7944264e30f12241317fc6d",
"after": "a78cc98873e93834bf07b47bbf5d92da888dfbccef06551aa6ac3c8f6e9f29f8"
},
"src/renderer/src/components/browser-pane/host-guest/browser-page-registration-ownership.test.ts": {
"current": "992b4730cd5546720b8b52366955386ad900d057f95c13beb6b92e5369bf5c1e",
"observed": "ebc38ebb195295170e8c355ecda539f3f2e75bb80548dbf4e97a17a7cd88c661"
}
},
"before": {
"exitCode": 1,
"passed": 6,
"failed": 10,
"after1000ClosedGuests": {
"liveWebviews": 0,
"registeredGuestIds": 1000,
"lateAnnotationSyncs": 1000,
"unregisterCalls": 1000
},
"failedCases": [
"renderer registration completion ownership does not restore 1000 closed IDs from delayed successful replies",
"renderer registration completion ownership keeps the replacement ID after an older reply arrives",
"renderer registration completion ownership keeps a new ID when the same DOM webview swaps its guest",
"renderer registration completion ownership a new session retries a disposed session registration on the same persistent guest and ref",
"renderer registration completion ownership a disposed listener cannot act after the same guest and ref are reused",
"renderer registration completion ownership does not restore a registry-removed guest before listener disposal runs",
"renderer registration completion ownership does not restore metadata when the current listener ref has moved",
"renderer registration completion ownership ignores a reply after reading the guest identity starts throwing",
"renderer registration completion ownership does not issue repair after a pending validation outlives its owner",
"renderer registration completion ownership does not clear a newer guest recovery error from a pending old repair reply"
]
},
"after": {
"exitCode": 0,
"passed": 16,
"failed": 0,
"after1000ClosedGuests": {
"liveWebviews": 0,
"registeredGuestIds": 0,
"lateAnnotationSyncs": 0,
"unregisterCalls": 1000
},
"failedCases": []
},
"passed": true
}
@@ -0,0 +1,44 @@
# Retired browser viewport operation ownership
The viewport operation captures a guest ID, then awaits CDP commands. Closing a tab deletes its viewport state, but the old continuation can subsequently recreate the UA-intent entry. A failed UA clear can also restore the old value over a replacement guest's completed desktop preset, or a late clear can delete the replacement's mobile intent.
The correction reuses that captured guest ID at three mutation boundaries: before publishing an applied preset's UA intent, before reading/deleting a cleared preset's intent, and before failed-clear rollback. Same-owner rollback, native UA profiles, navigation behavior, and the per-tab promise chain are preserved.
## Evidence
The regression fixture calls the actual manager, registration, unregistration, and viewport implementation. Electron WebContents and pending debugger replies are controlled ports; no native browser or window is launched.
- Baseline: **7 failing ownership regressions, 5 passing controls**.
- Fixed: **12/12 ownership cases**, plus **30 existing viewport, navigation, partial-failure, and UA cases**.
- Sixteen pending UA-clear rejections after `unregisterAll` leave **16 retired UA entries before, zero after**. Registration, preset, and promise maps remain empty.
- Other regressions cover closed-tab late success, failed-clear rollback, mobile/desktop replacement, and native-to-default profile replacement.
- Controls preserve ordinary serialized mobile/desktop/null operations, both native-profile presets, same-owner rollback, and the replacement promise tail while old queued operations settle.
- An independent reviewer ran all 12 candidate cases and reviewed the three mutation guards before promotion.
The retained entries are tab ID strings and booleans. This does **not** demonstrate retained native WebContents, a process RSS slope, or gigabyte-scale memory growth. In-flight CDP work still owns its continuation until it settles. Positive and negative post-close command replies are injected schedules, not an affected-host trace.
## Ordinary callers and compatibility
The renderer requests overrides when the user selects a viewport preset and on guest `dom-ready`, including null presets. The trusted IPC handler validates dimensions before calling this manager. Navigation later reads the UA-intent map, so stale replacement values can alter the standing mobile/desktop identity. The fixture does not execute the renderer or IPC producer.
Both local webview and host-side offscreen registrations use these maps. The correction changes no wire fields, protocol, execution-host ownership, native process lifecycle, folder/worktree handling, or UI layout. It only prevents an operation for a different guest from mutating the current registration's state.
`source-versions.json` records 11 paths at audit checkpoint `4a09b1d1`, independent main `291b4ddd`, and reported v1.4.198 `e0826956`. The viewport implementation, registration, registry declarations, IPC handler, and toolbar producer match all three. Ten sources match independent main and eight match v1.4.198. The guest-session producer contains an earlier audit fix; historical navigation and fixture sources differ. This is a current-dependency replay with the exact historical viewport source, not a historical app-binary replay.
The browsing activity in #19831 makes this path applicable in principle. The report does not establish the required overlap or tab count, and this small metadata mechanism does not account for its reported memory totals.
## Reproduction
From the worktree, run the fixed regression suite:
```sh
ORCA_BACKGROUND_LAUNCH=1 node node_modules/vitest/vitest.mjs run --config docs/audits/browser-viewport-owner-retention/vitest.config.mjs
```
Run the same tests with the exact baseline viewport implementation; exit status 1 and seven failed cases are expected:
```sh
ORCA_BACKGROUND_LAUNCH=1 ORCA_VIEWPORT_BASELINE=1 node node_modules/vitest/vitest.mjs run --config docs/audits/browser-viewport-owner-retention/vitest.config.mjs
```
The import overlay never rewrites product files. `baseline-source.txt` contains only the original viewport module; current support modules remain in use. `baseline-results.json`, `fixed-results.json`, and `validation.json` record the measured results and their scope.
@@ -0,0 +1,94 @@
{
"testFiles": 1,
"total": 12,
"passed": 5,
"failed": 7,
"cases": [
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership does not recreate closed-tab UA intent after a late touch completion",
"status": "failed",
"failures": [
"AssertionError: expected true to be undefined\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:109:37\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership does not restore closed-tab UA intent after a failed clear",
"status": "failed",
"failures": [
"AssertionError: expected true to be undefined\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:126:42\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership preserves replacement desktop intent after an old clear fails",
"status": "failed",
"failures": [
"AssertionError: expected true to be false // Object.is equality\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:142:42\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership preserves replacement mobile intent after an old clear resumes",
"status": "failed",
"failures": [
"AssertionError: expected false to be true // Object.is equality\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:160:42\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership same-owner clear failure still restores the legitimate earlier intent",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership old apply cannot overwrite a replacement guest desktop intent",
"status": "failed",
"failures": [
"AssertionError: expected true to be false // Object.is equality\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:185:41\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership an old native profile cannot write UA intent after replacement with a default profile",
"status": "failed",
"failures": [
"AssertionError: expected true to be undefined\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:197:49\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership old queued operations cannot remove or join a replacement promise tail",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership normal same-owner toggles preserve last-requested order and remove the promise tail",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership native UA mode remains unchanged with mobile=false",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership native UA mode remains unchanged with mobile=true",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership late rejected clears cannot repopulate all registries after unregisterAll",
"status": "failed",
"failures": [
"AssertionError: expected 16 to be +0 // Object.is equality\n at ./src/main/browser/browser-manager-viewport-ownership.test.ts:278:28\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://./node_modules/.pnpm/@vitest+runner@4.1.11/node_modules/@vitest/runner/dist/chunk-artifact.js:1903:20"
]
}
]
}
@@ -0,0 +1,219 @@
import { webContents } from 'electron'
import {
BROWSER_ANNOTATION_VIEWPORT_BRIDGE_WORLD_ID,
buildBrowserAnnotationViewportBridgeScript,
type BrowserAnnotationViewportBridgeOptions
} from '../../shared/browser-annotation-viewport-bridge'
import type { BrowserViewportOverride } from '../../shared/browser-workspace-types'
import { googleAuthUserAgent, isGoogleAuthUrl } from './browser-google-auth-ua'
import { BrowserManagerDownloadLifecycle } from './browser-manager-download-lifecycle'
export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifecycle {
// Why: guests are isolated from Orca's preload bridge, so main owns the devtools escape hatch after a tab→guest lookup.
async openDevTools(browserTabId: string): Promise<boolean> {
const webContentsId = this.webContentsIdByTabId.get(browserTabId)
if (!webContentsId) {
return false
}
const guest = webContents.fromId(webContentsId)
if (!guest || guest.isDestroyed()) {
// Why: a stale guest must clear every per-tab registry entry, not just the WebContents maps.
this.unregisterGuest(browserTabId)
return false
}
// Offscreen guests have no visible window on this desktop; detaching DevTools would open it
// on the host display with no route back to the remote client.
if (this.offscreenGuestIds.has(webContentsId)) {
return false
}
guest.openDevTools({ mode: 'detach' })
return true
}
// Why: emulate viewport via CDP; never detach the debugger here or the agent bridge's per-guest state is cleared.
async setViewportOverride(
browserTabId: string,
override: BrowserViewportOverride | null
): Promise<boolean> {
// Why: chain per-tab so rapid toggles don't interleave CDP commands and the last-requested override wins.
const expectedWebContentsId = this.webContentsIdByTabId.get(browserTabId)
if (expectedWebContentsId !== undefined) {
// Keep host panning available while CDP applies the requested dimensions. The guest id fence
// prevents this intent from leaking to a replacement guest; clearing the preset removes it.
this.viewportPresetActiveByTabId.set(browserTabId, {
guestWebContentsId: expectedWebContentsId,
active: override !== null
})
}
// The renderer resizes the host before CDP completes; discard the old geometry until it
// reports the new pane bounds so a pending preset cannot route wheel input using stale limits.
this.viewportScrollStateByTabId.delete(browserTabId)
const prev = this.viewportOpsByTabId.get(browserTabId) ?? Promise.resolve()
const next = prev
.catch(() => {})
.then(() => this.doSetViewportOverrideImpl(browserTabId, override, expectedWebContentsId))
this.viewportOpsByTabId.set(browserTabId, next)
try {
return await next
} finally {
// Why: only clear if we're still the tail; a later call may have replaced the entry, and deleting would break serialization.
if (this.viewportOpsByTabId.get(browserTabId) === next) {
this.viewportOpsByTabId.delete(browserTabId)
}
}
}
async setAnnotationViewportBridge(
browserTabId: string,
options: BrowserAnnotationViewportBridgeOptions,
resolveGuest: () => Electron.WebContents | null
): Promise<boolean> {
const prev = this.annotationViewportBridgeOpsByTabId.get(browserTabId) ?? Promise.resolve()
const next = prev
.catch(() => {})
.then(() => this.doSetAnnotationViewportBridgeImpl(options, resolveGuest))
this.annotationViewportBridgeOpsByTabId.set(browserTabId, next)
try {
return await next
} finally {
if (this.annotationViewportBridgeOpsByTabId.get(browserTabId) === next) {
this.annotationViewportBridgeOpsByTabId.delete(browserTabId)
}
}
}
// Why the caller resolves the guest: the same bridge serves browsing pages and workspace
// documents, which live in different halves of the page registry.
// Why a resolver and not the guest itself: this op may have waited behind another one, and a
// cross-process navigation meanwhile swaps the tab's contents without destroying the old one —
// injecting into the guest the request named would bridge a page nobody is looking at.
// Why no tab id: with teardown gone this reaches only the guest the resolver hands back, and
// taking an id it cannot act on would invite the next reader to act on it.
protected async doSetAnnotationViewportBridgeImpl(
options: BrowserAnnotationViewportBridgeOptions,
resolveGuest: () => Electron.WebContents | null
): Promise<boolean> {
// Why no teardown here: the resolver already unregisters a page whose guest died, and the only
// case it uniquely leaves is an ownership mismatch on a healthy page — where tearing down would
// cancel that page's in-flight downloads and grabs over a request that was merely misaddressed.
const guest = resolveGuest()
if (!guest || guest.isDestroyed()) {
return false
}
try {
// Why: run the scroll bridge in an isolated world so page scripts can't read the per-tab token or tamper with it.
await guest.executeJavaScriptInIsolatedWorld(
BROWSER_ANNOTATION_VIEWPORT_BRIDGE_WORLD_ID,
[{ code: buildBrowserAnnotationViewportBridgeScript(options) }],
false
)
return true
} catch {
return false
}
}
protected async doSetViewportOverrideImpl(
browserTabId: string,
override: BrowserViewportOverride | null,
expectedWebContentsId: number | undefined
): Promise<boolean> {
const webContentsId = this.webContentsIdByTabId.get(browserTabId)
if (!webContentsId || webContentsId !== expectedWebContentsId) {
return false
}
const guest = webContents.fromId(webContentsId)
if (!guest || guest.isDestroyed()) {
// Why: a stale guest must clear every per-tab registry entry, not just the WebContents maps.
this.unregisterGuest(browserTabId)
return false
}
try {
if (!guest.debugger.isAttached()) {
guest.debugger.attach('1.3')
}
} catch (err) {
// Why: attach throws if DevTools is open on the guest; log context so this failure mode is diagnosable.
console.warn('[browser-manager] setViewportOverride: failed to attach debugger', {
browserTabId,
webContentsId,
error: err instanceof Error ? err.message : String(err)
})
return false
}
const dbg = guest.debugger
try {
if (override) {
await dbg.sendCommand('Emulation.setDeviceMetricsOverride', {
width: override.width,
height: override.height,
deviceScaleFactor: override.deviceScaleFactor,
mobile: override.mobile
})
if (this.webContentsIdByTabId.get(browserTabId) === webContentsId) {
this.viewportPresetActiveByTabId.set(browserTabId, {
guestWebContentsId: webContentsId,
active: true
})
}
await dbg.sendCommand('Emulation.setTouchEmulationEnabled', {
enabled: override.mobile,
maxTouchPoints: override.mobile ? 5 : 0
})
// Why: viewport sizing must not override a profile's explicit native-UA identity.
if (this.userAgentModeByPageId.get(browserTabId) !== 'native') {
// Navigation must see the preset intent while the final CDP command is in flight.
this.viewportUaOverrideMobileByTabId.set(browserTabId, override.mobile)
// Why: same sender as the navigation path, so both resolve the tab's host identically.
await this.sendViewportUserAgentOverride(guest, override.mobile)
}
} else {
await dbg.sendCommand('Emulation.clearDeviceMetricsOverride', {})
if (this.webContentsIdByTabId.get(browserTabId) === webContentsId) {
this.viewportPresetActiveByTabId.set(browserTabId, {
guestWebContentsId: webContentsId,
active: false
})
}
await dbg.sendCommand('Emulation.setTouchEmulationEnabled', {
enabled: false,
maxTouchPoints: 0
})
const trackedMobile = this.viewportUaOverrideMobileByTabId.get(browserTabId)
// A navigation after this point must not re-install the override behind the clear.
this.viewportUaOverrideMobileByTabId.delete(browserTabId)
try {
if (this.authUserAgentOverrideStateByGuestId.has(guest.id)) {
const url = this.resolveTabNavigationUrl(guest)
const restored = await this.applyAuthUserAgentOverrideOverCdp(
guest,
false,
url,
isGoogleAuthUrl(url) ? googleAuthUserAgent() : guest.session.getUserAgent()
)
if (!restored) {
throw new Error('Failed to preserve auth user agent')
}
} else {
// Why: passing an empty string restores the session default UA.
await dbg.sendCommand('Emulation.setUserAgentOverride', { userAgent: '' })
}
} catch (error) {
if (trackedMobile !== undefined) {
this.viewportUaOverrideMobileByTabId.set(browserTabId, trackedMobile)
}
throw error
}
}
if (this.webContentsIdByTabId.get(browserTabId) !== webContentsId) {
return false
}
return true
} catch {
return false
}
}
}
@@ -0,0 +1,18 @@
diff --git a/src/main/browser/browser-manager-viewport.ts b/src/main/browser/browser-manager-viewport.ts
index ce31dbe37e..3f1fbb68fb 100644
--- a/src/main/browser/browser-manager-viewport.ts
+++ b/src/main/browser/browser-manager-viewport.ts
@@ -165,0 +166,3 @@ export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifec
+ if (this.webContentsIdByTabId.get(browserTabId) !== webContentsId) {
+ return false
+ }
@@ -184,0 +188,3 @@ export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifec
+ if (this.webContentsIdByTabId.get(browserTabId) !== webContentsId) {
+ return false
+ }
@@ -205 +211,4 @@ export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifec
- if (trackedMobile !== undefined) {
+ if (
+ trackedMobile !== undefined &&
+ this.webContentsIdByTabId.get(browserTabId) === webContentsId
+ ) {
@@ -0,0 +1,260 @@
{
"testFiles": 4,
"total": 42,
"passed": 42,
"failed": 0,
"cases": [
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride returns false when the tab is not registered",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride applies device metrics, touch emulation, and a mobile UA for mobile presets",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride keeps the session UA for native-mode profiles when mobile=false",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride keeps the session UA for native-mode profiles when mobile=true",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride presents the Firefox UA for a preset applied on a Google auth host (mobile=false)",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride presents the Firefox UA for a preset applied on a Google auth host (mobile=true)",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride re-issues the standing UA override when navigating onto and back off an auth host",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not leave the Chrome preset UA standing when a mobile preset lands mid-navigation onto an auth host",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not leave the Firefox UA standing when a preset lands mid-navigation off an auth host",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride falls back to the committed URL once a navigation commits or fails",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not let a superseded navigation failure revert a newer target",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride switches identity for a server redirect and restores it if the redirect fails",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride preserves the auth identity when a viewport preset is cleared after a redirect",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not inherit a mobile owner UA in a desktop popup",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride reapplies a preset when navigation starts during its final UA write",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not reinstall a preset while its final UA clear is in flight",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride keeps tracking the standing override when the CDP clear fails",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride does not touch the UA override on navigation when no preset is standing",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride stops re-issuing the UA override once the preset is cleared",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride leaves the UA override alone on navigation for native-UA profiles",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride clears device metrics and disables touch for override=null",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride attaches the debugger if not already attached and does not detach after",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-override.test.ts",
"title": "browserManager setViewportOverride returns false when debugger.attach throws (e.g. DevTools already open)",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership does not recreate closed-tab UA intent after a late touch completion",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership does not restore closed-tab UA intent after a failed clear",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership preserves replacement desktop intent after an old clear fails",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership preserves replacement mobile intent after an old clear resumes",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership same-owner clear failure still restores the legitimate earlier intent",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership old apply cannot overwrite a replacement guest desktop intent",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership an old native profile cannot write UA intent after replacement with a default profile",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership old queued operations cannot remove or join a replacement promise tail",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership normal same-owner toggles preserve last-requested order and remove the promise tail",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership native UA mode remains unchanged with mobile=false",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership native UA mode remains unchanged with mobile=true",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-ownership.test.ts",
"title": "browser viewport operation ownership late rejected clears cannot repopulate all registries after unregisterAll",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-partial-failure.test.ts",
"title": "browserManager viewport partial failure keeps wheel routing active when follow-up setup fails after metrics apply",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-manager-viewport-partial-failure.test.ts",
"title": "browserManager viewport partial failure keeps host panning available when metrics setup fails",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-viewport-user-agent.test.ts",
"title": "buildViewportUserAgentOverride presents the Firefox UA on Google auth hosts regardless of the preset",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-viewport-user-agent.test.ts",
"title": "buildViewportUserAgentOverride keeps the clean desktop UA off the auth hosts",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-viewport-user-agent.test.ts",
"title": "buildViewportUserAgentOverride splices the real Chrome major into the mobile UA and its client hints",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-viewport-user-agent.test.ts",
"title": "buildViewportUserAgentOverride falls back to a known Chrome major when the base UA carries none",
"status": "passed",
"failures": []
},
{
"file": "src/main/browser/browser-viewport-user-agent.test.ts",
"title": "buildViewportUserAgentOverride treats an unparseable URL as a non-auth host",
"status": "passed",
"failures": []
}
]
}
@@ -0,0 +1,98 @@
{
"refs": {
"audit": "4a09b1d108cfd8b57ffcc5d727b3a3bd71ae71fb",
"main": "291b4ddd6f1c1af480169885e0fda7f9c78ff053",
"v1.4.198": "e0826956fcfc532f5a1e55b5e081f2e57e553c43"
},
"canonicalLineEndings": "LF",
"sources": [
{
"path": "src/main/browser/browser-manager-viewport.ts",
"sha256": {
"audit": "701f9ea1a4310e509f409e08ee4ea0539f820bd209ff68f41001b8c91e5470e4",
"main": "701f9ea1a4310e509f409e08ee4ea0539f820bd209ff68f41001b8c91e5470e4",
"v1.4.198": "701f9ea1a4310e509f409e08ee4ea0539f820bd209ff68f41001b8c91e5470e4"
}
},
{
"path": "src/main/browser/browser-manager-registration.ts",
"sha256": {
"audit": "12c149b1fb87b67b4192b6c2f23bd7f45b7df2d0c60368182f33b5a47b56f96f",
"main": "12c149b1fb87b67b4192b6c2f23bd7f45b7df2d0c60368182f33b5a47b56f96f",
"v1.4.198": "12c149b1fb87b67b4192b6c2f23bd7f45b7df2d0c60368182f33b5a47b56f96f"
}
},
{
"path": "src/main/browser/browser-manager-navigation.ts",
"sha256": {
"audit": "a4c0e169ac35725b02d050c95843721e4274775a5bcb9ef2a5b7c835c4d8c234",
"main": "a4c0e169ac35725b02d050c95843721e4274775a5bcb9ef2a5b7c835c4d8c234",
"v1.4.198": "c93c060896351b4bc23db628a732ef4db4acd5b26760e4565e6bf029d5cf7531"
}
},
{
"path": "src/main/browser/browser-manager-guest-policy.ts",
"sha256": {
"audit": "6d567b0c675091ec0a59d39ffe701fd36098ec2730b61ef0b80ab53962da6144",
"main": "6d567b0c675091ec0a59d39ffe701fd36098ec2730b61ef0b80ab53962da6144",
"v1.4.198": "6d567b0c675091ec0a59d39ffe701fd36098ec2730b61ef0b80ab53962da6144"
}
},
{
"path": "src/main/browser/browser-manager-state.ts",
"sha256": {
"audit": "ea6e847afe227b9acbf432d68a6f21f0d15247748fc8b4f769ab73498d6a13c5",
"main": "ea6e847afe227b9acbf432d68a6f21f0d15247748fc8b4f769ab73498d6a13c5",
"v1.4.198": "ea6e847afe227b9acbf432d68a6f21f0d15247748fc8b4f769ab73498d6a13c5"
}
},
{
"path": "src/main/browser/browser-manager-types.ts",
"sha256": {
"audit": "4d28f7c397aa82b067971ff769a44fff74adadceaac66774345a615f799bb64f",
"main": "4d28f7c397aa82b067971ff769a44fff74adadceaac66774345a615f799bb64f",
"v1.4.198": "4d28f7c397aa82b067971ff769a44fff74adadceaac66774345a615f799bb64f"
}
},
{
"path": "src/main/browser/browser-manager-viewport-test-fixtures.ts",
"sha256": {
"audit": "0ef61054ae131db056b5268c6ed4f417b4486784d58d2c0d7d2285ad785666ee",
"main": "0ef61054ae131db056b5268c6ed4f417b4486784d58d2c0d7d2285ad785666ee",
"v1.4.198": "36d29f1d78bd559af3b235acc9be8dafe763e3dfb9ea4367272db04449eca707"
}
},
{
"path": "src/main/browser/browser-manager-test-harness.ts",
"sha256": {
"audit": "66de235be9285ec2cd2a6d783c8e8a046d6106d4171aaf808b5d263c08a72038",
"main": "66de235be9285ec2cd2a6d783c8e8a046d6106d4171aaf808b5d263c08a72038",
"v1.4.198": "66de235be9285ec2cd2a6d783c8e8a046d6106d4171aaf808b5d263c08a72038"
}
},
{
"path": "src/main/ipc/browser-guest-view-ipc.ts",
"sha256": {
"audit": "da4a441ce5f0851f1a1c6ec38c872c187ec191a9acad340f05ac557534731959",
"main": "da4a441ce5f0851f1a1c6ec38c872c187ec191a9acad340f05ac557534731959",
"v1.4.198": "da4a441ce5f0851f1a1c6ec38c872c187ec191a9acad340f05ac557534731959"
}
},
{
"path": "src/renderer/src/components/browser-pane/host-guest/browser-page-webview-guest-session.ts",
"sha256": {
"audit": "a78cc98873e93834bf07b47bbf5d92da888dfbccef06551aa6ac3c8f6e9f29f8",
"main": "2fe17f0fe4f8ef6ca7cff7ca5731d9d725dbf4b5e7944264e30f12241317fc6d",
"v1.4.198": "6c089c0c8285b21b3f3c0b3e06bd297a25d941cfa8a4849d03fbd08a6c89c139"
}
},
{
"path": "src/renderer/src/components/browser-pane/assemble-chrome/BrowserToolbarMenu.tsx",
"sha256": {
"audit": "e05e7d8d532d2f43f13a8dc1035540638ab513c9f3d6bdbd1531cf3e0e9462e1",
"main": "e05e7d8d532d2f43f13a8dc1035540638ab513c9f3d6bdbd1531cf3e0e9462e1",
"v1.4.198": "e05e7d8d532d2f43f13a8dc1035540638ab513c9f3d6bdbd1531cf3e0e9462e1"
}
}
]
}
@@ -0,0 +1,36 @@
{
"scope": "Actual manager and lifecycle; controlled Electron/CDP ports; no native guest, heap/RSS or incident attribution",
"baseline": {
"total": 12,
"failedOwnershipCases": 7,
"passedControls": 5
},
"fixed": {
"total": 42,
"passed": 42,
"testFiles": 4
},
"independentReview": {
"candidateTestsPassed": 12,
"findings": "No blocker; three map mutation guards preserve current guest and promise ownership"
},
"typecheck": {
"node": "passed after correcting fixture-only protected-map reads and array typing",
"cli": "passed",
"web": "passed"
},
"quality": {
"fullFileScans": 5,
"codeFiles": 3,
"newDiagnostics": 0
},
"sourceSha256": {
"src/main/browser/browser-manager-viewport.ts": "a839fd89cc5e687782323036ca3a8dd9de79bd838e77dd863a5e1ae101424b4c",
"src/main/browser/browser-manager-viewport-ownership.test.ts": "ec17f2acd6b766166d13cafc2ecd33f1d4f0de7a4b4e8896d0746b2d528341da"
},
"limitations": [
"Pending CDP response schedules are injected, not an affected-host capture",
"Retired map values are booleans; native objects and process RSS were not measured",
"Historical viewport source is exact; surrounding dependencies execute current audit versions"
]
}
@@ -0,0 +1,36 @@
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import base from '../../../config/vitest.config.ts'
if (process.env.ORCA_BACKGROUND_LAUNCH !== '1') {
throw new Error('Set ORCA_BACKGROUND_LAUNCH=1 for the viewport ownership replay')
}
const target = fileURLToPath(
new URL('../../../src/main/browser/browser-manager-viewport.ts', import.meta.url)
).replaceAll('\\', '/')
export default {
...base,
test: {
...base.test,
include: ['src/main/browser/browser-manager-viewport-ownership.test.ts']
},
plugins:
process.env.ORCA_VIEWPORT_BASELINE === '1'
? [
{
name: 'viewport-owner-baseline',
enforce: 'pre',
transform(_source, id) {
return id.replaceAll('\\', '/').split('?')[0] === target
? {
code: readFileSync(new URL('./baseline-source.txt', import.meta.url), 'utf8'),
map: null
}
: null
}
}
]
: []
}
@@ -0,0 +1,64 @@
# Retained Claude background-task text
The actual Claude task tracker retained oversized input strings through its
512-character description/name slices. Its live tasks, settled tasks, and
recently removed tasks can each retain those slices. The fix uses the existing
`ownRetainedString` at the shared text boundary; normalization, UTF-16 clipping,
task identity, publication, and lifecycle behavior stay the same.
This extends [ML-018 / #20960](https://github.com/stablyai/orca/pull/20960).
## Reproduce
```sh
ORCA_BACKGROUND_LAUNCH=1 node --expose-gc docs/audits/claude-task-retention/reproduce.cjs
```
The script bundles the actual tracker and its retention classes. Its baseline
removes only the new copy call in memory. It exercises flat strings, concatenated
strings, and JSON-parsed SDK-style frames; each input has a distinct task owner.
It measures after GC, then clears the tracker and yields before measuring cleanup.
[Results and bundle hashes](./results.json) preserve the complete run.
| JSON-parsed case | Input per task | Tasks | Visible text | Heap before | Heap after |
| ------------------------- | ---------------: | ----: | ----------------: | ----------: | ---------: |
| Live | 64 Ki characters | 32 | 16,384 characters | 2,125,672 | 43,536 |
| Settled | 64 Ki characters | 32 | 16,384 characters | 2,127,072 | 44,296 |
| Removed, awaiting outcome | 64 Ki characters | 32 | 0 characters | 2,108,136 | 25,360 |
| Live | 4 Mi characters | 8 | 4,096 characters | 33,562,624 | 11,048 |
| Settled | 4 Mi characters | 8 | 4,096 characters | 33,563,960 | 11,656 |
| Removed, awaiting outcome | 4 Mi characters | 8 | 0 characters | 33,558,584 | 7,008 |
Captured with Node v26.6.0 on macOS. Cleanup returned near the initial heap for
every case. Six regression tests retain the actual tracker through these three
lifetimes for both descriptions and names. Text behavior tests preserve whitespace
normalization, fallback names, and a clipped surrogate pair.
## Reachability and limits
`claude-stream-json-connection.ts` forwards SDK messages to the structured adapter,
whose `emit` calls `backgroundTasks.observe`. Installed SDK 0.3.251 uses Node
`readline` to assemble stdout records, parses each record with `JSON.parse`, then
yields it. The inspected path imposes no record or description length limit;
native read-chunk size does not cap an assembled JSON field. Descriptions are
declared as plain strings in `SDKTaskStartedMessage`.
The description slice and this SDK version also exist in `v1.4.198`; that tag
keeps the reader inline in `claude-background-task-tracker.ts`. The separate
settled/recently-removed retention and name-reader paths describe current code.
The current maps are count-bounded: at most 256 live, 256 settled, and 256 recently
removed entries per tracker. Settled context clears when no visible work remains;
recently removed context awaits an outcome, eviction, or explicit clearing.
Session end/close clears the tracker. Copy work is at most 512 UTF-16 code units
per retained field, and it does not reduce temporary parsing allocation.
These are synthetic oversized task fields, not evidence that an affected user
received such fields. The path concerns structured Claude sessions, not ordinary
terminal output or stderr. Neither #19831 nor #19768 establishes this trigger.
The separate digest-bounded subagent ID was also checked at actual consumers.
The mobile response sanitizer can temporarily retain the original until JSON
serialization flattens its concatenated ID. Worker transcript bounding already
serializes for its byte budget and released that parent in the probe. No durable
ID-owner leak was established, so that helper is unchanged.

Some files were not shown because too many files have changed in this diff Show More