starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-30 00:03:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into brennanb2025/claude-subagent-worklog

Browse Source 
Union of the Claude subagent lane with the merged Codex-lane carrier:
- worker-transcript-payload: main's TranscriptBoundState/64-cap arm plus the
  branch's digest-based roster-id bound (boundEntryId); removed the duplicate
  subagent-group arm and MAX_WORKER_TRANSCRIPT_SUBAGENTS the auto-merge left
- native-chat RPC: converged on main's native-chat-rpc-block-sanitize module,
  grafting boundSubagentEntryId for the roster key; dropped the branch's
  parallel native-chat-rpc-block-bounds fork and ported its tests
- worker-output: main's shared worker-transcript-text rendering subsumes the
  branch's [subagents] summary line
- claude-subagent-group-row: twin sentence now comes from the shared
  subagentGroupFallbackText (no frozen live count), matching main's design
- journal translation: both sides' additions (roster wiring + activity feed),
  with claudeOutputEnvelope/appendUnmodeledClaudeContent extracted to their
  domain modules to stay under the line cap
This commit is contained in:
Merge Sim
2026-09-07 11:20:11 -07:00
parent a1713afc34 83b1558ecc
commit 8f71bdf3dc
2358 changed files with 146752 additions and 25268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+14 -1
View File
@@ -4,11 +4,24 @@
/config/scripts/**/*.mjs text eol=lf
/skill-guides/*.md text eol=lf
/skill-stubs/*.md text eol=lf
/skill-stubs/_shared/*.md text eol=lf
/skills/*/SKILL.md text eol=lf
/src/cli/bundled-skill-guides.ts text eol=lf
# Bundled plugin trees are byte-hashed; CRLF checkout would break the pinned hash.
/resources/plugins/** text eol=lf
# pnpm hashes every patch byte-for-byte, so a CRLF checkout breaks the install.
# Relay assets are copied verbatim into the bundle and hashed byte-for-byte into
# .version, which names the immutable remote install dir. A CRLF checkout makes a
# Windows-built client disagree with a mac/Linux-built one on the same release,
# so one host ends up with two relay trees (#17886 review).
/config/relay-assets/** text eol=lf
# Pin the bytes so a patch reads and diffs identically on every host. It is NOT
# what makes the hash right: pnpm hashes a patch LF-normalized, so a CRLF checkout
# cannot change it. Believing otherwise put a hand-computed raw digest in the
# lockfile twice and broke every install (#17886).
# These files are stored LF, which is not always the encoding they were written
# against -- @vscode/windows-process-tree ships CRLF sources -- so any code that
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
/config/patches/*.patch -text
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
# source patch under xterm-src/ instead. The sibling patches stay diffable.
.github/actions/install-node-dependencies/action.yml
+16 -8
View File
@@ -77,14 +77,6 @@ runs:
;;
esac
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
if: runner.os == 'Linux' && inputs.native-runtime != 'none'
shell: bash
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Prepare dependency install
shell: bash
run: |
@@ -175,6 +167,22 @@ runs:
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
if: runner.os == 'Linux' && inputs.native-runtime != 'none'
shell: bash
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }}
run: |
# A cache hit can contain unusable addons; probe before skipping the rebuild toolchain.
if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] &&
node config/scripts/ensure-native-runtime.mjs --check-only; then
exit 0
fi
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Prepare native runtime
if: inputs.native-runtime != 'none'
shell: bash
.github/actions/setup-wsl-test-runtime/action.yml
+8
View File
@@ -0,0 +1,8 @@
name: Set up WSL test runtime
description: Install a checksum-pinned Ubuntu WSL1 guest with executable Node and Git for real terminal tests.
runs:
using: composite
steps:
- name: Provision Ubuntu WSL1
shell: pwsh
run: '& "${{ github.action_path }}/setup.ps1"'
.github/actions/setup-wsl-test-runtime/setup.ps1
+32
View File
@@ -0,0 +1,32 @@
$ErrorActionPreference = 'Stop'
if (-not $IsWindows) { throw 'WSL test provisioning requires a Windows runner' }
$rootfs = Join-Path $env:RUNNER_TEMP 'noble-rootfs.tar.gz'
Invoke-WebRequest 'https://releases.ubuntu.com/24.04.4/ubuntu-24.04.4-wsl-amd64.wsl' -OutFile $rootfs
if ((Get-FileHash $rootfs -Algorithm SHA256).Hash.ToLowerInvariant() -ne '9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5') { throw 'Ubuntu rootfs checksum mismatch' }
$distroDir = Join-Path $env:RUNNER_TEMP 'orca-wsl-ubuntu'
wsl.exe --import Ubuntu $distroDir $rootfs --version 1
if ($LASTEXITCODE -ne 0) { throw "WSL import failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/true
if ($LASTEXITCODE -ne 0) { throw "WSL guest did not start: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get update
if ($LASTEXITCODE -ne 0) { throw "WSL apt update failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get install --yes git curl xz-utils
if ($LASTEXITCODE -ne 0) { throw "WSL git install failed: $LASTEXITCODE" }
$kernelMsi = Join-Path $env:RUNNER_TEMP 'wsl_update_x64.msi'
Invoke-WebRequest 'https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi' -OutFile $kernelMsi
if ((Get-FileHash $kernelMsi -Algorithm SHA256).Hash.ToLowerInvariant() -ne '4d09c776c8d45f70a202281d18e19be1118f53159b0c217a5274a31ce18525fe') { throw 'WSL kernel installer checksum mismatch' }
$installer = Start-Process msiexec.exe -ArgumentList @('/i', $kernelMsi, '/quiet', '/norestart') -Wait -PassThru
if ($installer.ExitCode -ne 0) { throw "WSL kernel installation failed: $($installer.ExitCode)" }
wsl.exe --status
if ($LASTEXITCODE -ne 0) { throw "WSL status failed: $LASTEXITCODE" }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/curl --fail --silent --show-error --location https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz --output /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0) { throw 'Node download failed' }
$nodeHash = wsl.exe --distribution Ubuntu --user root --exec /usr/bin/sha256sum /tmp/orca-node.tar.xz
if ($LASTEXITCODE -ne 0 -or -not ($nodeHash -match '^69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec')) { throw 'Node checksum mismatch' }
wsl.exe --distribution Ubuntu --user root --exec /usr/bin/tar -xJf /tmp/orca-node.tar.xz -C /usr/local --strip-components=1
if ($LASTEXITCODE -ne 0) { throw 'Node extraction failed' }
wsl.exe --distribution Ubuntu --user root --exec /usr/local/bin/node --version
if ($LASTEXITCODE -ne 0) { throw 'Node cannot execute in WSL' }
wsl.exe --list --verbose
if ($LASTEXITCODE -ne 0) { throw "WSL enumeration failed: $LASTEXITCODE" }
.github/scripts/e2e-with-window-manager.sh
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
openbox --sm-disable > /tmp/orca-e2e-window-manager.log 2>&1 &
wm_pid=$!
cleanup() {
kill "$wm_pid" 2>/dev/null || true
wait "$wm_pid" 2>/dev/null || true
}
trap cleanup EXIT
ready=false
for attempt in {1..100}; do
if xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | rg -q 'window id # 0x[1-9a-fA-F]'; then
ready=true
break
fi
if ! kill -0 "$wm_pid" 2>/dev/null; then
cat /tmp/orca-e2e-window-manager.log
exit 1
fi
sleep 0.1
done
if [ "$ready" != true ]; then
echo 'Window manager did not acquire the Xvfb root window' >&2
exit 1
fi
"$@"
.github/workflows/adhoc-mac-build.yml
+8 -2
View File
@@ -127,9 +127,12 @@ jobs:
esac
# Bare: a work-tree repo refuses to fetch over its own checked-out
# branch. tree:0 keeps the fetch to the commit graph — no trees, no
# blobs — so this stays cheap next to the build it fronts.
# blobs — so this stays cheap next to the build it fronts. reftable
# because this repo has branches that differ only in casing, and the
# files backend cannot store both on a case-insensitive runner disk —
# it fails the entire fetch, not just the one ref.
scratch="$RUNNER_TEMP/vet-requested-ref"
git init -q --bare "$scratch"
git init -q --bare --ref-format=reftable "$scratch"
git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
# Branch first to keep actions/checkout's old tie-break: bare
# rev-parse would prefer the tag when a branch shares its name.
@@ -157,6 +160,9 @@ jobs:
- name: Checkout the requested ref
uses: actions/checkout@v6
env:
# Full-history checkout must also preserve case-twin branch and tag names.
GIT_DEFAULT_REF_FORMAT: reftable
with:
# Why an input at all rather than just github.ref: the whole point is to
# build code that has not landed, and the workflow definition itself
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+7 -5
View File
@@ -433,13 +433,13 @@ jobs:
# result's generation is authoritative either way.
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --mode isolate)"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --mode drain
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -501,6 +501,7 @@ jobs:
--rollback-image "${DESIRED_IMAGE}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
| jq -e '.changes == 2' >/dev/null
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -526,7 +527,8 @@ jobs:
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -613,7 +615,7 @@ jobs:
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --mode activate)"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
echo "${ACTIVATE_RESULT}"
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
<<< "${ACTIVATE_RESULT}")"
@@ -652,7 +654,7 @@ jobs:
test "${MUTATION_STARTED:-false}" = true || exit 0
ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --mode isolate)"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# The isolate result carries the authoritative post-isolate generation;
# fixed offsets are wrong whenever an earlier isolate was a no-op.
.github/workflows/cloud-operate-relay-production-rehome-job.yml
+4
View File
@@ -14,6 +14,7 @@ on:
not-before: { required: true, type: string }
rate-per-minute: { required: true, type: string }
preference-max-age-ms: { required: true, type: string }
host-cooldown-ms: { required: true, type: string }
drain-grace-ms: { required: true, type: string }
confirmation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
@@ -54,6 +55,7 @@ jobs:
NOT_BEFORE: ${{ inputs.not-before }}
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
CONFIRMATION: ${{ inputs.confirmation }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -128,6 +130,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -299,6 +302,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
.github/workflows/cloud-operate-relay-production-rehome.yml
+6
View File
@@ -52,6 +52,11 @@ on:
required: true
default: '86400000'
type: string
host-cooldown-ms:
description: Minimum gap between two rehomes of the same host
required: true
default: '604800000'
type: string
drain-grace-ms:
description: Per-host source drain grace
required: true
@@ -99,6 +104,7 @@ jobs:
not-before: ${{ inputs.not-before }}
rate-per-minute: ${{ inputs.rate-per-minute }}
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
drain-grace-ms: ${{ inputs.drain-grace-ms }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
.github/workflows/cloud-push-deploy.yml
+364
View File
@@ -0,0 +1,364 @@
name: Deploy Push Gateway Production
on:
workflow_dispatch:
inputs:
source_sha:
description: Full reviewed commit SHA to build (feature may remain unmerged)
required: true
type: string
confirmation:
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
required: true
type: string
permissions:
contents: read
id-token: write
# The gateway applies its own schema at startup against the shared Cloud SQL instance, so a
# deploy is a connection-budget rollout and belongs in the same serialized group as the relay.
concurrency:
group: production-cloud-sql-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: >-
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
SERVICE_NAME: orca-cloud-push
REPOSITORY_ID: orca-cloud
IMAGE_NAME: push
PUSH_ORIGIN: https://push.onorca.dev
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
# Scaling the serving revision must already hold, matching push_min_instances and
# push_max_instances. Terraform owns both, and the candidate inherits them from the
# service, so this deploy never passes a scaling flag: doing so would write a
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
# `push_max_instances` raise would then be reverted by every deploy. These two values
# are the expected shape, asserted before the candidate is created and again on the
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
PUSH_MIN_INSTANCES: 1
PUSH_MAX_INSTANCES: 2
CONFIRMATION: ${{ inputs.confirmation }}
SOURCE_SHA: ${{ inputs.source_sha }}
steps:
- uses: actions/checkout@v4
- name: Require the explicit deploy confirmation
shell: bash
run: |
set -euo pipefail
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
- name: Fetch the immutable gateway source
shell: bash
run: |
set -euo pipefail
git fetch --no-tags origin "${SOURCE_SHA}"
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
mkdir -p "${RUNNER_TEMP}/push-source"
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
| tar -x -C "${RUNNER_TEMP}/push-source"
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
- uses: docker/setup-buildx-action@v3
- name: Configure Docker auth
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
# Why: the build runs before the lease. Artifact Registry is not the Cloud SQL instance,
# and a multi-minute image build inside the lease blocks every relay deploy and rehome for
# its duration. The lease below covers exactly the connection-budget window: deploy, probe,
# shift.
- name: Build and publish the immutable gateway image
shell: bash
run: |
set -euo pipefail
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
docker build -f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
docker push "${image_tag}"
digest="$(gcloud artifacts docker images describe "${image_tag}" \
--format='value(image_summary.digest)')"
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
>> "${GITHUB_ENV}"
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
# applies its schema while the new revision starts, so the revision is the schema step.
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/cloud-sql-rollout/production.lock
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
# drifted below the floor would hand the candidate a cold start on every notification, and
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
# rollout lease was taken for. Refuse to inherit either rather than latch it.
- name: Record the serving revision and require its Terraform-owned scaling
shell: bash
run: |
set -euo pipefail
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${serving}"
floor="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
exit 1
fi
ceiling="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
# No traffic and a per-revision tag: the candidate boots, applies schema, and is probed on
# its own URL while every phone and desktop still reaches the previous revision.
- name: Deploy the candidate revision with no traffic
shell: bash
run: |
set -euo pipefail
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "CANDIDATE_TAG=${tag}" >> "${GITHUB_ENV}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--image "${IMAGE}" \
--tag "${tag}" \
--revision-suffix "${tag}" \
--no-traffic \
--quiet
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er --arg tag "${tag}" \
'[.status.traffic[] | select(.tag == $tag)]
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
# candidate and the serving revision each draw up to the ceiling during the probe window.
# The lease is taken for exactly that doubling; a candidate that inherited a wider ceiling
# would exceed it, so the inherited scaling is asserted here too.
- name: Require the candidate to serve the exact image and inherited scaling
shell: bash
run: |
set -euo pipefail
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${served}" = "${IMAGE}"
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
- name: Probe the candidate readiness endpoint
shell: bash
run: |
set -euo pipefail
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
for attempt in $(seq 1 30); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
if test "${code}" = 200; then
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2' > /dev/null
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: /ready returned ${code}"
sleep 5
done
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
exit 1
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
# runtime account's FCM grant end to end without delivering anything: validate_only stops
# Google before any push, and the deliberately invalid token means a healthy credential
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
#
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
# nothing about the credential, so it is retried rather than treated as either answer; a
# denied credential still fails on the first attempt, without burning the retries.
- name: Prove the runtime identity can reach FCM
shell: bash
run: |
set -euo pipefail
token="$(gcloud auth print-access-token \
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
test -n "${token}"
echo "::add-mask::${token}"
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
for attempt in $(seq 1 5); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
--data "${body}" || true)"
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
test "${code}" = 401 || test "${code}" = 403; then
break
fi
sleep 5
done
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
echo "the push runtime identity cannot send through FCM" >&2
exit 1
fi
test "${status}" = INVALID_ARGUMENT
- name: Shift all traffic to the verified candidate
shell: bash
run: |
set -euo pipefail
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${CANDIDATE_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${CANDIDATE_REVISION}"
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
# Why: the summary is written before the origin check, not after it. Once traffic has
# moved, the rollback target is the single thing an operator needs, and a summary that only
# appeared on success would be missing in exactly the run that needs it.
- name: Publish the rollout summary
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
shell: bash
run: |
set -euo pipefail
{
echo '### Push gateway rollout'
echo
echo "Source: ${SOURCE_SHA}"
echo
echo "Revision: \`${CANDIDATE_REVISION}\`"
echo
echo "Image: \`${IMAGE_DIGEST}\`"
echo
echo "Rollback: \`gcloud run services update-traffic ${SERVICE_NAME}" \
"--region ${GCP_REGION} --to-revisions ${ROLLBACK_REVISION}=100\`"
} >> "${GITHUB_STEP_SUMMARY}"
- name: Verify the public origin after the shift
shell: bash
run: |
set -euo pipefail
for attempt in $(seq 1 30); do
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
"${PUSH_ORIGIN}/ready" || true)"
if test "${code}" = 200; then
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2' > /dev/null
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
sleep 5
done
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
exit 1
# Why: everything after the shift runs with production on the candidate. A failure there
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
# is undone here rather than left to whoever reads the run.
- name: Roll traffic back to the previous revision
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' }}
shell: bash
run: |
set -euo pipefail
test -n "${ROLLBACK_REVISION:-}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${ROLLBACK_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${ROLLBACK_REVISION}"
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
{
echo
echo '### Push gateway rolled back'
echo
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
"\`${CANDIDATE_REVISION}\` no longer serves."
} >> "${GITHUB_STEP_SUMMARY}"
# Why: a candidate that never took traffic is a revision holding a warm floor and a Cloud
# SQL pool for nothing. Its tag comes off first, because Cloud Run refuses to delete a
# revision a traffic target still names, and clearing CANDIDATE_TAG makes the always() tag
# step below a no-op rather than a second failure.
- name: Delete the rejected candidate revision
if: ${{ (failure() || cancelled()) && (env.TRAFFIC_SHIFT_ATTEMPTED != 'true' || env.TRAFFIC_ROLLED_BACK == 'true') }}
shell: bash
run: |
set -euo pipefail
test -n "${CANDIDATE_REVISION:-}" || exit 0
if test -n "${CANDIDATE_TAG:-}"; then
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
fi
gcloud run revisions delete "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--quiet
echo "deleted the candidate revision ${CANDIDATE_REVISION}"
- name: Drop the candidate traffic tag
if: always()
shell: bash
run: |
set -euo pipefail
test -n "${CANDIDATE_TAG:-}" || exit 0
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
.github/workflows/cloud-verify.yml
+5 -4
View File
@@ -25,9 +25,10 @@ defaults:
working-directory: cloud
jobs:
# Public-repository hosted runners preserve Blacksmith allowance for macOS.
security:
name: Secret scan
runs-on: blacksmith-2vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
@@ -53,7 +54,7 @@ jobs:
# Compiles the workspace. No Postgres service: nothing here reaches a
# database, and the service container costs ~13s of startup.
build:
runs-on: blacksmith-4vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
@@ -73,7 +74,7 @@ jobs:
# package it needs through the relay pretest hook, so it does not depend on
# `pnpm build` having run.
test:
runs-on: blacksmith-4vcpu-ubuntu-2204
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:16-alpine
@@ -107,7 +108,7 @@ jobs:
# Fork pull requests reach this job, so it never configures a backend, never plans, and never
# holds a credential. Only the relay root ships here; foundation and apps stay private.
terraform:
runs-on: blacksmith-2vcpu-ubuntu-2204
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
.github/workflows/dev-channel-win-build.yml
+5 -2
View File
@@ -149,9 +149,12 @@ jobs:
fi
# Reachability is the trust test: GitHub serves PR-only commits by SHA,
# so resolving the object is not proof a branch or tag of this repo
# reaches it. Bare + tree:0 keeps this to the commit graph.
# reaches it. Bare + tree:0 keeps this to the commit graph; reftable
# because branches that differ only in casing cannot both be stored by
# the files backend on a case-insensitive runner disk, which fails the
# entire fetch rather than the one ref.
scratch="$RUNNER_TEMP/vet-requested-ref"
git init -q --bare "$scratch"
git init -q --bare --ref-format=reftable "$scratch"
git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
if ! git -C "$scratch" rev-parse --verify --quiet "$REQUESTED_SHA^{commit}" >/dev/null; then
echo "::error::Commit $REQUESTED_SHA is not in stablyai/orca."
.github/workflows/e2e.yml
+105 -9
View File
@@ -27,6 +27,10 @@ on:
description: Ref to check out (defaults to the workflow ref)
required: false
type: string
test_files:
description: JSON array of specs to run; empty runs the full suite
required: false
type: string
schedule:
# Why: GitHub cron uses UTC; these slots map to 10am and 3pm
# America/Phoenix for the default-branch E2E run.
@@ -146,7 +150,7 @@ jobs:
# Native cache misses need the compiler, Electron needs Xvfb, and paired
# Quick Open needs ripgrep. Install them in one apt transaction per shard.
- name: Install native build and headless UI tools
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
@@ -167,7 +171,7 @@ jobs:
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
# launches but never creates a BrowserWindow.
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
@@ -201,7 +205,7 @@ jobs:
# unbounded inventory fallback; the paired fixture exercises that real boundary.
# Why openssh-client: the Docker-SSH fixture shells out to ssh/ssh-keygen, and this
# lane now receives those specs from pr.yml's SSH source mapping.
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
@@ -223,6 +227,11 @@ jobs:
mapfile -t TEST_FILES < <(jq -r '.[] | select(
. != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
. != "tests/e2e/local-ssh-browser-routing.spec.ts" and
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
. != "tests/e2e/ssh-localhost.spec.ts" and
. != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
)' <<<"$TEST_FILES_JSON")
if [ "${#TEST_FILES[@]}" -eq 0 ]; then
@@ -241,7 +250,7 @@ jobs:
if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then
E2E_PROJECT_ARGS+=(--project=electron-headful)
fi
xvfb-run --auto-servernum env "${E2E_ENV[@]}" \
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \
pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"
- name: Upload Playwright traces
@@ -258,12 +267,15 @@ jobs:
needs: [build, prepare-native-cache]
# effect of one route listing a startup-readiness spec — pruning that spec would have
# silently retired the whole lane. The signal is now derived from the SSH routes directly.
# The two spec clauses stay for their honest purpose: changed-e2e hands these specs to this
# The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this
# lane, so editing one must still run it here.
if: >-
inputs.test_files == '' ||
inputs.ssh_source_changed == 'true' ||
contains(inputs.test_files, 'tests/e2e/local-ssh-browser-routing.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts')
runs-on: ubuntu-latest
# Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average
@@ -278,7 +290,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install native build and headless UI tools
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 xvfb zsh
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
@@ -293,7 +305,7 @@ jobs:
# Why: this is the release-path proof that the deployed Linux relay keeps
# its PTY and explorer live across a real watcher SIGSEGV.
- name: Run Docker SSH watcher isolation E2E
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
# Why: Playwright empties test-results/ when it starts, so each step here used to
# destroy the previous step's traces. Only the last lane's failure was ever
@@ -310,7 +322,7 @@ jobs:
# readiness across live SSH, headed paired, and headless serve topologies.
- name: Run Docker SSH terminal parking + startup readiness E2E
if: always()
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
- name: Keep terminal-parking traces
if: always()
@@ -326,7 +338,7 @@ jobs:
# legible as an SSH-named failure.
- name: Run remaining Docker SSH E2E
if: always()
run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker
- name: Keep remaining-ssh-docker traces
if: always()
@@ -344,3 +356,87 @@ jobs:
path: e2e-traces/
retention-days: 7
if-no-files-found: ignore
ssh-browser-network-route:
name: ssh browser network route
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts')
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
- name: Install SSH client
run: sudo apt-get update && sudo apt-get install -y openssh-client
- name: Run Docker SSH browser network route journeys
env:
ORCA_BACKGROUND_LAUNCH: '1'
ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
ssh-localhost:
name: localhost SSH terminal and hooks
needs: [build, prepare-native-cache]
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts')
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install SSH server and headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- uses: actions/download-artifact@v8
with:
name: e2e-build-out
path: out/
- name: Start isolated localhost SSH server
shell: bash
run: |
# Bare shells install Pi extensions only for an existing agent home.
mkdir -p "$HOME/.pi/agent"
fixture="$RUNNER_TEMP/orca-localhost-sshd"
mkdir -p "$fixture"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key"
cat > "$fixture/sshd_config" <<EOF
Port 22222
ListenAddress 127.0.0.1
HostKey $fixture/host_key
PidFile $fixture/sshd.pid
AuthorizedKeysFile $fixture/client_key.pub
StrictModes no
PasswordAuthentication no
KbdInteractiveAuthentication no
UsePAM yes
AllowUsers $(id -un)
Subsystem sftp internal-sftp
EOF
sudo mkdir -p /run/sshd
sudo /usr/sbin/sshd -f "$fixture/sshd_config" -E "$fixture/sshd.log"
ssh -i "$fixture/client_key" -p 22222 -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 127.0.0.1 true || { sudo cat "$fixture/sshd.log"; exit 1; }
{
echo "ORCA_E2E_SSH_PORT=22222"
echo "ORCA_E2E_SSH_USER=$(id -un)"
echo "ORCA_E2E_SSH_IDENTITY_FILE=$fixture/client_key"
} >> "$GITHUB_ENV"
- name: Run localhost SSH terminal and hook journey
env:
SKIP_BUILD: '1'
ORCA_E2E_SSH_LOCALHOST: '1'
ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
with:
name: localhost-ssh-traces
path: test-results/
retention-days: 7
if-no-files-found: ignore
.github/workflows/golden-e2e-experiment.yml
+5
View File
@@ -98,12 +98,17 @@ jobs:
$env:SKIP_BUILD = '1'
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
pnpm run --if-present test:e2e:workspace-session-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
pnpm run --if-present test:e2e:windows-fresh-startup-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) {
pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
}
pnpm run --if-present test:e2e:source-control-golden
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
- name: Upload Playwright traces
if: failure()
.github/workflows/hourly-mac-build.yml
+49 -41
View File
@@ -26,7 +26,7 @@ name: Hourly macOS Dev Build
# HOURLY_RELEASE_APP_ID the App's numeric id
# HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key
#
# Installation tokens live one hour, which is why this mints twice. Install and
# Installation tokens live one hour, so the build job mints twice. Install and
# build need no token at all, and notarization can hold the publish step for tens
# of minutes; minting again once the build is done starts the clock at the first
# call that actually uses it rather than burning a third of it on `pnpm install`.
@@ -60,33 +60,15 @@ env:
HOURLY_RETAIN_COUNT: 72
jobs:
build-hourly-mac:
# Avoid occupying the limited Mac pool when main has not moved.
preflight:
if: github.repository == 'stablyai/orca'
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
tag: ${{ steps.release.outputs.tag }}
version: ${{ steps.hourly.outputs.version }}
should_build: ${{ steps.freshness.outputs.should_build }}
head_sha: ${{ steps.freshness.outputs.head_sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
runs-on: blacksmith-6vcpu-macos-15
# Why 150: it must exceed the worst case the retry budgets below can produce
# (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
# the job is killed mid-retry and no cleanup step runs at all. A typical run
# is far shorter — this is the notary queue's tail, not its median.
timeout-minutes: 150
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
# build step is compromised (zizmor: artipacked).
persist-credentials: false
- name: Mint hourly repo token
id: app_token
uses: actions/create-github-app-token@v2
@@ -95,18 +77,19 @@ jobs:
private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
owner: stablyai
repositories: orca-hourly
permission-contents: read
# Why: main is often idle overnight. Rebuilding an unchanged commit burns a
# runner hour and adds a redundant tag to the retention window.
- name: Check whether main moved since the last hourly
id: freshness
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
MAIN_REPO_TOKEN: ${{ github.token }}
FORCED: ${{ github.event_name == 'workflow_dispatch' && inputs.force }}
run: |
set -euo pipefail
head_sha="$(git rev-parse HEAD)"
head_sha="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)"
[[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; }
echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT"
if [[ "$FORCED" == "true" ]]; then
echo "should_build=true" >>"$GITHUB_OUTPUT"
@@ -133,21 +116,55 @@ jobs:
echo "main moved to $head_sha (last hourly built $last_sha); building."
fi
build-hourly-mac:
needs: preflight
if: needs.preflight.outputs.should_build == 'true'
outputs:
tag: ${{ steps.release.outputs.tag }}
version: ${{ steps.hourly.outputs.version }}
head_sha: ${{ needs.preflight.outputs.head_sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
runs-on: blacksmith-6vcpu-macos-15
# Why 150: it must exceed the worst case the retry budgets below can produce
# (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
# the job is killed mid-retry and no cleanup step runs at all. A typical run
# is far shorter — this is the notary queue's tail, not its median.
timeout-minutes: 150
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ needs.preflight.outputs.head_sha }}
fetch-depth: 0
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
# build step is compromised (zizmor: artipacked).
persist-credentials: false
- name: Mint hourly repo token
id: app_token
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
owner: stablyai
repositories: orca-hourly
- name: Setup pnpm
if: steps.freshness.outputs.should_build == 'true'
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
if: steps.freshness.outputs.should_build == 'true'
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
uses: actions/cache@v5
with:
path: |
@@ -158,7 +175,6 @@ jobs:
electron-builder-mac-
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
@@ -169,7 +185,6 @@ jobs:
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
if: steps.freshness.outputs.should_build == 'true'
run: node config/scripts/verify-macos-release-env.mjs
env:
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -180,7 +195,6 @@ jobs:
- name: Compute hourly version
id: hourly
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -211,7 +225,7 @@ jobs:
node config/scripts/hourly-build-version.mjs \
>"$RUNNER_TEMP/hourly-identity.txt"
grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt"
# Why check rather than trust: the checkout above pins `ref: main`, but a
# Why check rather than trust: the checkout above pins the resolved main commit, but a
# workflow_dispatch runs this file from whatever branch was dispatched. A
# branch that edits this step while main still has the old script yields
# an empty name and an untitled release — silent, and only visible once
@@ -223,7 +237,6 @@ jobs:
cat "$RUNNER_TEMP/hourly-identity.txt" >>"$GITHUB_OUTPUT"
- name: Build app
if: steps.freshness.outputs.should_build == 'true'
run: pnpm build:release
env:
NODE_OPTIONS: --max-old-space-size=4096
@@ -239,7 +252,6 @@ jobs:
# part the full budget.
- name: Re-mint hourly repo token for publish
id: app_token_publish
if: steps.freshness.outputs.should_build == 'true'
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
@@ -249,13 +261,12 @@ jobs:
- name: Create hourly release
id: release
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
TAG: v${{ steps.hourly.outputs.version }}
NAME: ${{ steps.hourly.outputs.name }}
SHA: ${{ steps.freshness.outputs.head_sha }}
SHA: ${{ needs.preflight.outputs.head_sha }}
run: |
set -euo pipefail
# Kept at 12 even though the title shows 7: the freshness check above
@@ -291,7 +302,6 @@ jobs:
echo "tag=$TAG" >>"$GITHUB_OUTPUT"
- name: Publish hourly macOS artifacts
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
# Why 45 like the release pipeline: an attempt is pack + notarize +
@@ -322,7 +332,6 @@ jobs:
# release missing that manifest is a tag the picker offers and the download
# 404s on, so fail loudly instead of leaving a broken entry.
- name: Verify update manifest published
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
@@ -352,7 +361,6 @@ jobs:
# means the picker can never offer a release whose assets are incomplete.
- name: Publish the verified release
id: publish_live
if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
.github/workflows/mobile-android-release.yml
+37 -1
View File
@@ -104,11 +104,47 @@ jobs:
--clobber \
android/app/build/outputs/apk/release/*.apk
else
# Why: release tags live on side branches, so GitHub's automatic
# previous-tag detection reaches back several releases; that body
# already exceeds the 125000-character API limit and grows each
# release. Pin the comparison base and cap the size.
notes_file="$RUNNER_TEMP/android-release-notes.md"
previous_tag="$(
gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
| grep '^mobile-android-v' | grep -Fxv "$tag" | sort -V | tail -1 || true
)"
if [ -n "$previous_tag" ]; then
# Why: gh writes the JSON error body to stdout on an HTTP error, so a
# non-empty file is not proof of success — gate on exit status.
if ! gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" -X POST \
-f tag_name="$tag" \
-f target_commitish="$GITHUB_SHA" \
-f previous_tag_name="$previous_tag" \
--jq .body > "$notes_file"; then
: > "$notes_file"
fi
fi
if [ ! -s "$notes_file" ]; then
printf 'Orca Mobile Android %s\n' "$tag" > "$notes_file"
fi
# Why: reuse the desktop release path's character-safe truncation so a
# multi-byte character cannot be split at the cap.
NOTES_FILE="$notes_file" \
NOTES_MODULE="$GITHUB_WORKSPACE/config/scripts/create-draft-release.mjs" \
node --input-type=module -e '
const { readFileSync, writeFileSync } = await import("node:fs")
const { pathToFileURL } = await import("node:url")
const { truncateReleaseBody } = await import(pathToFileURL(process.env.NOTES_MODULE).href)
const file = process.env.NOTES_FILE
writeFileSync(file, truncateReleaseBody(readFileSync(file, "utf8")))
'
gh release create "$tag" \
--repo "$GITHUB_REPOSITORY" \
--title "Orca Mobile Android $tag" \
--prerelease \
--latest=false \
--generate-notes \
--notes-file "$notes_file" \
android/app/build/outputs/apk/release/*.apk
fi
.github/workflows/mobile.yml
+6 -21
View File
@@ -15,8 +15,13 @@ on:
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
- '.github/actions/install-node-dependencies/**'
- '.github/workflows/mobile-ios-release.yml'
concurrency:
group: mobile-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
@@ -35,10 +40,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v6
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- uses: ./.github/actions/install-node-dependencies
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
@@ -50,23 +52,6 @@ jobs:
bundler-cache: true
working-directory: mobile
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
# Why: the mobile typecheck imports shared types from ../src/shared, and
# some of those files import runtime deps (tweetnacl, ws) resolved from
# the repo-root node_modules. Without a root install, tsc fails with
# "Cannot find module 'tweetnacl'/'ws'". Mobile is a separate pnpm project
# (not in the root workspace), so this is a distinct install.
# --ignore-scripts skips the root postinstall (Electron native-module
# rebuild) which is irrelevant to a type-only check and would only add
# time and failure surface on this ubuntu mobile runner.
- name: Install root dependencies
working-directory: .
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install dependencies
run: pnpm install --frozen-lockfile
.github/workflows/packaged-browser-e2e.yml
+74
View File
@@ -0,0 +1,74 @@
name: Packaged browser compatibility
on:
workflow_dispatch:
inputs:
ref:
description: Commit SHA or ref to validate (defaults to the selected revision)
type: string
required: false
schedule:
- cron: '20 8 * * 1'
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
jobs:
compatibility:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Download pinned old release
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca"
python3 - <<'PYVERIFY'
import base64,hashlib,os,pathlib,subprocess
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
package=root/'orca-ide_1.4.188_amd64.deb'
expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g=='
assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected
extracted=root/'extracted'
subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
executable=extracted/'opt'/'Orca'/'orca-ide'
assert executable.is_file() and os.access(executable,os.X_OK)
with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(executable)+'\n')
print('Verified old package:',executable)
PYVERIFY
- name: Build current Electron app
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:relay
pnpm exec electron-vite build --mode e2e
pnpm run build:web-from-renderer
- name: Run both mixed-version directions
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
run: >-
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
pnpm exec playwright test --config tests/playwright.config.ts
tests/e2e/packaged-mixed-version-browser-placement.spec.ts
--project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
- name: Require all six compatibility executions
if: always()
run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
- uses: actions/upload-artifact@v7
if: always()
with:
name: packaged-mixed-version-audit
path: test-results/
retention-days: 3
.github/workflows/performance-contracts.yml
+63
View File
@@ -0,0 +1,63 @@
name: Performance contracts
on:
schedule:
- cron: '15 9 * * *'
workflow_dispatch:
pull_request:
paths:
- '.github/workflows/performance-contracts.yml'
- 'config/vitest.performance.config.ts'
- 'config/oxlint-performance-audit.json'
- 'config/oxlint-plugins/*performance.mjs'
- 'config/oxlint-plugins/quadratic-buffer-concat.mjs'
- 'config/scripts/*-plugin.test.mjs'
# Keep in sync with the contract list in config/vitest.performance.config.ts;
# without these a rename lands green and only breaks the next nightly.
- 'src/main/sqlite/sync-database.test.ts'
- 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
- 'src/relay/fs-path-metadata-symlink-concurrency.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
- 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
- 'src/renderer/src/components/terminal-pane/agent-completion-coordinator-queued-inspection-disposal.test.ts'
- 'src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler-queue-retention.test.ts'
permissions:
contents: read
concurrency:
group: performance-contracts-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
contracts:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Run operation-count and retention contracts
run: pnpm test:perf:contracts --reporter=default --reporter=json --outputFile=performance-contracts.json
# Source-only scan: identical on every OS, so run it once.
- name: Audit production performance patterns
if: always() && matrix.os == 'ubuntu-latest'
shell: bash
run: pnpm --silent audit:perf > performance-audit.json
- uses: actions/upload-artifact@v7
if: always()
with:
name: performance-contracts-${{ matrix.os }}
path: performance-contracts.json
if-no-files-found: error
- uses: actions/upload-artifact@v7
if: always() && matrix.os == 'ubuntu-latest'
with:
name: performance-audit
path: performance-audit.json
if-no-files-found: error
.github/workflows/pr.yml
+74 -62
View File
@@ -41,6 +41,11 @@ jobs:
managed_hook_node18: ${{ steps.filter.outputs.managed_hook_node18 }}
package: ${{ steps.filter.outputs.package }}
package_windows: ${{ steps.filter.outputs.package_windows }}
e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }}
test_files: ${{ steps.e2e_filter.outputs.test_files }}
ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }}
native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
wsl_source_changed: ${{ steps.e2e_filter.outputs.wsl_source_changed }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -66,6 +71,41 @@ jobs:
printf '%s\n' "$CHANGED"
printf '%s\n' "$CHANGED" | node config/scripts/pr-code-change-scope.mjs | tee -a "$GITHUB_OUTPUT"
# Reuse the path-detector checkout instead of queuing another runner.
- name: Filter changed E2E specs
id: e2e_filter
if: github.event.pull_request.draft != true && steps.filter.outputs.should_run == 'true'
run: |
set -euo pipefail
BASE="${{ github.event.pull_request.base.sha }}"
HEAD="${{ github.event.pull_request.head.sha }}"
CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
# Source routes are executable contracts so a test can prove exact
# authorities, exclusions, and sentinels without evaluating workflow shell.
TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
# Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
# spec name surviving in a route's list. Same routes, so the two cannot drift.
SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "SSH source changed: $SSH_SOURCE_CHANGED"
# Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
# trigger on IME source rather than on a spec name in some route's list.
NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR --merge-base "$BASE" "$HEAD")"
WSL_SOURCE_CHANGED="$(printf '%s\n' "$WSL_CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source)"
echo "wsl_source_changed=$WSL_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)"
if [ "$SHOULD_RUN" = true ]; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Changed E2E specs: $TEST_FILES_JSON"
else
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "No specs requiring the reusable E2E workflow"
fi
static_analysis:
name: static analysis
needs: [code_paths]
@@ -712,7 +752,11 @@ jobs:
- name: Package unpacked app
env:
ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1'
run: pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish never
# PR artifacts are only inspected locally; gzip avoids release-size xz compression.
run: >-
pnpm exec electron-builder --config config/electron-builder.config.cjs
--linux AppImage deb rpm --x64 --publish never
--config.deb.compression=gz --config.rpm.compression=gzip
- name: Verify root-package marker payloads
run: |
@@ -792,10 +836,13 @@ jobs:
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
# vitest runs here directly rather than through `pnpm test`, so the addon
# assertions only hold once install-node-dependencies has rebuilt natives.
- name: Test Windows-specific boundaries
run: >-
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts
src/main/browser/browser-route-tcp-egress.electron.test.ts
src/main/browser/browser-route-webrtc-egress.electron.test.ts
@@ -804,9 +851,15 @@ jobs:
src/main/providers/windows-conpty-wide-char-duplication.node-pty.test.ts
src/main/providers/pty-repaint-wide-char-buffer.node-pty.test.ts
src/shared/child-process/windows-command-line.win32.test.ts
src/shared/child-process/windows-cmd-shim-resolution.test.ts
src/shared/child-process/windows-cmd-shim-resolution.win32.test.ts
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-msys-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
src/main/windows/windows-process-tree-command-line-patch.test.ts
src/main/windows/windows-process-table-native-addon.win32.test.ts
src/main/windows-live-tree-kill.win32.test.ts
src/main/wsl/wsl-runner.test.ts
src/main/wsl/wsl-guest-environment.test.ts
@@ -815,14 +868,18 @@ jobs:
src/main/wsl/wsl-w1-w3-contract.test.ts
src/shared/source-scan/source-tree-scan.test.ts
src/main/cli/wsl-cli-powershell-boundary.test.ts
src/main/computer/desktop-script-runtime-host.win32.test.ts
src/main/cursor/hook-service.test.ts
src/main/orca-profiles/profile-index-store.test.ts
src/main/startup/windows-install-dir-acl-repair.win32.test.ts
src/main/runtime/repo-worktree-admin-fingerprint.test.ts
src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts
src/shared/secure-file-fsync-flags.test.ts
src/shared/secure-path-windows-acl.win32.test.ts
src/main/runtime/unreadable-secret-store-preservation.win32.test.ts
src/main/ipc/pty-codex-account-attribution.test.ts
src/main/ipc/pty-spawn-env-codex-resume-provenance.test.ts
src/relay/windows-port-scan.win32.test.ts
# Why the :parallel variant: identical to build:release except the three
# electron-vite targets overlap instead of running back to back. The Linux package
@@ -861,65 +918,10 @@ jobs:
- name: Smoke packaged CLI
run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/win-unpacked
# Why: PR E2E is advisory and only validates changed specs; scheduled and
# release runs retain full-suite coverage.
e2e-paths:
name: detect changed e2e specs
needs: [code_paths]
runs-on: ubuntu-latest
if: github.event.pull_request.draft != true && needs.code_paths.outputs.should_run == 'true'
# Why: detector only needs to read the checkout; do not inherit repo defaults.
permissions:
contents: read
outputs:
should_run: ${{ steps.filter.outputs.should_run }}
test_files: ${{ steps.filter.outputs.test_files }}
ssh_source_changed: ${{ steps.filter.outputs.ssh_source_changed }}
native_ime_source_changed: ${{ steps.filter.outputs.native_ime_source_changed }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
# Why blob:none: full history is needed for the merge-base diff, but historical
# file contents are not. Blobs are ~89% of this repo's pack, and Git fetches the
# few this job actually reads on demand.
fetch-depth: 0
filter: blob:none
persist-credentials: false
- name: Filter changed E2E specs
id: filter
run: |
set -euo pipefail
BASE="${{ github.event.pull_request.base.sha }}"
HEAD="${{ github.event.pull_request.head.sha }}"
CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
# Source routes are executable contracts so a test can prove exact
# authorities, exclusions, and sentinels without evaluating workflow shell.
TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
# Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
# spec name surviving in a route's list. Same routes, so the two cannot drift.
SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "SSH source changed: $SSH_SOURCE_CHANGED"
# Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
# trigger on IME source rather than on a spec name in some route's list.
NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
if [ "$TEST_FILES_JSON" != '[]' ]; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Changed E2E specs: $TEST_FILES_JSON"
else
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "No changed E2E specs"
fi
e2e:
name: e2e
needs: e2e-paths
if: needs.e2e-paths.outputs.should_run == 'true'
needs: code_paths
if: needs.code_paths.outputs.e2e_should_run == 'true'
# Why: reusable e2e.yml only checkouts, builds, and uploads artifacts.
permissions:
contents: read
@@ -928,8 +930,8 @@ jobs:
# The synthetic pull-request merge ref can disappear while this reusable
# workflow is queued. The head SHA is immutable and works for every PR.
ref: ${{ github.event.pull_request.head.sha }}
test_files: ${{ needs.e2e-paths.outputs.test_files }}
ssh_source_changed: ${{ needs.e2e-paths.outputs.ssh_source_changed }}
test_files: ${{ needs.code_paths.outputs.test_files }}
ssh_source_changed: ${{ needs.code_paths.outputs.ssh_source_changed }}
# Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability
# is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It
@@ -939,13 +941,23 @@ jobs:
# require `success || skipped` outside the strict loop — see the note on `e2e`.
terminal_ime_native:
name: real IME
needs: e2e-paths
if: needs.e2e-paths.outputs.native_ime_source_changed == 'true'
needs: code_paths
if: needs.code_paths.outputs.native_ime_source_changed == 'true'
# Why: the reusable workflow only checks out, builds, and uploads artifacts.
permissions:
contents: read
uses: ./.github/workflows/terminal-ime-e2e.yml
windows_wsl:
name: real WSL terminal
needs: code_paths
if: needs.code_paths.outputs.wsl_source_changed == 'true'
permissions:
contents: read
uses: ./.github/workflows/windows-wsl-e2e.yml
with:
ref: ${{ github.event.pull_request.head.sha }}
verify:
if: always()
needs:
.github/workflows/release-cut.yml
+146 -30
View File
@@ -809,13 +809,7 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ needs.cut.outputs.tag }}
run: |
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "Release $TAG already exists."
exit 0
fi
node config/scripts/create-draft-release.mjs "$TAG"
run: node config/scripts/create-draft-release.mjs "$TAG"
terminal-rendering-golden:
needs: cut
@@ -858,16 +852,17 @@ jobs:
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: Linux terminal golden E2E uses the same native install path as
# release CI, which needs pnpm to bypass its non-executable gyp_main.py.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1074,16 +1069,17 @@ jobs:
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: keep the non-blocking evidence lane on the same Linux native
# install path as the blocking golden and release build jobs.
- name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1425,6 +1421,17 @@ jobs:
command: ${{ matrix.release_command }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Why: the NSIS uninstaller only exists inside electron-builder's
# uninstaller pass, which deletes it right after embedding it. The sign
# hook in config/scripts/windows-uninstaller-signing.cjs copies it out
# here so it can ride the inner-binaries SignPath request below.
# Why runner.temp and never the workspace: `files` in
# config/electron-builder.config.cjs is all-negation, so app-builder
# prepends `**/*` and packs whatever is left in the checkout root. This
# step retries up to 3 times; attempt 1 writes the file after packing,
# but attempts 2 and 3 would then pack the unsigned uninstaller into
# app.asar - the exact defect this chain exists to remove.
ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe
- name: Verify Windows node-pty ConPTY runtime
if: matrix.platform == 'win' && github.run_attempt == 1
@@ -1451,7 +1458,10 @@ jobs:
# Why: SignPath cannot deep-sign inside NSIS installers, so inner PE
# files (Orca.exe, node-pty *.node, DLLs) are signed via a separate zip
# request, then the installer is rebuilt from the signed tree before the
# existing installer signing request below. Every step in this chain is
# existing installer signing request below. The NSIS uninstaller rides
# this same request (it is the MDE update cluster: old-uninstaller.exe /
# Uninstall Orca.exe), captured through electron-builder's sign hook and
# swapped back in during the rebuild — no third approval wait. Every step is
# fail-open (continue-on-error + outcome gating): any failure ships the
# original installer with unsigned inner binaries, exactly like releases
# did before this chain existed. Rehearsed end to end in run 28988432001
@@ -1498,6 +1508,36 @@ jobs:
Write-Host "Skipped $($skipped.Count) already-signed files:"
$skipped | ForEach-Object { Write-Host " $_" }
# Why the uninstaller rides this request: it is the file MDE flagged in
# the whole update cluster (old-uninstaller.exe / Uninstall Orca.exe),
# and folding it in here costs no extra approval wait. Why it is kept
# out of inner-signing-list.txt: that list drives the copy-back into
# dist/win-unpacked, and the uninstaller does not live there — it is
# re-injected through the sign hook during the rebuild instead.
# Why this name and not "Uninstall Orca.exe": the restore loop below
# matches staged files by suffix (`-like "*$relative"`) and takes the
# first hit, so any staged path ending in "Orca.exe" is separated from
# the real Orca.exe only by Get-ChildItem's enumeration order. That
# order happens to favour the root file today, but it is not a
# documented guarantee; a name that cannot suffix-match is.
# Why the whole block is caught rather than just Test-Path'd: this
# step's outcome gates the upload of every inner binary, so a locked
# file or a full disk here would cost all of them their signatures -
# worse than shipping no uninstaller signature at all.
try {
$exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe'
if (Test-Path -LiteralPath $exportedUninstaller) {
$uninstallerStagePath = Join-Path $stage.FullName 'uninstaller\orca-uninstaller.exe'
New-Item -ItemType Directory -Force -Path (Split-Path $uninstallerStagePath) -ErrorAction Stop | Out-Null
Copy-Item -LiteralPath $exportedUninstaller -Destination $uninstallerStagePath -Force -ErrorAction Stop
Write-Host 'Staged the NSIS uninstaller for signing: uninstaller\orca-uninstaller.exe'
} else {
Write-Host "::warning::No exported NSIS uninstaller at $exportedUninstaller; this release ships an unsigned uninstaller (fail-open)."
}
} catch {
Write-Host "::warning::Could not stage the NSIS uninstaller ($_); this release ships an unsigned uninstaller (fail-open)."
}
- name: Upload unsigned inner binaries for SignPath
id: upload-unsigned-inner
if: matrix.platform == 'win' && github.run_attempt == 1 && steps.stage-inner.outcome == 'success'
@@ -1642,6 +1682,31 @@ jobs:
throw "Signed inner artifact did not round-trip cleanly ($($failures.Count) failures)."
}
# Why gated separately from the inner restore above: if SignPath's
# windows-inner-binaries-zip artifact configuration does not (yet) cover the
# uninstaller/ directory, the uninstaller comes back missing. That must cost
# only the uninstaller signature — the rebuild below still runs and still
# ships the signed inner binaries, exactly as it does today.
- name: Restore signed uninstaller for the installer rebuild
id: restore-signed-uninstaller
if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
continue-on-error: true
shell: pwsh
run: |
$signed = Get-ChildItem -Path signed-inner -Recurse -File -Filter 'orca-uninstaller.exe' |
Select-Object -First 1
if ($null -eq $signed) {
throw 'SignPath did not return uninstaller/orca-uninstaller.exe; check the windows-inner-binaries-zip artifact configuration covers it.'
}
$signature = Get-AuthenticodeSignature -FilePath $signed.FullName
if ($null -eq $signature.SignerCertificate) {
throw 'The returned NSIS uninstaller carries no signature.'
}
$signedDir = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed'
New-Item -ItemType Directory -Force -Path $signedDir | Out-Null
Copy-Item -LiteralPath $signed.FullName -Destination (Join-Path $signedDir 'orca-uninstaller.exe') -Force
Write-Host ("{0,-14} uninstaller <{1}>" -f $signature.Status, $signature.SignerCertificate.Subject)
# Why this step exists: electron-builder's CopyElevateHelper re-copies a
# pristine elevate.exe from its download cache over resources\elevate.exe
# on EVERY nsis pack — including the --prepackaged rebuild below — which
@@ -1651,9 +1716,12 @@ jobs:
# no-op. Known quirk: the cache persists across releases via actions/cache,
# so later runs may see elevate.exe as already signed and skip staging it —
# that is fine (the signature is timestamped) and the evidence gate checks
# elevate.exe in the shipped installer unconditionally. If this ever causes
# trouble, delete this step; the only effect is elevate.exe shipping
# unsigned again, which the evidence gate will flag.
# elevate.exe in the shipped installer unconditionally.
#
# The cache lookup lives in a script because the inline path this step used
# (`<cache>\nsis`) matches no app-builder-lib layout, and `SilentlyContinue`
# plus `exit 0` turned that miss into a green step — v1.4.193 and v1.4.194
# shipped an unsigned elevate.exe that way. A miss now fails the step.
- name: Replace cached elevate.exe with the signed copy
id: sign-elevate-cache
if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
@@ -1665,20 +1733,26 @@ jobs:
Write-Host '::warning::No elevate.exe in win-unpacked resources; nothing to protect from the rebuild clobber.'
exit 0
}
# Why this guard stays: windows-signing-rehearsal.yml shares the
# electron-builder-win-<lockfile hash> cache key with this workflow, so a
# test-certificate elevate.exe must never be staged into a release cache.
$signature = Get-AuthenticodeSignature -FilePath $signed
$subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
if ($signature.Status -ne 'Valid' -or $subject -notlike '*CN=SignPath Foundation*') {
Write-Host "::warning::win-unpacked elevate.exe is not SignPath-signed ($($signature.Status), $subject); skipping cache swap."
exit 0
}
$cached = @(Get-ChildItem "$env:LOCALAPPDATA\electron-builder\Cache\nsis" -Recurse -Filter elevate.exe -ErrorAction SilentlyContinue)
if ($cached.Count -eq 0) {
Write-Host '::warning::No cached elevate.exe found (electron-builder cache layout changed?); the rebuild will pack the unsigned copy and the evidence gate will flag it.'
exit 0
}
foreach ($file in $cached) {
Copy-Item -Path $signed -Destination $file.FullName -Force
Write-Host "Replaced $($file.FullName) with the SignPath-signed copy."
node config/scripts/replace-cached-nsis-elevate.mjs $signed
if ($LASTEXITCODE -ne 0) {
$message = 'Cached elevate.exe swap found nothing to replace; the rebuilt installer ships an unsigned UAC elevation helper (issue #7785).'
if ($env:GITHUB_STEP_SUMMARY) {
try {
Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value "**Windows elevate.exe cache swap:** FAILED — $message" -ErrorAction Stop
} catch {
Write-Host "::warning::Could not write the elevate.exe swap verdict to the job summary: $_"
}
}
throw $message
}
- name: Rebuild NSIS installer from signed unpacked app
@@ -1686,6 +1760,11 @@ jobs:
if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
continue-on-error: true
shell: pwsh
env:
# Why unconditional: the sign hook keys off the file existing, which it
# only does when the restore step above succeeded. A missing file logs a
# warning and embeds the freshly built unsigned uninstaller instead.
ORCA_WIN_UNINSTALLER_SIGNED_PATH: ${{ runner.temp }}\uninstaller-signing\signed\orca-uninstaller.exe
run: |
# Why: keep the pre-rebuild artifacts so a failed rebuild can fall
# back to shipping them unchanged (fail-open).
@@ -1716,6 +1795,7 @@ jobs:
with:
name: orca-windows-unsigned-${{ needs.cut.outputs.tag }}
path: dist/orca-windows-setup.exe
compression-level: 0
if-no-files-found: error
# Why: SignPath Foundation production certificates require manual review,
@@ -1876,6 +1956,7 @@ jobs:
env:
ORCA_WINDOWS_INNER_SIGNATURE_REQUIRED: 'false'
INNER_SIGNING_COMPLETED: ${{ steps.rebuild-nsis-signed.outcome == 'success' }}
UNINSTALLER_SIGNING_COMPLETED: ${{ steps.restore-signed-uninstaller.outcome == 'success' }}
run: |
$required = $env:ORCA_WINDOWS_INNER_SIGNATURE_REQUIRED -eq 'true'
@@ -1956,6 +2037,39 @@ jobs:
if ($targets -notcontains 'resources\elevate.exe') {
$targets += 'resources\elevate.exe'
}
# Why the uninstaller is not in $targets: NSIS embeds it in its own
# compressed data section (`File /oname=${UNINSTALL_FILENAME}` in
# app-builder-lib templates/nsis/include/installer.nsh), not in the
# app 7z payload extracted above - the bundled 7za cannot see it.
# What the receipt proves and does not: the digest comparison is
# equal by construction (the hook digests the bytes it copied from
# this same file), so the real signal is that the receipt exists at
# all - the import leg ran, and these are the bytes it embedded. The
# signature check below is the part with teeth. The shipped-artifact
# check lives in windows-signing-rehearsal.yml, which installs the
# installer and inspects the uninstaller it drops on disk.
if ($env:UNINSTALLER_SIGNING_COMPLETED -eq 'true') {
$signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe'
$receipt = "$signedUninstaller.embedded-sha256"
if (-not (Test-Path -LiteralPath $receipt)) {
$failures.Add('the sign hook did not embed the signed uninstaller into the rebuilt installer')
} else {
$embedded = (Get-Content -LiteralPath $receipt -Raw).Trim()
$actual = (Get-FileHash -LiteralPath $signedUninstaller -Algorithm SHA256).Hash.ToLowerInvariant()
$signature = Get-AuthenticodeSignature -FilePath $signedUninstaller
$subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
$line = "{0,-14} {1} <{2}>" -f $signature.Status, 'Uninstall Orca.exe (embedded)', $subject
$report.Add($line)
Write-Host $line
if ($embedded -ne $actual) {
$failures.Add("the rebuilt installer embedded different uninstaller bytes than the signed one ($embedded vs $actual)")
} elseif ($signature.Status -ne 'Valid' -or $subject -notlike '*CN=SignPath Foundation*') {
$failures.Add("not signed by SignPath Foundation: Uninstall Orca.exe ($($signature.Status), $subject)")
}
}
} else {
Write-Host '::warning::The NSIS uninstaller was not signed on this run; it is excluded from the evidence gate (fail-open).'
}
foreach ($relative in $targets) {
$path = Join-Path $root $relative
if (-not (Test-Path $path)) {
@@ -1988,7 +2102,9 @@ jobs:
Add-GateEvidence "VERDICT: FAILED — $message"
Add-GateSummary "FAILED — $message"
} else {
$ok = "All $($targets.Count) inner binaries in the shipped installer are signed by SignPath Foundation."
# $report, not $targets: the embedded uninstaller is reported but
# is not one of the extracted payload targets.
$ok = "All $($report.Count) checked binaries are signed by SignPath Foundation."
Add-GateEvidence "VERDICT: PASSED — $ok"
Add-GateSummary "PASSED — $ok"
Write-Host $ok
.github/workflows/release-ref-validation.yml
+38
View File
@@ -0,0 +1,38 @@
name: Release ref validation
on:
pull_request:
paths:
- '.github/workflows/adhoc-mac-build.yml'
- '.github/workflows/dev-channel-win-build.yml'
- '.github/workflows/release-ref-validation.yml'
- 'config/scripts/workflow-ref-reachability.test.mjs'
- 'config/scripts/workflow-ref-mirror-case-safety.test.mjs'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: release-ref-validation-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
validate:
strategy:
fail-fast: false
matrix:
os: [macos-15, windows-2022]
runs-on: ${{ matrix.os }}
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Verify case-twin refs and release trust boundary
run: >-
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/workflow-ref-reachability.test.mjs
config/scripts/workflow-ref-mirror-case-safety.test.mjs
config/scripts/dev-channel-windows-workflow-contract.test.mjs
.github/workflows/skill-update-roundtrip.yml
+6
View File
@@ -22,6 +22,10 @@ on:
- main
paths: *skill-roundtrip-paths
concurrency:
group: skill-roundtrip-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
roundtrip:
strategy:
@@ -41,7 +45,9 @@ jobs:
steps:
- uses: actions/checkout@v6
with:
# Historical skill snapshots need tags, but only their blobs are read.
fetch-depth: 0
filter: blob:none
persist-credentials: false
- uses: actions/setup-node@v6
with:
.github/workflows/terminal-ime-e2e.yml
+39 -16
View File
@@ -38,23 +38,9 @@ jobs:
xfwm4
xvfb
- name: Setup Node.js
uses: actions/setup-node@v6
- uses: ./.github/actions/install-node-dependencies
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Use external node-gyp to avoid pnpm bundled copy
run: |
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
- name: Install dependencies
run: pnpm install --frozen-lockfile
native-runtime: electron
- name: Build Electron app for E2E
run: pnpm exec electron-vite build --mode e2e
@@ -84,3 +70,40 @@ jobs:
path: test-results/
retention-days: 7
if-no-files-found: ignore
linux-wayland:
name: Linux Wayland Hangul terminating digit
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install native build, nested compositor and IME tools
run: >-
sudo apt-get update && sudo apt-get install -y
build-essential python3 fonts-noto-cjk dbus-x11 dconf-gsettings-backend
ibus ibus-hangul gnome-shell gnome-settings-daemon libglib2.0-bin
xdotool xvfb x11-utils imagemagick
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Build Electron app for E2E
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:relay
pnpm exec electron-vite build --mode e2e
pnpm run build:web-from-renderer
- name: Run native Wayland Hangul terminating digit
env:
SKIP_BUILD: '1'
run: node config/scripts/run-terminal-ibus-hangul-e2e.mjs --nested-wayland
- name: Upload Wayland terminal IME evidence
if: always()
uses: actions/upload-artifact@v7
with:
name: terminal-wayland-ime-evidence
path: test-results/
retention-days: 7
if-no-files-found: error
.github/workflows/terminal-perf.yml
+6 -5
View File
@@ -67,16 +67,17 @@ jobs:
- name: Install native build tools and xvfb
run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb zsh
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
- name: Setup pnpm
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
# Why: this scheduled/manual workflow uses the same native install path as
# PR and E2E CI, which needs pnpm to bypass its bundled gyp_main.py.
- name: Use external node-gyp to avoid pnpm's bundled copy
.github/workflows/windows-signing-rehearsal.yml
+204 -12
View File
@@ -3,9 +3,11 @@
# Why: SignPath cannot deep-sign inside NSIS installers, so shipping signed
# inner binaries (Orca.exe, node-pty *.node, DLLs — see issue #7785) requires
# a two-request flow: sign the unpacked PE files first, then build the NSIS
# installer from the signed tree, then sign the installer. This workflow
# rehearses that entire flow from a branch, end to end, without publishing
# anything — so the release pipeline on main is never at risk while we verify.
# installer from the signed tree, then sign the installer. The NSIS uninstaller
# rides that same first request — it is captured through electron-builder's sign
# hook and swapped back in during the rebuild — so it adds no third approval.
# This workflow rehearses that entire flow from a branch, end to end, without
# publishing anything — so the release pipeline on main is never at risk.
#
# Runs only via manual dispatch. Use the test-signing policy for iteration
# (auto-approved test certificate) and release-signing to rehearse the
@@ -81,15 +83,27 @@ jobs:
env:
NODE_OPTIONS: --max-old-space-size=4096
- name: Package unpacked Windows app
# Why a full --win build and not --dir: the NSIS uninstaller only exists
# inside the installer build, and it is the file the MDE update cluster
# flags. --dir would never produce it, so the rehearsal would not rehearse
# the uninstaller leg at all. This mirrors release-cut's first Windows pass.
- name: Package Windows app and export the NSIS uninstaller
shell: pwsh
env:
# runner.temp, never the workspace: the all-negation `files` list in
# config/electron-builder.config.cjs packs whatever is left in the
# checkout root into app.asar.
ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe
run: |
node config/scripts/ensure-native-runtime.mjs --runtime=electron
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
pnpm exec electron-builder --config config/electron-builder.config.cjs --win --dir --publish never
pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
if (-not (Test-Path 'dist/win-unpacked/Orca.exe')) {
throw 'electron-builder --dir did not produce dist/win-unpacked/Orca.exe'
throw 'electron-builder --win did not produce dist/win-unpacked/Orca.exe'
}
if (-not (Test-Path -LiteralPath $env:ORCA_WIN_UNINSTALLER_EXPORT_PATH)) {
throw "The sign hook did not export the NSIS uninstaller to $env:ORCA_WIN_UNINSTALLER_EXPORT_PATH"
}
# Why: only unsigned PE files go to SignPath. Files that already carry a
@@ -132,6 +146,17 @@ jobs:
Write-Host "Skipped $($skipped.Count) already-signed files:"
$skipped | ForEach-Object { Write-Host " $_" }
# Why kept out of inner-signing-list.txt: that list drives the copy-back
# into dist/win-unpacked, and the uninstaller does not live there — it is
# re-injected through the electron-builder sign hook during the rebuild.
# No catch here, unlike the release job: the rehearsal exists to prove
# the flow, so a staging failure must fail it loudly.
$exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe'
$uninstallerStagePath = Join-Path $stage.FullName 'uninstaller\orca-uninstaller.exe'
New-Item -ItemType Directory -Force -Path (Split-Path $uninstallerStagePath) | Out-Null
Copy-Item -LiteralPath $exportedUninstaller -Destination $uninstallerStagePath -Force
Write-Host 'Staged the NSIS uninstaller for signing: uninstaller\orca-uninstaller.exe'
- name: Upload unsigned inner binaries for SignPath
id: upload-unsigned-inner
uses: actions/upload-artifact@v7
@@ -200,8 +225,27 @@ jobs:
throw "Signed inner artifact did not round-trip cleanly ($($failures.Count) failures)."
}
- name: Restore signed uninstaller for the installer rebuild
shell: pwsh
run: |
$signed = Get-ChildItem -Path signed-inner -Recurse -File -Filter 'orca-uninstaller.exe' |
Select-Object -First 1
if ($null -eq $signed) {
throw 'SignPath did not return uninstaller/orca-uninstaller.exe; check the inner-binaries artifact configuration covers it.'
}
$signature = Get-AuthenticodeSignature -FilePath $signed.FullName
if ($null -eq $signature.SignerCertificate) {
throw 'The returned NSIS uninstaller carries no signature.'
}
$signedDir = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed'
New-Item -ItemType Directory -Force -Path $signedDir | Out-Null
Copy-Item -LiteralPath $signed.FullName -Destination (Join-Path $signedDir 'orca-uninstaller.exe') -Force
Write-Host ("{0,-14} uninstaller <{1}>" -f $signature.Status, $signature.SignerCertificate.Subject)
- name: Build NSIS installer from signed unpacked app
shell: pwsh
env:
ORCA_WIN_UNINSTALLER_SIGNED_PATH: ${{ runner.temp }}\uninstaller-signing\signed\orca-uninstaller.exe
run: |
pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never --prepackaged "$env:GITHUB_WORKSPACE\dist\win-unpacked"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
@@ -215,6 +259,7 @@ jobs:
with:
name: orca-windows-installer-unsigned-${{ github.run_id }}
path: dist/orca-windows-setup.exe
compression-level: 0
if-no-files-found: error
- name: Submit Windows installer signing request
@@ -288,20 +333,33 @@ jobs:
run: |
$report = New-Object System.Collections.Generic.List[string]
$failures = New-Object System.Collections.Generic.List[string]
$advisories = New-Object System.Collections.Generic.List[string]
$requireValid = $env:SIGNING_POLICY -eq 'release-signing'
function Test-Signature([string]$label, [string]$path) {
# -Advisory records a problem without failing the run. It exists for
# exactly one file (resources\elevate.exe, below) and must not be
# widened casually: the point of this workflow is to fail when signing
# is broken.
function Test-Signature([string]$label, [string]$path, [switch]$Advisory) {
$signature = Get-AuthenticodeSignature -FilePath $path
$subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
$line = "{0,-14} {1} <{2}>" -f $signature.Status, $label, $subject
$script:report.Add($line)
Write-Host $line
$problem = $null
if ($null -eq $signature.SignerCertificate -or $signature.Status -eq 'NotSigned') {
$script:failures.Add("unsigned: $label")
$problem = "unsigned: $label"
} elseif ($script:requireValid -and $signature.Status -ne 'Valid') {
$script:failures.Add("not Valid under release-signing: $label ($($signature.Status))")
$problem = "not Valid under release-signing: $label ($($signature.Status))"
} elseif ($script:requireValid -and $subject -notlike '*CN=SignPath Foundation*') {
$script:failures.Add("unexpected signer: $label ($subject)")
$problem = "unexpected signer: $label ($subject)"
}
if ($null -eq $problem) { return }
if ($Advisory) {
$script:advisories.Add($problem)
Write-Host "::warning::$problem - known pre-existing issue, not failing the rehearsal"
} else {
$script:failures.Add($problem)
}
}
@@ -323,21 +381,155 @@ jobs:
& $7za x 'dist/orca-windows-setup.exe' '-oextracted-app' -y | Out-Null
$root = Resolve-Path 'extracted-app'
# The receipt only proves the import leg ran; it cannot prove what NSIS
# embedded, because the uninstaller lives in a compressed NSIS data
# section rather than the app 7z payload above and the bundled 7za has
# no NSIS handler. So the rehearsal - unlike the release job, which
# must not mutate the runner it publishes from - goes all the way: it
# installs the installer silently and inspects the uninstaller the
# installer actually wrote to disk. That is the file MDE flags.
$signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe'
$receipt = "$signedUninstaller.embedded-sha256"
if (-not (Test-Path -LiteralPath $receipt)) {
$failures.Add('the sign hook did not embed the signed uninstaller into the rebuilt installer')
} else {
Test-Signature 'relayed: orca-uninstaller.exe' $signedUninstaller
}
# Why a full 7-Zip attempt first: it is non-invasive. The runner image
# ships the complete 7z.exe, which - unlike the reduced 7za - has an
# NSIS handler. If it cannot read the section either, fall back to a
# real silent install.
$installedUninstaller = $null
$installedVia = $null
$expectedDigest = if (Test-Path -LiteralPath $receipt) { (Get-Content -LiteralPath $receipt -Raw).Trim() } else { $null }
$full7z = 'C:\Program Files\7-Zip\7z.exe'
if (Test-Path -LiteralPath $full7z) {
New-Item -ItemType Directory -Path nsis-extract -Force | Out-Null
& $full7z x -tnsis 'dist/orca-windows-setup.exe' '-onsis-extract' -y 2>&1 | Out-Null
$installedUninstaller = Get-ChildItem -Path nsis-extract -Recurse -File -Filter 'Uninstall*.exe' -ErrorAction SilentlyContinue |
Select-Object -First 1
# Why the digest guard before trusting this route: 7-Zip's NSIS
# handler emits partial or garbled output on some NSIS builds, and a
# truncated extract would score NotSigned and fail the rehearsal as
# "the shipped uninstaller is unsigned" when nothing is wrong. Only
# trust it when it reproduces the bytes the relay embedded; otherwise
# fall through to the install route, which is ground truth. A name
# miss (the handler labelling the entry by its source name) falls
# through the same way.
if ($null -ne $installedUninstaller -and $null -ne $expectedDigest -and
(Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant() -ne $expectedDigest) {
Write-Host "7-Zip's NSIS output did not match the relayed digest; falling back to a silent install."
$installedUninstaller = $null
}
if ($null -ne $installedUninstaller) {
$installedVia = "7-Zip's NSIS handler"
Write-Host "Read the embedded uninstaller with 7-Zip's NSIS handler: $($installedUninstaller.FullName)"
} else {
Write-Host "7-Zip's NSIS handler did not yield a usable uninstaller; falling back to a silent install."
}
}
if ($null -eq $installedUninstaller) {
# Nothing here is published, so mutating this runner is free.
# Why -PassThru and a bounded wait rather than -Wait: a bare -Wait on
# an installer that ever prompts hangs to the job's 360-minute cap.
$installerProcess = Start-Process -FilePath (Resolve-Path 'dist/orca-windows-setup.exe') -ArgumentList '/S' -PassThru
if (-not $installerProcess.WaitForExit(300000)) {
$installerProcess | Stop-Process -Force -ErrorAction SilentlyContinue
$failures.Add('the silent install did not exit within 5 minutes; it is likely prompting')
}
# Why a poll rather than one Stop-Process: the oneClick installer
# launches the app as it finishes, so Orca.exe can appear *after* the
# installer process exits. A single silenced Stop-Process would miss
# it and leave Orca plus orca-terminal-daemon.exe holding handles
# under %LOCALAPPDATA%\Programs for the rest of the job.
for ($attempt = 0; $attempt -lt 20; $attempt++) {
$running = @(Get-Process -Name 'Orca' -ErrorAction SilentlyContinue)
if ($running.Count -gt 0) {
$running | Stop-Process -Force -ErrorAction SilentlyContinue
break
}
Start-Sleep -Milliseconds 500
}
Get-Process -Name 'orca-terminal-daemon' -ErrorAction SilentlyContinue |
Stop-Process -Force -ErrorAction SilentlyContinue
$installedUninstaller = Get-ChildItem -Path "$env:LOCALAPPDATA\Programs" -Recurse -File -Filter 'Uninstall*.exe' -ErrorAction SilentlyContinue |
Where-Object { $_.FullName -like '*Orca*' } |
Select-Object -First 1
if ($null -ne $installedUninstaller) { $installedVia = 'a silent install' }
}
if ($null -eq $installedUninstaller) {
$failures.Add('could not obtain the uninstaller the installer ships; neither 7-Zip nor a silent install produced it')
} else {
# Why this digest comparison is the point of the whole rehearsal:
# unlike the release job's, it hashes a file NSIS itself wrote out
# rather than the file the hook copied, so it is the only check that
# proves the shipped installer embedded the SignPath-signed bytes. On
# the 7-Zip route the guard above already forced equality; on the
# install route this is the first time it is tested.
if ($null -ne $expectedDigest) {
$shippedDigest = (Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
if ($shippedDigest -ne $expectedDigest) {
$failures.Add("the uninstaller the installer ships is not the relayed one (via $installedVia): $shippedDigest vs $expectedDigest")
}
}
Test-Signature "shipped: Uninstall Orca.exe (via $installedVia)" $installedUninstaller.FullName
}
foreach ($relative in Get-Content 'inner-signing-list.txt') {
$path = Join-Path $root $relative
if (-not (Test-Path $path)) {
$failures.Add("missing from installer payload: $relative")
continue
}
Test-Signature "installed: $relative" $path
# Why elevate.exe alone is advisory: app-builder-lib re-copies the
# pristine cached elevate.exe over resources\elevate.exe on EVERY nsis
# pack - AppPackageHelper.packArch calls elevateHelper.copy() before
# buildAppPackage (nsisUtil.js), and CopyElevateHelper.copy does
# `copyFile(elevatePath, outFile, false)` then `signIf(outFile)`, which
# signs nothing because this build configures no certificate. So the
# signed copy restored into win-unpacked is clobbered by the rebuild.
# This predates the uninstaller relay and is not caused by it: with no
# `sign` hook, signIf already returned false at "no signing info
# identified" (windowsSignToolManager.js), so no signtool call was
# displaced. release-cut.yml mitigates it separately by pre-seeding the
# electron-builder cache ("Replace cached elevate.exe with the signed
# copy"); this workflow has no such step, which is why the clobber is
# visible here and not there. Mirroring that step here would not help:
# it only swaps when the copy is already Valid and SignPath-signed, so
# it no-ops under the test certificate.
#
# DO NOT relax that Valid + SignPath-signed guard to make this
# rehearsal go green. This workflow and release-cut.yml share the
# cache key `electron-builder-win-<lockfile hash>`, and that guard is
# the only thing stopping a test certificate from being seeded into
# the cache a real release restores from. Shipping users a binary
# signed by "Test certificate for 'Orca agent ide [OSS]'" is worse
# than shipping it unsigned.
#
# Fixing elevate.exe belongs in its own PR - it is a UAC elevation
# helper, and it deserves more scrutiny than a footnote in an
# uninstaller change.
if ($relative -eq 'resources\elevate.exe') {
Test-Signature "installed: $relative" $path -Advisory
} else {
Test-Signature "installed: $relative" $path
}
}
if ($advisories.Count -gt 0) {
$report.Add('')
$report.Add('ADVISORY (known pre-existing, did not fail this run):')
$advisories | ForEach-Object { $report.Add(" $_") }
}
Set-Content -Path 'signing-evidence.txt' -Value ($report -join "`n")
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Host "::error::$_" }
throw "Signing rehearsal failed with $($failures.Count) problems."
}
Write-Host "All $((Get-Content 'inner-signing-list.txt').Count) inner binaries plus the installer are signed."
Write-Host "All checked binaries are signed, including the uninstaller the installer writes to disk ($($advisories.Count) advisory)."
- name: Upload rehearsal evidence and installer
if: always()
.github/workflows/windows-wsl-e2e.yml
+74
View File
@@ -0,0 +1,74 @@
name: Windows WSL terminal E2E
on:
workflow_dispatch:
inputs:
ref:
description: Commit to validate
type: string
required: false
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
concurrency:
group: windows-wsl-e2e-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
wsl-terminal:
runs-on: windows-2022
timeout-minutes: 30
env:
NODE_OPTIONS: --max-old-space-size=4096
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- uses: ./.github/actions/setup-wsl-test-runtime
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Build relay and Electron
run: |
pnpm run build:relay
if ($LASTEXITCODE -ne 0) { throw 'Relay build failed' }
pnpm exec electron-vite build --mode e2e
if ($LASTEXITCODE -ne 0) { throw 'Electron build failed' }
- name: Exercise real WSL launch and paste
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/wsl-results.json
run: >-
pnpm exec playwright test
tests/e2e/golden-tab-bar-agent-launch.spec.ts
tests/e2e/terminal-windows-shell-paste-ownership.spec.ts
--config tests/playwright.config.ts
--project=electron-headless
--grep "WSL"
--repeat-each=3
--workers=1
--reporter=list,json
- name: Require all nine WSL executions
if: always()
run: node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json
- name: Upload WSL participation report
uses: actions/upload-artifact@v7
if: always()
with:
name: windows-wsl-participation-report
path: test-results/wsl-results.json
retention-days: 3
- uses: actions/upload-artifact@v7
if: failure()
with:
name: windows-wsl-terminal-traces
path: test-results/
retention-days: 7
.gitignore
+2
View File
@@ -110,6 +110,8 @@ docs/**
!docs/reference/macos-press-and-hold.md
!docs/reference/orcad-operations.md
!docs/reference/relay-grace-time-reconfiguration.md
!docs/reference/windows-cmd-shim-resolution.md
!docs/reference/windows-daemon-host-relocation.md
!docs/reference/windows-edr-posture.md
!docs/reference/windows-process-enumeration.md
!docs/reference/wsl-runner-verification.md
.oxlintrc.json
+5
View File
@@ -2,6 +2,10 @@
"$schema": "./node_modules/oxlint/configuration_schema.json",
"plugins": ["typescript", "react", "react-hooks", "react-perf", "unicorn"],
"jsPlugins": [
{
"name": "sort-comparator-performance",
"specifier": "./config/oxlint-plugins/sort-comparator-performance.mjs"
},
{
"name": "mobile-pairing",
"specifier": "./config/oxlint-plugins/mobile-pairing-qrcode-import.mjs"
@@ -23,6 +27,7 @@
"correctness": "error"
},
"rules": {
"sort-comparator-performance/no-repeated-collator": "warn",
"app-store-performance/require-selector": "error",
"app-store-performance/no-identity-selector": "error",
"app-store-performance/no-fresh-selector-result": "error",
AGENTS.md
+8 -1
View File
@@ -4,6 +4,12 @@ All UI work — layout, color, typography, spacing, component selection, UX beha
## Electron UI Validation
Always run tests and agent-launched apps in the background with `ORCA_BACKGROUND_LAUNCH=1`.
Never steal monitor focus or reveal test windows: no `show()`, `showInactive()`, `bringToFront()`,
`app.focus()`, or OS activation. Use CDP screenshots of hidden renderers. Keep native-focus and
visible-window tests paused on the user's desktop; run them on an isolated display or CI.
Rebuild modified launch-policy code before running an app; stale build wrappers are not safe.
Use the `$electron` skill and Playwright CDP for rendered Orca UI checks. Do not use computer-use for Orca UI validation.
# Style
@@ -47,8 +53,9 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **Shortcut labels in UI**: Display `⌘` / `⇧` on Mac and `Ctrl+` / `Shift+` on other platforms.
- **File paths**: Use `path.join` or Electron/Node path utilities — never assume `/` or `\`.
- **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import.
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import. Recognised npm/pnpm `.cmd` shims are resolved to their real target so the spawn skips `cmd.exe` entirely; see [`docs/reference/windows-cmd-shim-resolution.md`](./docs/reference/windows-cmd-shim-resolution.md) before adding a shim shape or debugging one.
- **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
- **Windows daemon-host relocation**: the terminal daemon runs from a copy of the app runtime under `%LOCALAPPDATA%`, which is what survives an auto-update. Before touching that copy, its exe name, or the NSIS uninstall macro, read [`docs/reference/windows-daemon-host-relocation.md`](./docs/reference/windows-daemon-host-relocation.md).
- **Windows EDR signal**: don't add `-ExecutionPolicy Bypass`, `-EncodedCommand`, `cmd.exe /c` with escaped free text, per-operation interpreter spawning, or runtime `Add-Type` compilation without reading [`docs/reference/windows-edr-posture.md`](./docs/reference/windows-edr-posture.md) first — behavioural EDR scores each of those, and being signed does not clear them.
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
- **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.
README.md
+2 -2
View File
@@ -36,7 +36,7 @@
Monitor and steer your agents from your phone — get notified when an agent finishes and send follow-ups from anywhere.
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.47](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.47/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
</td>
<td width="50%">
@@ -230,7 +230,7 @@ yay -S stably-orca-bin
Pair with your desktop app to monitor and steer your agents from your phone.
- **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) or [join TestFlight](https://testflight.apple.com/join/YjeGMQBA)
- **Android:** [Download APK 0.0.47](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.47/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
- **Android:** [Download APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
---
cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
+25
View File
@@ -72,6 +72,7 @@ function sample(): IncidentSample {
expectedSelector: selector,
cells: [{
cellId: 'production-gce-c1',
region: 'us-central1',
runtimeKnown: true,
powered: true,
expectedAdmissionState: 'existing-only'
@@ -253,6 +254,30 @@ describe('relay incident live preflight', () => {
)).rejects.toThrow('cloud-monitoring/threshold_max')
})
// Why: a frozen wave has to name what froze it without re-reading the sample.
it('names the signal and its numbers in the failure message', async () => {
const slowCell = sample()
slowCell.sources['active-probe']!.signals[
'cell.production-gce-c1.latency_ms'
]!.value = 2_568
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: async () => slowCell }
)).rejects.toThrow(
'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
)
// A failure with no signal keeps the source/code token and drops the rest.
const stale = sample()
stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
await expect(runIncidentLivePreflight(
['--state-file', stateFile()],
{ now: () => now, collect: async () => stale }
)).rejects.toThrow(
'relay live preflight failed: active-probe/source_stale observed=60001 threshold=60000'
)
})
it('enforces the signed migration policy', async () => {
const inactiveTarget = sample()
inactiveTarget.sources['director-admin']!.signals[
cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
+13 -1
View File
@@ -9,6 +9,7 @@ import {
evaluateIncidentSample,
FRESHNESS_FAILURE_CODES,
preDrainDryRunPassed,
type IncidentFailure,
type IncidentSample
} from './incident-monitor.js'
import { createIncidentSampleCollector } from './incident-monitor-sources.js'
@@ -69,6 +70,17 @@ const PreflightStateSchema = z.object({
}
})
// Keep the source/code prefix other tooling matches on, then name the signal and
// its numbers so a frozen wave is attributable without re-reading the sample.
function describeFailure(failure: IncidentFailure): string {
const detail = [
failure.signal,
failure.observed === undefined ? null : `observed=${failure.observed}`,
failure.threshold === undefined ? null : `threshold=${failure.threshold}`
].filter((part): part is string => part !== null && part !== undefined)
return [`${failure.source}/${failure.code}`, ...detail].join(' ')
}
export async function runIncidentLivePreflight(
argv: string[],
dependencies: {
@@ -175,7 +187,7 @@ export async function runIncidentLivePreflight(
if (!freshnessOnly || attempt === attempts || budgetExhausted) {
throw new Error(
`relay live preflight failed: ${evaluation.failures
.map((failure) => `${failure.source}/${failure.code}`)
.map(describeFailure)
.join(',')}`
)
}
cloud/apps/relay-ops/src/incident-monitor-cli.test.ts
+1
View File
@@ -44,6 +44,7 @@ function sample(at: number): IncidentSample {
expectedSelector: selector,
cells: [{
cellId,
region: 'us-central1',
runtimeKnown: true,
powered: true,
expectedAdmissionState: 'general'
cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
+43
View File
@@ -317,6 +317,49 @@ describe('incident monitor sources', () => {
}, endAt)).toBeNull()
})
// Why: the per-region cell latency bar is only correct if the tfvars region
// reaches the evaluator on every cell expectation.
it('carries the configured region onto every cell expectation', async () => {
const gcloud: GcloudClient = {
accessToken: async () => 'unused',
identityToken: async () => 'unused'
}
const selector = {
generation: 1,
membership: {
existingOnly: [],
migrationOnly: [],
general: productionCells
}
}
const fetchImpl: typeof fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
if (body.cellId) {
return Response.json({
status: {
enabled: true,
connectionCapacity: { hardCap: 600 },
runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
}
})
}
return Response.json({
blocked: 0,
blockedExpiredUnregistered: 0,
registeredTargetInactive: 0
})
}
const result = await directorSignals('production', selector, gcloud, now, fetchImpl)
const regionById = new Map(result.cells.map((cell) => [cell.cellId, cell.region]))
expect(regionById.get('production-gce-c1')).toBe('us-central1')
expect(regionById.get('production-gce-c27')).toBe('asia-east2')
expect(result.cells).toHaveLength(productionCells.length)
for (const cell of RELAY_OPS_ENVIRONMENTS.production.cells) {
expect(regionById.get(cell.cellId)).toBe(cell.region)
}
})
it('aggregates admin state without returning tokens or response identities', async () => {
const identityToken = 'secret.header.signature'
const sensitiveIdentity = 'user@example.test'
cloud/apps/relay-ops/src/incident-monitor-sources.ts
+1
View File
@@ -558,6 +558,7 @@ export async function directorSignals(
selector,
cells: statuses.map(({ cell }) => ({
cellId: cell.cellId,
region: cell.region,
runtimeKnown: true,
powered: true,
expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId)
cloud/apps/relay-ops/src/incident-monitor.test.ts
+49
View File
@@ -33,6 +33,7 @@ function healthySample(at = startedAt): IncidentSample {
expectedSelector: selector,
cells: [{
cellId: 'production-gce-c1',
region: 'us-central1',
runtimeKnown: true,
powered: true,
expectedAdmissionState: 'general'
@@ -151,6 +152,52 @@ describe('incident monitor evaluator', () => {
})
})
// Why: an asia-east2 cell's /ready reaches auth and Cloud SQL in us-central1, so
// from the US runner it measures p50 0.88 s / max 2.7 s and the flat 2 000 bar
// froze three healthy gates on 2026-09-05 (c27 at 2568/2668/2685 ms).
it('holds cell endpoint latency to a per-region bar', () => {
const asiaTail = healthySample()
asiaTail.cells[0]!.region = 'asia-east2'
asiaTail.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
signal(2_685)
expect(evaluateIncidentSample(asiaTail, startedAt)).toMatchObject({
status: 'green',
failures: []
})
const asiaIncident = healthySample()
asiaIncident.cells[0]!.region = 'asia-east2'
asiaIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
signal(4_001)
expect(evaluateIncidentSample(asiaIncident, startedAt)).toMatchObject({
status: 'freeze',
failures: [
expect.objectContaining({
code: 'threshold_max',
source: 'active-probe',
signal: 'cell.production-gce-c1.latency_ms',
observed: 4_001,
threshold: 4_000
})
]
})
const usIncident = healthySample()
usIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
signal(2_001)
expect(evaluateIncidentSample(usIncident, startedAt)).toMatchObject({
status: 'freeze',
failures: [
expect.objectContaining({
code: 'threshold_max',
signal: 'cell.production-gce-c1.latency_ms',
observed: 2_001,
threshold: 2_000
})
]
})
})
it('allows missing auth readiness and legacy existing-only connections', () => {
const sample = healthySample()
const legacySelector = {
@@ -401,12 +448,14 @@ describe('incident monitor evaluator', () => {
] = signal(0)
sample.cells.push({
cellId: 'production-gce-c2',
region: 'us-central1',
runtimeKnown: true,
powered: true,
expectedAdmissionState: 'general'
})
sample.cells.push({
cellId: 'production-gce-c3',
region: 'us-central1',
runtimeKnown: true,
powered: true,
expectedAdmissionState: 'general'
cloud/apps/relay-ops/src/incident-monitor.ts
+12 -1
View File
@@ -1,3 +1,4 @@
import type { RelayOpsRegion } from './environment-config.js'
import {
exactAdmissionSelector,
type AdmissionSelector,
@@ -30,6 +31,15 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
relayLogMaxAgeMs: 180_000,
heartbeatMaxAgeMs: 45_000,
endpointLatencyMs: 2_000,
// Why: a cell's /ready fetches the auth JWKS and runs SELECT 1 against Cloud SQL,
// both in us-central1, so from the US runner asia-east2 cells measure p50 0.88 s /
// max 2.7 s against 0.08-0.5 s for us-central1. The flat 2 000 bar froze three
// healthy 15-minute gates on 2026-09-05 (c27 at 2568/2668/2685 ms); hard faults
// are still caught by the .health/.ready equal-1 checks and the 8 s fetch timeout.
cellEndpointLatencyMs: {
'us-central1': 2_000,
'asia-east2': 4_000
} as const satisfies Record<RelayOpsRegion, number>,
cloudSqlCpuUtilization: 0.8,
cloudSqlMemoryUtilization: 0.9,
// Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
@@ -126,6 +136,7 @@ export type IncidentSource = {
export type IncidentCellExpectation = {
cellId: string
region: RelayOpsRegion
runtimeKnown: boolean
powered: boolean
expectedAdmissionState: AdmissionState
@@ -405,7 +416,7 @@ function checkCell(
'active-probe',
probe,
`cell.${cell.cellId}.latency_ms`,
INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs,
INCIDENT_MONITOR_THRESHOLDS.cellEndpointLatencyMs[cell.region],
'max'
],
[
cloud/apps/relay/src/app.ts
+7 -1
View File
@@ -606,8 +606,9 @@ export function createRelayApp(
const source = await operations.assignments.cellDeploymentStatus(
body.data.sourceCellId
)
// Any cell that can be drained can be a rehome source, in either
// direction, so the probe is gated on the protocol and not on a region.
if (
source.region !== RELAY_DEFAULT_REGION ||
!source.runtime ||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
!source.runtime.ready ||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
cloud/apps/relay/src/assignment-inventory-snapshot.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export type CellInventorySnapshotRow = {
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
return {
cells: cellRows.map((row) => ({
cellId: asText(row, 'cell_id'),
region: optionalText(row, 'region') ?? 'us-central1',
region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
admissionState: optionalText(row, 'admission_state') ?? 'unset',
enabled: asInteger(row, 'enabled') === 1,
capacityRequests: asInteger(row, 'capacity_requests'),
cloud/apps/relay/src/assignment-store.ts
+107 -28
View File
@@ -30,6 +30,9 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
import type { RelayCellConfig } from './config.js'
import type {
RelayDatabase,
@@ -121,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
export type RegionalRehomeAttempt = AssignmentIdentity & {
attemptId: string
preferredRegion: 'asia-east2'
preferredRegion: RelayRegion
sourceCellId: string
sourceCellUrl: string
sourceCellIncarnation: string
@@ -151,6 +154,7 @@ export type RegionalRehomeControl = {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}
@@ -4921,6 +4925,7 @@ export class RelayAssignmentStore {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}): Promise<RegionalRehomeControl> {
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
@@ -4939,6 +4944,13 @@ export class RelayAssignmentStore {
) {
throw new Error('invalid_regional_rehome_preference_age')
}
if (
!Number.isSafeInteger(input.hostCooldownMs) ||
input.hostCooldownMs < 60_000 ||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
) {
throw new Error('invalid_regional_rehome_host_cooldown')
}
if (
!Number.isSafeInteger(input.drainGraceMs) ||
input.drainGraceMs < 60_000 ||
@@ -4967,14 +4979,15 @@ export class RelayAssignmentStore {
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = ?, not_before = ?,
rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
updated_at = ?
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
drain_grace_ms = ?, updated_at = ?
WHERE control_id = 'global'`,
[
input.enabled ? 1 : 0,
input.notBefore,
input.ratePerMinute,
input.preferenceMaxAgeMs,
input.hostCooldownMs,
input.drainGraceMs,
now
]
@@ -5006,10 +5019,17 @@ export class RelayAssignmentStore {
await database.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
ON CONFLICT (control_id) DO NOTHING`,
[now, 24 * 60 * 60_000, 60 * 60_000, now]
[
now,
24 * 60 * 60_000,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
60 * 60_000,
now
]
)
}
@@ -5017,6 +5037,9 @@ export class RelayAssignmentStore {
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
}
// The rehome fleet is every general cell that can be drained: those are the
// sources and, because a host must be movable back out again, the only legal
// targets. The region join stays so a cell with no region row is excluded.
private async readRegionalRehomeFleetSafety(
database: RelayDatabase,
now: number
@@ -5037,10 +5060,7 @@ export class RelayAssignmentStore {
ON safety.cell_id = runtime.cell_id
AND safety.cell_incarnation = runtime.cell_incarnation
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
AND (
region.region = 'asia-east2' OR
(region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
)`
AND capability.regional_rehome_protocol >= 1`
)
const valid = rows.filter(
(row) =>
@@ -5119,6 +5139,10 @@ export class RelayAssignmentStore {
}
const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
// A host that was rehomed recently is left alone whichever way its
// preference now points: a flapping region probe must not walk one host
// back and forth across an ocean.
const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
@@ -5284,9 +5308,8 @@ export class RelayAssignmentStore {
JOIN relay_cell_capabilities capability
ON capability.cell_id = runtime.cell_id
AND capability.cell_incarnation = runtime.cell_incarnation
WHERE preference.preferred_region = 'asia-east2'
WHERE preference.preferred_region <> region.region
AND preference.observed_at >= ?
AND region.region = 'us-central1'
AND admission.admission_state = 'general'
AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
AND capability.regional_rehome_protocol >= 1
@@ -5306,9 +5329,38 @@ export class RelayAssignmentStore {
AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
)
AND NOT EXISTS (
SELECT 1 FROM relay_region_rehome_attempts recent
WHERE recent.user_id = preference.user_id
AND recent.relay_host_id = preference.relay_host_id
AND recent.created_at > ?
)
AND EXISTS (
SELECT 1 FROM relay_cell_regions target_region
JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
JOIN relay_cell_admission target_admission
ON target_admission.cell_id = target_region.cell_id
JOIN relay_cell_runtime target_runtime
ON target_runtime.cell_id = target_region.cell_id
JOIN relay_cell_capabilities target_capability
ON target_capability.cell_id = target_runtime.cell_id
AND target_capability.cell_incarnation = target_runtime.cell_incarnation
WHERE target_region.region = preference.preferred_region
AND target_cell.enabled = 1
AND target_admission.admission_state = 'general'
AND target_runtime.ready = 1
AND target_runtime.last_heartbeat_at > ?
AND target_capability.regional_rehome_protocol >= 1
)
ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
LIMIT 10`,
[preferenceCutoff, now - this.heartbeatTtlMs, now]
[
preferenceCutoff,
now - this.heartbeatTtlMs,
now,
cooldownCutoff,
now - this.heartbeatTtlMs
]
)
candidatesTotal = candidates.length
for (const candidate of candidates) {
@@ -5320,6 +5372,7 @@ export class RelayAssignmentStore {
sourceCellId: text(candidate, 'source_cell_id'),
assignmentEpoch: integer(candidate, 'assignment_epoch'),
preferenceCutoff,
cooldownCutoff,
drainGraceMs: integer(control, 'drain_grace_ms'),
processSafety: effectiveProcessSafety,
worker,
@@ -5374,6 +5427,7 @@ export class RelayAssignmentStore {
sourceCellId: string
assignmentEpoch: number
preferenceCutoff: number
cooldownCutoff: number
drainGraceMs: number
processSafety: RegionalRehomeSafetySnapshot
worker: SqlRow
@@ -5397,14 +5451,11 @@ export class RelayAssignmentStore {
[input.identity.userId, input.identity.relayHostId]
)
)[0]
if (
!preference ||
text(preference, 'preferred_region') !== 'asia-east2' ||
integer(preference, 'observed_at') < input.preferenceCutoff
) {
if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const preferredRegion = relayRegion(preference, 'preferred_region')
const activeMigration = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
@@ -5415,6 +5466,18 @@ export class RelayAssignmentStore {
input.skips.push({ reason: 'candidate_stale' })
return null
}
// Re-read under the claim: an attempt committed between the scan and here
// would otherwise start a second move for the same host.
const recentAttempt = await transaction.query(
`SELECT 1 FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
LIMIT 1`,
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
)
if (recentAttempt.length > 0) {
input.skips.push({ reason: 'host_cooldown' })
return null
}
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
assertAssignmentActivityCounts(assignment, activityLeases, 0)
const cells = await this.lockCellInventory(transaction, 'nowait')
@@ -5469,11 +5532,17 @@ export class RelayAssignmentStore {
)
return null
}
// The preference read under lock can now agree with the cell the host is
// already on: nothing to move, in either direction.
if (regions.get(input.sourceCellId) === preferredRegion) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
if (
!source ||
integer(source, 'enabled') !== 1 ||
admission.get(input.sourceCellId) !== 'general' ||
regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
regions.get(input.sourceCellId) === undefined ||
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
@@ -5502,17 +5571,25 @@ export class RelayAssignmentStore {
return null
}
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
// A target must be drainable too, or the host lands somewhere it can never
// be rehomed out of again -- the trap this bidirectional move exists to undo.
const eligibleTargets = cells.filter((row) => {
const cellId = text(row, 'cell_id')
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
const capability = capabilities.find(
(candidate) => text(candidate, 'cell_id') === cellId
)
return (
cellId !== input.sourceCellId &&
integer(row, 'enabled') === 1 &&
admission.get(cellId) === 'general' &&
regions.get(cellId) === 'asia-east2' &&
regions.get(cellId) === preferredRegion &&
runtime !== undefined &&
integer(runtime, 'ready') === 1 &&
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
capability !== undefined &&
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const targetIsClean = (row: SqlRow): boolean => {
@@ -5668,12 +5745,13 @@ export class RelayAssignmentStore {
drain_grace_ms, send_attempts, last_send_attempt_at,
drain_receipt_at, drain_outcome, completed_at, aborted_at,
created_at, updated_at)
VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
NULL, NULL, NULL, NULL, ?, ?)`,
[
attemptId,
input.identity.userId,
input.identity.relayHostId,
preferredRegion,
input.sourceCellId,
text(sourceRuntime, 'cell_incarnation'),
targetCellId,
@@ -5688,7 +5766,7 @@ export class RelayAssignmentStore {
return {
...input.identity,
attemptId,
preferredRegion: 'asia-east2',
preferredRegion,
sourceCellId: input.sourceCellId,
sourceCellUrl: text(source, 'cell_url'),
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
@@ -8101,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
attemptId: text(row, 'attempt_id'),
userId: text(row, 'user_id'),
relayHostId: text(row, 'relay_host_id'),
preferredRegion: 'asia-east2',
preferredRegion: relayRegion(row, 'preferred_region'),
sourceCellId: text(row, 'source_cell_id'),
sourceCellUrl: text(row, 'source_cell_url'),
sourceCellIncarnation: text(row, 'source_cell_incarnation'),
@@ -8122,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
notBefore: integer(row, 'not_before'),
ratePerMinute: integer(row, 'rate_per_minute'),
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
hostCooldownMs: integer(row, 'host_cooldown_ms'),
drainGraceMs: integer(row, 'drain_grace_ms')
}
}
@@ -8161,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
return (
integer(row, 'enabled') === 1 &&
input.admission.get(cellId) === 'general' &&
(input.regions.get(cellId) === 'asia-east2' ||
(input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1))
input.regions.get(cellId) !== undefined &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const valid = required.flatMap((row) => {
@@ -8231,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
type RegionalRehomeCandidateSkip = {
reason:
| 'candidate_stale'
| 'host_cooldown'
| 'source_ineligible'
| 'source_unclean'
| 'source_control_inactive'
cloud/apps/relay/src/cell-heartbeat-client.ts
+2 -1
View File
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto'
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
v: 1,
cellId: config.cellId,
cellUrl: config.cellUrl,
region: config.region ?? 'us-central1',
region: config.region ?? RELAY_DEFAULT_REGION,
cellIncarnation,
startedAt,
ready,
cloud/apps/relay/src/database-postgres-timeout.test.ts
+56 -2
View File
@@ -34,7 +34,11 @@ vi.mock('pg', () => ({
}
}))
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
import {
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
relayPostgresStatementTimeoutMs
} from './database.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
const SCHEMA_POOL = {
@@ -118,7 +122,9 @@ describe('PostgreSQL relay deadlines', () => {
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
expect(
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
await database.close()
@@ -263,6 +269,54 @@ describe('PostgreSQL schema startup', () => {
expect(query).toHaveBeenCalledTimes(2)
})
it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
// repeat 42710, so a re-run and a concurrent startup both move on.
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi
.fn<(statement: string) => Promise<unknown>>()
.mockRejectedValueOnce(error)
.mockResolvedValue(undefined)
const pause = vi.fn(async () => undefined)
await applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
query,
{ wait: pause }
)
expect(pause).not.toHaveBeenCalled()
expect(query).toHaveBeenCalledTimes(2)
expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
})
it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
// Guards the statement text against the pattern that classifies it.
const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
statement.includes('ADD CONSTRAINT')
)
expect(shipped.length).toBeGreaterThan(0)
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await applyPostgresSchema(shipped, query, { wait: async () => undefined })
expect(query).toHaveBeenCalledTimes(shipped.length)
})
it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
const error = Object.assign(new Error('check violation'), { code: '23514' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await expect(
applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
query,
{ wait: async () => undefined }
)
).rejects.toBe(error)
})
it.each([
['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
['42710', 'CREATE TABLE test'],
cloud/apps/relay/src/database.test.ts
+41 -1
View File
@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
import {
openInMemoryRelayDatabase,
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
const temporaryDirectories: string[] = []
@@ -142,6 +147,41 @@ describe('relay database', () => {
await second.close()
})
it('renders every region check from the shared region list', async () => {
// Derived, not hand-written: a third region must not leave one column
// rejecting a value the rest of the relay already accepts.
const database = await openInMemoryRelayDatabase()
const checked = await database.query(
`SELECT name, sql FROM sqlite_master
WHERE type = 'table'
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
'relay_region_rehome_attempts')
ORDER BY name`
)
const list = `IN ('us-central1', 'asia-east2')`
expect(checked.map((row) => row.name)).toEqual([
'relay_assignment_region_preferences',
'relay_cell_regions',
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
await database.close()
})
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
`SELECT sql FROM sqlite_master
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
)
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
await database.close()
})
it('indexes region preference expiry by observation time', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
cloud/apps/relay/src/database.ts
+37 -4
View File
@@ -3,6 +3,7 @@ import { performance } from 'node:perf_hooks'
import { join } from 'node:path'
import { DatabaseSync } from 'node:sqlite'
import pg from 'pg'
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
import {
emptyPostgresPoolPressureCounts,
PostgresPoolPressure,
@@ -24,6 +25,14 @@ function setLocalLockTimeout(milliseconds: number): string {
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
}
// Region CHECK lists come from the contract so a new region cannot leave a
// column rejecting values the rest of the relay already accepts.
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
// A host that was just moved is not a candidate again for this long, so a
// desktop whose region probe flips cannot walk itself back and forth.
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
export type SqlRow = Record<string, unknown>
export type RelayLockOptions = {
failIfUnavailable?: boolean
@@ -181,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL
CHECK (preferred_region IN ('us-central1', 'asia-east2')),
CHECK (preferred_region IN (${REGION_LIST})),
observed_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id)
);
@@ -204,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
);
@@ -212,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
preferred_region TEXT NOT NULL
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
@@ -234,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
CREATE TABLE IF NOT EXISTS relay_cells (
cell_id TEXT PRIMARY KEY,
@@ -248,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
CREATE TABLE IF NOT EXISTS relay_cell_regions (
cell_id TEXT PRIMARY KEY,
region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
);
CREATE TABLE IF NOT EXISTS relay_cell_admission (
@@ -580,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
`
// Rehoming is bidirectional, but tables created before that carry the
// original single-region column check. The old constraint is the one Postgres
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
]
function postgresSql(sql: string): string {
let index = 0
return sql.replace(/\?/g, () => `$${++index}`)
@@ -1009,7 +1039,10 @@ async function applySchemaOnUntimedPool(
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
[
...SCHEMA.split(';').filter((statement) => statement.trim()),
...POSTGRES_SCHEMA_MIGRATIONS
],
async (statement) => await database.query(statement)
)
} finally {
cloud/apps/relay/src/host-session-client-accept.test.ts
+590
View File
@@ -0,0 +1,590 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { CredentialReservation, RelayCredentialStore } from './credential-store.js'
import {
CONTROL_LEASE_JITTER_MS,
CONTROL_LEASE_MS,
HostSessionRegistry
} from './host-session-registry.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
// Incident 2026-09-04 ~01:05Z: the phone's dial bound ran out while the cell was
// still inside acceptClient's serialized Postgres phase (cell-inventory lock
// contention). The cell then finished the work for a socket nobody held, holding
// an activity lease for the 10s attach deadline before its timer unwound it, and
// logged `host_data_reservation_already_bound`.
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSING = 2
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number, reason?: string) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(reason ?? ''))
})
readonly terminate = vi.fn(() => {
this.readyState = this.CLOSED
this.emit('close')
})
}
const config = {
port: 8080,
publicUrl: 'https://relay-c3.example.com',
cellUrl: 'https://relay-c3.example.com',
authIssuer: 'https://auth.example.com',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.com/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c3',
cells: [{ id: 'production-gce-c3', url: 'https://relay-c3.example.com', capacityRequests: 4_000 }],
adminAudience: 'https://relay-c3.example.com/v1/admin/drain',
deployServiceAccount: 'deploy@example.com',
runtimeServiceAccount: 'runtime@example.com',
adminJwksUrl: 'https://auth.example.com/admin-jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './test-data'
} satisfies RelayConfig
const identity = {
sub: 'user-1',
prof: 'profile-1',
relayHostId: 'abcdefghijklmnop',
purpose: 'host-control',
exp: 4_102_444_800
} satisfies RelayTokenClaims
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void
const promise = new Promise<T>((next) => (resolve = next))
return { promise, resolve }
}
const reservation: CredentialReservation = {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'resume',
relayDeviceId: 'device-1',
tokenHash: 'hash',
reservationId: 'reservation-1',
leaseExpiresAt: Date.now() + 60_000,
acceptedCredentialVersion: 2,
acceptedAs: 'current'
}
function harness(options: { random?: () => number; now?: () => number } = {}) {
const acquireActivity = vi.fn().mockResolvedValue(undefined)
const releaseActivity = vi.fn().mockResolvedValue(true)
const assignments = {
activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity,
renewControlActivity: vi.fn().mockResolvedValue(undefined),
releaseActivity
} as unknown as RelayAssignmentStore
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined),
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
deactivateBasis: vi.fn().mockResolvedValue(undefined)
}
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordClientAcceptAbandoned: vi.fn(),
recordClientAcceptCompleted: vi.fn(),
recordControlRtt: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
vi.fn(),
store as unknown as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer,
options.now,
options.random
)
const activate = (
registry as unknown as {
activate: (
socket: WebSocket,
identity: RelayTokenClaims,
existing: null,
generation: number,
rebind: boolean,
assignmentEpoch: number,
appVersion: string
) => Promise<void>
}
).activate.bind(registry)
return { registry, store, assignments, acquireActivity, releaseActivity, observer, activate }
}
async function activeHost(h: ReturnType<typeof harness>): Promise<FakeSocket> {
const control = new FakeSocket()
await h.activate(control as unknown as WebSocket, identity, null, 1, false, 1, '1.4.197')
return control
}
describe('client accept abandoned mid-DB-phase', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
const slowAcquire = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slowAcquire.promise)
const capacity = { bind: vi.fn(), release: vi.fn() }
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
expect(h.acquireActivity).toHaveBeenCalledOnce()
// The phone's 12s bound fires while the cell still waits on Postgres.
client.close(1000, 'client bound')
capacity.release()
slowAcquire.resolve()
await accepting
// No conn-open reached the desktop; nothing pending; the lease it just took is
// released instead of leaking to expiry cleanup; bind never throws.
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })
expect(session?.pendingConns.size).toBe(0)
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
expect(h.releaseActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
expect.stringMatching(/^confirmation:/)
)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'activity',
expect.any(Number)
)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('orca_relay_client_accept_abandoned')
)
expect(line).toBeDefined()
expect(JSON.parse(line!)).toMatchObject({ stage: 'activity' })
expect(line).not.toContain(identity.relayHostId)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('stops after a slow credential reservation without acquiring an activity lease', async () => {
const h = harness()
await activeHost(h)
const slowReserve = deferred<CredentialReservation>()
h.store.reserveCredential.mockReturnValueOnce(slowReserve.promise)
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowReserve.resolve(reservation)
await accepting
expect(h.acquireActivity).not.toHaveBeenCalled()
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'credential',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('stops after a slow resume lookup before starting the invite and assignment lookups', async () => {
const h = harness()
await activeHost(h)
const store = h.store as typeof h.store & { resolveInviteForMove: ReturnType<typeof vi.fn> }
store.resolveInviteForMove = vi.fn().mockResolvedValue(null)
const slowResume = deferred<null>()
h.store.resolveResume.mockReturnValueOnce(slowResume.promise)
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
.resolve
resolveAssignment.mockClear()
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowResume.resolve(null)
await accepting
expect(store.resolveInviteForMove).not.toHaveBeenCalled()
expect(resolveAssignment).not.toHaveBeenCalled()
expect(h.store.reserveCredential).not.toHaveBeenCalled()
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'assignment',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('stops after a slow same-cell assignment resolve, before reserving a credential', async () => {
const h = harness()
await activeHost(h)
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
.resolve
const slowResolve = deferred<{ cellId: string }>()
resolveAssignment.mockReturnValueOnce(slowResolve.promise)
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
// A correct, same-cell assignment: only the closed socket stops the accept.
slowResolve.resolve({ cellId: config.cellId })
await accepting
// Proves the accept reached the third guard, not the first.
expect(resolveAssignment).toHaveBeenCalled()
expect(h.store.reserveCredential).not.toHaveBeenCalled()
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'assignment',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('still opens the connection when the phone is holding on', async () => {
const h = harness()
const control = await activeHost(h)
const capacity = { bind: vi.fn(), release: vi.fn() }
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
expect(capacity.bind).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptAbandoned).not.toHaveBeenCalled()
expect(client.close).not.toHaveBeenCalled()
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
describe('successful client accept timing', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
let now = 1_700_000_000_000
const h = harness({ now: () => now })
const control = await activeHost(h)
h.store.resolveResume.mockImplementationOnce(async () => {
now += 5
return { userId: identity.sub }
})
h.store.reserveCredential.mockImplementationOnce(async () => {
now += 7
return reservation
})
h.acquireActivity.mockImplementationOnce(async () => {
now += 11
})
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
now += 3
})
const client = new FakeSocket()
const hostData = new FakeSocket()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
const connOpen = JSON.parse(
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
connOpen.connTicket,
1
)
expect(accepted).toBe(true)
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
})
const line = log.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
expect(line).toBeDefined()
const event = JSON.parse(line!) as {
role: string
cellId: string
region: string
credentialKind: string
stageMs: Record<string, number>
totalMs: number
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
'activity',
'assignment',
'attach',
'basis',
'credential'
])
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
// The stages tile the accept end to end: every millisecond is attributed.
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
expect(summed).toBe(event.totalMs)
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(line).not.toContain(identity.relayHostId)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
// echo the registry will time.
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const ping = control.send.mock.calls
.filter((call) => String(call[0]).includes('"type":"ping"'))
.at(-1)!
return (JSON.parse(String(ping[0])) as { t: number }).t
}
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('logs a host once at the fourth sample and not again within the hour', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const rttLines = (): string[] =>
log.mock.calls
.map((call) => String(call[0]))
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
const roundTrip = async (): Promise<void> => {
const pingAt = await advanceToPing(control, clock)
clock.now += 40
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
}
try {
for (let round = 0; round < 3; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
expect(rttLines()).toHaveLength(0)
await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
expect(rttLines()).toHaveLength(1)
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
event: 'orca_relay_host_control_rtt',
role: 'cell',
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
// Later samples keep feeding the fleet metric, but stay silent for an hour.
for (let round = 0; round < 8; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
expect(rttLines()).toHaveLength(1)
const elapsedStart = clock.now
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
expect(rttLines()).toHaveLength(2)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('ignores a pong that answers no outstanding ping', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
try {
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
control.emit('message', JSON.stringify({ type: 'pong' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
const pingAt = await advanceToPing(control, clock)
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
clock.now += 10
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
} finally {
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('records one sample per ping however many pongs a host floods', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
const pingAt = await advanceToPing(control, clock)
clock.now += 12
for (let flood = 0; flood < 5_000; flood++) {
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
}
// One answered ping is one process-wide sample and one per-session sample, so
// neither the metric window nor the hourly log line can be flooded.
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
expect(
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
).toHaveLength(0)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
describe('control lease jitter', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('grants a lease uniformly around its mean so cohorts drift apart at the same mean rate', async () => {
const now = 1_700_000_000_000
const helloAck = (socket: FakeSocket) =>
JSON.parse(
String(socket.send.mock.calls.find((call) => String(call[0]).includes('host-hello-ack'))![0])
) as { leaseExpiresAt: number }
const shortest = harness({ now: () => now, random: () => 0 })
const shortestAck = helloAck(await activeHost(shortest))
const centered = harness({ now: () => now, random: () => 0.5 })
const centeredAck = helloAck(await activeHost(centered))
const longestRoll = 0.999999
const longest = harness({ now: () => now, random: () => longestRoll })
const longestAck = helloAck(await activeHost(longest))
// Pinned, not bounded: a jitter clamped to one side still satisfies an upper
// bound, so only the exact top of the band proves it is symmetric.
const longestOffset = Math.floor((longestRoll * 2 - 1) * CONTROL_LEASE_JITTER_MS)
expect(shortestAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS - CONTROL_LEASE_JITTER_MS)
expect(centeredAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS)
expect(longestAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + longestOffset)
shortest.registry.drain(0)
centered.registry.drain(0)
longest.registry.drain(0)
vi.advanceTimersByTime(0)
})
it('rebinds re-roll the jitter instead of pinning the cohort phase', async () => {
const now = 1_700_000_000_000
let roll = 0
const h = harness({ now: () => now, random: () => roll })
const first = await activeHost(h)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const firstLease = session.leaseExpiresAt
roll = 0.75
const rebind = new FakeSocket()
await (
h.registry as unknown as {
activate: (...args: unknown[]) => Promise<void>
}
).activate(rebind as unknown as WebSocket, identity, session, 1, true, 1, '1.4.197')
expect(session.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS / 2)
expect(session.leaseExpiresAt).not.toBe(firstLease)
expect(first.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound')
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
cloud/apps/relay/src/host-session-registry.test.ts
+113
View File
@@ -3,6 +3,7 @@ import {
ASSIGNMENT_LIMITS,
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
}
})
})
describe('host hello ack pending connections', () => {
const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
function newRegistry(): ReturnType<typeof createRegistry> {
return createRegistry(
vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
)
}
function addPendingConnection(session: HostSession): void {
session.pendingConns.set('conn-1', {
...LEGACY_ENTRY,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
relayDeviceId: 'device-1'
},
client: new FakeSocket() as unknown as WebSocket,
attachTimer: setTimeout(() => {}, 60_000),
credentialActivityId: null
} as unknown as Parameters<typeof session.pendingConns.set>[1])
}
function sentAck(socket: FakeSocket): Record<string, unknown> {
const acks = socket.send.mock.calls
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
.filter((message) => message.type === 'host-hello-ack')
return acks.at(-1)!
}
function sessionOf(registry: HostSessionRegistry): HostSession {
return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
}
async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
const { registry, activate } = newRegistry()
const socket = new FakeSocket()
registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
capabilities ?? new Set()
)
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
socket.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
return sentAck(socket)
}
async function ackAfterRebind(
first: ReadonlySet<string>,
successor: ReadonlySet<string>
): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
const { registry, activate } = newRegistry()
const opening = new FakeSocket()
registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
opening.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
const rebound = new FakeSocket()
registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
return { opening: sentAck(opening), rebound: sentAck(rebound) }
}
it('states the pending kind and device to a host that advertised it can read them', async () => {
const ack = await ackFor(DETAILS)
expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
})
it('restates only the identifiers to a host that never advertised the capability', async () => {
// A shipped host parses these entries strictly, so an unannounced key fails
// the whole ack parse and kills a control that was working.
const ack = await ackFor()
expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
})
it('downgrades the restated entry when the successor control drops the capability', async () => {
// The capability belongs to the socket, not the session: a rebind can land a
// control whose decoder is older than the one that opened the session.
const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
})
it('upgrades the restated entry when the successor control adds the capability', async () => {
const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
cloud/apps/relay/src/host-session-registry.ts
+210 -11
View File
@@ -1,6 +1,7 @@
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
import {
ASSIGNMENT_LIMITS,
RELAY_DEFAULT_REGION,
AuthRefreshSchema,
buildHostChallengePlaintext,
buildHostProofMacInput,
@@ -13,9 +14,11 @@ import {
HostChallengeAckSchema,
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason
type RelayHostCloseReason,
type RelayRegion
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import type WebSocket from 'ws'
@@ -29,7 +32,12 @@ import {
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
import {
percentile,
type RelayClientAcceptStage,
type RelayClientAcceptTimedStage,
type RelayRuntimeObserver
} from './relay-observability.js'
import type { PendingHostDataReservation } from './relay-connection-ledger.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
@@ -45,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
// so a short window is enough to ride out one stalled ping.
const CONTROL_RTT_WINDOW = 8
const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
// A pong claiming a multi-minute round trip is clock skew, not distance.
const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
// Wall clock can step backwards mid-accept; a negative latency would poison the
// percentiles it feeds.
function nonNegativeMs(elapsedMs: number): number {
return Math.max(0, elapsedMs)
}
const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
// Preserve the existing 75s renewal runway after doubling the successful-call interval.
const CONTROL_ACTIVITY_LEASE_MS =
@@ -68,6 +90,10 @@ export type HostSession = {
orphanTimer: ReturnType<typeof setTimeout> | null
heartbeatTimer: ReturnType<typeof setInterval> | null
lastPongAt: number
// The `t` of the ping still waiting for its echo; null once one has answered it.
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -95,6 +121,15 @@ type PendingConnection = {
attachTimer: ReturnType<typeof setTimeout>
credentialActivityId: string | null
capacityReservation?: PendingHostDataReservation
timing: ClientAcceptTiming
}
// Carries the phone-side accept clock across to the desktop's data leg, which
// lands in a separate call and is the only place the accept is known to succeed.
type ClientAcceptTiming = {
startedAt: number
connOpenAt: number
stageMs: Record<RelayClientAcceptStage, number>
}
function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
@@ -129,6 +164,16 @@ function send(socket: WebSocket, type: string, message: object): void {
// stalled predecessor only accumulates doomed sockets.
const ACTIVATION_QUEUE_WAIT_MS = 30_000
// Why: this lease bounds how long a host lingers on a cell after a missed drain,
// and rebinding it is the only passive rebalancing we have, so it has to stay
// finite. 6h keeps both properties while cutting control-activation traffic on
// the contended cell-inventory lock ~6x; the relay JWT (5 min, refreshed by the
// desktop) and the 75s silence watchdog are enforced separately, so a longer
// grant authorizes nothing extra. Symmetric jitter walks same-minute reconnect
// cohorts apart across cycles without changing the mean rebind rate.
export const CONTROL_LEASE_MS = 6 * 60 * 60 * 1000
export const CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000
export class HostSessionRegistry {
private readonly sessions = new Map<string, HostSession>()
private readonly activationQueues = new Map<string, Promise<void>>()
@@ -136,6 +181,7 @@ export class HostSessionRegistry {
// but a signed-out desktop never comes back, so the phone that asks minutes
// later would otherwise find nothing to explain its rejection with.
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
constructor(
@@ -145,9 +191,16 @@ export class HostSessionRegistry {
private readonly assignments: RelayAssignmentStore,
private readonly queuedByteBudget: ProcessQueuedByteBudget,
private readonly observer: RelayRuntimeObserver,
private readonly now: () => number = Date.now
private readonly now: () => number = Date.now,
private readonly random: () => number = Math.random
) {}
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
private controlLeaseExpiresAt(): number {
const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
return this.now() + CONTROL_LEASE_MS + offset
}
async acceptClient(
socket: WebSocket,
hostId: string,
@@ -159,10 +212,42 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
return
}
// Why: the accept runs several serialized Postgres calls behind the contended
// cell-inventory lock, and phones bound their dial. Finishing the work for a
// phone that already hung up took an activity lease held for the 10s attach
// deadline, then failed at bind with host_data_reservation_already_bound.
const acceptStartedAt = this.now()
const abandonedByClient = (stage: RelayClientAcceptStage, cleanup?: () => void): boolean => {
if (socket.readyState === socket.OPEN) return false
capacityReservation?.release()
cleanup?.()
const elapsedMs = this.now() - acceptStartedAt
this.observer.recordClientAcceptAbandoned?.(stage, elapsedMs)
console.warn(
JSON.stringify({ event: 'orca_relay_client_accept_abandoned', stage, elapsedMs })
)
return true
}
const stageMs: Record<RelayClientAcceptStage, number> = {
assignment: 0,
credential: 0,
activity: 0
}
let stageCursor = acceptStartedAt
const markStage = (stage: RelayClientAcceptStage): void => {
const at = this.now()
stageMs[stage] = at - stageCursor
stageCursor = at
}
if (this.config.role === 'cell') {
const outerIdentity =
(await this.store.resolveResume(hostId, credential)) ??
(await this.store.resolveInviteForMove(hostId, credential))
// Each lookup is its own pooled round trip; stop between them once the phone
// has left instead of running the rest of the chain for nobody.
let outerIdentity = await this.store.resolveResume(hostId, credential)
if (abandonedByClient('assignment')) return
if (!outerIdentity) {
outerIdentity = await this.store.resolveInviteForMove(hostId, credential)
if (abandonedByClient('assignment')) return
}
const assignment = outerIdentity
? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId })
: null
@@ -172,7 +257,9 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
if (abandonedByClient('assignment')) return
}
markStage('assignment')
const reservation = await this.store.reserveCredential(hostId, credential)
if (!reservation) {
capacityReservation?.release()
@@ -181,6 +268,8 @@ export class HostSessionRegistry {
return
}
this.observer.recordAuth(true)
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
markStage('credential')
const sessionKey = this.key(reservation.userId, hostId)
const session = this.sessions.get(sessionKey)
if (
@@ -227,6 +316,15 @@ export class HostSessionRegistry {
return
}
}
if (
abandonedByClient('activity', () => {
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
})
) {
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
capacityReservation?.release()
@@ -241,7 +339,10 @@ export class HostSessionRegistry {
client: socket,
attachTimer,
credentialActivityId,
capacityReservation
capacityReservation,
// Attach starts where the activity stage ended, so the conn-open send is
// charged to it and no wall-clock gap goes unattributed.
timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
}
capacityReservation?.bind(connId)
session.pendingConns.set(connId, pending)
@@ -288,6 +389,7 @@ export class HostSessionRegistry {
return false
}
this.observer.recordAuth(true)
const attachedAt = this.now()
clearTimeout(pending.attachTimer)
session.pendingConns.delete(connId)
session.activeConnIds.add(connId)
@@ -361,6 +463,7 @@ export class HostSessionRegistry {
close()
return false
}
const helloAt = this.now()
send(pending.client, 'relay-hello', {
ok: true,
credentialKind: pending.reservation.credentialKind,
@@ -376,14 +479,92 @@ export class HostSessionRegistry {
}
: {})
})
this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
return true
}
// The stages tile the whole accept, so their sum is the total minus only the
// clamping above: `basis` is the splice lease and connection-basis writes that
// land between the host data leg and relay-hello.
private recordClientAcceptCompleted(
session: HostSession,
pending: PendingConnection,
attachedAt: number,
helloAt: number
): void {
const stageMs: Record<RelayClientAcceptTimedStage, number> = {
assignment: nonNegativeMs(pending.timing.stageMs.assignment),
credential: nonNegativeMs(pending.timing.stageMs.credential),
activity: nonNegativeMs(pending.timing.stageMs.activity),
attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
basis: nonNegativeMs(helloAt - attachedAt)
}
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
console.log(
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
})
)
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
return {
role: this.config.role,
cellId: this.config.cellId,
region: this.config.region ?? RELAY_DEFAULT_REGION
}
}
// Every desktop build already echoes the ping's `t`, so a pong is only timed when
// it answers the outstanding ping: at most one sample per ping this cell sent,
// however many a host floods. A pong that lost the race to the next ping is
// dropped here but still counts as proof of life for the silence watchdog.
private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
session.pendingPingAt = null
const now = this.now()
const rttMs = now - echoedPingAt
if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
this.observer.recordControlRtt?.(rttMs)
const samples = session.controlRttSamplesMs
samples.push(rttMs)
if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
if (
session.controlRttLoggedAt !== null &&
now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
) {
return
}
session.controlRttLoggedAt = now
console.log(
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
})
)
}
acceptControl(
socket: WebSocket,
identity: RelayTokenClaims,
connectionInclusionWatermark?: number
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
if (this.draining) {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
return
@@ -740,8 +921,9 @@ export class HostSessionRegistry {
existing.socket = socket
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
existing.appVersion = appVersion
existing.leaseExpiresAt = this.now() + 55 * 60 * 1000
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
@@ -791,10 +973,13 @@ export class HostSessionRegistry {
appVersion,
state: 'active',
socket,
leaseExpiresAt: this.now() + 55 * 60 * 1000,
leaseExpiresAt: this.controlLeaseExpiresAt(),
orphanTimer: null,
heartbeatTimer: null,
lastPongAt: this.now(),
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -852,6 +1037,7 @@ export class HostSessionRegistry {
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
if (parsed.type === 'pong') {
session.lastPongAt = this.now()
this.recordControlRtt(session, parsed.t)
return
}
if (parsed.type === 'auth-refresh') {
@@ -999,11 +1185,18 @@ export class HostSessionRegistry {
session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
return
}
session.pendingPingAt = now
send(session.socket, 'ping', { t: now })
}
private sendHelloAck(session: HostSession): void {
if (!session.socket) return
// Without these a host that missed the conn-open cannot dial the pending
// connection: it would have to guess the pairing kind and the device the
// relay authorized. Only sent to a host that said it can read them.
const details = this.hostCapabilities
.get(session.socket)
?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
send(session.socket, 'host-hello-ack', {
v: 1,
generation: session.generation,
@@ -1012,7 +1205,13 @@ export class HostSessionRegistry {
activeConnIds: [...session.activeConnIds],
pendingConns: [...session.pendingConns.values()].map((pending) => ({
connId: pending.connId,
connTicket: pending.connTicket
connTicket: pending.connTicket,
...(details
? {
kind: pending.reservation.credentialKind,
relayDeviceId: pending.reservation.relayDeviceId
}
: {})
}))
})
}
cloud/apps/relay/src/postgres-schema-startup.ts
+15
View File
@@ -49,6 +49,20 @@ function concurrentCreateCollision(
return false
}
const ALTER_TABLE_ADD_CONSTRAINT =
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
// this is terminal, not transient: retrying only repeats it, so the statement
// counts as applied.
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
return (
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
(error as { code?: unknown }).code === '42710'
)
}
function retryableSchemaError(error: unknown, statement: string): boolean {
const value = error as { code?: unknown; constraint?: unknown }
return (
@@ -73,6 +87,7 @@ export async function applyPostgresSchema(
await query(statement)
break
} catch (error) {
if (constraintAlreadyApplied(error, statement)) break
const code = String((error as { code?: unknown }).code)
const remainingMs = deadlineAt - now()
const retryable = retryableSchemaError(error, statement)
cloud/apps/relay/src/regional-host-drain-app.test.ts
+81
View File
@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
notBefore: 100,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
})
it('probes a source cell in any region, not only the default one', async () => {
// Rehoming moves hosts in both directions, so an asia-east2 cell is a
// source too and its trust has to be provable the same way.
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 1
}
})
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: (async () =>
Response.json({
v: 1,
outcome: 'host-not-connected',
sharedRuntimeIdentityRejected: true
})) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
})
it('still refuses a trust probe against a cell without the drain protocol', async () => {
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 0
}
})
const sourceFetch = vi.fn<typeof fetch>()
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
})
it('restricts trust probes to deploy authorization and strict input', async () => {
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
cloud/apps/relay/src/regional-rehome-constraint-migration-postgres.test.ts
+195
View File
@@ -0,0 +1,195 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import {
openRelayDatabase,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
type RelayDatabase
} from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_rehome_constraint_migration_test'
// The shape shipped before rehoming became bidirectional: a single-region
// column check that Postgres auto-names.
const LEGACY_ATTEMPTS_TABLE = `
CREATE TABLE relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
send_attempts BIGINT NOT NULL,
last_send_attempt_at BIGINT,
drain_receipt_at BIGINT,
drain_outcome TEXT CHECK (
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
),
completed_at BIGINT,
aborted_at BIGINT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
UNIQUE (user_id, relay_host_id, assignment_epoch)
)`
// The control row as it shipped before the per-host cooldown existed.
const LEGACY_CONTROL_TABLE = `
CREATE TABLE relay_region_rehome_control (
control_id TEXT PRIMARY KEY,
generation BIGINT NOT NULL,
enabled BIGINT NOT NULL,
observation_started_at BIGINT NOT NULL,
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)`
const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
attemptId,
'user-1',
'abcdefghijklmnop',
preferredRegion,
'cell-source',
'11111111-1111-4111-8111-111111111111',
'cell-target',
'22222222-2222-4222-8222-222222222222',
1,
Number(attemptId.at(-1)),
0,
0,
1_000_000,
1_000_000
]
const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
describePostgres('PostgreSQL regional rehome constraint migration', () => {
let scopedUrl = ''
async function withClient(
operation: (client: pg.Client) => Promise<void>
): Promise<void> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await operation(client)
} finally {
await client.end()
}
}
beforeEach(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
await client.query(`SET search_path = ${schema}`)
await client.query(LEGACY_ATTEMPTS_TABLE)
await client.query(LEGACY_CONTROL_TABLE)
await client.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
)
// Production data the replacement constraint has to validate.
await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
})
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
scopedUrl = url.toString()
})
afterAll(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
})
})
it('upgrades a legacy single-region constraint in place', async () => {
const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
try {
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
await expect(
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
).rejects.toMatchObject({ code: '23514' })
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'
ORDER BY conname`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
// The existing control row keeps its tuning and gains the cooldown.
const control = await client.query(
`SELECT generation, preference_max_age_ms, host_cooldown_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
)
expect(control.rows).toEqual([
{
generation: '3',
preference_max_age_ms: '86400000',
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
}
])
})
} finally {
await database.close()
}
})
it('upgrades once across concurrent startups', async () => {
const results = await Promise.allSettled(
Array.from(
{ length: 5 },
async (): Promise<RelayDatabase> =>
await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
)
)
const databases = results.flatMap((result) =>
result.status === 'fulfilled' ? [result.value] : []
)
await Promise.all(databases.map(async (database) => await database.close()))
expect(
results.flatMap((result) =>
result.status === 'rejected'
? [
{
code: (result.reason as { code?: unknown }).code,
message: String(result.reason)
}
]
: []
)
).toEqual([])
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
})
}, 60_000)
})
cloud/apps/relay/src/regional-rehome-postgres.test.ts
+191 -10
View File
@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
expect(await context.store.claimRegionalRehome()).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
const context = await fixture({
sourceRegion: 'asia-east2',
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
// A move this host already made, whichever way it went.
await primary.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
completed_at, created_at, updated_at)
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
[
`pg-rehome-cooldown-${context.identity.relayHostId}`,
context.identity.userId,
context.identity.relayHostId,
context.target.id,
'22222222-2222-4222-8222-222222222222',
context.source.id,
'11111111-1111-4111-8111-111111111111',
context.now(),
context.now() - 3 * 24 * 60 * 60_000 + 1,
context.now()
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
hostCooldownMs: 3 * 24 * 60 * 60_000
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 1,
migrations: 0
})
// One millisecond past the window the same host is a candidate again.
await primary.query(
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
})
it('leaves a host whose preferred region holds no drainable cell', async () => {
// A cell that cannot be drained cannot be a target: the host would land
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('skips an unclean cell without latching the control off', async () => {
const context = await fixture()
await primary.query(
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'44444444-4444-4444-8444-444444444444',
0,
1,
context.now()
)
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
}): Promise<{ attempts: number; migrations: number }> {
const attempts = await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
const migrations = await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
return {
attempts: Number(attempts[0]!.count),
migrations: Number(migrations[0]!.count)
}
}
async function controlAccounting(identity: {
userId: string
relayHostId: string
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
async function fixture() {
async function fixture(options: FixtureOptions = {}) {
sequence++
let now = 1_000_000
const suffix = String(sequence)
const source = cell(suffix, 'source', 'us-central1')
const target = cell(suffix, 'target', 'asia-east2')
const sourceRegion = options.sourceRegion ?? 'us-central1'
const targetRegion = options.targetRegion ?? 'asia-east2'
const preferredRegion = options.preferredRegion ?? targetRegion
const source = cell(suffix, 'source', sourceRegion)
const target = cell(suffix, 'target', targetRegion)
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
await store.inspectRegionalRehomeControl()
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
store,
target,
'22222222-2222-4222-8222-222222222222',
0,
options.targetProtocol ?? 1,
900_000
)
const identity = {
userId: `pg-rehome-user-${suffix}`,
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const assignment = await store.assign(identity, undefined, sourceRegion)
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await store.assign(identity, 'asia-east2')
await store.assign(identity, preferredRegion)
return {
preferredRegion,
store,
competingStore,
identity,
@@ -500,7 +672,16 @@ const storeOptions = {
heartbeatTtlMs: 45_000
}
function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
type Region = 'us-central1' | 'asia-east2'
type FixtureOptions = {
sourceRegion?: Region
targetRegion?: Region
preferredRegion?: Region
targetProtocol?: number
hostCooldownMs?: number
}
function cell(suffix: string, role: string, region: Region) {
return {
id: `pg-rehome-cell-${suffix}-${role}`,
url: `https://pg-rehome-${suffix}-${role}.example.test`,
cloud/apps/relay/src/regional-rehome-store.test.ts
+295 -8
View File
@@ -81,6 +81,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).rejects.toThrow('regional_rehome_generation_mismatch')
await expect(context.store.applyRegionalRehomeControl({
@@ -89,6 +90,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).resolves.toMatchObject({ generation: 3, enabled: true })
await context.database.close()
@@ -207,7 +209,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 1,
reconnects: 3,
@@ -226,6 +228,23 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('counts only drainable cells as the rehome fleet, in every region', async () => {
// The fleet whose health gates a rehome is exactly the cells that can be a
// source or a target, and both roles require the drain protocol.
const context = await setup({ targetProtocol: 0 })
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 1,
missingCells: 0
})
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 2,
missingCells: 0
})
await context.database.close()
})
it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
const context = await setup()
const baseline = {
@@ -238,7 +257,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitMsMax: 1
}
await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
@@ -324,6 +343,204 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
await activateReversePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
userId: identity.userId,
relayHostId: identity.relayHostId,
preferredRegion: 'us-central1',
sourceCellId: target.id,
sourceCellIncarnation: targetIncarnation,
targetCellId: source.id,
targetCellIncarnation: sourceIncarnation,
previousEpoch: 1,
assignmentEpoch: 2,
sendAttempts: 1
})
expect(
await context.database.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts`
)
).toEqual([{
preferred_region: 'us-central1',
source_cell_id: target.id,
target_cell_id: source.id
}])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
await context.database.close()
})
it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
const context = await setup()
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
// A disabled cell is not a target, and the scan must say so: leaving it to
// the claim would burn a slot of the candidate batch on a certain skip.
await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('names the skip when the last target is lost between scan and claim', async () => {
const database = await openInMemoryRelayDatabase()
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
})
})
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
])
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('leaves a host alone until its cooldown expires, then moves it back', async () => {
const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const targetControl = await completeRehomeToTarget(context, identity)
// Past the dispatch interval the earlier claim charged, so the next tick
// really does scan and the cooldown is the only thing holding this host.
context.advance(10_000)
// The desktop's region probe now says us-central1 again.
await context.store.assign(identity, 'us-central1')
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
context.advance(3 * 24 * 60 * 60_000)
await freshHeartbeats(context)
await context.store.renewControlActivity(identity, {
activityId: targetControl,
cellId: target.id,
expiresAt: context.now() + 90_000
})
await context.store.assign(identity, 'us-central1')
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: target.id,
targetCellId: source.id
})
await context.database.close()
})
it('rejects a host whose attempt lands between the scan and the claim', async () => {
const database = await openInMemoryRelayDatabase()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
[
identity.userId,
identity.relayHostId,
source.id,
sourceIncarnation,
target.id,
targetIncarnation,
context.now(),
context.now()
]
)
})
})
await activatePreferredSource(context, identity)
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'host_cooldown', candidates: 1 }] }
])
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('does not scan a candidate whose preferred region has no drainable cell', async () => {
// A cell without the drain protocol cannot be a target: the host would land
// where no later rehome could move it out again. The candidate query drops
// it, so the tick stays idle instead of paying for an inventory scan.
const context = await setup({ targetProtocol: 0 })
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('skips an unclean cell without latching the control off', async () => {
const context = await setup()
await activatePreferredSource(context, {
@@ -457,7 +674,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
@@ -477,6 +694,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
const retry = await context.store.claimRegionalRehome()
@@ -547,7 +765,7 @@ describe('regional rehome assignment state', () => {
await activatePreferredSource(context, identity)
await context.store.claimRegionalRehome()
context.advance(6 * 60_000)
await heartbeat(context.store, target, targetIncarnation, 0, 2)
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
expect(await context.store.abortExpiredEvacuations()).toBe(0)
@@ -1549,10 +1767,16 @@ function collectDisableWarnings() {
}
async function setup(
options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {}
options: {
sourceProtocol?: number
targetProtocol?: number
hostCooldownMs?: number
database?: RelayDatabase
wrap?: (database: RelayDatabase) => RelayDatabase
} = {}
) {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const database = options.database ?? (await openInMemoryRelayDatabase())
const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
@@ -1565,11 +1789,12 @@ async function setup(
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, target])
await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
await heartbeat(store, target, targetIncarnation, 0)
await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
return {
database,
store,
@@ -1671,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
}
// The clock doubles as a strictly-increasing connection inclusion watermark.
await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
}
async function activatePreferredSource(
@@ -1688,6 +1913,68 @@ async function activatePreferredSource(
return control
}
// Runs a hook inside the claim transaction, right after the candidate scan, so
// a scan-versus-claim race is deterministic instead of timing-dependent.
function hookAfterCandidateScan(
database: RelayDatabase,
hook: (transaction: RelayDatabase) => Promise<void>
): RelayDatabase {
let fired = false
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
query: async (sql, params) => {
const rows = await delegate.query(sql, params)
if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
fired = true
await hook(delegate)
}
return rows
},
queryLocked: async (sql, params, lockOptions) =>
await delegate.queryLocked(sql, params, lockOptions),
transaction: async (operation, transactionOptions) =>
await delegate.transaction(
async (transaction) => await operation(decorate(transaction)),
transactionOptions
),
close: async () => undefined
})
return decorate(database)
}
async function completeRehomeToTarget(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const sourceControl = await activatePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
const targetControl = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(identity, sourceControl)
await context.store.completeReadyRegionalRehomes()
return targetControl
}
async function activateReversePreferredSource(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const assignment = await context.store.assign(identity, undefined, 'asia-east2')
const control = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await context.store.assign(identity, 'us-central1')
return control
}
async function activateSource(
context: Context,
identity: { userId: string; relayHostId: string }
cloud/apps/relay/src/regional-rehome-target-selection.test.ts
+9 -8
View File
@@ -36,6 +36,7 @@ async function setup() {
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
cloud/apps/relay/src/relay-observability.test.ts
+173 -5
View File
@@ -1,7 +1,9 @@
import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
RelayObservability,
type RelayProcessCounts
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
databasePoolWaitMsMax: 1_250
}
// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
// is keyed by stage name and one stage is `credential`. Rename those exact keys in
// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
// else still trips the guard below.
const SCHEMA_KEY_ALIASES: Record<string, string> = {
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
}
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
return JSON.stringify(
entries.map((entry) =>
Object.fromEntries(
Object.entries(entry).map(([key, value]) => [
SCHEMA_KEY_ALIASES[key] ?? key,
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
])
)
)
)
}
function renameStageKeys(bucket: unknown): unknown {
if (bucket === null || typeof bucket !== 'object') return bucket
return Object.fromEntries(
Object.entries(bucket).map(([stage, count]) => [
stage === 'credential' ? 'stageTwo' : stage,
count
])
)
}
describe('relay observability', () => {
it('emits safe readiness dependency outcomes', () => {
const entries: Array<Record<string, unknown>> = []
@@ -106,14 +139,31 @@ describe('relay observability', () => {
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
selectedRegionsDelta: { 'us-central1': 1 },
regionFallbacksDelta: { 'asia-east2': 1 },
unavailableRegionsDelta: { 'asia-east2': 1 }
unavailableRegionsDelta: { 'asia-east2': 1 },
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 1,
selectedRegionUsCentral1Delta: 1,
selectedRegionAsiaEast2Delta: 0
})
expect(entries[1]).toMatchObject({
requestedRegionsDelta: {},
selectedRegionsDelta: {},
regionFallbacksDelta: {},
unavailableRegionsDelta: {}
unavailableRegionsDelta: {},
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 0,
selectedRegionUsCentral1Delta: 0,
selectedRegionAsiaEast2Delta: 0
})
// A region added to the contract has to reach the flat keys, or the skew alert's
// denominator silently misses it.
for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
}
expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
})
it('emits bounded aggregate runtime signals without identities or credentials', () => {
@@ -181,7 +231,7 @@ describe('relay observability', () => {
controlActivityRecoveryFailuresDelta: 0,
httpLatencyMsMax: 0
})
expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('aggregates control and splice closes as bounded per-reason deltas', () => {
@@ -195,19 +245,137 @@ describe('relay observability', () => {
observability.recordControlClose(4402)
observability.recordSpliceClose('host-oversize-frame')
observability.recordSpliceClose('queue-limit')
observability.recordClientAcceptAbandoned('activity', 14_250.4)
observability.recordClientAcceptAbandoned('activity', 2_000)
observability.recordClientAcceptAbandoned('credential', 3_000)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
controlClosesByCodeDelta: { 1006: 2, 4402: 1 },
spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 }
spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 },
clientAcceptsAbandonedByStageDelta: { activity: 2, credential: 1 },
clientAcceptAbandonedMsMax: 14_250.4
})
expect(entries[1]).toMatchObject({
controlClosesByCodeDelta: {},
spliceClosesByTriggerDelta: {}
spliceClosesByTriggerDelta: {},
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
})
it('summarises completed client accepts and control round trips per window', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordClientAcceptCompleted({
totalMs: 812.4567,
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
})
observability.recordClientAcceptCompleted({
totalMs: 6_400,
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
})
observability.recordControlRtt(28)
observability.recordControlRtt(240)
observability.recordControlRtt(31)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
clientAcceptCompletedDelta: 2,
clientAcceptTotalMsP50: 812.457,
clientAcceptTotalMsP95: 6_400,
clientAcceptTotalMsMax: 6_400,
clientAcceptAssignmentMsP95: 4_100,
clientAcceptCredentialMsP95: 95,
clientAcceptActivityMsP95: 60,
clientAcceptAttachMsP95: 2_000,
clientAcceptBasisMsP95: 145,
controlRttSamplesDelta: 3,
controlRttMsP50: 31,
controlRttMsP95: 240,
controlRttMsMax: 240
})
// Only-add: the pre-existing fields still read the same after the extension.
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
metricVersion: 2,
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
// An empty window publishes counts only: a zero percentile point is
// indistinguishable from a real zero once Cloud Logging aggregates it.
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
for (const omitted of [
'clientAcceptTotalMsP50',
'clientAcceptTotalMsP95',
'clientAcceptTotalMsMax',
'clientAcceptAssignmentMsP95',
'clientAcceptCredentialMsP95',
'clientAcceptActivityMsP95',
'clientAcceptAttachMsP95',
'clientAcceptBasisMsP95',
'controlRttMsP50',
'controlRttMsP95',
'controlRttMsMax'
]) {
expect(entries[1]).not.toHaveProperty(omitted)
expect(entries[0]).toHaveProperty(omitted)
}
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('caps the control round-trip reservoir and reports what it dropped', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
for (let sample = 0; sample < flooded; sample++) {
observability.recordControlRtt(10 + (sample % 40))
}
observability.flush(counts)
// Dropped is observed minus retained, so this pins the retained window at the cap.
expect(entries[0]).toMatchObject({
controlRttSamplesDelta: flooded,
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
})
// The kept samples are real observations, not a truncated or synthesised window.
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
observability.flush(counts)
expect(entries[1]).toMatchObject({
controlRttSamplesDelta: 0,
controlRttSamplesDroppedDelta: 0
})
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
})
it('samples the whole flooded window rather than its first samples', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
observability.flush(counts)
// Keeping the first N instead would publish a window of nothing but 10s. Each
// reservoir slot ends up drawn from the late half with ~1/2 probability, so
// fewer than the 5% the p95 needs is out of reach of this suite.
expect(entries[0]!.controlRttMsP95).toBe(900)
expect(entries[0]!.controlRttMsMax).toBe(900)
})
it('observes successful and failed database calls including transactions', async () => {
const recordSql = vi.fn()
const underlying: RelayDatabase = {
cloud/apps/relay/src/relay-observability.ts
+145 -13
View File
@@ -1,5 +1,5 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import type { RelayRegion } from '@orca-cloud/relay-contract'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
@@ -64,6 +64,30 @@ export interface RelayRuntimeObserver {
}): void
recordControlClose?(code: number): void
recordSpliceClose?(trigger: string): void
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
recordControlRtt?(rttMs: number): void
}
// Which serialized accept step the phone had already hung up behind.
export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
// The attach window and the basis writes that follow it are only measurable once
// the host data leg lands, so they join the serialized pre-attach steps on
// completed accepts only.
export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
'assignment',
'credential',
'activity',
'attach',
'basis'
] as const satisfies readonly RelayClientAcceptTimedStage[]
export type RelayClientAcceptSample = {
totalMs: number
stageMs: Record<RelayClientAcceptTimedStage, number>
}
type RelayMetricDeltas = {
@@ -87,12 +111,22 @@ type RelayMetricDeltas = {
unavailableRegions: Record<string, number>
controlClosesByCode: Record<string, number>
spliceClosesByTrigger: Record<string, number>
clientAcceptsAbandonedByStage: Record<string, number>
clientAcceptAbandonedMsMax: number
clientAcceptTotalsMs: number[]
clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
controlRttSamplesMs: number[]
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
// A host chooses how often it answers a ping, so the process-wide window is a
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
type MetricWriter = (entry: Record<string, unknown>) => void
const emptyDeltas = (): RelayMetricDeltas => ({
@@ -116,18 +150,44 @@ const emptyDeltas = (): RelayMetricDeltas => ({
unavailableRegions: {},
controlClosesByCode: {},
spliceClosesByTrigger: {},
clientAcceptsAbandonedByStage: {},
clientAcceptAbandonedMsMax: 0,
clientAcceptTotalsMs: [],
clientAcceptStageSamplesMs: {
assignment: [],
credential: [],
activity: [],
attach: [],
basis: []
},
controlRttSamplesMs: [],
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
function percentile(values: number[], percentileRank: number): number {
export function percentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
function roundMs(value: number): number {
return Number(value.toFixed(3))
}
// Spreading a window into Math.max blows the stack once a busy cell samples
// enough of it, so the maximum is folded instead.
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
return {
p50: roundMs(percentile(samples, 0.5)),
p95: roundMs(percentile(samples, 0.95)),
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
export class RelayObservability implements RelayRuntimeObserver {
private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
private deltas = emptyDeltas()
@@ -228,6 +288,33 @@ export class RelayObservability implements RelayRuntimeObserver {
(this.deltas.spliceClosesByTrigger[trigger] ?? 0) + 1
}
recordClientAcceptAbandoned(stage: RelayClientAcceptStage, elapsedMs: number): void {
increment(this.deltas.clientAcceptsAbandonedByStage, stage)
this.deltas.clientAcceptAbandonedMsMax = Math.max(
this.deltas.clientAcceptAbandonedMsMax,
elapsedMs
)
}
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
}
}
recordControlRtt(rttMs: number): void {
const samples = this.deltas.controlRttSamplesMs
const observedBefore = this.deltas.controlRttObserved++
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
samples.push(rttMs)
return
}
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
const slot = Math.floor(Math.random() * (observedBefore + 1))
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
}
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
if (this.timer) return
this.eventLoop.enable()
@@ -261,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
}
this.deltas = emptyDeltas()
const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -285,13 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
requestedRegionsDelta: deltas.requestedRegions,
selectedRegionsDelta: deltas.selectedRegions,
...regionCounterFields('requestedRegion', deltas.requestedRegions),
...regionCounterFields('selectedRegion', deltas.selectedRegions),
regionFallbacksDelta: deltas.regionFallbacks,
unavailableRegionsDelta: deltas.unavailableRegions,
controlClosesByCodeDelta: deltas.controlClosesByCode,
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
// Accepts are sparse: publishing a zero percentile for every empty window
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
...(deltas.clientAcceptTotalsMs.length === 0
? {}
: {
clientAcceptTotalMsP50: acceptTotals.p50,
clientAcceptTotalMsP95: acceptTotals.p95,
clientAcceptTotalMsMax: acceptTotals.max,
clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
clientAcceptCredentialMsP95: acceptStageP95('credential'),
clientAcceptActivityMsP95: acceptStageP95('activity'),
clientAcceptAttachMsP95: acceptStageP95('attach'),
clientAcceptBasisMsP95: acceptStageP95('basis')
}),
// Every round trip observed in the window, including the ones the reservoir
// above declined to keep; the percentiles summarise only what it kept.
controlRttSamplesDelta: deltas.controlRttObserved,
controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
...(deltas.controlRttSamplesMs.length === 0
? {}
: {
controlRttMsP50: controlRtt.p50,
controlRttMsP95: controlRtt.p95,
controlRttMsMax: controlRtt.max
}),
sqlQueriesDelta: deltas.sqlQueries,
sqlFailuresDelta: deltas.sqlFailures,
sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
@@ -299,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
controlRenewalLatencyMsP50: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
),
controlRenewalLatencyMsP95: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
),
controlRenewalLatencyMsMax: Number(
Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
),
httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
eventLoopDelayMsP99: Number(p99.toFixed(3))
@@ -316,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
}
}
// Flat siblings of the nested region maps, always emitted for every region including zeros.
// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
// path, and an absent key would drop a series out of the inner join the region-skew alert does.
// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
function regionCounterFields(
prefix: 'requestedRegion' | 'selectedRegion',
counts: Record<string, number>
): Record<string, number> {
return Object.fromEntries(
Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
`${prefix}${segment}Delta`,
counts[region] ?? 0
])
)
}
function increment(counts: Record<string, number>, key: string): void {
counts[key] = (counts[key] ?? 0) + 1
}
cloud/apps/relay/src/relay-server.ts
+7 -2
View File
@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
import {
hasAdmissionCapacity,
HostDataAuthSchema,
parseRelayHostCapabilities,
RELAY_ADMISSION_BUDGETS,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_CLOSE_CODE,
RELAY_DEFAULT_REGION,
RELAY_PROTOCOL_LIMITS,
@@ -88,6 +90,7 @@ export function createRelayServer(
database: RelayDatabase,
options: {
now?: () => number
random?: () => number
connectionLedgerLimits?: { hardCap: number; controlReserve: number }
cellIncarnation?: string
} = {}
@@ -123,7 +126,8 @@ export function createRelayServer(
assignments,
queuedBytes,
observability,
options.now
options.now,
options.random
)
const app = createRelayApp(config, {
store,
@@ -484,7 +488,8 @@ export function createRelayServer(
sessions.acceptControl(
webSocket,
identity,
controlUpgrade?.inclusionWatermark
controlUpgrade?.inclusionWatermark,
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
)
})
} catch {
cloud/apps/relay/src/relay.blackbox.test.ts
+73 -2
View File
@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
import {
buildHostProofMacInput,
HOST_CHALLENGE_PLAINTEXT_DOMAIN
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
previousGeneration?: number
keyPair?: nacl.BoxKeyPair
assignmentEpoch?: number
capabilities?: string
}): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
const keyPair = input?.keyPair ?? nacl.box.keyPair()
const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
headers: {
authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
...(input?.capabilities
? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
: {})
},
perMessageDeflate: false
})
await new Promise<void>((resolveOpen, reject) => {
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
expect(result.reason).not.toContain('http')
})
it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
// The one link the unit tests cannot reach: an upgrade that really carries
// x-orca-host-capabilities must reach acceptControl and change the ack. A
// typo in the header name here passes every other test in the suite.
const host = await openHostControl()
const hostId = createHash('sha256')
.update(host.keyPair.publicKey)
.digest('base64url')
.slice(0, 16)
const inviteResponse = nextMessage(host.socket)
host.socket.send(
JSON.stringify({
type: 'invite-create',
reqId: 'capability-invite',
relayDeviceId: 'capability-device'
})
)
const invite = await inviteResponse
const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
headers: forwardedHeaders()
})
await new Promise<void>((resolveOpen, reject) => {
phone.once('open', resolveOpen)
phone.once('error', reject)
})
const connectionPromise = nextMessage(host.socket)
phone.send(
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
)
// Never attached: the connection stays pending, which is what the ack restates.
const connection = await connectionPromise
expect(connection.type).toBe('conn-open')
const capable = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(host.ack.controlResumeSecret),
previousGeneration: 1,
capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
})
expect(capable.ack.pendingConns).toEqual([
{
connId: connection.connId,
connTicket: connection.connTicket,
kind: 'invite',
relayDeviceId: 'capability-device'
}
])
const legacy = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(capable.ack.controlResumeSecret),
previousGeneration: 1
})
// A shipped host parses these entries strictly, so an unannounced key would
// fail the whole ack and kill a control that was working.
expect(legacy.ack.pendingConns).toEqual([
{ connId: connection.connId, connTicket: connection.connTicket }
])
phone.close()
legacy.socket.close()
})
it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
const host = await openHostControl()
const hostId = createHash('sha256')
cloud/dev/fixtures/terraform-root-partition/families.json
+3
View File
@@ -133,14 +133,17 @@
"google_logging_metric.relay_snapshot",
"google_monitoring_alert_policy.relay_assignment_5xx",
"google_monitoring_alert_policy.relay_assignment_edge_429",
"google_monitoring_alert_policy.relay_cell_control_rtt",
"google_monitoring_alert_policy.relay_cell_process_exit",
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
"google_monitoring_alert_policy.relay_cloud_sql_backends",
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
"google_monitoring_alert_policy.relay_cloud_sql_disk",
"google_monitoring_alert_policy.relay_custom",
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
"google_monitoring_alert_policy.relay_gce_connection_headroom",
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
"google_monitoring_alert_policy.relay_region_hint_skew",
"google_monitoring_dashboard.relay_incident",
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs
+2
View File
@@ -283,6 +283,8 @@ export const LEASED_WORKFLOWS = named([
'operate-relay-production-rehome.yml',
production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
],
// The gateway applies its schema at startup, so its deploy revision is the schema step.
['push-deploy.yml', production()],
['deploy-relay-asia-topology.yml', eitherEnvironment()],
['operate-relay-asia-admission.yml', eitherEnvironment()],
['deploy-relay-staging.yml', staging()],
cloud/dev/scripts/operate-relay-regional-rehome.mjs
+24
View File
@@ -45,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'not-before',
'rate-per-minute',
'preference-max-age-ms',
'host-cooldown-ms',
'drain-grace-ms',
'confirmation'
]
@@ -117,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'--preference-max-age-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
hostCooldownMs: integer(
values['host-cooldown-ms'],
'--host-cooldown-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
minimum: 60_000,
maximum: 60 * 60_000
@@ -147,6 +153,11 @@ function assertControl(control, expected) {
!Number.isSafeInteger(control.notBefore) ||
!Number.isSafeInteger(control.ratePerMinute) ||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
// A director predating the per-host cooldown does not report it. Reading
// the control and both emergency brakes must keep working against that
// image; only enable requires the field.
(control.hostCooldownMs !== undefined &&
!Number.isSafeInteger(control.hostCooldownMs)) ||
!Number.isSafeInteger(control.drainGraceMs)
) throw new Error('director returned an invalid regional rehome control')
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
@@ -155,6 +166,12 @@ function assertControl(control, expected) {
return control
}
// Echo the cooldown only when the director already reports it: a legacy
// director rejects the unknown key outright and would refuse every brake.
function cooldownField(before, value) {
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
}
async function verifiedDisabledControl(post, generation) {
return assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
@@ -171,6 +188,7 @@ async function applyDisabledControl(post, before) {
notBefore: before.notBefore,
ratePerMinute: before.ratePerMinute,
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
...cooldownField(before, before.hostCooldownMs),
drainGraceMs: before.drainGraceMs,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})).control, { generation: before.generation + 1, enabled: false })
@@ -270,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
throw new Error('regional rehome is already paused')
}
const enabled = config.mode === 'enable'
if (enabled && before.hostCooldownMs === undefined) {
throw new Error(
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
)
}
const applied = await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'apply',
@@ -278,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
notBefore: config.notBefore,
ratePerMinute: config.ratePerMinute,
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
...cooldownField(before, config.hostCooldownMs),
drainGraceMs: config.drainGraceMs,
confirmation: enabled
? 'ENABLE_REGIONAL_REHOMING'
cloud/dev/scripts/operate-relay-regional-rehome.test.mjs
+105
View File
@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
'--not-before', '2000000000000',
'--rate-per-minute', '10',
'--preference-max-age-ms', '86400000',
'--host-cooldown-ms', '604800000',
'--drain-grace-ms', '60000',
'--confirmation', confirmation
])
@@ -40,10 +41,29 @@ function control(generation, enabled) {
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
}
}
// The control a director predating the per-host cooldown reports.
function legacyControl(generation, enabled) {
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
return rest
}
function legacyDirector(controls) {
const requests = []
const post = async (path, body) => {
requests.push({ path, body })
if (path === '/v1/admin/admission-selector/status') {
return { selector: { generation: 11, membership } }
}
return { v: 1, control: controls.shift() }
}
return { requests, post }
}
test('parses exact selector and typed control confirmation', () => {
const parsed = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
assert.equal(parsed.expectedSelectorGeneration, 11)
assert.equal(parsed.expectedControlGeneration, 4)
assert.equal(parsed.ratePerMinute, 10)
assert.equal(parsed.hostCooldownMs, 604_800_000)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
(value, index, all) =>
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/complete durable control shape/
)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 0,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
...control
}))
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
}
})
assert.equal(result.control.generation, 5)
assert.equal(result.control.hostCooldownMs, 604_800_000)
assert.deepEqual(requests[2].body, {
v: 1,
action: 'apply',
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
})
})
test('inspects a director that predates the per-host cooldown', async () => {
const director = legacyDirector([legacyControl(4, true)])
const config = parseRegionalRehomeArguments(
argumentsFor('inspect'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 4)
assert.equal(result.control.hostCooldownMs, undefined)
})
for (const [mode, confirmation, enabledBefore] of [
['pause', 'PAUSE_REGIONAL_REHOMING', true],
['disable', 'DISABLE_REGIONAL_REHOMING', false]
]) {
test(`${mode} still brakes a director that predates the cooldown`, async () => {
const director = legacyDirector([
legacyControl(4, enabledBefore),
legacyControl(5, false),
legacyControl(5, false)
])
const config = parseRegionalRehomeArguments(
argumentsFor(mode, confirmation),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 5)
// The unknown key would be refused by that director's strict schema.
assert.equal('hostCooldownMs' in director.requests[2].body, false)
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
})
}
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
const requests = []
let current = legacyControl(7, true)
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
requests.push(body)
if (body.action === 'inspect') return { control: current }
current = legacyControl(8, false)
return { control: current }
})
assert.equal(result.control.generation, 8)
assert.equal('hostCooldownMs' in requests[1], false)
})
test('refuses to enable a director that does not report the cooldown', async () => {
const director = legacyDirector([legacyControl(4, false)])
const config = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
await assert.rejects(
operateRegionalRehome(config, { post: director.post }),
/per-host rehome cooldown/
)
// Read-only: selector status and the control inspect, and nothing else.
assert.equal(director.requests.length, 2)
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
})
test('fails closed on selector drift before reading or mutating control', async () => {
let calls = 0
const config = parseRegionalRehomeArguments(
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs
+12 -1
View File
@@ -6,6 +6,7 @@ import {
membershipWithStates,
selectorCellState
} from './relay-admission-selector.mjs'
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export const PRODUCTION_CAPACITY_CELL_IDS = [
@@ -31,6 +32,9 @@ function cellOrigin(cellId) {
return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev`
}
// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
export function parseProductionCapacityCellArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
@@ -42,8 +46,15 @@ export function parseProductionCapacityCellArguments(argv) {
if (!['isolate', 'drain', 'activate'].includes(values.mode)) {
throw new Error('--mode must be isolate, drain, or activate')
}
const approvedList = values['approved-cells']
if (approvedList !== undefined && !APPROVED_CELL_LISTS[approvedList]) {
throw new Error('--approved-cells is not a known allowlist')
}
const approvedCellIds = approvedList === undefined
? PRODUCTION_CAPACITY_CELL_IDS
: APPROVED_CELL_LISTS[approvedList]
const cellId = values['cell-id']
if (!PRODUCTION_CAPACITY_CELL_IDS.includes(cellId)) {
if (!approvedCellIds.includes(cellId)) {
throw new Error('production capacity target is not approved')
}
const expectedCellOrigin = cellOrigin(cellId)
cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs
+41
View File
@@ -104,6 +104,47 @@ describe('production Relay capacity cell admission', () => {
'--cell-id', 'production-gce-c7',
'--mode', 'isolate'
]), /origin is not exact/)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c27.relay.onorca.dev',
'--cell-id', 'production-gce-c27',
'--mode', 'isolate'
]), /not approved/)
})
it('admits the same-cap Asia cells only under the same-cap allowlist', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
const hostname = cellId.slice('production-gce-'.length)
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', `https://${hostname}.relay.onorca.dev`,
'--cell-id', cellId,
'--approved-cells', 'same-cap',
'--mode', 'isolate'
]), {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname}.relay.onorca.dev`,
cellId,
mode: 'isolate'
})
}
for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
const hostname = cellId.slice('production-gce-'.length)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', `https://${hostname}.relay.onorca.dev`,
'--cell-id', cellId,
'--approved-cells', 'same-cap',
'--mode', 'isolate'
]), /not approved/)
}
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c27.relay.onorca.dev',
'--cell-id', 'production-gce-c27',
'--approved-cells', 'every-cell',
'--mode', 'isolate'
]), /not a known allowlist/)
})
it('isolates only the selected cell without depending on its runtime', async () => {
cloud/dev/scripts/probe-relay-rehome-trust.mjs
+3 -1
View File
@@ -1,7 +1,9 @@
import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
// Every general cell that carries the rehome identity: the sixteen US cells and the
// three asia-east2 cells that drain mis-homed hosts back the other way.
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
+20
View File
@@ -111,3 +111,23 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
)
assert.equal(calls, 2)
})
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
const parsed = parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
)
assert.equal(parsed.cellId, cellId)
}
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
assert.throws(
() =>
parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
),
/--cell-id is not approved/
)
}
})
cloud/dev/scripts/relay-production-identity-boundaries.test.mjs
+2 -1
View File
@@ -32,7 +32,8 @@ test('no workflow names the retired generic production deploy identity', async (
'deploy-relay-production.yml',
'operate-relay-asia-admission.yml',
'operate-relay-production-rehome-job.yml',
'publish-relay-production.yml'
'publish-relay-production.yml',
'push-deploy.yml'
].map((name) => relayWorkflowFile(name)).sort())
})
cloud/dev/scripts/relay-public-workflow-contract.test.mjs
+1 -1
View File
@@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml')
const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
test('the copy carries every relay workflow', () => {
assert.equal(relayWorkflows().length, 24)
assert.equal(relayWorkflows().length, 25)
})
// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming
cloud/dev/scripts/relay-region-hint-metrics.test.mjs
+84
View File
@@ -0,0 +1,84 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of
// actual placements. Both shares are sums over one log-based metric per region, and the region
// list is written out by hand in Terraform. A region added to the contract without matching
// metrics would silently drop out of both denominators and move the ratio the alert fires on.
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
const collapse = (text) => text.replaceAll(/\s+/g, ' ')
const contractRegions = (() => {
const source = read('../../packages/relay-contract/src/relay-regions.ts')
const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source)
assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts')
return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1])
})()
const terraform = read('../../infra/terraform/relay-observability.tf')
const terraformRegions = (() => {
const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform)
assert.ok(literal, 'relay_region_keys not found in relay-observability.tf')
return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])
})()
// Both sides now spell the field-name segments out, so the test compares the two declared maps
// rather than two source expressions. Reformatting either file cannot break this, and a literal
// expected value below still catches an identical wrong edit made to both.
const declaredSegments = (source, open, close) => {
const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open)))
return Object.fromEntries(
[...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [
match[1],
match[2]
])
)
}
const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}')
const contractSegments = declaredSegments(
read('../../packages/relay-contract/src/relay-regions.ts'),
'RELAY_REGION_METRIC_SEGMENTS = {',
'}'
)
test('terraform covers exactly the regions the contract can hint or select', () => {
assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort())
})
test('terraform and the contract declare the same flat field segments', () => {
assert.deepEqual(terraformSegments, contractSegments)
// Pinned literally so the same wrong edit applied to both sides still fails.
assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' })
assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort())
})
test('the skew query compares a catalogued region against itself', () => {
const columns = terraformRegions.map((region) => region.replaceAll('-', '_'))
const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform)
const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform)
assert.ok(hint && placement, 'skew query share columns not found')
assert.equal(hint[1], placement[1], 'the two shares must be about the same region')
assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`)
})
test('the skew condition never divides by the placement share', () => {
// A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form
// silences exactly the case the alert exists for.
assert.ok(
!/hint_share \/ placement_share/.test(terraform),
'cross-multiply instead: hint_share > 2 * placement_share'
)
assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/)
})
test('the unhinted bucket stays out of the skew denominators', () => {
assert.ok(
!terraformRegions.includes('unhinted'),
'unhinted requests are a client-side choice, not a region; including them moves the share'
)
})
cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs
+4 -1
View File
@@ -73,7 +73,10 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(job, /host-drain \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/)
assert.match(
job,
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/
)
assert.match(job, /resume requires the isolated migration-only cell/)
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+241
View File
@@ -0,0 +1,241 @@
import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { describe, it } from 'node:test'
import { parseProductionCapacityCellArguments } from './prepare-relay-production-capacity-canary.mjs'
import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
import { validateCapacityPlan } from './validate-relay-capacity-plan.mjs'
const workflow = readRelayWorkflow('deploy-relay-production-same-cap-job.yml')
const capacityWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
const production = readFileSync(
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
'utf8'
)
const REHOME_SOURCE_CELLS = rehomeSourceCells()
const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
// The startup template emits rehome trust only for cells in this list, so it is what decides
// whether a cell's plan may carry those lines at all.
function rehomeSourceCells() {
const start = production.indexOf('relay_region_rehome_source_cell_ids = [')
assert.notEqual(start, -1, 'production.tfvars has no rehome source cell list')
const end = production.indexOf(']', start)
assert.notEqual(end, -1, 'the rehome source cell list is unterminated')
return new Set(
[...production.slice(start, end).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell)
)
}
function startupScript({ cap, image, trusted }) {
return [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
...(trusted ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${DIRECTOR_IDENTITY}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${AUDIENCE}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${image.split('@')[1]}'`,
`docker pull '${image}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${image}'`
].join('\n')
}
// The exact shape the apply step's plan has: template replaced, MIG rebound to it.
function rollPlan({ cellId, cap, protocol }) {
return {
configuration: {
root_module: {
resources: [{
address: 'google_compute_instance_group_manager.relay_gce_cell',
expressions: {
version: [{
instance_template: {
references: [
'google_compute_instance_template.relay_gce_cell',
'each.key'
]
},
name: { constant_value: 'primary' }
}]
}
}]
}
},
resource_changes: [
{
address: `google_compute_instance_template.relay_gce_cell[${JSON.stringify(cellId)}]`,
change: {
actions: ['create', 'delete'],
before: {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol === 1
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol === 1
}),
self_link: null
},
after_unknown: { self_link: true }
}
},
{
address: `google_compute_instance_group_manager.relay_gce_cell[${JSON.stringify(cellId)}]`,
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
]
}
}
function hostname(cellId) {
return cellId.slice('production-gce-'.length)
}
// The job resolves cap and region from the cell id before any admin call; run that block alone.
function resolveCellShape(cellId) {
const start = workflow.indexOf(' TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"')
assert.notEqual(start, -1, 'the same-cap cell shape block is missing')
const end = workflow.indexOf('\n esac\n', start)
assert.notEqual(end, -1, 'the same-cap cell shape block has no esac')
const script = workflow.slice(start, end + '\n esac'.length).replace(/^ {10}/gm, '')
return spawnSync('bash', [
'-euo',
'pipefail',
'-c',
`${script}\necho "\${EXPECTED_REGION} \${EXPECTED_HARD_CAP}"`
], { env: { ...process.env, TARGET_CELL_ID: cellId }, encoding: 'utf8' })
}
describe('same-cap roll scripts accept every same-cap cell', () => {
it('parses every wave cell through the same-cap canary allowlist', () => {
for (const cellId of SAME_CAP_CELLS) {
for (const mode of ['isolate', 'drain', 'activate']) {
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', `https://${hostname(cellId)}.relay.onorca.dev`,
'--cell-id', cellId,
'--approved-cells', 'same-cap',
'--mode', mode
]), {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
cellId,
mode
})
}
}
})
it('resolves a cap and region for every wave cell and refuses anything else', () => {
for (const cellId of SAME_CAP_CELLS) {
const resolved = resolveCellShape(cellId)
assert.equal(resolved.status, 0, `${cellId}: ${resolved.stderr}`)
assert.match(resolved.stdout.trim(), /^(us-central1 1000|asia-east2 3000)$/)
}
assert.equal(resolveCellShape('production-gce-c17').status, 1)
assert.equal(resolveCellShape('production-gce-c30').status, 1)
})
it('passes the same-cap allowlist on every canary invocation the job runs', () => {
const invocations = workflow.split('prepare-relay-production-capacity-canary.mjs').slice(1)
assert.equal(invocations.length, 4)
for (const invocation of invocations) {
const lines = invocation.split('\n')
const end = lines.findIndex((line) => !line.endsWith('\\'))
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--approved-cells same-cap/)
assert.match(call, /--mode (isolate|drain|activate)/)
}
})
it('passes this cell\'s rehome protocol on every plan validation the job runs', () => {
const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
assert.equal(invocations.length, 2)
for (const invocation of invocations) {
const lines = invocation.split('\n')
const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
const call = lines.slice(0, end + 1).join(' ')
assert.match(call, /--mode same-cap-cell/)
assert.match(call, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
}
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: Number(cap),
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: String(protocol)
}
const plan = rollPlan({ cellId, cap, protocol })
assert.deepEqual(
validateCapacityPlan(plan, config),
{ mode: 'same-cap-cell', changes: 2 },
cellId
)
// The other protocol must reject the same plan, or the flag decides nothing.
assert.throws(
() => validateCapacityPlan(plan, {
...config,
regionalRehomeProtocol: String(1 - protocol)
}),
/reviewed image and capacity/,
cellId
)
}
})
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
const cellId = 'production-gce-c17'
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
/reviewed image and capacity/
)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+32 -8
View File
@@ -4,7 +4,18 @@ import { pathToFileURL } from 'node:url'
const SERVICE_ACCOUNT_EMAIL =
/^[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com$/
function parseArguments(argv) {
const REHOME_CONFIG =
/^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) {
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
}
return Number(regionalRehomeProtocol)
}
export function parseCapacityPlanArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
@@ -31,8 +42,12 @@ function parseArguments(argv) {
values.mode === 'same-cap-cell' &&
(!values['rollback-image'] ||
!values['rehome-director-service-account'] ||
!values['rehome-audience'])
!values['rehome-audience'] ||
!['0', '1'].includes(values['regional-rehome-protocol']))
) throw new Error('same-cap validation requires rollback image and rehome trust config')
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
}
if (values.mode === 'same-cap-image' && !values['rollback-image']) {
throw new Error('same-cap image validation requires a rollback image')
}
@@ -51,7 +66,8 @@ function parseArguments(argv) {
capacityServiceAccount: values['capacity-service-account'],
rollbackImage: values['rollback-image'],
rehomeDirectorServiceAccount: values['rehome-director-service-account'],
rehomeAudience: values['rehome-audience']
rehomeAudience: values['rehome-audience'],
regionalRehomeProtocol: values['regional-rehome-protocol']
}
}
@@ -175,15 +191,13 @@ function normalizedStartupScript(
/^ printf 'ORCA_RELAY_CELL_CONNECTION_(?:HARD_CAP|UNOBSERVED_BOUND)=%s\\n' '[0-9]+'$/
const capacityIdentity =
/^ printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/
const rehomeConfig =
/^ printf 'ORCA_RELAY_REHOME_(?:DIRECTOR_SERVICE_ACCOUNT|AUDIENCE)=%s\\n' '[^'\n]+'$/
return script
.split('\n')
.filter(
(line) =>
(preserveCapacity || !capacityAssignment.test(line)) &&
(!stripCapacityIdentity || !capacityIdentity.test(line)) &&
(!stripRehomeConfig || !rehomeConfig.test(line))
(!stripRehomeConfig || !REHOME_CONFIG.test(line))
)
.join('\n')
.replaceAll(image, '<relay-image>')
@@ -213,7 +227,8 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
])
}
if (config.mode === 'same-cap-cell') {
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1
if (rehomeTrusted) {
expected.push(
[
/^ printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '[^'\n]+'$/,
@@ -225,9 +240,15 @@ function requireDesiredStartupScript(script, config) {
]
)
}
// A protocol-0 cell is not a rehome source, so gaining any rehome trust line is real drift.
const unexpectedRehome =
config.mode === 'same-cap-cell' &&
!rehomeTrusted &&
lines.some((line) => REHOME_CONFIG.test(line))
if (
typeof script !== 'string' ||
relayImage(script) !== config.image ||
unexpectedRehome ||
expected.some(([pattern, line]) => !hasExactSingleAssignment(lines, pattern, line))
) {
throw new Error('cell plan does not contain the reviewed image and capacity')
@@ -450,6 +471,9 @@ export function validateCapacityPlan(plan, config) {
) {
throw new Error('capacity Terraform plan has an invalid service account')
}
if (config.mode === 'same-cap-cell') {
rehomeProtocol(config)
}
if (
config.mode === 'same-cap-cell' &&
(!SERVICE_ACCOUNT_EMAIL.test(config.rehomeDirectorServiceAccount ?? '') ||
@@ -504,7 +528,7 @@ export function validateCapacityPlan(plan, config) {
}
export function main(argv = process.argv.slice(2)) {
const config = parseArguments(argv)
const config = parseCapacityPlanArguments(argv)
const plan = JSON.parse(readFileSync(0, 'utf8'))
process.stdout.write(`${JSON.stringify({ event: 'relay_capacity_plan_verified', ...validateCapacityPlan(plan, config) })}\n`)
}
cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
+137 -2
View File
@@ -1,6 +1,9 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { validateCapacityPlan as validateCapacityPlanRaw } from './validate-relay-capacity-plan.mjs'
import {
parseCapacityPlanArguments,
validateCapacityPlan as validateCapacityPlanRaw
} from './validate-relay-capacity-plan.mjs'
const config = {
cellId: 'staging-gce-c3',
@@ -466,7 +469,8 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
image,
rollbackImage,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience
rehomeAudience: audience,
regionalRehomeProtocol: '1'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
@@ -644,3 +648,134 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
{ mode: 'same-cap-image', changes: 1, changeKind: 'manager-convergence' }
)
})
test('protocol-0 same-cap cells roll without rehome trust lines', () => {
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
const audience = 'https://relay.example.com/v1/admin/host-drain'
const startup = ({ selectedImage, trust = false }) => [
` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
` printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
...(trust ? [
` printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
` printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
] : []),
`printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
`docker pull '${selectedImage}'`,
'docker run --detach \\',
' --name orca-relay \\',
` '${selectedImage}'`
].join('\n')
const template = {
address: 'google_compute_instance_template.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['create', 'delete'],
before: { metadata_startup_script: startup({ selectedImage: rollbackImage }) },
after: { metadata_startup_script: startup({ selectedImage: image }), self_link: null },
after_unknown: { self_link: true }
}
}
const manager = {
address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c27"]',
change: {
actions: ['update'],
before: { target_size: 1, version: [{ instance_template: 'old' }] },
after: { target_size: 1, version: [{ instance_template: null }] },
after_unknown: { version: [{ instance_template: true }] }
}
}
const asiaConfig = {
cellId: 'production-gce-c27',
hardCap: 3_000,
unobservedBound: 60,
mode: 'same-cap-cell',
image,
rollbackImage,
rehomeDirectorServiceAccount: directorIdentity,
rehomeAudience: audience,
regionalRehomeProtocol: '0'
}
assert.deepEqual(
validateCapacityPlan({ resource_changes: [template, manager] }, asiaConfig),
{ mode: 'same-cap-cell', changes: 2 }
)
const gainsTrust = structuredClone(template)
gainsTrust.change.after.metadata_startup_script = startup({
selectedImage: image,
trust: true
})
assert.throws(
() => validateCapacityPlan({ resource_changes: [gainsTrust, manager] }, asiaConfig),
/reviewed image and capacity/
)
// Under protocol 1 that same script is the reviewed roll: trust is added, not drift.
assert.deepEqual(
validateCapacityPlan(
{ resource_changes: [gainsTrust, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
{ mode: 'same-cap-cell', changes: 2 }
)
// A protocol-1 cell whose script has no rehome lines is the pre-existing failure, unchanged.
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: '1' }
),
/reviewed image and capacity/
)
for (const protocol of [undefined, '', '2', 'yes']) {
assert.throws(
() => validateCapacityPlan(
{ resource_changes: [template, manager] },
{ ...asiaConfig, regionalRehomeProtocol: protocol }
),
/invalid regional rehome protocol/
)
}
})
test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
const sameCapArguments = (...extra) => [
'--mode', 'same-cap-cell',
'--cell-id', 'production-gce-c27',
'--hard-cap', '3000',
'--unobserved-bound', '60',
'--image', image,
'--rollback-image', rollbackImage,
'--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
'--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
...extra
]
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '0'))
.regionalRehomeProtocol,
'0'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
)
for (const protocol of ['', '2', 'true']) {
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', protocol)),
/requires rollback image and rehome trust config/
)
}
assert.throws(
() => parseCapacityPlanArguments([
'--mode', 'bootstrap-cell',
'--cell-id', 'staging-gce-c3',
'--hard-cap', '1000',
'--unobserved-bound', '60',
'--image', image,
'--capacity-service-account', 'orca-cap@onorca-cloud.iam.gserviceaccount.com',
'--regional-rehome-protocol', '0'
]),
/applies only to same-cap-cell validation/
)
})
cloud/docs/orca-relay-operations.md
+12
View File
@@ -464,6 +464,18 @@ Once a target control is registered, do not force the pre-registration rollback.
After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch.
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
every flip, since the preference age never expires while the host keeps reconnecting.
## Game-day matrix
Run and record each scenario in staging before launch:
cloud/docs/relay-improvement-checklist-2026-09.md
+192
View File
@@ -0,0 +1,192 @@
# Relay improvement: implementation checklist, lanes, and disruption
Companion to [`relay-improvement-roadmap-2026-09.md`](./relay-improvement-roadmap-2026-09.md) (item numbers
match). This file answers three questions per item: what are the concrete steps, what can run in parallel,
and will a user notice.
## Status as of 2026-09-06 16:30Z
Three buckets. "Merged" means the code is on `main` and nothing in production has changed yet. "Deployed" means users are already getting it. "Awaiting owner" means I will not touch production without a go.
**Deployed to production**
- Roll 2 relay image `4916ed67` (stablyai/orca #18959 + #18722 + #18720 flag unset): director since 2026-09-06 01:02Z, all 19 general cells by 16:29Z. Control lease 6 h ± 30 min, accept abandonment, per-cell inventory locks, pool `statement_timeout`. Record: findings doc, "Roll 2" section.
- Auth instance cap 20 + dead-family audit fix (orca-cloud #474) as revision `orca-cloud-auth-00031-tox`.
- Dynamic NAT ports in both regions (stablyai/orca #18693). Zero drops and zero proxy dial errors since.
- Nine alert policies with log metrics: 4 auth (#475), 3 relay Cloud SQL/NAT (#18693), 1 cell process-exit (#18717), all on the relay Slack channel.
**Merged, not yet live**
- Cells dial Cloud SQL with `--private-ip` when configured (#18720). Deployed in Roll 2 with the flag unset; inert until 2.1 applies.
- Phone shows a clear "sign in on the desktop again" state when the desktop is signed out (#18698).
**Merged, ships with the next auth deploy**
- Refresh rotation grace window (orca-cloud #478). Startup adds one nullable column (brief exclusive lock on `refresh_tokens`).
- Pruning job code (orca-cloud #476) is in the image; the job itself is Terraform-disabled until 1.2.
**Merged, ships with the next desktop release**
- Never replay a refresh token after a timeout; ±10 % jitter on relay lease renewal (#18719).
- Renderer learns when a cloud session is revoked (#18694).
**Merged, not applied**
- Incident dashboard (#18717) blocked behind the runtime-metric label drift (5.x first item).
- Monitor probe fix (#18723) is live in the workflow; the same-cap roll gate has not yet produced a green dry-run since.
**Awaiting owner go (production mutations)**
1. Roll 1 cell image roll (1.1): dry-run gate, then c8 canary, then batches.
2. Auth deploy carrying #478 (3.1): quiet minute for the column add.
3. orca-cloud #477 private IP (2.1): merge arms an instance restart and a one-way door. Recommendation: hold.
4. Runtime-metric `region` label drift (5.x): intentional replacement of 21 metrics, or drop the label.
5. Enable pruning (1.2): first budget 20k rows; needs a Terraform apply.
6. Paging channel for auth alerts (5.2): needs the destination from you.
**Open code follow-ups (no gate, nobody assigned)**
- Monitor summary Markdown does not render `tolerated: true` continuity events (added by #18798); the state artifact has them, the checkpoint table does not.
- Relay container boot races the `cloud-sql-proxy` sidecar: c13's fresh container exited twice (`applyPostgresSchema` connection timeout, 2 s each) before the proxy was listening. Make schema apply wait for the proxy or order the containers.
- `cloud-deploy-relay-production-capacity-job.yml` (~line 416) has the same wave-0 single-shot preflight carve-out that #18778 removes from the same-cap job; its single-evidence path never retries freshness-only failures.
- `cloud/package.json` `test` names every dev-script test file explicitly; an unregistered `*.test.mjs` is silently never run in CI (found by #18769). Needs a glob or a ratchet that fails on an unlisted test file.
- Same-cap job's verify step uses bare `curl --fail-with-body` against the just-rolled cell; one 503 at the LB warm-up edge failed c8 canary #2 (run 33935407461) after the transition verifier had already passed. Needs a bounded retry, same rule as #18723/#18740.
- `verify-mutation` in `cloud-deploy-relay-production.yml`, the multi-target workflow, and the capacity workflow still binds to an exact commit; same exposure #18754 fixed for the same-cap and rehome paths.
- `incident-live-preflight-cli.ts` reports only `source/code` (`active-probe/threshold_max`) with no signal name or observed value, so a failed mutation preflight (c27 recovery #3, run 33986948522) cannot be attributed to an endpoint without an out-of-band probe. Print the signal and observed/threshold pair. Related: the 2 000 ms `endpointLatencyMs` bar is shared by US and Asia cells while Asia /health round trips from a US runner sit at 0.7–1.3 s idle; consider a per-region bar or the p50 of the gate window instead of one shot. Gates #44 and #45 (2026-09-05) both froze on `cell.production-gce-c27.latency_ms` at 2.6–2.7 s with c28 showing the identical tail under operator probes; the bar is now blocking Asia rolls. **Fix: stablyai/orca #18877** (per-region `cellEndpointLatencyMs`, us-central1 2 000 / asia-east2 4 000, plus signal/observed/threshold in preflight messages). Residual: `probeEndpointHealth` in `resource-inventory.ts` still uses the flat 2 000 bar to decide whether to retry after the 10 s readiness-cache wait, so a healthy Asia cell over 2 s costs one extra probe per sample (latency, not verdict); thread the region bar into the retry decision.
- The root oxlint config ignores `cloud/**`, so `check:code-quality:changed` never inspects relay-ops or the cloud dev scripts; typecheck + vitest is the only gate there.
- Monitor bars that froze on non-health today: `directorInstancesMin: 5` with `latest-sum` (one-minute instance recycle), `endpointLatencyMs: 2000` on a US-runner probe to asia-east2, `cloudDataMaxAgeMs: 180000` vs Cloud Monitoring publish lag up to 255 s. Recalibrate with a week of data.
- `parsed()` in `resource-inventory.ts` still returns null on a 200 with a malformed MIG body; a second path to `runtime_power_unknown`.
- Deploy script strips `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` on every release (3.1 first item).
- `assignOnce` placement lock still global (4.1 remainder).
- Region preference (4.2), retries-bar recalibration after a week of Roll 2 data (4.4), pruner `stopReason` alert (1.5).
- Full apps-root apply for 4 unrelated drifts (1.4), from a host with the 1Password account.
## Uplift ranking (reliability gained per unit of effort)
| Rank | Item | Why it ranks here |
|---|---|---|
| 1 | 1.1 cell image roll | Removes the only crash mode we have seen in production. 22 of 23 cells still have it. One afternoon. |
| 2 | 3.1 refresh rotation grace window | Turns the entire "slow auth → mass sign-out" class into a slowdown. One day. |
| 3 | 4.1 inventory lock contention | The floor under every 503 and slow phone accept, every day, not just incidents. One week. |
| — | 2.2 relay/auth database split | **Deferred 2026-09-04** to ~2026-11-01. Biggest structural fix, but the concrete cause is fixed and alerts now page; see roadmap 2.2 for re-open triggers. |
| 4 | 1.2 + 1.3 pruning and reclaim | Defuses the 63 M-row time bomb. Low effort, mostly waiting. |
| 5 | 5.1 + 5.2 crash alert, page a human | Cheapest detection uplift; today's incident ran 4 h unpaged. |
| 6 | 2.1 private IP | Durable version of a fix that already landed (dynamic NAT ports). Do it on the existing instance. |
| 7 | 4.3 + 3.2 desktop hardening | Small, ride the normal desktop release. |
| 8 | 4.2, 4.4, 5.4, 1.4, 1.5 | Housekeeping and quality-of-life. |
## The shared bottleneck: cell rolls
Every change to what runs on a cell (image, proxy flag, env, relay code) needs a same-cap roll: drain →
recreate → verify, one wave at a time, gated by the 15-minute monitor, about an afternoon. Each wave forces
the desktops on that cell to re-dial (c7 canary: 807 controls re-dialed in ~10 s) and phones on those
desktops reconnect on their normal retry. Users see a few seconds of "reconnecting" per wave.
So batch. Two rolls, not five:
- **Roll 1 (now):** current image only (1.1). Do not wait for anything else.
- **Roll 2 (week 2–3):** proxy `--private-ip` (2.1) + relay pool `statement_timeout` (2.3) + lock-contention
fix (4.1), all in one image/template. Prerequisite: 2.1's peering and private IP exist first.
## Lanes (independent; different people can own them)
```
Lane A data plane 1.1 roll ──────────────────► Roll 2 (2.1 flag + 2.3 + 4.1) ──► 4.4 recalibrate
Lane B auth/DB 1.2 enable pruning ──(10 d)──► 1.3 reclaim 3.1 grace window (any time)
Lane C network 2.1 peering + private IP ─────┐ (feeds Roll 2) (2.2 DB split deferred)
Lane D desktop 3.2 no same-token retry, 4.3 lease jitter (any release; wire-compatible)
Lane E observability 1.5, 5.1, 5.2, 5.4 (Terraform only, any time)
Lane F director 4.2 region preference (Cloud Run deploy, any time)
Misc 1.4 full apps-root apply (any time; see its check)
```
Hard dependencies: Roll 2 waits on 2.1's network work; 1.3 waits on 1.2 finishing. Everything else is
independent. (2.2 deferred; if revived, do it after 2.1 so the new instance is private from day one.)
## Disruption summary
| Item | User-visible? | What they see | Mitigation |
|---|---|---|---|
| 1.1 / Roll 2 | **Yes, transient** | Per wave, desktops on that cell reconnect within seconds; phones follow on retry. | Waves gated by the monitor; run in the US night. Already rehearsed on c7. |
| 1.2 pruning | No | Background deletes, 5k rows per batch. | Small first budget; watch `stopReason` and Cloud SQL write throughput. Stop the scheduler if checkpoint alerts fire. |
| 1.3 reclaim | **Depends on tool** | `VACUUM FULL` takes an exclusive lock on `refresh_tokens`: sign-in and refresh block for its duration (minutes to tens of minutes on 16 GB). `pg_repack` holds only brief locks. | Use `pg_repack`. If VACUUM FULL, announce a maintenance window. |
| 1.4 full apps apply | Should be none, **verify** | Terraform will create a new auth revision (env added). Traffic is pinned to `00031-tox` by name, so the new revision should receive 0 %. | Confirm in the plan that no `traffic` change appears. If it does, stop: the Terraform image variable is not the serving image. |
| 1.5, 5.x alerts | No | | |
| 2.1 private IP | **Yes, certain** | Google: "Configuring an existing Cloud SQL instance to use private IP causes the instance to restart, resulting in downtime." No in-place path, HA does not avoid it. Expect 1–2 min DB unavailability: sign-in fails, relay renewals retry. **One-way door**: private IP cannot be disabled and the VPC link cannot be removed once set. The proxy flag change rides Roll 2. | Off-peak; only after Roll 1 (old image dies on a 2 min DB blip). Owner decision required before the foundation apply. |
| 2.2 DB split (deferred) | **Yes, scheduled** | Relay unavailable for the cutover (drain all cells → copy relay tables → flip `DATABASE_URL` → restart). Minutes if rehearsed. Desktops and phones reconnect automatically after. | Rehearse on staging; do it in the US night; announce. |
| 2.3 statement timeout | No beyond Roll 2 | | |
| 3.1 grace window | No | Auth deploys are no-traffic candidate → smoke → promote. | Security trade-off: a stolen token replayed inside the window is served once instead of revoking. 60 s is the usual choice. |
| 3.2, 4.3 desktop | No | Normal app update. | |
| 4.1 lock fix | No beyond Roll 2 | | Verify against real Postgres on 55440 with concurrent probes before shipping. |
| 4.2 region preference | **Minor, Asia users** | Phones that start being placed in Asia reconnect once to a nearer cell. | Roll out behind the existing region-preference flag. |
| 4.4 | No | | |
## Checklists
### 1.1 Cell image roll (Roll 1)
- [x] Confirm fleet is quiet: 15-min monitor dry-run passes. #19 green 23:07:53Z (run 33927238469). Canary then failed the evidence provenance check because main moved during the gate; re-gating with a same-commit chain.
- [x] Confirm director is on 519f4914 and c7 on 85bf6799 (confirmed 2026-09-04 via instance-template census; 20 serving cells still on `5aedbca5`) (`verify` mode of the same-cap workflow).
- [x] Dispatch `cloud-deploy-relay-production-same-cap` waves per the plan in the findings doc; one wave, verify, next. Done 2026-09-05 01:14Z–22:27Z: c8 canary, US batches c9–c10, c13–c16, c19–c26 at protocol 1, then Asia c27 (recovered via `mode=rollback` re-entry after gate freezes on the flat latency bar, fixed by #18877), c28, c29 as single-cell canaries at protocol 0.
- [x] After each wave: the transition verifier passed at migration-only and again at general on every cell (assignments carried, heartbeat fresh, hard cap 3 000); no `container die` fleet-wide across the whole roll. The 4408/1006 burst per wave was not measured separately; the verifier's assignment count before and after each restart is the recovery evidence recorded.
- [x] Record image census in the findings doc. 2026-09-05 22:27Z: all 19 general cells on `519f4914` except c7 on `85bf6799`; existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched on their older images by design. Selector at gen 148.
### 1.2 Enable pruning
- [x] `auth_token_pruner_image` = digest of `orca-cloud-auth-00031-tox` (`343a0915…`; it contains the entrypoint). orca-cloud #479 merged.
- [x] `auth_token_pruner_enabled = true`, `auth_token_pruner_max_rows_per_run = 20000` for the first day (orca-cloud #479).
- [x] Targeted plan asserted 9 create / 0 change / 0 destroy. Applied 2026-09-05 02:06Z.
- [x] Trigger one run by hand; read the summary event. 02:18Z: `time-budget`, 73 batches, 365k scanned, 1 040 deleted (1 021 revoked, 19 expired), no errors. Scan-bound.
- [ ] Raise the budget to the default 200k after a clean day; watch Cloud SQL write MB/s and the checkpoint alert.
- [ ] 1.5: log metric + policy on `stopReason != complete`.
### 1.3 Reclaim
- [ ] Wait for steady-state runs deleting ~0 rows.
- [ ] `pg_repack -t refresh_tokens` off-peak (needs the extension; check `pg_available_extensions`). Not `VACUUM FULL` without a window.
- [ ] Confirm table + index size and `disk/utilization` dropped.
### 1.4 Full apps-root apply
- [ ] Run from CI or a host with the 1Password account (local plan fails on the Cloudflare data source).
- [ ] Plan shows exactly the four known drifts and **no traffic change** on `google_cloud_run_v2_service.auth`.
- [ ] Apply; confirm `status.traffic` still pins `00031-tox` at 100 %.
### 2.1 Private IP (PRs open: orca-cloud #477 foundation, stablyai/orca #18720 relay flag)
- [ ] **Owner decision**: the foundation apply restarts the instance and is irreversible on Google's side. Merging #477 arms the next foundation apply; hold the merge until the window is chosen.
- [ ] Director is out of scope: it uses the Cloud Run built-in connector (managed Google path, not the relay VPC NAT), so it consumed none of the exhausted ports; moving it needs Direct VPC egress + a separate DSN secret. Own PR if ever wanted.
- [ ] Step 7 (`ipv4_enabled=false`) is blocked until humans have IAP/bastion access and the director is moved; it breaks both today.
- [ ] Allocate a `/24` private services range on the relay VPC; `google_service_networking_connection`.
- [ ] Add `ip_configuration.private_network` to `google_sql_database_instance.auth` (foundation root). Plan must show update, not replace.
- [ ] Apply off-peak; expect a possible restart. Watch auth 5xx alert and relay `sqlFailures`.
- [ ] Cell template: proxy args add `--private-ip` (code merged #18720; flag not set). Director: Direct VPC egress or connector, then the same flag. Both ride Roll 2.
- [ ] After Roll 2: NAT `port_usage` for relay gateways drops to ~0; then consider `ipv4_enabled = false` (removes the public IP; breaks the local `cloud-sql-proxy --token` workflow unless it also goes private).
### 2.2 Database split (deferred to ~2026-11-01; checklist kept for when it is revived)
- [ ] New `google_sql_database_instance.relay` (private IP from day one, its own size and flags). Staging first.
- [ ] Relay schema applies cleanly to an empty instance (it does at startup).
- [ ] Rehearsal on staging: drain → `pg_dump` relay tables → restore → flip `relay_database_url` secret → restart director + cells → phones/desktops reconnect. Time it.
- [ ] Production: announce a window; same steps; verify `orca_relay_runtime_metrics` controls recover to pre-cutover count.
- [ ] Update `production-cloud-sql-app-consumers` budget test and both alert policies' `database_id`.
### 2.3 Relay pool statement timeout (deployed in Roll 2, 2026-09-06)
- [x] `statement_timeout` on the relay `pg.Pool` (5 s, env-configurable; schema pool untimed; `57014` retryable), below the control-renewal deadline; DDL on an untimed connection (same pattern as auth #476).
- [x] Postgres test on 55440: a held lock fails the query fast and the bounded retry takes over.
- [x] Deployed fleet-wide in Roll 2 (`4916ed67`), 2026-09-06.
### 3.1 Refresh rotation grace window (orca-cloud #478 merged 2026-09-04; deploy pending owner go)
- [ ] Fix the deploy-script env strip for `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` (pre-existing; found by #478).
- [x] `rotateRefreshToken`: if `rotated_at` within 60 s and not revoked, return the existing successor (idempotent), no revoke, no audit.
- [x] Outside the window or a third presentation: unchanged (revoke + audit).
- [x] Tests: replay inside window returns same successor; outside revokes; concurrent double-present yields one successor.
- [x] Deploy via `deploy-auth-production` (candidate → smoke → promote). Deployed 2026-09-04 23:15Z as `orca-cloud-auth-00035-gos`, cap 20 kept, 0 5xx; `successor_material` column present; sealed successors being written. (candidate → smoke → promote).
### 3.2 / 4.3 Desktop (merged stablyai/orca #18719; ships next desktop release; relay side of 4.3 deployed in Roll 2)
- [x] 3.2: on refresh timeout, re-read stored session before retrying; do not re-send a token already rotated locally.
- [x] 4.3: ±10 % jitter on control lease renewal; unit test on the distribution; wire-compatible (server accepts early renewals already).
- [x] 4.3 relay side: control lease 55 min → 6 h ± 30 min (#18959), deployed in Roll 2, 2026-09-06.
### 4.1 Lock contention (partial: stablyai/orca #18722 deployed in Roll 2, 2026-09-06)
- [x] Replace the global `FOR UPDATE` over `relay_cells` with per-cell row locks; counters delta-only. Remaining: `assignOnce` placement lock is still global (optimistic snapshot follow-up). with per-cell row locks or `pg_advisory_xact_lock(cell)`; counters delta-only.
- [x] Postgres tests on 55440 with concurrent probes (in #18722). Staging load run still owed; `postgres_retries` per hour drops in staging load run.
- [x] Shipped in Roll 2 (2026-09-06). Director retries first 6 h on the new image: 13 vs 85 on the predecessor's prior 6 h.
- [ ] 4.4: recalibrate the retries bar from a week of data (after 2026-09-13).
### 4.2 Region preference
- [ ] Director: honor requested region when the preferred region has headroom, else sticky. Behind the existing flag.
- [ ] Measure with `orca_relay_runtime_metrics` region counters before/after.
### 5.x Observability
- [x] **Relay-root runtime-metric drift**: resolved by dropping the `region` label to match live state (stablyai/orca #18734). Applied 2026-09-04 23:11Z: 8 never-applied `control_*` renewal metrics + the incident dashboard created, 0 destroyed, 21 live metrics untouched.
- [x] 5.1 `container die` log metric per cell (`relay_cell_process_exit`, applied 2026-09-04 via #18717), > 3 / 15 min, relay channel.
- [ ] 5.2 Add a paging channel (**needs owner input**: destination) to `auth_alert_notification_channels` for refresh rejections + latency.
- [x] 5.4 One dashboard (applied 2026-09-04 23:11Z): `orca_relay_cloud_sql_wal_checkpoint`, NAT drops, `orca_auth_refresh_401`, summed `controls`.
cloud/docs/relay-improvement-roadmap-2026-09.md
+67
View File
@@ -0,0 +1,67 @@
# Relay improvement roadmap (written 2026-09-04, after the auth/relay outage)
Owner-facing list of what is left to make the relay more robust, in priority order. Evidence and history
for every item is in [`relay-reconnect-2026-09-findings.md`](./relay-reconnect-2026-09-findings.md)
(Findings 1–13). Everything already landed on 2026-09-04 is listed at the end so this file is complete on
its own.
## 1. Finish what 2026-09-04 started (this week)
| # | Item | Why | How | Size |
|---|---|---|---|---|
| 1.1 | **Roll all 23 cells onto the current relay image** | Every cell still runs the image that exits the whole process on a Postgres connect timeout (Finding 6). The fixed image runs only on the director and c7. Any future DB stall repeats the 200-crashes-in-48h pattern. | `cloud-deploy-relay-production-same-cap` waves, gated by the 15-min monitor. Roll inputs and canary results are in the findings doc ("Roll inputs", "Canary blast radius"). | one afternoon |
| 1.2 | **Enable the refresh_tokens pruning job** (orca-cloud #476, merged, off) | `refresh_tokens` is 63 M rows / 26 GB and grows forever; its size is what turned a slow disk into a sign-out storm (Finding 13). | Build an auth image from main (the 21:04Z deploy already contains the entrypoint: `orca-cloud-auth-00031-tox`, digest `343a0915…`), set `auth_token_pruner_enabled = true` and the image digest in `infra/terraform-apps/environments/production.tfvars`, apply targeted. First run with a small `auth_token_pruner_max_deleted_rows`. Watch the run summary's `stopReason`, not the exit code. ~48 M rows drain in ~10 days at 200k/hour. | 1 hour + 10 days of watching |
| 1.3 | **Reclaim the disk after pruning** | Deletes leave dead tuples; the 16 GB table does not shrink on its own. | `pg_repack` (or `VACUUM FULL` in a maintenance window; it takes an exclusive lock) on `refresh_tokens` off-peak, after 1.2 finishes. | 1 evening |
| 1.4 | **Full Terraform apply of the orca-cloud apps root** | The production plan carries four drifts from other merged work: `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` env on the auth service (#476), a skill-share log exclusion filter change, skill pressure threshold 16→8, an artifacts bucket lifecycle rule. Locally it also fails on the 1Password Cloudflare data source. | Run from CI or a machine with the 1Password account; review the four drifts as ordinary changes. | 30 min |
| 1.5 | **Alert on the pruning job** | A run that only ever times out exits 0 and reads as green. | Log metric on the job's summary event where `stopReason != "complete"`, policy on the relay channel. | 1 hour |
## 2. Remove the shared fate between auth and relay (2.1 and 2.3 this quarter; 2.2 deferred)
| # | Item | Why | How | Size |
|---|---|---|---|---|
| 2.1 | **Private IP for Cloud SQL, `--private-ip` on the cell proxies** (do this on the existing shared instance; do not wait for 2.2) | Cells reach the database's public IP through Cloud NAT. Dynamic port allocation (landed) raised the ceiling from 64 to 4096 ports per VM, but the NAT is still in the path and its logs are still the only place port exhaustion shows up (Finding 11). | Add a private IP to `orca-cloud-auth-db` (foundation root, orca-cloud), peer the relay VPC, switch the proxy flag in the cell template, roll. | 1–2 days |
| 2.2 | **Split the relay database from the auth database** — *DEFERRED 2026-09-04 (owner decision): revisit ~2026-11-01 once pruning is done and there is a month of alert history* | One Cloud SQL instance serves `orca_auth`, `orca_relay`, `orca_push`, `orca_skills`. The auth table's growth stalled the relay for a day (Findings 10, 13). Deferral rationale: the concrete cause is fixed (disk 250 GB, WAL 16 GB, index, pruning), 2.3 + 1.1 turn a future stall into retries, and the checkpoint/disk/headroom alerts now page. Re-open if the checkpoint-loop or connection-headroom alert fires, or a large new auth-side table is planned. | New instance for `orca_relay`; migrate with a short relay drain. Relay state is small so the cutover is minutes. | 1–2 weeks incl. rehearsal on staging |
| 2.3 | **Statement timeouts on the relay pool** (the auth pool got one in #476) | A relay query stuck behind a checkpoint fsync should fail fast and let the bounded retry take over rather than hold a pool slot for seconds. | `statement_timeout` on the relay `pg.Pool` in `cloud/apps/relay`, tuned under the lease renewal deadline. | half a day |
## 3. Make the desktop refresh path forgiving (next 2 weeks)
| # | Item | Why | How | Size |
|---|---|---|---|---|
| 3.1 | **Refresh-token rotation grace window** | The server revokes the whole family the first time a just-rotated token is presented again. On 2026-09-04 that turned a 30 s server slowdown into 21,605 sign-outs. A short window (e.g. 60 s) where the immediately-previous token is still accepted, returning the same new token, is standard practice. | In `apps/auth/src/tokens/refresh-tokens.ts`: accept `rotated_at` within the window, return the successor instead of revoking. Keep true reuse (outside the window, or a third presentation) as revocation. | 1 day incl. tests |
| 3.2 | **Do not retry `/refresh` with the same token on timeout** | Desktop's 30 s `CLOUD_REQUEST_TIMEOUT_MS` expiring is treated like a network error and retried with a token the server may already have rotated. | In `src/main/orca-profiles/profile-cloud-session-refresh.ts`: on timeout, re-read the stored session first, and prefer a longer single attempt for the refresh call specifically. | half a day |
| 3.3 | **Un-revoke is impossible; make sign-out recovery obvious instead** | Server-side un-revoke does not help because the desktop deletes its local token on the 401. Landed: desktop notices immediately (#18694) and the phone says "desktop signed out" (#18698). | Nothing more unless we want a re-auth deep link from the phone to the desktop. | — |
## 4. Chronic relay issues already characterised
| # | Item | Why | How | Size |
|---|---|---|---|---|
| 4.1 | **Cell-inventory lock contention** (partial: PR #18722 narrowed the remaining non-placement sites; `assignOnce` placement lock is the follow-up) | `postgres_retries` is a global `FOR UPDATE` over the 23-row `relay_cells` table with a 1 s `lock_timeout`; it is the floor under every 503 and every slow phone accept (Findings 2, 5; memory `relay-cell-inventory-lock-contention`). | Per-cell row locks or an advisory lock keyed by cell; move capacity counters to delta writes. Verify against real Postgres on 55440. | 1 week |
| 4.2 | **Region preference is mostly inert** | Phones request an Asia cell on ~19 % of attempts and get one ~6 % of the time; the sticky lane wins silently, so Asia users ride the US path more than intended (memory `relay-region-preference-mostly-inert`). | Let a region preference override stickiness when the preferred region has headroom; measure with `orca_relay_runtime_metrics` region counters. | 2–3 days |
| 4.3 | **Desktop lease-rotation waves** | A cell recreate seeds a fleet-wide 1006/4408 reconnect burst ~54 min later, every ~54 min (Finding 3). | Jitter the desktop control lease renewal by ±10 % so the cohort spreads out. | half a day, desktop + wire-compatible |
| 4.4 | **Raise `postgres_retries` gate calibration** | The 300 bar was recalibrated (PR #18580) but should track the post-lock-fix baseline once 4.1 lands. | Re-derive from a week of `orca_relay_postgres_transaction_retry` counts. | 1 hour |
## 5. Observability still missing
| # | Item | Why | How |
|---|---|---|---|
| 5.1 | **Cell crash-rate alert** | 201 process exits in 48 h with no page (Finding 6). | Log metric on `container die` for `resource.type="gce_instance"` relay cells, > 3 per 15 min per cell. In `cloud/infra/terraform/relay-observability.tf`. |
| 5.2 | **Page a person for auth alerts** | Today's four auth policies (orca-cloud #475) route to the relay Slack channel only. A repeat of 2026-09-04 deserves a page. | Add a PagerDuty/phone notification channel to `auth_alert_notification_channels` for refresh rejections and latency. |
| 5.3 | **Pruning job alert** | See 1.5. | |
| 5.4 | **Dashboard that puts the four signals side by side** | Diagnosis took hours because checkpoint state, NAT drops, auth 401 rate, and fleet controls live in four consoles. | One Cloud Monitoring dashboard: `orca_relay_cloud_sql_wal_checkpoint`, NAT `dropped_sent_packets_count`, `orca_auth_refresh_401`, summed `controls`. |
## Landed on 2026-09-04 (for completeness)
- Auth service cap 2 → 20 (service-level manual scaling removed); Cloud SQL disk 49 → 250 GB PD-SSD;
`max_wal_size` 16384; partial index `refresh_tokens_family_unrevoked` built concurrently by hand.
- orca-cloud #474: the above in Terraform + deploy workflow; replayed dead token answers 401 without
re-revoking or re-auditing. Deployed as `orca-cloud-auth-00031-tox` 21:04Z.
- orca-cloud #475: auth alerts (refresh 401 > 100/5 min, 429 > 20/5 min, 5xx > 10/5 min, p99 > 10 s). Applied.
- orca-cloud #476: batched `refresh_tokens` pruner (disabled), auth pool `statement_timeout` 10 s, schema
DDL on an untimed connection.
- stablyai/orca #18693: both relay NATs on dynamic port allocation 64..4096 (applied US 21:01Z, Asia 21:05Z);
alerts for Cloud SQL WAL-checkpoint loop, disk > 70 %, NAT `OUT_OF_RESOURCES` drops. Applied.
- stablyai/orca #18694: desktop learns of a revoked session immediately, panes re-fetch on mount, pairing
notice says "Sign in again to use Orca Relay".
- stablyai/orca #18698: phone shows "Desktop signed out — sign in to Orca on your desktop to reconnect" via
the WebSocket close reason (only additive slot old phones tolerate).
- Director on image 519f4914; c7 on 85bf6799; other 22 cells still on the old image (see 1.1).
cloud/docs/relay-incident-monitor.md
+73
View File
@@ -121,6 +121,79 @@ durably marked consumed before mutation and cannot authorize another run.
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
heartbeats, and matching live admission.
## Region placement alert policies
Cloud Monitoring alert policies, not monitor freeze bars: these page from
`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in
`relay_alert_notification_channels`, and they do not gate any workflow. All
three exist because US desktops sat on asia-east2 cells for weeks in 2026-08
with every existing bar green.
| Alert policy | Condition |
| --- | ---: |
| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts |
| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples |
| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests |
Threshold basis:
- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a
cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and
well under the far-cell floor. The 20-accept minimum keeps one slow accept
on a quiet cell off the pager. The p95 is the published value, so the
window aggregate is its median, not its max.
- Control round trip. In-region is tens of milliseconds; a US desktop on an
asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes
the pong on its main thread, so the published p95 and max track renderer
stalls rather than distance. 500 samples per hour is about two
continuously connected hosts at the 15-second control ping. Tuning risk: EU
desktops on us-central1 sit at 100-130 ms, so a cell whose population is
mostly European can approach the bar while correctly homed. Check where the
hosts are before reading a first breach as mis-homing.
- Region hint skew. This compares two shares of the same hour rather than
testing one absolute share, because an absolute bar is wrong at both ends.
Measured over twelve hours on 2026-09-07, while the desktop region probe
was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests
and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of
25.9 points. A fixed 40% bar would have stayed silent through that, and
once the probe is fixed the genuine APAC share climbs past any such bar and
pages forever on the correct end state. The 2x and 15-point bars sit inside
the broken state and outside a healthy one. `unhinted` requests are
excluded from the denominator: they were 27% of all requests, so a client
change that always sends a hint would move the number with no behaviour
change at all. The two bars are cross-multiplied rather than divided. An
hour that placed nobody in the region is the most extreme skew there is,
and it happens whenever the region is drained, fenced, or at capacity, but
dividing by that zero placement share makes MQL drop the row and lose the
series before any other clause runs.
Expect the skew alert to stay lit after a client fix until the mis-homed
backlog is rehomed. Sticky assignment never re-consults the hint, so a
desktop already on an asia cell keeps being placed there whatever it now
asks for; the ratio clears only once the rehome sweep has drained.
All three conditions are written in MQL rather than the metric filters the
other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and
the only scalar aligners a filter condition can apply to one are percentiles;
each of these alerts needs the sum of the extracted values as a volume floor,
which is `sum(value.<metric>)` in MQL and unreachable otherwise. None of the
metrics they read exists in the project yet, so what was checked against
production is the query shape: the same MQL run over existing metrics of the
same kind confirmed the distribution sum, the join arity, the unit literals,
and the condition clause.
The skew shares are built from one log-based metric per region for hints and
one per region for placements. They read flat `requestedRegion<Region>Delta`
and `selectedRegion<Region>Delta` fields that the relay publishes as zeros in
every interval, not the nested region maps: a log-based metric would need a
quoted field path to reach a hyphenated map key, and an absent key would drop
a series out of the inner join. The region list lives in Terraform as
`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by
`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field
name segments out as literal maps rather than deriving them, so the same test
compares the two declarations directly. Adding a region to the contract
without its segment is a compile error in relay-contract, not a silent gap.
## Implementation log
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
cloud/docs/relay-reconnect-2026-09-findings.md
+1032
View File
File diff suppressed because it is too large Load Diff
cloud/docs/relay-roll2-plan-2026-09.md
+159
View File
@@ -0,0 +1,159 @@
# Relay Roll 2 and close-out plan (2026-09-05)
Owner-approved scope 2026-09-05: finish the relay reliability work with one more cell image roll,
deferring the Cloud SQL private-IP move (2.1, orca-cloud #477) to a separate owner decision. Roll 1
is complete (see `relay-reconnect-2026-09-findings.md`, "Roll 1 complete"); every serving cell runs
`519f4914` except c7 on `85bf6799`.
Estimate: about two working days of effort over one week of calendar time. The cell roll itself is
6 to 7 hours of mostly unattended wall clock, run in the US night.
## Phase 0. Land the code (half a day, no production change)
### 0a. Split PR #18565
The branch mixes three relay/mobile/desktop fixes with the operator record. Split so the record
lands regardless of how the code review goes.
- **Docs PR** (new branch off main): `relay-reconnect-2026-09-findings.md`,
`relay-improvement-checklist-2026-09.md`, `relay-improvement-roadmap-2026-09.md`, this file.
Docs only, merge on CI green.
- **Code PR** (rebase #18565 onto main, resolve two conflicts):
- `cloud/apps/relay/src/host-session-registry.ts`: conflict with #18698 (signed-out signal).
Keep both; the accept-abandonment and lease changes are orthogonal to the signed-out path.
- `src/main/runtime/relay/relay-origin-pool.ts`: **drop this branch's version**. #18719 already
merged the desktop early-window jitter (1 to 6 min). Also drop
`relay-session-broker.test.ts` additions that only exercise the dropped change.
- Keep: relay accept abandonment (`orca_relay_client_accept_abandoned` event), relay-side lease
jitter, mobile direct-probe fail-fast, and their tests.
### 0b. Lengthen the control lease (same code PR)
In `cloud/apps/relay/src/host-session-registry.ts`:
```
CONTROL_LEASE_MS = 6 * 60 * 60 * 1000 // was 55 min
CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000 // was 5 min
```
Why 6 h: the lease bounds how long a host stays on a cell after a missed drain and is the only
passive rebalancing; 6 h keeps both and cuts control-activation traffic on the inventory lock by
about 6x. Nothing else depends on it: the relay JWT (5 min) is refreshed by the desktop on its own
schedule and liveness is the 75 s silence watchdog. Wire-safe: the relay sends `leaseExpiresAt` in
the hello ack and old desktops schedule from that value.
Update the comment above the constants and the three assertions in
`host-session-client-accept.test.ts` that pin the lease arithmetic. Check that nothing in
`cloud/apps/relay-ops` or the monitor thresholds assumes a 55 min rotation period (grep
`55`, `CONTROL_LEASE`, `rotation`).
### 0c. Review and merge
Review rounds per the standing process (Opus review, then Codex pass). Merge order: docs PR first
(no dependency), then the code PR. Record the merge SHA of the code PR; that is the Roll 2 image
source.
## Phase 1. Build and stage the image (half a day)
Roll 2 image = code PR merge SHA. It carries, relative to `519f4914`:
| Change | PR | Effect |
|---|---|---|
| Per-cell inventory locks, delta counters | #18722 | Removes the global `relay_cells FOR UPDATE` behind the phone accept hang |
| Relay pool `statement_timeout` 5 s | #18722 | A relay query can no longer hang a cell |
| Accept abandonment | #18565 | Cell stops finishing accepts for phones that already closed |
| Control lease 6 h ± 30 min | #18565 | Fewer, spread-out rebinds |
| `--private-ip` proxy flag support | #18720 | Code only; flag stays unset until 2.1 |
Steps, in order (from the findings doc's post-merge dispatch plan):
1. `gh workflow run cloud-publish-relay-production.yml --ref main -f mode=publish`. Resolve the
digest by tag, not from the log:
`gcloud artifacts docker images describe us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay:sha-<merge-sha> --format='value(image_summary.digest)'`.
2. Staging: `cloud-deploy-relay-staging.yml` with the new digest; paired phone plus desktop smoke
(connect, background, reconnect). Confirm `orca_relay_client_accept_abandoned` appears only when
a client closes early, and that `sqlLatencyMsMax` no longer pins at the lock timeout.
3. Director: `cloud-deploy-relay-production-director.yml -f image-digest=<new>
-f regional-placement-mode=preserve -f prune-incompatible-revisions=false
-f expected-rehome-generation=12 -f bootstrap-runtime-identity=false
-f predecessor-image-digest=<serving digest>`. Blue/green; prior revision stays as rollback.
Watch director `orca_relay_postgres_transaction_retry` per minute before and after. The director
goes first so the per-cell locks are live before any cell restart burst.
4. Same-cap `verify` mode against c7 with target=<new>, rollback=`519f4914`. Read-only.
Go/no-go for Phase 2: director serving the new image for at least 30 min, retries per minute at or
below the pre-deploy baseline, no `container die`, no auth 5xx.
## Phase 2. Roll the cells (one US night, mostly unattended)
Same machinery as Roll 1: `cloud-monitor-relay-production.yml` dry-run gate, then
`cloud-deploy-relay-production-same-cap.yml`. Cells roll one at a time by design (exact selector
assertions, single Terraform state, and one cell's ~1.2k-host reconnect burst per restart). Do not
add parallelism for this roll.
Inputs: target=<new digest>, rollback=`519f4914` (c7: rollback=`85bf6799`). Selector membership is
unchanged from the end of Roll 1 (gen 148; existing-only c1–c6, c11, c12; migration-only c17, c18).
Order:
1. **c7 canary** (`canary-apply`, protocol 1). c7 is the rehearsal cell and the only one not on
`519f4914`.
2. **c8 canary**, then **batch c9, c10, c13, c14**.
3. **c15 canary**, then **batch c16, c19, c20, c21**.
4. **c22 canary**, then **batch c23, c24, c25, c26**.
5. **Asia c27, c28, c29** as three single canaries at protocol 0 (`PROTO=0`). Batch mode cannot
take Asia cells yet and needs at least two cells.
Each batch needs a same-commit canary authority; each wave needs a fresh 15 min gate. Use the
chain script pattern from Roll 1 (wait gate green, check trusted-path ancestry, dispatch within 5 min,
log `CANARY <run> <status>`) under `caffeinate -i`. Budget: 11 to 13 min per cell plus 15 min per gate,
about 6 to 7 h total.
Per wave checks (same as Roll 1): transition verifier passes at migration-only and again at general
with assignments carried; no `container die` fleet-wide; selector generation advances by exactly 2
per cell. After the Asia cells: image census from MIG templates; every general cell on the new digest.
Failure handling: a failed canary re-enters through `mode=rollback` with rollback-digest = desired
image (Roll 1 c27 pattern). A gate freeze on an Asia latency probe despite the 4 000 ms bar is a
stop-and-investigate, not a retry. Monitor-side freezes (freshness, continuity deadline) re-gate
after a 2 min back-off; the chain does this on its own.
Record every gate and wave in the findings doc as in Roll 1.
## Phase 3. After the roll (spread over the following week)
- **4.4 Recalibrate the retries bar.** After one week of `orca_relay_postgres_transaction_retry`
on the new image, re-derive the `postgres_retries` monitor threshold from the new baseline
(PR against `cloud/apps/relay-ops/src/incident-monitor.ts` thresholds). About 2 h.
- **1.2 Pruner budget.** Raise `auth_token_pruner_max_rows_per_run` to the default 200k after a
clean day; watch Cloud SQL write MB/s and the checkpoint alert. Then **1.5** log metric plus
policy on `stopReason != complete`.
- **1.3 Reclaim.** Once pruner runs delete ~0 rows: `pg_repack -t refresh_tokens` off-peak (check
`pg_available_extensions` first; not `VACUUM FULL`). Confirm table, index, and `disk/utilization`
dropped.
- **Monitor residuals** already in the checklist: `probeEndpointHealth` retry decision still uses the
flat 2 000 ms bar; operator protocol unbound for Asia; `probe-relay-rehome-trust` regex.
- **Same-cap job residuals found in Roll 2** (three of eleven mutating runs needed the resume path):
the post-apply `admin_post target-runtime` read has no transient-5xx tolerance and failed twice on a
one-request 503 `unconditional drop overload` from the edge ~80 s after readiness (c26, c21); and
`probe-relay-rehome-trust` prints only the status on a 409, so the transient c13 failure left no
reason on record. Retry both once and print the error body.
- Update the checklist status header; tick 2.3, 4.1, 4.3 relay-side as deployed.
## Deferred, owner decision required
- **2.1 Private IP** (orca-cloud #477). One-way door with a Cloud SQL restart. When chosen: apply the
foundation off-peak, then a template-only change that sets the `--private-ip` proxy flag. That is
another cell roll unless bundled with a future image.
- **5.2 Paging channel** for auth alerts: needs a destination.
- **Parallel cell rolls** (2 or 3 at a time): about 1.5 days (relax exact-selector assertions to
"exact except in-flight", single coordinator Terraform apply, parallel job shape, tests). Only
worth building if more image rolls are planned after Roll 2, and only once the per-cell locks are
live so a multi-cell reconnect burst is safe.
- **2.2 Database split**: deferred to ~2026-11-01.
## Not in this plan
Desktop and mobile changes already merged (#18719 desktop early-window jitter and no same-token
refresh retry; #18565 mobile fail-fast once merged) ship with the next desktop and mobile releases
on their own schedules. No relay action needed.
cloud/infra/terraform/environments/production.tfvars
+5 -1
View File
@@ -402,7 +402,11 @@ relay_region_rehome_source_cell_ids = [
"production-gce-c23",
"production-gce-c24",
"production-gce-c25",
"production-gce-c26"
"production-gce-c26",
# Asia cells carry the same trust so mis-homed hosts can be drained back off them.
"production-gce-c27",
"production-gce-c28",
"production-gce-c29"
]
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
cloud/infra/terraform/relay-gce-cells.tf
+3 -2
View File
@@ -82,14 +82,15 @@ check "relay_gce_fixed_one_topology" {
assert {
condition = alltrue([
# Region is not asserted here: the director's own rehome source and target predicates
# own eligibility, so this pins only cell shape.
for cell_id in var.relay_region_rehome_source_cell_ids : try(
var.relay_gce_cells[cell_id].region == var.region &&
var.relay_gce_cells[cell_id].connection_hard_cap != null &&
!contains(var.relay_gce_fenced_cells, cell_id),
false
)
])
error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits."
error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits."
}
assert {
cloud/infra/terraform/relay-observability.tf
+215 -3
View File
@@ -65,6 +65,20 @@ locals {
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." }
control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." }
control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." }
control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." }
control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." }
control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." }
control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." }
client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." }
client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." }
client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." }
client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." }
client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." }
client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." }
client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." }
client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." }
client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." }
heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." }
event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." }
forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." }
@@ -80,6 +94,80 @@ locals {
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
}
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
# dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below
# against the emitter. A region missing here drops out of both shares the skew alert compares.
relay_region_keys = ["us-central1", "asia-east2"]
# Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need
# a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in
# every interval so no series can drop out of the alert's inner join. Spelled out rather than
# derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared
# directly; reformatting either side cannot break the check and neither can drift alone.
relay_region_field_segments = {
"us-central1" = "UsCentral1"
"asia-east2" = "AsiaEast2"
}
relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") }
relay_region_share_metrics = merge(
{
for key in local.relay_region_keys :
"requested_regions_${local.relay_region_columns[key]}" => {
field = "requestedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignment requests that hinted ${key}."
}
},
{
for key in local.relay_region_keys :
"selected_regions_${local.relay_region_columns[key]}" => {
field = "selectedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignments that placed a host in ${key}."
}
}
)
relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"])
relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"])
# MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar
# aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of
# the extracted values, which is `sum(value.<metric>)` in MQL and unreachable otherwise.
relay_region_hint_skew_query = join("\n", concat(
["{"],
flatten([
for index, entry in [
for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" }
] : [
index == 0 ? "" : ";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}",
" | align delta(1h) | every 1h",
" | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]"
]
]),
flatten([
for key in local.relay_region_keys : [
";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}",
" | align delta(1h) | every 1h",
" | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]"
]
]),
[
"}",
"| join",
"| value [",
" hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),",
" placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),",
" hinted_requests: ${local.relay_region_hinted_total}",
" ]",
# Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the
# region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so
# the whole series vanishes before the other clauses run. That hour is the worst skew there
# is - every desktop asking for a region the director is putting nobody in - and it happens
# whenever the region is drained, fenced, or at capacity. Both forms were run read-only
# against production surrogates with a zero denominator: the ratio returned no rows, this
# returned the series with the condition true.
"| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'"
]
))
relay_custom_alerts = {
connection_headroom = {
pages_oncall = true
@@ -201,7 +289,9 @@ locals {
}
resource "google_logging_metric" "relay_snapshot" {
for_each = local.relay_runtime_metrics
# Region-request metrics ride the same event and shape; merging adds map entries only, so the
# existing metric instances are untouched (a label change, not a new key, is what recreates them).
for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics)
project = var.project_id
name = "orca_relay_${each.key}"
@@ -211,14 +301,14 @@ resource "google_logging_metric" "relay_snapshot" {
label_extractors = {
role = "EXTRACT(jsonPayload.role)"
cell_id = "EXTRACT(jsonPayload.cellId)"
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
# No region label: adding one replaces all 42 live metrics (label change = delete+create),
# which resets history and blanks the relay alert policies during the swap.
}
metric_descriptor {
metric_kind = "DELTA"
value_type = "DISTRIBUTION"
unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
labels {
key = "role"
@@ -672,6 +762,128 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
depends_on = [google_logging_metric.relay_incident]
}
# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two
# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region
# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor,
# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles.
# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an
# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance`
# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need
# its own arm here. None of the metrics these query exist in the project yet, so what was checked
# against production is the query shape: the same MQL run over existing metrics of the same kind
# confirmed the distribution sum, the join arity, the unit literals, and the condition clause.
resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" {
project = var.project_id
display_name = "Orca Relay: far-cell phone accept latency"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Phone accept p95 above 2 s for 15 minutes"
condition_monitoring_query_language {
# percentile(..., 50) over the window, not max: the published value is already a p95, so the
# median of the interval p95s reads as sustained slowness instead of one bad 30-second flush.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)]
}
| join
| condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_cell_control_rtt" {
project = var.project_id
display_name = "Orca Relay: cell control round trip"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Control ping p50 above 150 ms for an hour"
condition_monitoring_query_language {
# p50 only. The desktop echoes the pong on its main thread, so the published p95 and max
# track renderer stalls, not distance; the median is the only column that reads as distance.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50
| align delta(1h) | every 1h
| group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples
| align delta(1h) | every 1h
| group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)]
}
| join
| condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_region_hint_skew" {
project = var.project_id
display_name = "Orca Relay: region hint skew"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "asia-east2 hint share above 2x its placement share for an hour"
condition_monitoring_query_language {
query = local.relay_region_hint_skew_query
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
resource "google_monitoring_dashboard" "relay_incident" {
project = var.project_id
cloud/infra/terraform/variables.tf
+1 -1
View File
@@ -245,7 +245,7 @@ variable "relay_regional_placement_enabled" {
variable "relay_region_rehome_source_cell_ids" {
type = set(string)
description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol."
description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol."
default = []
}
cloud/package.json
+2 -2
View File
@@ -20,8 +20,8 @@
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"typecheck": "pnpm -r typecheck"
},
"devDependencies": {
cloud/packages/relay-contract/src/contract.test.ts
+48 -1
View File
@@ -6,7 +6,10 @@ import {
HostChallengeSchema,
HostDataAuthSchema,
HostHelloAckSchema,
HostHelloSchema
HostHelloSchema,
parseRelayHostCapabilities,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from './control-messages.js'
import {
DeviceCredentialInstallSchema,
@@ -345,3 +348,47 @@ describe('relay protocol contract', () => {
).toBe(false)
})
})
describe('pending connection details capability', () => {
it('reads a pending entry with or without the stated kind and device', () => {
const ack = {
v: 1 as const,
generation: 3,
controlResumeSecret: 'R'.repeat(43),
leaseExpiresAt: 1_800_000_000_000,
activeConnIds: []
}
const identifiers = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
expect(HostHelloAckSchema.safeParse({ ...ack, pendingConns: [identifiers] }).success).toBe(true)
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }]
}).success
).toBe(true)
// Still strict otherwise: an unannounced key must not slip through as data.
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, reservationId: 'injected' }]
}).success
).toBe(false)
})
it('pins the header and token the desktop mirrors by hand', () => {
// The desktop cannot import this package; drift silently disables the
// feature, so both literals are asserted on each side.
expect(RELAY_HOST_CAPABILITIES_HEADER).toBe('x-orca-host-capabilities')
expect(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS).toBe('pending-conn-details')
})
it('reads the advertised capabilities from a control upgrade header', () => {
expect(
parseRelayHostCapabilities(` ${RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS} , future-thing`)
).toEqual(new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, 'future-thing']))
// A host that predates the header sends nothing; absence is never capable.
expect(parseRelayHostCapabilities(undefined).size).toBe(0)
expect(parseRelayHostCapabilities('').size).toBe(0)
expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0)
})
})
cloud/packages/relay-contract/src/control-messages.ts
+28 -1
View File
@@ -44,8 +44,35 @@ export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema })
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
cloud/packages/relay-contract/src/relay-regions.ts
+9
View File
@@ -8,6 +8,15 @@ export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
config/electron-builder.config.cjs
+12 -3
View File
@@ -19,6 +19,7 @@ const {
} = require('./scripts/verify-packaged-node-pty-job-ownership.cjs')
const { verifySkillsCliRuntime } = require('./scripts/verify-skills-cli-runtime.cjs')
const { verifyStaticAppImagePackage } = require('./scripts/static-appimage-package-contract.cjs')
const { signWindowsUninstallerViaSignPath } = require('./scripts/windows-uninstaller-signing.cjs')
// Why: dev-channel builds must carry the *release* identity — same bundle id,
// Developer ID signature, and notarization ticket — or Squirrel.Mac refuses to
@@ -401,9 +402,17 @@ module.exports = {
// name is absent. An unsigned build that still claimed 'SignPath Foundation'
// would therefore reject its own channel's next build — and its way back to
// stable with it. Dropping it is what makes dev→dev and dev→stable work.
...(isWinDevChannel
? { verifyUpdateCodeSignature: false }
: { signtoolOptions: { publisherName: 'SignPath Foundation' } }),
// Why a sign hook on a build that does not sign: it is the only moment
// electron-builder exposes the NSIS uninstaller (built in its own makensis
// pass, embedded, then deleted). The hook signs nothing — it relays the file
// to and from the CI SignPath request, and is inert when the relay env vars
// are unset, so local and dev builds are unaffected. publisherName stays on
// its existing channel split above.
signtoolOptions: {
sign: signWindowsUninstallerViaSignPath,
...(isWinDevChannel ? {} : { publisherName: 'SignPath Foundation' })
},
...(isWinDevChannel ? { verifyUpdateCodeSignature: false } : {}),
extraResources: [
...commonExtraResources,
...createPackagedRuntimeNodeModuleResources('win32'),
config/nsis/orca-installer-hooks.nsh
+35 -9
View File
@@ -49,22 +49,48 @@
; ---------------------------------------------------------------------------
; Clean up the relocated terminal daemon on a REAL uninstall.
;
; Why: the daemon host is deliberately copied to a distinct image name
; (orca-terminal-daemon.exe) under %LOCALAPPDATA%\Orca\daemon-host so that app
; UPDATES cannot kill it — that relocation is what keeps terminals alive across
; updates. The same design means a normal uninstall's process sweep and file
; removal both miss it, leaving an orphaned daemon plus its runtime copy behind.
; Why: the daemon host is deliberately copied OUT of the install dir into
; %LOCALAPPDATA%\Orca\daemon-host so that app UPDATES cannot kill it —
; electron-builder's kill sweep selects processes whose image path is under
; $INSTDIR, and that relocation is what keeps terminals alive across updates.
; The same design means a normal uninstall's process sweep and file removal both
; miss it, leaving an orphaned daemon plus its runtime copy behind.
;
; The ${isUpdated} guard is essential: electron-builder runs this uninstaller as
; part of uninstallOldVersion on EVERY update, and killing the daemon there would
; defeat the whole feature. Only clean up on a genuine uninstall.
;
; The image name and the LOCALAPPDATA folder name must stay in sync with
; DAEMON_HOST_EXE_NAME and LOCAL_HOST_ROOT_NAME in
; src/main/daemon/daemon-host-relocation.ts.
; The LOCALAPPDATA folder name must stay in sync with LOCAL_HOST_ROOT_NAME in
; src/main/daemon/daemon-host-relocation.ts. See
; docs/reference/windows-daemon-host-relocation.md.
!macro customUnInstall
${ifNot} ${isUpdated}
nsExec::Exec 'taskkill /F /IM orca-terminal-daemon.exe'
Push $0
Push $1
Push $2
; The host exe is a verbatim copy of the app exe, so the app's own image name
; reaches it; the second name covers hosts left by builds that renamed the copy.
; Filtered to the current user like upstream's per-user KILL_PROCESS, so an
; elevated machine-wide uninstall cannot reach another logged-on user's session.
; NSIS expands USERNAME itself: routing through cmd.exe only to get %USERNAME%
; would add two interpreter spawns to the uninstall path for nothing.
ReadEnvStr $1 USERNAME
${if} $1 == ""
; Measured: taskkill rejects an empty filter value outright ("The search filter
; cannot be recognized") and kills nothing, so with no USERNAME to scope by,
; kill unfiltered rather than not at all. USERNAME is set in every session an
; uninstaller runs in, so this is a backstop, not the expected path.
StrCpy $2 ""
${else}
StrCpy $2 '/FI "USERNAME eq $1"'
${endIf}
nsExec::Exec 'taskkill /F /IM "${APP_EXECUTABLE_FILENAME}" $2'
Pop $0
nsExec::Exec 'taskkill /F /IM "orca-terminal-daemon.exe" $2'
Pop $0
Pop $2
Pop $1
Pop $0
; Give the OS a moment to release the image lock before removing the tree.
Sleep 500
RMDir /r "$LOCALAPPDATA\Orca\daemon-host"
config/oxlint-performance-audit.json
+36
View File
@@ -0,0 +1,36 @@
{
"$schema": "../node_modules/oxlint/configuration_schema.json",
"plugins": [],
"categories": {
"correctness": "off",
"suspicious": "off",
"pedantic": "off",
"perf": "off",
"style": "off",
"restriction": "off",
"nursery": "off"
},
"jsPlugins": [
{
"name": "app-store-performance",
"specifier": "../config/oxlint-plugins/app-store-performance.mjs"
},
{
"name": "quadratic-buffer-concat",
"specifier": "../config/oxlint-plugins/quadratic-buffer-concat.mjs"
},
{
"name": "sort-comparator-performance",
"specifier": "../config/oxlint-plugins/sort-comparator-performance.mjs"
}
],
"rules": {
"app-store-performance/require-selector": "warn",
"app-store-performance/no-identity-selector": "warn",
"app-store-performance/no-fresh-selector-result": "warn",
"app-store-performance/no-nested-fresh-under-shallow": "warn",
"quadratic-buffer-concat/no-loop-carried-concat": "warn",
"sort-comparator-performance/no-repeated-collator": "warn"
},
"ignorePatterns": ["**/node_modules", "**/dist", "**/out", "**/*.test.*", "**/*.spec.*"]
}
config/oxlint-plugins/app-store-performance.mjs
+204 -68
View File
@@ -8,6 +8,19 @@ const ALLOCATING_METHODS = new Set([
'toSpliced',
'with'
])
const ALLOCATING_OBJECT_STATICS = new Set([
'assign',
'create',
'entries',
'fromEntries',
'keys',
'values'
])
const FUNCTION_NODES = new Set([
'ArrowFunctionExpression',
'FunctionDeclaration',
'FunctionExpression'
])
function identifierName(node) {
return node?.type === 'Identifier' ? node.name : null
@@ -25,8 +38,12 @@ function propertyName(node) {
: null
}
function functionNode(node) {
return FUNCTION_NODES.has(node?.type) ? node : null
}
function returnedExpressions(selector) {
if (selector?.type !== 'ArrowFunctionExpression' && selector?.type !== 'FunctionExpression') {
if (!functionNode(selector)) {
return []
}
if (selector.body.type !== 'BlockStatement') {
@@ -37,10 +54,7 @@ function returnedExpressions(selector) {
if (!node || typeof node !== 'object') {
return
}
if (
node !== selector.body &&
['ArrowFunctionExpression', 'FunctionDeclaration', 'FunctionExpression'].includes(node.type)
) {
if (node !== selector.body && FUNCTION_NODES.has(node.type)) {
return
}
if (node.type === 'ReturnStatement') {
@@ -76,10 +90,7 @@ function unwrapShallowSelector(selector, shallowHooks) {
}
function isIdentitySelector(selector) {
if (selector?.type !== 'ArrowFunctionExpression' && selector?.type !== 'FunctionExpression') {
return false
}
const parameter = selector.params[0]
const parameter = functionNode(selector)?.params[0]
if (parameter?.type !== 'Identifier') {
return false
}
@@ -88,14 +99,23 @@ function isIdentitySelector(selector) {
)
}
function isAllocatingExpression(expression) {
if (expression?.type === 'ConditionalExpression') {
return (
isAllocatingExpression(expression.consequent) || isAllocatingExpression(expression.alternate)
)
}
if (expression?.type === 'LogicalExpression') {
return isAllocatingExpression(expression.left) || isAllocatingExpression(expression.right)
/**
* `everyBranch` decides how a conditional counts. An inline selector is flagged
* when ANY branch allocates; a helper the selector delegates to must allocate on
* EVERY branch, so the `cache.get(k) ?? build(state)` identity-caching shape is
* not a false positive.
*/
function allocates(expression, everyBranch) {
const branches =
expression?.type === 'ConditionalExpression'
? [expression.consequent, expression.alternate]
: expression?.type === 'LogicalExpression'
? [expression.left, expression.right]
: null
if (branches) {
return everyBranch
? branches.every((branch) => allocates(branch, true))
: branches.some((branch) => allocates(branch, false))
}
if (
expression?.type === 'ArrayExpression' ||
@@ -107,44 +127,104 @@ function isAllocatingExpression(expression) {
if (expression?.type !== 'CallExpression') {
return false
}
const method = propertyName(expression.callee)
if (method && ALLOCATING_METHODS.has(method)) {
return true
}
const callee = expression.callee
const method = propertyName(callee)
return (
callee.type === 'MemberExpression' &&
identifierName(callee.object) === 'Object' &&
['assign', 'create', 'entries', 'fromEntries', 'keys', 'values'].includes(propertyName(callee))
ALLOCATING_METHODS.has(method) ||
(identifierName(callee.object) === 'Object' && ALLOCATING_OBJECT_STATICS.has(method))
)
}
function importedLocalName(specifier, importedName) {
if (specifier.type !== 'ImportSpecifier' || identifierName(specifier.imported) !== importedName) {
return null
function isAllocatingExpression(expression) {
return allocates(expression, false)
}
// Project-local zustand hooks follow the use<Name>Store convention; React's
// useSyncExternalStore matches that shape but is not a store subscription.
const STORE_HOOK_NAME = /^use[A-Z][A-Za-z0-9]*Store$/
const NON_STORE_HOOKS = new Set(['useSyncExternalStore'])
function isLocalModuleSource(source) {
return typeof source === 'string' && (source.startsWith('.') || source.startsWith('@/'))
}
/** Module scope only: a component-local helper must not shadow a same-named import. */
function isModuleScope(node) {
const parent = node.parent
return (
parent?.type === 'Program' ||
(parent?.type === 'ExportNamedDeclaration' && parent.parent?.type === 'Program')
)
}
/** Records module-scope `const selectX = (state) => ...` so identifier selectors resolve. */
function recordNamedSelector(node, state) {
if (!isModuleScope(node)) {
return
}
return identifierName(specifier.local)
const declared =
node.type === 'FunctionDeclaration'
? [[node.id, node]]
: node.declarations.map((declarator) => [declarator.id, declarator.init])
for (const [id, initializer] of declared) {
const name = identifierName(id)
if (name && functionNode(initializer)) {
state.namedSelectors.set(name, initializer)
}
}
}
/** Inline function, or a module-scope selector referenced by name. */
function resolveSelector(argument, state) {
return functionNode(argument) ?? state.namedSelectors.get(identifierName(argument)) ?? null
}
/**
* One hop: a selector that delegates to a module-scope helper is the idiomatic
* shape here, and neither the inline-body check nor a reviewer reading the call
* site can see what that helper returns. An unresolvable helper is left alone.
*/
function expandThroughNamedHelper(expression, state) {
const helper =
expression?.type === 'CallExpression'
? state.namedSelectors.get(identifierName(expression.callee))
: undefined
const returned = helper ? returnedExpressions(helper) : []
return returned.length > 0 && returned.every((entry) => allocates(entry, true))
? returned
: [expression]
}
function createRuleState() {
return {
appStoreHooks: new Set(),
shallowHooks: new Set()
shallowHooks: new Set(),
namedSelectors: new Map(),
deferredCalls: []
}
}
function recordImports(node, state) {
if (node.source?.value === 'zustand/react/shallow') {
for (const specifier of node.specifiers) {
const localName = importedLocalName(specifier, 'useShallow')
if (localName) {
state.shallowHooks.add(localName)
}
}
}
const source = node.source?.value
for (const specifier of node.specifiers) {
const localName = importedLocalName(specifier, 'useAppStore')
if (localName) {
if (specifier.type !== 'ImportSpecifier') {
continue
}
const imported = identifierName(specifier.imported)
const localName = identifierName(specifier.local)
if (!imported || !localName) {
continue
}
if (source === 'zustand/react/shallow' && imported === 'useShallow') {
state.shallowHooks.add(localName)
}
// useAppStore is the app store wherever it is re-exported from; sibling
// stores are trusted by naming convention only when they come from this codebase.
if (
STORE_HOOK_NAME.test(imported) &&
!NON_STORE_HOOKS.has(imported) &&
(imported === 'useAppStore' || isLocalModuleSource(source))
) {
state.appStoreHooks.add(localName)
}
}
@@ -176,52 +256,107 @@ function requireSelectorRule() {
}
}
function noIdentitySelectorRule() {
/**
* Selector arguments are collected during traversal and judged at Program:exit so a
* selector hoisted below its call site still resolves.
*/
function deferredSelectorRule(inspect) {
const state = createRuleState()
return {
ImportDeclaration(node) {
recordImports(node, state)
},
FunctionDeclaration(node) {
recordNamedSelector(node, state)
},
VariableDeclaration(node) {
recordNamedSelector(node, state)
},
CallExpression(node) {
if (!isAppStoreCall(node, state)) {
return
if (isAppStoreCall(node, state)) {
state.deferredCalls.push(node)
}
const { selector } = unwrapShallowSelector(node.arguments[0], state.shallowHooks)
if (isIdentitySelector(selector)) {
this.report({
node: selector,
message:
'Select the smallest required fields instead of subscribing to the entire app store.'
},
'Program:exit'() {
for (const node of state.deferredCalls) {
const { selector: argument, shallow } = unwrapShallowSelector(
node.arguments[0],
state.shallowHooks
)
const report = inspect({
selector: resolveSelector(argument, state),
shallow,
state
})
if (report) {
this.report(report)
}
}
}
}
}
function noIdentitySelectorRule() {
return deferredSelectorRule(({ selector }) =>
isIdentitySelector(selector)
? {
node: selector,
message:
'Select the smallest required fields instead of subscribing to the entire app store.'
}
: null
)
}
function noFreshSelectorResultRule() {
const state = createRuleState()
return {
ImportDeclaration(node) {
recordImports(node, state)
},
CallExpression(node) {
if (!isAppStoreCall(node, state)) {
return
}
const { selector, shallow } = unwrapShallowSelector(node.arguments[0], state.shallowHooks)
if (shallow) {
return
}
const freshResult = returnedExpressions(selector).find(isAllocatingExpression)
if (freshResult) {
this.report({
return deferredSelectorRule(({ selector, shallow, state }) => {
if (shallow || !selector) {
return null
}
const freshResult = returnedExpressions(selector)
.flatMap((expression) => expandThroughNamedHelper(expression, state))
.find(isAllocatingExpression)
return freshResult
? {
node: freshResult,
message:
'This selector returns a fresh reference on every store write; select a stable field, cache the result, or use useShallow.'
})
}
}
}
: null
})
}
/** useShallow compares one level deep, so a fresh reference nested inside its result never matches. */
function nestedFreshValues(expression) {
if (expression?.type === 'ObjectExpression') {
return expression.properties
.map((property) => (property.type === 'Property' ? property.value : null))
.filter(Boolean)
}
if (expression?.type === 'ArrayExpression') {
return expression.elements.filter(Boolean)
}
return []
}
function noNestedFreshUnderShallowRule() {
return deferredSelectorRule(({ selector, shallow, state }) => {
if (!shallow || !selector) {
return null
}
const nestedFresh = returnedExpressions(selector)
.flatMap((expression) => expandThroughNamedHelper(expression, state))
.flatMap(nestedFreshValues)
.flatMap((expression) => expandThroughNamedHelper(expression, state))
.find(isAllocatingExpression)
return nestedFresh
? {
node: nestedFresh,
message:
'useShallow compares only one level deep, so this nested fresh reference changes on every store write and defeats the memo; project the primitives the component actually renders.'
}
: null
})
}
function bindContext(createVisitors) {
@@ -239,6 +374,7 @@ export default {
rules: {
'require-selector': { create: bindContext(requireSelectorRule) },
'no-identity-selector': { create: bindContext(noIdentitySelectorRule) },
'no-fresh-selector-result': { create: bindContext(noFreshSelectorResultRule) }
'no-fresh-selector-result': { create: bindContext(noFreshSelectorResultRule) },
'no-nested-fresh-under-shallow': { create: bindContext(noNestedFreshUnderShallowRule) }
}
}
config/oxlint-plugins/sort-comparator-performance.mjs
+60
View File
@@ -0,0 +1,60 @@
const FUNCTION_TYPES = new Set([
'ArrowFunctionExpression',
'FunctionExpression',
'FunctionDeclaration'
])
function propertyName(node) {
if (node?.type !== 'MemberExpression') {
return null
}
if (!node.computed && node.property.type === 'Identifier') {
return node.property.name
}
return node.property.type === 'Literal' ? node.property.value : null
}
function isInlineSortComparator(node) {
for (let parent = node.parent; parent; parent = parent.parent) {
if (!FUNCTION_TYPES.has(parent.type)) {
continue
}
const call = parent.parent
return (
call?.type === 'CallExpression' &&
call.arguments[0] === parent &&
['sort', 'toSorted'].includes(propertyName(call.callee))
)
}
return false
}
function isCollatorConstruction(node) {
return (
node.callee?.object?.type === 'Identifier' &&
node.callee.object.name === 'Intl' &&
propertyName(node.callee) === 'Collator'
)
}
function createRule(context) {
function inspect(node) {
const optionedComparison =
node.type === 'CallExpression' &&
propertyName(node.callee) === 'localeCompare' &&
node.arguments.length >= 3
if ((optionedComparison || isCollatorConstruction(node)) && isInlineSortComparator(node)) {
context.report({
node,
message:
'Create one Intl.Collator before sorting and reuse its compare method; resolving collation options inside the comparator repeats setup for every comparison. Preserve the locale, options, and tie-breaker.'
})
}
}
return { CallExpression: inspect, NewExpression: inspect }
}
export default {
meta: { name: 'sort-comparator-performance' },
rules: { 'no-repeated-collator': { create: createRule } }
}
config/packaged-runtime-node-modules.cjs
+31 -8
View File
@@ -14,6 +14,7 @@ const projectDir = resolve(__dirname, '..')
const requireFromProject = createRequire(join(projectDir, 'package.json'))
const PACKAGED_RUNTIME_PACKAGE_ROOTS = [
'@anthropic-ai/claude-agent-sdk',
'@electron-toolkit/utils',
'@linear/sdk',
'@parcel/watcher',
@@ -56,6 +57,11 @@ const ELECTRON_ARCHITECTURE_BY_ENUM = {
4: 'universal'
}
const PACKAGED_NATIVE_ARCHITECTURES = new Set(['ia32', 'x64', 'arm', 'arm64'])
const PACKAGED_MAIN_REQUIRED_FILES = [
'out/main/index.js',
'out/main/agent-hooks/managed-agent-hook-controls.js'
]
const PACKAGED_MAIN_SOURCE_RE = /^out\/main\/.+\.js$/
const TYPE_DECLARATION_ARTIFACT_RE = /\.d\.(?:c|m)?ts(?:\.map)?$/
const JS_SOURCE_MAP_ARTIFACT_RE = /\.(?:c|m)?js\.map$/
const VERSIONED_ONNXRUNTIME_DYLIB_RE = /^libonnxruntime\.\d[\d.]*\.dylib$/
@@ -223,22 +229,39 @@ function verifyPackagedMainRuntimeDeps(resourcesDir, asar = require('@electron/a
return
}
const mainFiles = ['out/main/index.js', 'out/main/agent-hooks/managed-agent-hook-controls.js']
const entries = asar.listPackage(asarPath)
const missing = new Set()
for (const file of mainFiles) {
const entry = findAsarEntry(entries, file)
if (!entry) {
for (const file of PACKAGED_MAIN_REQUIRED_FILES) {
if (!findAsarEntry(entries, file)) {
throw new Error(`Packaged main file ${file} was not found in ${asarPath}`)
}
}
const missing = new Set()
// Why every emitted main file rather than the entry points alone: rolldown hoists
// modules shared by two entries into out/main/chunks, so an entry's own bare imports
// move out from under a fixed file list and silently stop being checked.
for (const entry of entries) {
if (!PACKAGED_MAIN_SOURCE_RE.test(normalizeAsarEntryPath(entry))) {
continue
}
// Why: @electron/asar lists entries with host separators; Windows returns
// backslashes, and extractFile expects that same host-style path.
const internalPath = entry.replace(/^[\\/]+/, '')
const source = asar.extractFile(asarPath, internalPath).toString('utf8')
for (const match of source.matchAll(/require\(["']([^"']+)["']\)/g)) {
const specifier = match[1]
// Why the lookbehind: Orca has its own registry methods named `require`, so a
// minified `registry.require('some-id')` must not read as a bare specifier.
// Why it readmits `...`: a dot that ends a spread is not member access, and
// the two error directions are not symmetric -- a false positive fails the
// release build loudly, a false negative is this guard going blind.
// Known limit: a specifier inside an embedded source string counts too, and
// ssh-relay-deploy's remote probe names node-pty that way. A remote-only
// dependency added to that script would fail desktop packaging here; telling
// the two apart needs a parser, not a wider pattern.
for (const match of source.matchAll(
/(?:(?<![.\w])|(?<=\.\.\.))(?:require|import)\s*\(\s*(["'`])([^"'`$]+)\1\s*\)/g
)) {
const specifier = match[2]
if (!isPackagedExternalSpecifier(specifier)) {
continue
}
config/patches/@vscode__windows-process-tree@0.8.0.patch
+403 -4
View File
@@ -1,5 +1,5 @@
diff --git a/binding.gyp b/binding.gyp
index 855bd4b86f0a3c18c7594212c0e42b6e35bc4001..33774e7ae296f0de39dd94156673c9e773638bf4 100644
index 855bd4b86f0a3c18c7594212c0e42b6e35bc4001..0bb2af7923b6e6f1f0da40cae8067304cd1fea14 100644
--- a/binding.gyp
+++ b/binding.gyp
@@ -3,7 +3,6 @@
@@ -10,7 +10,8 @@ index 855bd4b86f0a3c18c7594212c0e42b6e35bc4001..33774e7ae296f0de39dd94156673c9e7
],
"conditions": [
['OS=="win"', {
@@ -15,12 +14,11 @@
@@ -14,13 +13,12 @@
"src/process_worker.cc",
"src/process_commandline.cc"
],
- "include_dirs": [],
@@ -26,11 +27,111 @@ index 855bd4b86f0a3c18c7594212c0e42b6e35bc4001..33774e7ae296f0de39dd94156673c9e7
"AdditionalOptions": [
"/guard:cf",
"/sdl",
diff --git a/lib/index.js b/lib/index.js
index 9747a7402600cd252859144d32580ed45c8c93f7..001e81fa8bc89091971d06aaf9d051ba20906615 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -7,11 +7,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
exports.getAllProcesses = exports.getProcessTree = exports.getProcessCpuUsage = exports.getProcessList = exports.filterProcessList = exports.buildProcessTree = exports.ProcessDataFlag = void 0;
const util_1 = require("util");
const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
+exports.supportedProcessDataFlags = native === undefined ? undefined : native.supportedProcessDataFlags;
var ProcessDataFlag;
(function (ProcessDataFlag) {
ProcessDataFlag[ProcessDataFlag["None"] = 0] = "None";
ProcessDataFlag[ProcessDataFlag["Memory"] = 1] = "Memory";
ProcessDataFlag[ProcessDataFlag["CommandLine"] = 2] = "CommandLine";
+ ProcessDataFlag[ProcessDataFlag["CreationTime"] = 4] = "CreationTime";
})(ProcessDataFlag = exports.ProcessDataFlag || (exports.ProcessDataFlag = {}));
// requestInProgress is used for any function that uses CreateToolhelp32Snapshot, as multiple calls
// to this cannot be done at the same time.
@@ -66,11 +68,12 @@ function buildProcessTree(rootPid, processList, maxDepth = MAX_FILTER_DEPTH) {
// • the properties are inlined/splatted
// • the 'ppid' field is omitted
// • the depth of the tree is limited by `maxDepth`
- const buildNode = ({ info: { pid, name, memory, commandLine }, children }, depth) => ({
+ const buildNode = ({ info: { pid, name, memory, commandLine, creationTimeMs }, children }, depth) => ({
pid,
name,
memory,
commandLine,
+ creationTimeMs,
children: depth > 0 ? children.map(c => buildNode(c, depth - 1)) : [],
});
return buildNode(root, maxDepth);
diff --git a/lib/index.ts b/lib/index.ts
index f9aa005d9ced9e42885b8a976de5eb5bd61899ee..1b509af0b9065918bcb5cb75f2d7f23821d4a56a 100644
--- a/lib/index.ts
+++ b/lib/index.ts
@@ -6,12 +6,15 @@
import { promisify } from 'util';
const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
+/** The flag bits this compiled addon reports; undefined off win32. */
+export const supportedProcessDataFlags: number | undefined = native?.supportedProcessDataFlags;
import { IProcessInfo, IProcessTreeNode, IProcessCpuInfo } from '@vscode/windows-process-tree';
export enum ProcessDataFlag {
None = 0,
Memory = 1,
- CommandLine = 2
+ CommandLine = 2,
+ CreationTime = 4
}
type RequestCallback = (processList: IProcessInfo[]) => void;
@@ -81,11 +84,12 @@ export function buildProcessTree(rootPid: number, processList: Iterable<IProcess
// • the properties are inlined/splatted
// • the 'ppid' field is omitted
// • the depth of the tree is limited by `maxDepth`
- const buildNode = ({ info: { pid, name, memory, commandLine }, children }: IProcessInfoNode, depth: number): IProcessTreeNode => ({
+ const buildNode = ({ info: { pid, name, memory, commandLine, creationTimeMs }, children }: IProcessInfoNode, depth: number): IProcessTreeNode => ({
pid,
name,
memory,
commandLine,
+ creationTimeMs,
children: depth > 0 ? children.map(c => buildNode(c, depth - 1)) : [],
});
diff --git a/src/addon.cc b/src/addon.cc
index 9214aff281251e797a70ecb9f6e0b52932a0503f..722edd42ddb4740296bfc47582a181bd6d00c464 100644
--- a/src/addon.cc
+++ b/src/addon.cc
@@ -53,6 +53,10 @@ void GetProcessCpuUsage(const Napi::CallbackInfo& args) {
Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports.Set("getProcessList", Napi::Function::New(env, GetProcessList));
exports.Set("getProcessCpuUsage", Napi::Function::New(env, GetProcessCpuUsage));
+ // Lets a caller prove THIS BINARY understands CREATIONTIME. The JS enum is
+ // patched source and says nothing about what the .node was compiled from.
+ exports.Set("supportedProcessDataFlags",
+ Napi::Number::New(env, MEMORY | COMMANDLINE | CREATIONTIME));
return exports;
}
diff --git a/src/process.cc b/src/process.cc
index 3eea92077c4d1d433119361d5c432881859131e9..1998f4addd4d7e9aba946ea6f7f7a4a5d13291bc 100644
index 3eea92077c4d1d433119361d5c432881859131e9..22a47421da919c76e2194280974d39c2287b098d 100644
--- a/src/process.cc
+++ b/src/process.cc
@@ -37,7 +37,7 @@ uint32_t GetRawProcessList(std::vector<ProcessInfo>& process_info,
@@ -21,7 +21,8 @@ uint32_t GetRawProcessList(std::vector<ProcessInfo>& process_info,
if (Process32First(snapshot_handle, &process_entry)) {
do {
if (process_entry.th32ProcessID != 0) {
- ProcessInfo pinfo;
+ // Value-initialize: `memory` is otherwise stack garbage when the flag is unset.
+ ProcessInfo pinfo{};
pinfo.pid = process_entry.th32ProcessID;
pinfo.ppid = process_entry.th32ParentProcessID;
@@ -33,23 +34,51 @@ uint32_t GetRawProcessList(std::vector<ProcessInfo>& process_info,
GetProcessCommandLine(pinfo);
}
+ if (CREATIONTIME & process_data_flags) {
+ GetProcessCreationTime(pinfo);
+ }
+
strcpy(pinfo.name, process_entry.szExeFile);
process_info.push_back(std::move(pinfo));
process_count++;
}
@@ -39,3 +140,301 @@ index 3eea92077c4d1d433119361d5c432881859131e9..1998f4addd4d7e9aba946ea6f7f7a4a5
}
CloseHandle(snapshot_handle);
return process_count;
}
+void GetProcessCreationTime(ProcessInfo& process_info) {
+ HANDLE hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, process_info.pid);
+ if (hProcess == NULL) {
+ return;
+ }
+
+ FILETIME creationTime, exitTime, kernelTime, userTime;
+ if (GetProcessTimes(hProcess, &creationTime, &exitTime, &kernelTime, &userTime)) {
+ ULARGE_INTEGER timestamp;
+ timestamp.LowPart = creationTime.dwLowDateTime;
+ timestamp.HighPart = creationTime.dwHighDateTime;
+ constexpr ULONGLONG WINDOWS_EPOCH_OFFSET_100NS = 116444736000000000ULL;
+ constexpr ULONGLONG HUNDRED_NS_PER_MILLISECOND = 10000ULL;
+ if (timestamp.QuadPart >= WINDOWS_EPOCH_OFFSET_100NS) {
+ process_info.creationTimeMs =
+ (timestamp.QuadPart - WINDOWS_EPOCH_OFFSET_100NS) / HUNDRED_NS_PER_MILLISECOND;
+ }
+ }
+
+ CloseHandle(hProcess);
+}
+
void GetProcessMemoryUsage(ProcessInfo& process_info) {
DWORD pid = process_info.pid;
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
- hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid);
+ // PROCESS_VM_READ is never used here -- GetProcessMemoryInfo reads counters the
+ // kernel keeps, not the address space -- and acquiring it is what EDR scores.
+ hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid);
if (hProcess == NULL) {
return;
@@ -81,7 +110,8 @@ void GetCpuUsage(Cpu& cpu_info, bool first_pass) {
DWORD pid = cpu_info.pid;
HANDLE hProcess;
- hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid);
+ // GetProcessTimes needs no more than PROCESS_QUERY_LIMITED_INFORMATION.
+ hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid);
if (hProcess == NULL) {
return;
diff --git a/src/process.h b/src/process.h
index 82f8e4bcfa742551e5d874a7632736a7611d7aa7..78d1d2c3b2360ed06fd624b4cb2f5042510f7a77 100644
--- a/src/process.h
+++ b/src/process.h
@@ -22,18 +22,22 @@ struct ProcessInfo {
DWORD ppid;
DWORD memory; // Reported in bytes
std::string commandLine;
+ ULONGLONG creationTimeMs;
};
enum ProcessDataFlags {
NONE = 0,
MEMORY = 1,
- COMMANDLINE = 2
+ COMMANDLINE = 2,
+ CREATIONTIME = 4
};
uint32_t GetRawProcessList(std::vector<ProcessInfo>& process_info, DWORD flags);
void GetProcessMemoryUsage(ProcessInfo& process_info);
+void GetProcessCreationTime(ProcessInfo& process_info);
+
void GetCpuUsage(Cpu& cpu_info, bool first_run);
#endif // SRC_PROCESS_H_
diff --git a/src/process_commandline.cc b/src/process_commandline.cc
index ea822b120e8038a4803e34647042f08f4aaf5ca1..25907c0bf542bed6c72b1b462b19bcf3210c3cfd 100644
--- a/src/process_commandline.cc
+++ b/src/process_commandline.cc
@@ -7,61 +7,119 @@
#include "process_commandline.h"
#include <windows.h>
#include <winternl.h>
-#include <iostream>
+#include <vector>
-bool GetProcessCommandLine(ProcessInfo& process_info) {
- HINSTANCE ntdll = GetModuleHandleW(L"ntdll.dll");
+namespace {
+
+// Windows 8.1 and later hand back a process's command line as a UNICODE_STRING
+// the kernel builds, needing only PROCESS_QUERY_LIMITED_INFORMATION.
+//
+// There is deliberately no PEB fallback. Reading the command line out of the
+// target's address space -- opening it for VM reads and then chaining
+// memory reads across every pid on a timer -- is the credential-dumping
+// primitive this reader exists to not perform, so it is absent from the binary
+// rather than one anomalous NTSTATUS away. Electron's floor is Windows 10, so
+// every OS Orca supports has this class; if a hooked ntdll refuses it anyway,
+// the command line comes back empty, which callers already handle, instead of
+// silently reinstating the primitive on exactly the instrumented machines this
+// reader was written for.
+const ULONG kProcessCommandLineInformation = 60;
+
+const NTSTATUS kStatusInfoLengthMismatch = static_cast<NTSTATUS>(0xC0000004L);
+const NTSTATUS kStatusBufferTooSmall = static_cast<NTSTATUS>(0xC0000023L);
+
+// A command line is a UNICODE_STRING, whose Length is a USHORT, so the kernel
+// can never need more than the header plus 64 KiB. Refusing anything larger
+// keeps a bogus size from throwing bad_alloc out of a scan that has already
+// walked most of the table.
+const ULONG kMaxCommandLineBytes = sizeof(UNICODE_STRING) + 0xFFFF + sizeof(wchar_t);
+
+// winternl.h's PROCESSINFOCLASS does not name class 60 and its enumerator range
+// stops far short of it, so the class travels as a ULONG rather than a cast enum.
+typedef NTSTATUS(NTAPI* NtQueryInformationProcessFn)(HANDLE, ULONG, PVOID, ULONG, PULONG);
+
+// ntdll ships no import library for this entry point; it has to be resolved.
+NtQueryInformationProcessFn ResolveNtQueryInformationProcess() {
+ HMODULE ntdll = GetModuleHandleW(L"ntdll.dll");
if (!ntdll) {
+ return nullptr;
+ }
+ return reinterpret_cast<NtQueryInformationProcessFn>(
+ GetProcAddress(ntdll, "NtQueryInformationProcess"));
+}
+
+NtQueryInformationProcessFn NtQueryInformationProcessEntry() {
+ static NtQueryInformationProcessFn entry = ResolveNtQueryInformationProcess();
+ return entry;
+}
+
+bool StoreCommandLineUtf8(ProcessInfo& process_info, const wchar_t* data, size_t wide_length) {
+ if (wide_length == 0) {
+ return false;
+ }
+ int length = static_cast<int>(wide_length);
+ int charcount = WideCharToMultiByte(CP_UTF8, 0, data, length, NULL, 0, NULL, NULL);
+ if (!charcount) {
return false;
}
+ process_info.commandLine.resize(static_cast<size_t>(charcount));
+ WideCharToMultiByte(CP_UTF8, 0, data, length, &process_info.commandLine[0], charcount, NULL,
+ NULL);
+ return true;
+}
+
+} // namespace
- decltype(NtQueryInformationProcess)* nt_query_information_process =
- reinterpret_cast<decltype(NtQueryInformationProcess)*>(
- GetProcAddress(ntdll, "NtQueryInformationProcess"));
+bool GetProcessCommandLine(ProcessInfo& process_info) {
+ NtQueryInformationProcessFn query = NtQueryInformationProcessEntry();
+ if (!query) {
+ return false;
+ }
- if (!nt_query_information_process) {
+ HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, process_info.pid);
+ if (process == NULL) {
return false;
}
- PROCESS_BASIC_INFORMATION pbi{};
- PEB peb = {NULL};
- RTL_USER_PROCESS_PARAMETERS process_parameters = {NULL};
+ ULONG size = 0;
+ NTSTATUS status = query(process, kProcessCommandLineInformation, nullptr, 0, &size);
+ if (NT_SUCCESS(status)) {
+ // Nothing was written, so there is no command line to read.
+ CloseHandle(process);
+ return false;
+ }
+ if (status != kStatusInfoLengthMismatch && status != kStatusBufferTooSmall) {
+ CloseHandle(process);
+ return false;
+ }
+ if (size < sizeof(UNICODE_STRING) || size > kMaxCommandLineBytes) {
+ CloseHandle(process);
+ return false;
+ }
- // Get process handle
- DWORD pid = process_info.pid;
- HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid);
- if (hProcess == INVALID_HANDLE_VALUE) {
+ std::vector<unsigned char> buffer(size);
+ status = query(process, kProcessCommandLineInformation, &buffer[0], size, &size);
+ CloseHandle(process);
+ if (!NT_SUCCESS(status)) {
return false;
}
- // Get Process Environment Block (PEB)
- NTSTATUS status = nt_query_information_process(hProcess, ProcessBasicInformation, &pbi, sizeof(pbi), nullptr);
- if (NT_SUCCESS(status) && pbi.PebBaseAddress) {
- // Read PEB
- if (ReadProcessMemory(hProcess, pbi.PebBaseAddress, &peb, sizeof(peb), nullptr)) {
- // Read the processs parameters
- if (ReadProcessMemory(hProcess, peb.ProcessParameters, &process_parameters, sizeof(RTL_USER_PROCESS_PARAMETERS), nullptr)) {
- if (process_parameters.CommandLine.Length > 0) {
- std::wstring buffer;
- buffer.resize(process_parameters.CommandLine.Length / sizeof(wchar_t));
- if (ReadProcessMemory(hProcess, process_parameters.CommandLine.Buffer, &buffer[0], process_parameters.CommandLine.Length, nullptr)) {
- int wide_length = static_cast<int>(buffer.length());
- int charcount = WideCharToMultiByte(CP_UTF8, 0, buffer.data(), wide_length,
- NULL, 0, NULL, NULL);
- if (charcount) {
- process_info.commandLine.resize(static_cast<size_t>(charcount));
- WideCharToMultiByte(CP_UTF8, 0, buffer.data(), wide_length,
- &process_info.commandLine[0], charcount,
- NULL, NULL);
- }
- CloseHandle(hProcess);
- return true;
- }
- }
- }
- }
+ // Header and characters arrive in one allocation, but treat the header as
+ // untrusted: a hooked ntdll is the case this reader is written for, and an
+ // unchecked Buffer/Length here would be an over-read encoded straight into JS.
+ // Bound against buffer.size(), never `size` -- the second query overwrote it.
+ const UNICODE_STRING* command_line = reinterpret_cast<const UNICODE_STRING*>(&buffer[0]);
+ const unsigned char* begin = &buffer[0];
+ const unsigned char* end = begin + buffer.size();
+ const unsigned char* chars = reinterpret_cast<const unsigned char*>(command_line->Buffer);
+ if (chars == nullptr || chars < begin + sizeof(UNICODE_STRING) || chars > end ||
+ command_line->Length > static_cast<ULONG>(end - chars)) {
+ return false;
}
- CloseHandle(hProcess);
- return false;
+ // True only when a command line was actually stored, so "empty" and "not
+ // recovered" stay the same answer they were before this reader replaced the
+ // PEB read. `src/process.cc` discards the result either way.
+ return StoreCommandLineUtf8(process_info, command_line->Buffer,
+ command_line->Length / sizeof(wchar_t));
}
diff --git a/src/process_worker.cc b/src/process_worker.cc
index c9e3457a759c1acaa2644231a4917d45aed951f8..3f26a354477f062b34bd31fbd17be529e6a2fd7a 100644
--- a/src/process_worker.cc
+++ b/src/process_worker.cc
@@ -43,6 +43,11 @@ void GetProcessesWorker::OnOK() {
Napi::String::New(env, pinfo.commandLine));
}
+ if ((CREATIONTIME & process_data_flags_) && pinfo.creationTimeMs != 0) {
+ object.Set("creationTimeMs",
+ Napi::Number::New(env, static_cast<double>(pinfo.creationTimeMs)));
+ }
+
result.Set(i, object);
}
diff --git a/typings/windows-process-tree.d.ts b/typings/windows-process-tree.d.ts
index 08bdac2fdc5ead6f0fcfb5ee5a021e2298c7d523..458981566fc45c0084badff566b1e3791ec1b629 100644
--- a/typings/windows-process-tree.d.ts
+++ b/typings/windows-process-tree.d.ts
@@ -7,9 +7,17 @@ declare module '@vscode/windows-process-tree' {
export enum ProcessDataFlag {
None = 0,
Memory = 1,
- CommandLine = 2
+ CommandLine = 2,
+ CreationTime = 4
}
+ /**
+ * The flag bits the compiled addon actually understands, or undefined off
+ * win32. `ProcessDataFlag` above is source; this is what the binary reports,
+ * so it is the only way to tell a patched build from a stale prebuilt.
+ */
+ export const supportedProcessDataFlags: number | undefined;
+
export interface IProcessInfo {
pid: number;
ppid: number;
@@ -24,6 +32,9 @@ declare module '@vscode/windows-process-tree' {
* The string returned is at most 512 chars, strings exceeding this length are truncated.
*/
commandLine?: string;
+
+ /** Process creation time in Unix milliseconds. */
+ creationTimeMs?: number;
}
export interface IProcessCpuInfo extends IProcessInfo {
@@ -35,6 +46,7 @@ declare module '@vscode/windows-process-tree' {
name: string;
memory?: number;
commandLine?: string;
+ creationTimeMs?: number;
children: IProcessTreeNode[];
}
config/patches/node-pty@1.1.0.patch
+69 -38
View File
@@ -603,7 +603,7 @@ index 7b4b9e1f990fbf95b51528bb56dc9717f5b87532..2ae787c5bd4f3eba470584dc658a01a5
}
#endif
diff --git a/src/win/conpty.cc b/src/win/conpty.cc
index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c97209248e 100644
index 7b286d3d644c26141df516929703aa6e129df4b2..4b06d18576c807c3d1181a7bd714140c6678cf86 100644
--- a/src/win/conpty.cc
+++ b/src/win/conpty.cc
@@ -18,6 +18,7 @@
@@ -614,7 +614,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
#include <vector>
#include <Windows.h>
#include <strsafe.h>
@@ -44,12 +45,39 @@ struct pty_baton {
@@ -44,12 +45,40 @@ struct pty_baton {
HANDLE hOut;
HPCON hpc;
@@ -630,6 +630,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ // refused to create or assign one (an outer job without breakaway rights),
+ // in which case callers fall back to their pre-job behaviour.
+ HANDLE hJob = nullptr;
+ bool allowJobBreakaway = true;
+
+ // Orca: teardown needs BOTH the shell's death and an explicit kill() before
+ // the baton can be freed, so each side records that it has run. Whichever
@@ -655,7 +656,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
static volatile LONG ptyCounter;
static pty_baton* get_pty_baton(int id) {
@@ -102,8 +130,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
@@ -102,8 +131,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
// Get process exit code.
GetExitCodeProcess(baton->hShell, (LPDWORD)(&exit_event->exit_code));
// Clean up handles
@@ -689,7 +690,36 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
auto status = tsfn.BlockingCall(exit_event, callback); // In main thread
switch (status) {
@@ -409,6 +460,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -242,6 +294,20 @@
return HRESULT_FROM_WIN32(GetLastError());
}
+// Cygwin and MSYS request breakaway for every child whenever the job allows it,
+// so their shells need one that does not. The runtime DLL on the exe's search
+// path is the signal; Git for Windows ships bash.exe in bin\ beside usr\bin\.
+static bool usesCygwinRuntime(const std::wstring& shellpath) {
+ const size_t separator = shellpath.find_last_of(L"\\/");
+ if (separator == std::wstring::npos) return false;
+ const std::wstring directory = shellpath.substr(0, separator + 1);
+ for (const wchar_t* dll : {L"msys-2.0.dll", L"cygwin1.dll"}) {
+ if (path_util::file_exists(directory + dll) ||
+ path_util::file_exists(directory + L"..\\usr\\bin\\" + dll)) return true;
+ }
+ return false;
+}
+
static Napi::Value PtyStartProcess(const Napi::CallbackInfo& info) {
Napi::Env env(info.Env());
Napi::HandleScope scope(env);
@@ -303,6 +369,7 @@
marshal.Set("pty", Napi::Number::New(env, ptyId));
ptyHandles.emplace_back(
std::make_unique<pty_baton>(ptyId, hIn, hOut, hpc));
+ ptyHandles.back()->allowJobBreakaway = !usesCygwinRuntime(shellpath);
} else {
throw Napi::Error::New(env, "Cannot launch conpty");
}
@@ -409,6 +476,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "UpdateProcThreadAttribute failed");
}
@@ -705,7 +735,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
PROCESS_INFORMATION piClient{};
fSuccess = !!CreateProcessW(
nullptr,
@@ -416,7 +476,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -416,7 +492,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
nullptr, // lpProcessAttributes
nullptr, // lpThreadAttributes
false, // bInheritHandles VERY IMPORTANT that this is false
@@ -717,7 +747,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
envArg, // lpEnvironment
mutableCwd.get(), // lpCurrentDirectory
&siEx.StartupInfo, // lpStartupInfo
@@ -426,8 +489,47 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -426,8 +505,48 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "Cannot create process");
}
@@ -735,13 +765,14 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ // EXPLICIT teardown exact, not to redefine what a clean exit means.
+ HANDLE hJob = CreateJobObjectW(nullptr, nullptr);
+ if (hJob != nullptr) {
+ // Why BREAKAWAY_OK and not a bare job: with no limits set, a child asking
+ // for CREATE_BREAKAWAY_FROM_JOB is refused with ERROR_ACCESS_DENIED.
+ // Installers, msiexec and some updater and service-control paths spawn that
+ // way deliberately, so a bare job breaks them ONLY inside an Orca terminal.
+ // With this flag a child has to ask, so ordinary descendants stay owned.
+ // Native shells retain explicit breakaway for installers and updaters.
+ // Cygwin/MSYS shells take it automatically for ordinary children whenever
+ // this flag is present, so they get strict per-PTY membership instead.
+ // Explicit breakaway requests inside such a pane are consequently denied;
+ // ordinary backgrounding and clean shell exit remain supported.
+ JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobLimits{};
+ jobLimits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_BREAKAWAY_OK;
+ jobLimits.BasicLimitInformation.LimitFlags =
+ handle->allowJobBreakaway ? JOB_OBJECT_LIMIT_BREAKAWAY_OK : 0;
+ if (!SetInformationJobObject(hJob, JobObjectExtendedLimitInformation, &jobLimits, sizeof(jobLimits)) ||
+ !AssignProcessToJobObject(hJob, piClient.hProcess)) {
+ // Why tolerate failure: an outer job without JOB_OBJECT_LIMIT_BREAKAWAY_OK
@@ -767,7 +798,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
if (useConptyDll && fLoadedDll)
{
PFNRELEASEPSEUDOCONSOLE const pfnReleasePseudoConsole = (PFNRELEASEPSEUDOCONSOLE)GetProcAddress(
@@ -440,6 +542,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -440,6 +559,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
// Update handle
handle->hShell = piClient.hProcess;
@@ -776,11 +807,16 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
// Close the thread handle to avoid resource leak
CloseHandle(piClient.hThread);
@@ -544,29 +648,215 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
@@ -544,27 +665,213 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
int id = info[0].As<Napi::Number>().Int32Value();
const bool useConptyDll = info[1].As<Napi::Boolean>().Value();
- const pty_baton* handle = get_pty_baton(id);
-
- if (handle != nullptr) {
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
- bool fLoadedDll = hLibrary != nullptr;
- if (fLoadedDll)
+ // Orca: resolve the DLL BEFORE touching any baton state, for the same reason
+ // PtyConnect does it before creating anything. LoadConptyDll throws when
+ // conpty.dll is missing, and a throw after consoleClosed was set would strand
@@ -794,18 +830,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ (HMODULE)hLibrary,
+ useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
+ }
- if (handle != nullptr) {
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
- bool fLoadedDll = hLibrary != nullptr;
- if (fLoadedDll)
- {
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
- (HMODULE)hLibrary,
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
- if (pfnClosePseudoConsole)
- {
- pfnClosePseudoConsole(handle->hpc);
+
+ // Orca: the baton now outlives the shell, so this runs on a self-exited pty
+ // too -- that is the whole point. Take what we need under the lock: the
+ // watcher thread nulls hShell the moment the shell dies, and TerminateProcess
@@ -841,18 +866,26 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ const bool removed = remove_pty_baton(id);
+ assert(removed);
+ (void)removed;
}
+ }
+ // Else the shell is still running and the watcher frees the baton.
}
- if (useConptyDll) {
- TerminateProcess(handle->hShell, 1);
+ }
+ }
+
+ // Why outside the lock: ClosePseudoConsole blocks until the conout side has
+ // drained, and the watcher must be able to take the lock while it does.
+ if (owed) {
+ if (pfnClosePseudoConsole)
+ {
{
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
- (HMODULE)hLibrary,
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
- if (pfnClosePseudoConsole)
- {
- pfnClosePseudoConsole(handle->hpc);
- }
- }
- if (useConptyDll) {
- TerminateProcess(handle->hShell, 1);
+ pfnClosePseudoConsole(hpc);
+ }
+ if (hShellDup != nullptr) {
@@ -862,8 +895,8 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
}
return env.Undefined();
}
+}
+
+/**
+ * Orca: confirm a baton really is the pty the caller means.
+ *
@@ -1001,12 +1034,10 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ }
+ hHostJob = job;
+ return Napi::Boolean::New(env, true);
+}
+
}
/**
* Init
*/
@@ -577,6 +867,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
@@ -577,6 +884,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
exports.Set("resize", Napi::Function::New(env, PtyResize));
exports.Set("clear", Napi::Function::New(env, PtyClear));
exports.Set("kill", Napi::Function::New(env, PtyKill));
config/performance-audit.md
+38
View File
@@ -0,0 +1,38 @@
# Performance regression checks
`pnpm --silent audit:perf > performance-audit.json` scans production `src/` with
the existing app-store and buffer-concatenation rules plus the sort-comparator
rule. Warnings are advisory in this full inventory; tool/parser failures fail.
New warning findings on changed lines fail `pnpm check:code-quality:changed`.
Tests, generated files, `mobile/` and `cloud/` are outside this source audit.
The sort rule detects optioned `localeCompare` and `Intl.Collator` construction
inside inline `sort`/`toSorted` callbacks. Construct one collator outside the
callback, preserving locale, options and tie-breakers. If the locale changes at
runtime, reconstruct at the next sort or key the cache by locale. Bare comparisons
and standalone equality checks are allowed. There is no autofix or interprocedural
analysis: named comparators, aliases, custom methods and deferred callbacks need
manual review. A warning identifies repeated setup, not proof of visible lag.
`pnpm test:perf:contracts` runs the explicit selection in
`vitest.performance.config.ts`: SQLite statement reuse and schema parity, relay
filesystem concurrency, tokenizer rejection, highlighting cache, queued
cancellation, terminal backing-memory retention and detector fixtures. Missing
listed files fail configuration loading. Tests run serially, without retries,
and inherit the full suite's setup and forced-GC support. This makes existing
regression coverage easy to run and attribute; it does not create new workload
coverage by itself.
`.github/workflows/performance-contracts.yml` runs daily and manually on Linux,
macOS and Windows, and on PRs changing this tooling or any listed contract file.
It uploads per-OS JSON test results, plus the source inventory once from Linux
because that scan is OS-independent. Its schedule starts after merge. Run the existing
`test:e2e:terminal-perf:scale:report` for rendered typing/frame budgets and
`test:e2e:ssh-docker-perf` for real transport behavior. Relay unit tests do not
measure SSH RTT, WSL scheduling or a packaged Electron renderer.
To extend coverage, select a production-path regression with an operation-count,
identity, queue-admission or retained-memory oracle. Confirm it fails with the
old behavior. Use controlled, counterbalanced benchmark samples for timings;
avoid new machine-dependent millisecond gates in the normal unit suite. A green
source scan and these contracts cannot establish that the whole app is fast.
config/reliability-gates.jsonc
+900 -89
View File
File diff suppressed because one or more lines are too long
config/scripts/app-store-performance-plugin.test.mjs
+90 -1
View File
@@ -12,7 +12,8 @@ function lintSource(source) {
rules: {
'app-store-performance/require-selector': 'warn',
'app-store-performance/no-identity-selector': 'warn',
'app-store-performance/no-fresh-selector-result': 'warn'
'app-store-performance/no-fresh-selector-result': 'warn',
'app-store-performance/no-nested-fresh-under-shallow': 'warn'
}
})
}
@@ -52,4 +53,92 @@ describe('app store performance Oxlint plugin', () => {
expect(diagnostics).toEqual([])
})
it('resolves selectors referenced by name, including ones hoisted below the call', () => {
const diagnostics = lintSource(`
import { useAppStore } from '@/store'
const EarlyFresh = () => useAppStore(selectFreshRows)
const selectFreshRows = (state) => state.rows.filter(Boolean)
const Stable = () => useAppStore(selectActiveId)
const selectActiveId = (state) => state.activeId
`)
expect(diagnostics.map((diagnostic) => diagnostic.code)).toEqual([
'app-store-performance(no-fresh-selector-result)'
])
})
it('does not let a component-local helper resolve a same-named imported selector', () => {
const diagnostics = lintSource(`
import { useAppStore } from '@/store'
import { selectRows } from './selectors'
const Other = () => {
const selectRows = (state) => state.rows.map((row) => row.id)
return selectRows
}
const Imported = () => useAppStore(selectRows)
`)
expect(diagnostics).toEqual([])
})
it('covers sibling store hooks but not useSyncExternalStore', () => {
const diagnostics = lintSource(`
import { usePluginPanelsStore } from '@/store/plugin-panels'
import { useSyncExternalStore } from 'react'
const WholePanels = () => usePluginPanelsStore()
const FreshPanels = () => usePluginPanelsStore((state) => ({ open: state.open }))
const External = () => useSyncExternalStore(subscribe, () => ({ open: true }))
`)
expect(diagnostics.map((diagnostic) => diagnostic.code)).toEqual([
'app-store-performance(require-selector)',
'app-store-performance(no-fresh-selector-result)'
])
})
it('reports fresh references nested inside a useShallow projection', () => {
const diagnostics = lintSource(`
import { useAppStore } from '@/store'
import { useShallow } from 'zustand/react/shallow'
const NestedObject = () => useAppStore(useShallow((state) => ({ ids: state.rows.map((row) => row.id) })))
const NestedArray = () => useAppStore(useShallow((state) => [state.activeId, state.rows.filter(Boolean)]))
const Flat = () => useAppStore(useShallow((state) => ({ activeId: state.activeId, rows: state.rows })))
`)
expect(diagnostics.map((diagnostic) => diagnostic.code)).toEqual([
'app-store-performance(no-nested-fresh-under-shallow)',
'app-store-performance(no-nested-fresh-under-shallow)'
])
})
it('follows a selector one hop into a module-scope helper', () => {
const diagnostics = lintSource(`
import { useAppStore } from '@/store'
import { useShallow } from 'zustand/react/shallow'
const buildRows = (state) => state.rows.map((row) => row.id)
const Delegating = () => useAppStore((state) => buildRows(state))
const NestedDelegating = () => useAppStore(useShallow((state) => ({ ids: buildRows(state) })))
`)
expect(diagnostics.map((diagnostic) => diagnostic.code)).toEqual([
'app-store-performance(no-fresh-selector-result)',
'app-store-performance(no-nested-fresh-under-shallow)'
])
})
it('does not flag a helper that returns a cached reference on some branch', () => {
const diagnostics = lintSource(`
import { useAppStore } from '@/store'
import { useShallow } from 'zustand/react/shallow'
// The identity-caching shape: fresh only on a miss, cached otherwise.
const selectCachedRows = (state) => cache.get(state.key) ?? state.rows.filter(Boolean)
const Cached = () => useAppStore((state) => selectCachedRows(state))
const CachedNested = () => useAppStore(useShallow((state) => ({ rows: selectCachedRows(state) })))
// An unknown helper cannot be resolved, so it must not be guessed at.
const External = () => useAppStore((state) => externalBuild(state))
`)
expect(diagnostics).toEqual([])
})
})
config/scripts/benchmark-browser-tunnel-framing.mjs
+110
View File
@@ -0,0 +1,110 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { stripTypeScriptTypes } from 'node:module'
import { performance } from 'node:perf_hooks'
// Run from the worktree root: node config/scripts/benchmark-browser-tunnel-framing.mjs [base-ref]
const path = 'src/shared/browser-network-tunnel-stream-framing.ts'
const baselineRef = process.argv[2] ?? 'HEAD'
const beforeSource = execFileSync('git', ['show', `${baselineRef}:${path}`], {
encoding: 'utf8'
})
const afterSource = readFileSync(path, 'utf8')
const load = (source) =>
import(
`data:text/javascript;base64,${Buffer.from(
stripTypeScriptTypes(source, { mode: 'transform' })
).toString('base64')}`
)
const before = await load(beforeSource)
const after = await load(afterSource)
function measure(module, chunks, payload, repetitions) {
let frameCount = 0
let lastFrame
const onFrame = (frame) => {
frameCount++
lastFrame = frame
}
const onError = (error) => {
throw error
}
const run = () => {
const decoder = new module.BrowserNetworkTunnelStreamFrameDecoder(onFrame, onError)
for (const chunk of chunks) {
decoder.feed(chunk)
}
}
run()
assert.deepEqual(lastFrame, payload)
const samples = []
for (let sample = 0; sample < 5; sample++) {
const start = performance.now()
for (let iteration = 0; iteration < repetitions; iteration++) {
run()
}
samples.push((performance.now() - start) / repetitions)
}
assert.equal(frameCount, 1 + 5 * repetitions)
return samples.sort((a, b) => a - b)[2]
}
function countCopies(module, chunks) {
const originalSet = Uint8Array.prototype.set
const originalSlice = Uint8Array.prototype.slice
let copied = 0
Uint8Array.prototype.set = function (source, offset) {
copied += source.length
return originalSet.call(this, source, offset)
}
Uint8Array.prototype.slice = function (...args) {
const result = originalSlice.apply(this, args)
copied += result.length
return result
}
try {
const decoder = new module.BrowserNetworkTunnelStreamFrameDecoder(
() => {},
(error) => {
throw error
}
)
for (const chunk of chunks) {
decoder.feed(chunk)
}
} finally {
Uint8Array.prototype.set = originalSet
Uint8Array.prototype.slice = originalSlice
}
return copied
}
const rows = []
for (const [payloadBytes, chunkBytes, repetitions] of [
[1, 5, 10000],
[64 * 1024, 65540, 1000],
[64 * 1024, 4096, 100],
[64 * 1024, 256, 25],
[64 * 1024, 16, 5],
[64 * 1024, 1, 1]
]) {
const payload = Uint8Array.from({ length: payloadBytes }, (_, index) => index % 251)
const encoded = before.encodeBrowserNetworkTunnelStreamFrame(payload)
const chunks = []
for (let offset = 0; offset < encoded.length; offset += chunkBytes) {
chunks.push(encoded.subarray(offset, offset + chunkBytes))
}
const beforeMs = measure(before, chunks, payload, repetitions)
const afterMs = measure(after, chunks, payload, repetitions)
rows.push({
payloadBytes,
chunkBytes,
beforeMs: +beforeMs.toFixed(6),
afterMs: +afterMs.toFixed(6),
speedup: +(beforeMs / afterMs).toFixed(2),
beforeCopiedBytes: countCopies(before, chunks),
afterCopiedBytes: countCopies(after, chunks)
})
}
console.log(JSON.stringify({ node: process.version, baselineRef, rows }, null, 2))
config/scripts/benchmark-cli-error-imports.mjs
+121
View File
@@ -0,0 +1,121 @@
import assert from 'node:assert/strict'
import { createRequire } from 'node:module'
import { existsSync, realpathSync } from 'node:fs'
import { delimiter, join, resolve } from 'node:path'
// Emit each revision with tsc -p config/tsconfig.cli.json --outDir <dir> --composite false --incremental false.
// Run: node config/scripts/benchmark-cli-error-imports.mjs <before-dir> <after-dir>
const [beforeDir, afterDir] = process.argv.slice(2)
assert.ok(beforeDir && afterDir, 'Pass distinct before and after TypeScript output directories.')
assert.notEqual(
realpathSync(beforeDir),
realpathSync(afterDir),
'Do not compare a build to itself.'
)
const entries = {
before: join(resolve(beforeDir), 'cli', 'index.js'),
after: join(resolve(afterDir), 'cli', 'index.js')
}
for (const entry of Object.values(entries)) {
assert.ok(existsSync(entry), `Missing emitted CLI: ${entry}`)
}
const { runProcessSync } = createRequire(import.meta.url)(
join(resolve(afterDir), 'shared', 'child-process', 'run-process.js')
)
const child = String.raw`
const { performance } = require('node:perf_hooks')
const { writeSync } = require('node:fs')
const { createHash } = require('node:crypto')
const { basename } = require('node:path')
let stdout = '', stderr = ''
process.stdout.write = (text) => { stdout += text; return true }
process.stderr.write = (text) => { stderr += text; return true }
const started = performance.now()
const cli = require(process.argv[1])
const importMs = performance.now() - started
cli.main(JSON.parse(process.argv[2])).then(() => {
const totalMs = performance.now() - started
const modules = Object.keys(require.cache)
writeSync(1, JSON.stringify({
importMs, totalMs, modules: modules.length,
featureFormatters: modules.filter((file) => ['browser', 'terminal', 'project', 'automation', 'workspace', 'computer'].some((name) => basename(file) === name + '-format.js')),
stdout: createHash('sha256').update(stdout).digest('hex'),
stderr: createHash('sha256').update(stderr).digest('hex'),
exitCode: process.exitCode || 0
}))
process.exitCode = 0
}).catch((error) => { writeSync(2, String(error)); process.exitCode = 1 })
`
const cases = [
['--help'],
['help', 'terminal', 'read'],
['does-not-exist'],
['computer', 'click', '--does-not-exist'],
['does-not-exist', '--json']
]
const median = (values) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]
const summarize = (samples) => ({
importMs: median(samples.map((sample) => sample.importMs)),
totalMs: median(samples.map((sample) => sample.totalMs)),
modules: samples[0].modules
})
const rows = []
for (const args of cases) {
const samples = { before: [], after: [] }
let expected
for (let run = 0; run < 22; run++) {
for (const variant of run % 2 ? ['after', 'before'] : ['before', 'after']) {
const result = runProcessSync({
program: process.execPath,
args: ['-e', child, entries[variant], JSON.stringify(args)],
timeoutMs: 30_000,
env: {
...process.env,
NODE_PATH: [resolve('node_modules'), process.env.NODE_PATH]
.filter(Boolean)
.join(delimiter)
}
})
assert.equal(result.timedOut, false, 'CLI child timed out.')
assert.equal(result.code, 0, result.stderr)
const sample = JSON.parse(result.stdout)
const output = { stdout: sample.stdout, stderr: sample.stderr, exitCode: sample.exitCode }
expected ??= output
assert.deepEqual(output, expected, `${variant} output changed for ${args.join(' ')}`)
if (variant === 'after') {
assert.deepEqual(
sample.featureFormatters,
[],
'Help and syntax errors must skip feature formatters.'
)
}
if (run >= 2) {
samples[variant].push(sample)
}
}
}
assert.ok(samples.after[0].modules < samples.before[0].modules, 'Expected fewer loaded modules.')
rows.push({
args,
before: summarize(samples.before),
after: summarize(samples.after),
output: expected,
samples
})
}
console.log(
JSON.stringify(
{
node: process.version,
platform: process.platform,
measurement:
'Fresh-process import + main; excludes process creation; warmed filesystem; 2 warmups and 20 samples per variant, alternating order.',
entries,
rows
},
null,
2
)
)

Some files were not shown because too many files have changed in this diff Show More