starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-22 00:02:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into brennanb2025/impl-agent-sleep-wake

Browse Source 
# Conflicts:
#	src/main/providers/agent-foreground-process.test.ts
#	src/main/runtime/orca-runtime-register-pty.ts
#	src/main/runtime/orca-runtime-sync-window-graph.ts
#	src/main/runtime/orchestration/mailbox-delivery-target.ts
#	src/main/runtime/orchestration/mailbox-pointer-submit.ts
#	src/renderer/src/lib/agent-hibernation-pane-eligibility.ts
#	src/renderer/src/store/slices/agent-status-manual-sleep-capture.test.ts
#	src/shared/process-table-snapshot-reader.ts
This commit is contained in:
Brennan Benson
2026-09-21 16:14:14 -07:00
parent 9c7672adf1 6ee7e9511b
commit 46ecbb9e20
9759 changed files with 1300407 additions and 96321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+57
View File
@@ -23,14 +23,71 @@
# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
/config/patches/*.patch -text
# Same reason, and pnpm parses these too: a CRLF checkout makes the mobile
# patches unparseable, so Windows packaging dies on ERR_PNPM_INVALID_PATCH.
/mobile/patches/*.patch -text
# The xterm bundle hunks also make a diff nobody can read; review the hand-written
# source patch under xterm-src/ instead. The sibling patches stay diffable.
/config/patches/@xterm__xterm@*.patch -diff
/config/patches/xterm-src/*.patch text eol=lf
# pnpm parses these unified diffs during Windows installs; keep checkout bytes stable.
/mobile/patches/*.patch -text
# Generated wrapper fixtures: collapse them in the PR diff so they stop drowning
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
# Mobile web bundle source. Every text byte here is hashed into an asset digest and
# from there into buildId, so a CRLF checkout produces a different bundle id for the
# same commit (91af2897 vs 9d78435e). The PNG is -text because it must not be touched.
/src/mobile-web/index.html text eol=lf
/src/mobile-web/src/*.ts text eol=lf
/src/mobile-web/src/*.css text eol=lf
/src/mobile-web/src/*.png -text
# Mobile web page source. Same buildId hazard as src/mobile-web above: these bytes are
# hashed into the Phase C bundle, so a CRLF Windows checkout would ship a different
# buildId for identical source. web-entry/ does not exist yet; the pin lands ahead of it.
/mobile/src/** text eol=lf
/mobile/app/** text eol=lf
/mobile/web-entry/** text eol=lf
# The blanket pin above would mark a future binary as text; exempt the asset types an
# RN page actually carries, the same way src/mobile-web exempts its PNG.
/mobile/src/**/*.png -text
/mobile/src/**/*.jpg -text
/mobile/src/**/*.jpeg -text
/mobile/src/**/*.gif -text
/mobile/src/**/*.ico -text
/mobile/src/**/*.webp -text
/mobile/src/**/*.ttf -text
/mobile/src/**/*.otf -text
/mobile/src/**/*.woff -text
/mobile/src/**/*.woff2 -text
/mobile/app/**/*.png -text
/mobile/app/**/*.jpg -text
/mobile/app/**/*.jpeg -text
/mobile/app/**/*.gif -text
/mobile/app/**/*.ico -text
/mobile/app/**/*.webp -text
/mobile/app/**/*.ttf -text
/mobile/app/**/*.otf -text
/mobile/app/**/*.woff -text
/mobile/app/**/*.woff2 -text
/mobile/web-entry/**/*.png -text
/mobile/web-entry/**/*.jpg -text
/mobile/web-entry/**/*.jpeg -text
/mobile/web-entry/**/*.gif -text
/mobile/web-entry/**/*.ico -text
/mobile/web-entry/**/*.webp -text
/mobile/web-entry/**/*.ttf -text
/mobile/web-entry/**/*.otf -text
/mobile/web-entry/**/*.woff -text
/mobile/web-entry/**/*.woff2 -text
.github/CONTRIBUTING.md
+5
View File
@@ -21,6 +21,11 @@ pnpm install
pnpm dev
```
Ordinary installs include native optional dependencies for the current OS and CPU only.
Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and
arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants.
See [the install policy](../docs/reference/pnpm-install-policy.md).
## Branch Naming
Use a clear, descriptive branch name that reflects the change.
.github/actions/install-mobile-dependencies/action.yml
+22
View File
@@ -0,0 +1,22 @@
name: Install mobile dependencies
description: Frozen pnpm install for the mobile/ project, whose node_modules the mobile web bundle build and the mobile-aware lint passes resolve React Native and Expo from.
runs:
using: composite
steps:
# Why a separate install: mobile is its own pnpm project, so the root install leaves
# mobile/node_modules empty and every mobile import resolves to nothing.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates the
# gitignored terminal/mermaid webview engine modules that tracked source imports.
# The drift guard mirrors the root install so a stale mobile lockfile fails by name --
# mobile's lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
shell: bash
working-directory: mobile
run: |
pnpm install --frozen-lockfile
# Job containers can run composite steps from a source mirror without .git.
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
.github/actions/install-node-dependencies/action.yml
+13 -15
View File
@@ -10,6 +10,10 @@ inputs:
description: Node.js version override; defaults to the version declared in package.json.
required: false
default: ''
cache-dependency-path:
description: Lockfiles for the pnpm download store; include mobile/pnpm-lock.yaml only when the job installs mobile dependencies.
required: false
default: pnpm-lock.yaml
persist-native-cache:
description: Save restored native modules at job end. Set false when a later step overwrites the same path with a different ABI.
required: false
@@ -39,9 +43,7 @@ runs:
with:
install: false
# Why both lockfiles: setup-node keys the pnpm store on the root lockfile alone, so
# jobs that also install mobile restored a store with none of the React Native tree
# in it and re-downloaded the lot on every run.
# Desktop-only jobs should not miss their download cache when mobile dependencies change.
- name: Setup Node.js
id: default-node
if: inputs.node-version == ''
@@ -49,9 +51,7 @@ runs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Setup requested Node.js
id: requested-node
@@ -60,9 +60,7 @@ runs:
with:
node-version: ${{ inputs.node-version }}
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
cache-dependency-path: ${{ inputs.cache-dependency-path }}
- name: Validate native runtime
shell: bash
@@ -152,9 +150,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Restore compiled native modules without saving
id: native-cache-restore-only
@@ -163,9 +161,9 @@ runs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- name: Use external node-gyp
.github/pull_request_template.md
+4 -4
View File
@@ -1,14 +1,14 @@
## ELI5
<!-- Simple high-level explanation -->
<!-- Simple high-level explanation, in plain language. No jargon. -->
## What Changed
<!-- Describe the change clearly and keep scope tight. -->
<!-- Describe the change clearly and keep scope tight. Cover the before and after as the user experiences it, and the mechanism you changed — not just the symptom. -->
## Why
<!-- What problem does this solve, and why is this approach right? -->
<!-- What problem does this solve, and why is this approach better than the alternatives you considered? -->
## Linked Issue
@@ -47,7 +47,7 @@ Ensure no issues in: Security, Cross-platoform support (Linux, Windows, Mac), Re
## Checklist
- [ ] This PR is small and focused
- [ ] I explained what changed and why (including ELI5)
- [ ] I explained what changed and why (ELI5, the user-facing before/after, the mechanism, and why over the alternatives)
- [ ] Before/after screenshots or videos attached for UI changes, or `N/A` with reason
- [ ] Self-reviewed for correctness, security, and performance
- [ ] Cross-platform, SSH/remote, and path/shortcut impact considered (or N/A)
.github/workflows/adhoc-mac-build.yml
+20 -10
View File
@@ -160,16 +160,14 @@ jobs:
- name: Checkout the requested ref
uses: actions/checkout@v6
env:
# Full-history checkout must also preserve case-twin branch and tag names.
GIT_DEFAULT_REF_FORMAT: reftable
with:
# Why an input at all rather than just github.ref: the whole point is to
# build code that has not landed, and the workflow definition itself
# always comes from the dispatch ref — naming the branch here instead
# applies main's current copy of this file to an arbitrary branch.
ref: ${{ steps.vetted.outputs.sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from the release API.
fetch-depth: 1
# This job only reads stablyai/orca and never pushes; every write goes
# to the adhoc repo through a minted App token passed by env. Not
# persisting the checkout credential shrinks the blast radius if a build
@@ -186,6 +184,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -197,13 +198,19 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an adhoc build installable over an existing
# Orca, so a missing cert must fail here rather than after a 20-minute build.
@@ -235,11 +242,14 @@ jobs:
fi
done
echo "head_sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT"
# Why the main repo's tags: package.json on a branch is as stale as the
# main it forked from, and stable patches never merge back into it.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag, which still owns that number.
# Releases-only let adhoc sit on a number already taken, so the updater
# would not install it. Empty on failure — the script then falls back
# to package.json.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \
node config/scripts/adhoc-build-version.mjs \
>"$RUNNER_TEMP/adhoc-identity.txt"
.github/workflows/cloud-deploy-relay-production-director.yml
+13 -3
View File
@@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
image-digest:
description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
required: true
type: string
regional-placement-mode:
@@ -13,6 +13,11 @@ on:
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
@@ -62,6 +67,7 @@ jobs:
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
@@ -106,8 +112,11 @@ jobs:
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
@@ -218,7 +227,8 @@ jobs:
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}"
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+314 -56
View File
@@ -16,6 +16,8 @@ on:
expected-rehome-generation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
monitor-run-attempt: { required: true, type: string }
gate-override-reason: { required: false, type: string, default: '' }
gate-override-confirmation: { required: false, type: string, default: '' }
wave-index: { required: true, type: string }
permissions:
@@ -52,7 +54,12 @@ jobs:
WAVE_INDEX: ${{ inputs.wave-index }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
# ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
# 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
DRAIN_PACE_WINDOW_MS: '120000'
steps:
- name: Require exact reusable-workflow configuration
working-directory: .
@@ -67,17 +74,20 @@ jobs:
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
# The caller validated this too; re-check here so a malformed override
# can never reach a mutation through this reusable workflow.
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
test "${DEPLOY_MODE}" != verify
test "${GATE_OVERRIDE_CONFIRMATION}" = \
"SKIP_RELAY_MONITOR_GATE ${TARGET_IMAGE_DIGEST}"
[[ "${GATE_OVERRIDE_REASON}" =~ ^[[:print:]]{12,500}$ ]]
fi
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
echo "so recover each remaining cell with its own fresh monitor" >&2
@@ -109,14 +119,34 @@ jobs:
- uses: hashicorp/setup-terraform@v3
with: { terraform_wrapper: false }
# One approved-cell table, in the wave validator the dispatch gate already uses, so
# a cell's class and its wave's selector delta cannot drift apart between the two.
- name: Resolve this cell's admission class and wave selector delta
run: |
CELL_CLASS="$(node dev/scripts/relay-production-same-cap-wave.mjs cell-class \
--cell-id "${TARGET_CELL_ID}")"
ENTRY_ADMISSION="$(jq -er '.entryAdmission' <<< "${CELL_CLASS}")"
SELECTOR_WAVE_DELTA="$(jq -er '.selectorWaveDelta' <<< "${CELL_CLASS}")"
if test "${DEPLOY_MODE}" = verify; then
EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
else
EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION \
+ (SELECTOR_WAVE_DELTA * WAVE_INDEX)))"
fi
{
echo "ENTRY_ADMISSION=${ENTRY_ADMISSION}"
echo "SELECTOR_WAVE_DELTA=${SELECTOR_WAVE_DELTA}"
echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}"
} >> "${GITHUB_ENV}"
- name: Require fresh aggregate monitor evidence reference
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
[[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
- name: Download private aggregate monitor evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -125,7 +155,7 @@ jobs:
run-id: ${{ inputs.monitor-run-id }}
- name: Verify monitor evidence provenance
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
node dev/scripts/relay-monitor-evidence.mjs verify-authority \
--directory "${OUTPUT_DIRECTORY}" \
@@ -138,7 +168,7 @@ jobs:
--wave-index "${WAVE_INDEX}"
- name: Download this wave's single-use safety authority
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/download-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -147,7 +177,7 @@ jobs:
run-id: ${{ github.run_id }}
- name: Require safety evidence consumed by this workflow
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
run: |
# Mutations are single-dispatch: a fresh dispatch cannot resume a
# partial batch (the canary authority binds the batch-entry selector
@@ -184,9 +214,32 @@ jobs:
# Freshness-only failures are publish lag, not health, on every wave
# including the first; the CLI still caps the retry at the wave's
# evidence-age budget, so this cannot mutate on aged evidence.
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" --retry-freshness
#
# This live recheck runs on every mutating wave, including a
# break-glass one. With the aggregate gate overridden there is no
# sealed state to read, so the expected selector comes from the
# dispatch inputs the rehome inspect below verifies against the live
# director; every threshold the sample is judged against is unchanged.
if test -n "${GATE_OVERRIDE_CONFIRMATION}"; then
jq -n \
--arg existingOnly "${EXPECTED_EXISTING_ONLY_CELLS/none/}" \
--arg migrationOnly "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg general "${EXPECTED_GENERAL_CELLS/none/}" \
'{existingOnly:$existingOnly,migrationOnly:$migrationOnly,general:$general}
| map_values(split(",") | map(select(length > 0)))' \
> "${RUNNER_TEMP}/relay-same-cap-selector.json"
pnpm incident:relay-preflight -- \
--no-monitor-state \
--expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
--selector-membership-file "${RUNNER_TEMP}/relay-same-cap-selector.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
else
pnpm incident:relay-preflight -- \
--state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
--wave-index "${WAVE_INDEX}" \
--selector-wave-delta "${SELECTOR_WAVE_DELTA}" --retry-freshness
fi
- name: Require durable rehome disabled and exact selector
env:
@@ -213,10 +266,17 @@ jobs:
c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
EXPECTED_HARD_CAP=1000
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c17|c18)
EXPECTED_HARD_CAP=600
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
EXPECTED_DATABASE_POOL_MAX=16
;;
*) exit 1 ;;
esac
@@ -228,14 +288,14 @@ jobs:
SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
-var-file=environments/production.tfvars \
<<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
if test "${EXPECTED_REGION}" = us-central1; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_UNOBSERVED_BOUND}"
# The startup script emits a pool line only off the root default, so an unpinned cell
# must still be on that default or its plan would carry a line nothing reviews.
test "$(jq -r '.database_pool_max' <<< "${CURRENT_SHAPE}")" = \
"${EXPECTED_DATABASE_POOL_MAX:-10}"
TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
if test "${DEPLOY_MODE}" = rollback; then
@@ -249,6 +309,14 @@ jobs:
DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
fi
# The startup template emits rehome trust lines only for a declared source cell, so
# require membership exactly when either side of this roll expects those lines.
if test "${EXPECTED_REGION}" = us-central1 && {
test "${DESIRED_REHOME_PROTOCOL}" != 0 || test "${CURRENT_REHOME_PROTOCOL}" != 0
}; then
jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
<<< "${SOURCE_CELLS}" >/dev/null
fi
DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
--arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
@@ -264,6 +332,7 @@ jobs:
echo "MIG_NAME=${MIG_NAME}"
echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
echo "EXPECTED_DATABASE_POOL_MAX=${EXPECTED_DATABASE_POOL_MAX}"
echo "EXPECTED_REGION=${EXPECTED_REGION}"
echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
@@ -292,20 +361,66 @@ jobs:
}
CURRENT_RUNTIME="$(admin_post current-runtime \
"${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
# A rollback that failed between template apply and admission restore
# leaves the cell already on the rollback image; resume from that
# state instead of demanding the pre-rollback predecessor.
# Two different failures leave the cell on the rollback image, and the image
# alone cannot tell them apart. A rollback that failed between its template
# apply and its admission restore restarted the cell, so that cell is not
# draining and resumes. A wave that stopped after its drain and before its
# template apply never restarted anything, so its cell is still draining and
# is stranded: the drain flag only clears on a restart, so it has to be rolled.
LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
LIVE_DRAINING="$(jq -r '.draining' <<< "${CURRENT_RUNTIME}")"
if test "${DEPLOY_MODE}" = rollback \
&& test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
ROLLBACK_RESUME=true
if test "${LIVE_DRAINING}" = true; then
ROLLBACK_STAGE=stranded
else
ROLLBACK_STAGE=resume
fi
PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
else
ROLLBACK_RESUME=false
ROLLBACK_STAGE=roll
PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
fi
if test "${ROLLBACK_STAGE}" = resume; then
ROLLBACK_RESUME=true
else
ROLLBACK_RESUME=false
fi
# A stranded cell's template still carries the image the cell is serving, so that
# is the predecessor its plan is reviewed against. A template already moved on to
# the target is refused here rather than rolled backwards under a stale review.
if test "${ROLLBACK_STAGE}" = stranded; then
PLAN_ROLLBACK_IMAGE="${DESIRED_IMAGE}"
else
PLAN_ROLLBACK_IMAGE="${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}"
fi
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require the cell pristine in the
# class it is declared to serve in.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
else
PRECHECK_ADMISSION="${ENTRY_ADMISSION}"
fi
# Draining sheds connections, and a migration-only cell holds none, so the flag
# carries no precondition there. It also outlives a failed wave, because the drain
# that set it is followed by no restart, which is the state a failed canary leaves.
if test "${DEPLOY_MODE}" = rollback \
|| test "${ENTRY_ADMISSION}" = migration-only; then
PRECHECK_DRAINING=either
else
PRECHECK_DRAINING=forbidden
fi
# A resumed rollback already restarted, so its cell has to come back not draining;
# that is what separates it from a wave that stopped before its template apply.
if test "${PRECHECK_DRAINING}" = either && test "${ROLLBACK_RESUME}" != true; then
PREDECESSOR_DRAINING_OK=true
else
PREDECESSOR_DRAINING_OK=false
fi
RESTORED_MIGRATION_CELLS="$(jq -rn \
--arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
--arg target "${TARGET_CELL_ID}" \
@@ -326,8 +441,19 @@ jobs:
'$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
# A migration-only cell is already isolated and is handed back isolated, so both
# halves of its wave see exactly the membership it entered with.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORED_MIGRATION_CELLS="${ISOLATED_MIGRATION_CELLS}"
RESTORED_GENERAL_CELLS="${ISOLATED_GENERAL_CELLS}"
fi
{
echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
echo "ROLLBACK_STAGE=${ROLLBACK_STAGE}"
echo "PLAN_ROLLBACK_IMAGE=${PLAN_ROLLBACK_IMAGE}"
# The drain wait and the plan review both read the image this cell actually
# serves, which is the rollback image on a stranded cell and not the current one.
echo "PREDECESSOR_IMAGE_DIGEST=${PREDECESSOR_IMAGE_DIGEST}"
# The failsafe consumes these; deriving them here keeps them
# defined for a failure in any later step.
echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
@@ -346,8 +472,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'.role == "cell" and .cellId == $cell and .cellUrl == $origin and
(.region == $region or
($region == "us-central1" and $protocol == 0 and .region == null)) and
@@ -363,8 +488,7 @@ jobs:
--argjson hardCap "${EXPECTED_HARD_CAP}" \
--argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
--argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
--argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
&& test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
--argjson drainingOk "${PREDECESSOR_DRAINING_OK}" \
'[
if .role != "cell" then "role" else empty end,
if .cellId != $cell then "cellId" else empty end,
@@ -400,16 +524,6 @@ jobs:
fi
[[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
# Rollback is the documented recovery from a failed canary, which
# leaves the cell migration-only (and possibly still marked
# draining); apply and verify still require a pristine general cell.
if test "${DEPLOY_MODE}" = rollback; then
PRECHECK_ADMISSION=general-or-migration-only
PRECHECK_DRAINING=either
else
PRECHECK_ADMISSION=general
PRECHECK_DRAINING=forbidden
fi
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -424,9 +538,12 @@ jobs:
- name: Reversibly isolate and drain only the selected cell
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: drain
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
run: |
# Opens the window the report-only shadow health gate below judges this cell over.
echo "drain-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
# A cell isolated by a failed canary is already migration-only, so
# isolate is a no-op there that does not advance the selector; the
@@ -435,18 +552,29 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Isolating a migration-only cell must be a read-only no-op; a change here would
# mean the live class is not the one this wave planned around.
if test "${ENTRY_ADMISSION}" = migration-only; then
jq -e '.changed == false' <<< "${ISOLATE_RESULT}" >/dev/null
fi
# The director re-places hosts off a cell only when the isolate stamped it, so a
# script too old to ask for the stamp produces today's behaviour and the canary
# reads as "the fix did nothing" with nothing to tell that from a wrong premise.
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
--pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
# The wait has to outlast the pacing window as well as the leases it waits on.
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat either --admission migration-only --draining required \
--activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
--timeout-ms 900000
--activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
--timeout-ms 1020000
- id: capacity-auth
if: ${{ inputs.mode != 'verify' }}
@@ -459,16 +587,27 @@ jobs:
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
# Zero resource changes prove the prior run's apply completed and no
# restart will follow, keeping the incarnation check honest. Root
# outputs may lag a targeted apply, so judge resource_changes only.
# The backend service is targeted too, so its reviewed drain timeout
# and request logging can be the only thing left here; neither
# restarts an instance, so the validator below clears that on its
# own, without the template-and-MIG pair.
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_backend_service.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
if ! terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
@@ -491,7 +630,7 @@ jobs:
| select(.change.actions | any(. != "no-op" and . != "read"))
| .address] | join(","))'
echo 'requiring reviewed rollback-image drift'
terraform -chdir=infra/terraform show -json \
RESUME_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
@@ -499,40 +638,89 @@ jobs:
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--image "${DESIRED_IMAGE}" \
--rollback-image "${DESIRED_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
| jq -e '.changes == 2' >/dev/null
"${POOL_ARGUMENTS[@]}")"
echo "${RESUME_REVIEW}"
jq -e '.changes == 2
or (.changes == 0 and ((.backendUpdate // []) | length) > 0)' \
<<< "${RESUME_REVIEW}" >/dev/null
# changes == 0 here means the template and MIG are converged and this cell's
# reviewed backend update is the only thing left, so the resume is not complete:
# apply it, or the cell silently keeps the 300-second drain and no request
# logging and the operator reads that as a finished roll. The plan holds nothing
# else (the validator bounded it to this cell's backend, and the template and MIG
# are no-ops in it), and neither attribute restarts an instance, so the
# incarnation check downstream stays honest. Template-and-MIG drift still applies
# nothing, which is what a resume means.
if test "$(jq -er '.changes' <<< "${RESUME_REVIEW}")" = 0; then
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
fi
fi
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
- name: Apply only the selected same-cap template and MIG
if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
id: apply
shell: bash
env:
CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
run: |
# A cell on the root pool default emits no pool line, so pin one only where it exists.
POOL_ARGUMENTS=()
if test -n "${EXPECTED_DATABASE_POOL_MAX}"; then
POOL_ARGUMENTS=(--database-pool-max "${EXPECTED_DATABASE_POOL_MAX}")
fi
terraform -chdir=infra/terraform plan \
-var-file=environments/production.tfvars \
-var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
"-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
"-target=google_compute_backend_service.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
-out="${RUNNER_TEMP}/relay-same-cap.tfplan"
terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
PLAN_REVIEW="$(terraform -chdir=infra/terraform show -json \
"${RUNNER_TEMP}/relay-same-cap.tfplan" \
| node dev/scripts/validate-relay-capacity-plan.mjs \
--mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
--hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
--rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
--rollback-image "${PLAN_ROLLBACK_IMAGE}" \
--capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
--rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
--rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
"${POOL_ARGUMENTS[@]}")"
echo "${PLAN_REVIEW}"
# Stamped before the apply, not after it: the new container announces its listener while
# the MIG is still converging, so a bound taken at the end of this step is already past
# the announcement the shadow gate looks for.
echo "apply-started-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
terraform -chdir=infra/terraform apply -auto-approve \
"${RUNNER_TEMP}/relay-same-cap.tfplan"
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
# A stranded cell already runs the reviewed template, so the apply above replaces
# no instance and the drain flag, which only a restart clears, would survive the
# whole wave. Roll the MIG explicitly on exactly the policy a template change uses.
# Every field is passed: gcloud persists these into the MIG's update policy, and it
# defaults the method to substitute on a group with no stateful config, so omitting
# one drifts the policy off the reviewed one and fails every later targeted plan.
if test "${ROLLBACK_STAGE}" = stranded \
&& test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then
gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
--replacement-method recreate --max-surge 0 --max-unavailable 1
gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
--project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
fi
# Recorded for the operator comparing verdicts; the gate's boot search opens at the
# apply-started-at stamp above, not here.
echo "apply-completed-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- id: post-auth
if: ${{ inputs.mode != 'verify' }}
@@ -546,6 +734,7 @@ jobs:
- name: Verify new incarnation, exact image, protocol, and durable safety
if: ${{ inputs.mode != 'verify' }}
id: verify-target
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
@@ -597,9 +786,10 @@ jobs:
--expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
echo "verify-ended-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
@@ -607,34 +797,99 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
--cell-incarnation "${TARGET_INCARNATION}"
- name: Restore only the verified selected cell to general admission
- name: Restore only the verified selected cell to its entry admission
if: ${{ inputs.mode != 'verify' }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
# Activating a migration-only cell would promote it to a serving cell for good, so
# restore it with the idempotent isolate that reports the authoritative generation.
if test "${ENTRY_ADMISSION}" = migration-only; then
RESTORE_MODE=isolate
else
RESTORE_MODE=activate
fi
RESTORE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
echo "${ACTIVATE_RESULT}"
SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
<<< "${ACTIVATE_RESULT}")"
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode "${RESTORE_MODE}")"
echo "${RESTORE_RESULT}"
SELECTOR_GENERATION_AFTER_RESTORE="$(jq -er '.generation' \
<<< "${RESTORE_RESULT}")"
node dev/scripts/verify-relay-capacity-transition.mjs \
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
--unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
--heartbeat fresh --admission general --draining forbidden --activity allowed \
--heartbeat fresh --admission "${ENTRY_ADMISSION}" \
--draining forbidden --activity allowed \
--expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
--regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
node dev/scripts/operate-relay-regional-rehome.mjs \
--mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
--expected-selector-generation "${SELECTOR_GENERATION_AFTER_RESTORE}" \
--expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
--expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
--expected-general-cells "${RESTORED_GENERAL_CELLS}" \
--expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
| jq -e '.control.enabled == false' >/dev/null
# Report only: this evaluates the oracles an operator reads by hand after a cell and records
# what it would have called, so its verdicts can be compared with the operator's over a full
# roll before it is ever allowed to block. Two independent guarantees keep it inert: the
# script exits 0 on every verdict, and continue-on-error keeps even a crash off the job's
# outcome. The failsafe below therefore cannot fire on anything this step observes.
#
# It runs after the restore, not before it, for two reasons: the cell goes back into
# admission on exactly today's schedule rather than waiting out a minute of log reads, and
# the window it judges has closed by then, so Cloud Logging's ingestion lag is behind it.
# These are fleet oracles anyway; when this does gate, what it gates is the next cell.
- name: Shadow health gate (report only)
id: shadow-gate
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# continue-on-error bounds this step's contribution to the job outcome, not its clock,
# and its reads are serialised. A timed-out step is a failed step, which continue-on-error
# absorbs; without this bound a Logging 429 storm or an expired credential makes every
# read cost its full retry budget and can push the job past timeout-minutes, and a
# cancelled job takes the whole wave with it. A healthy gate is already minutes of
# serial reads, so both bounds sit above that: the script settles at seven minutes and
# reaching this eight is the pathological case. Eight on top of a ~14-minute cell still
# leaves the job's 75 minutes intact.
timeout-minutes: 8
env:
DRAIN_STARTED_AT: ${{ steps.drain.outputs.drain-started-at }}
APPLY_STARTED_AT: ${{ steps.apply.outputs.apply-started-at }}
APPLY_COMPLETED_AT: ${{ steps.apply.outputs.apply-completed-at }}
VERIFY_ENDED_AT: ${{ steps.verify-target.outputs.verify-ended-at }}
SHADOW_GATE_DIRECTORY: ${{ runner.temp }}/relay-same-cap-shadow-gate
SHADOW_GATE_NAME: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
run: |
mkdir -p "${SHADOW_GATE_DIRECTORY}"
node dev/scripts/relay-same-cap-shadow-gate.mjs \
--cell-id "${TARGET_CELL_ID}" \
--cell-host "${TARGET_HOSTNAME}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" \
--director-service orca-cloud-relay \
--drain-started-at "${DRAIN_STARTED_AT}" \
--apply-started-at "${APPLY_STARTED_AT}" \
--apply-completed-at "${APPLY_COMPLETED_AT}" \
--verify-ended-at "${VERIFY_ENDED_AT}" \
--summary-file "${GITHUB_STEP_SUMMARY}" \
--output-file "${SHADOW_GATE_DIRECTORY}/${SHADOW_GATE_NAME}"
- name: Publish the shadow health gate verdict
if: ${{ inputs.mode != 'verify' }}
continue-on-error: true
# One small JSON file; a retrying upload must not spend the wave's remaining minutes either.
timeout-minutes: 2
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-shadow-gate-${{ inputs.target-cell-id }}-${{ github.run_id }}.json
path: ${{ runner.temp }}/relay-same-cap-shadow-gate
if-no-files-found: warn
retention-days: 14
overwrite: true
- id: cleanup-auth
if: ${{ failure() && inputs.mode != 'verify' }}
uses: google-github-actions/auth@v2
@@ -656,6 +911,9 @@ jobs:
--director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
--cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
echo "${ISOLATE_RESULT}"
# Same reason as the isolate step above: a failed wave leaves this cell isolated
# deliberately, and the stamp is what lets its hosts leave.
jq -e '.rollIsolated == true' <<< "${ISOLATE_RESULT}" >/dev/null
# The isolate result carries the authoritative post-isolate generation;
# fixed offsets are wrong whenever an earlier isolate was a no-op.
FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+221 -11
View File
@@ -10,7 +10,7 @@ on:
type: choice
options: [verify, canary-apply, batch-apply, rollback]
cell-ids:
description: Ordered comma-separated serving cells; one canary or two to four batch cells
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
required: true
type: string
target-image-digest:
@@ -26,13 +26,13 @@ on:
required: true
default: '1'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
@@ -62,13 +62,24 @@ on:
required: false
type: string
canary-run-id:
description: Successful same-commit canary run required for batch-apply
description: Successful same-code canary in this rehome control generation; reusable across batches
required: false
type: string
confirmation:
description: Exact digest-and-cell-bound mutation confirmation
required: false
type: string
gate-override-reason:
description: >-
Break-glass only: why this wave may skip the aggregate 15-minute monitor
dry-run gate. The live per-wave preflight still runs.
required: false
type: string
gate-override-confirmation:
description: >-
Break-glass only: exactly "SKIP_RELAY_MONITOR_GATE <target-image-digest>"
required: false
type: string
permissions:
actions: read
@@ -111,11 +122,16 @@ jobs:
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
CONFIRMATION: ${{ inputs.confirmation }}
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
run: |
# Fails closed on a partial or mismatched override, before any mutation.
CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
--mode "${MODE}" --cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
--confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}")"
echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
@@ -123,6 +139,35 @@ jobs:
echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
fi
- name: Record the monitor gate override in the run summary
if: ${{ inputs.gate-override-confirmation != '' }}
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
MODE: ${{ inputs.mode }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
run: |
{
echo '## Aggregate monitor gate overridden (break-glass)'
echo
echo '| field | value |'
echo '| --- | --- |'
echo "| actor | ${ACTOR} |"
echo "| mode | ${MODE} |"
echo "| cells | ${CELL_IDS} |"
echo "| target digest | \`${TARGET_DIGEST}\` |"
echo "| reason | ${GATE_OVERRIDE_REASON} |"
echo "| confirmation | \`${GATE_OVERRIDE_CONFIRMATION}\` |"
echo
echo 'Skipped: the 15-minute aggregate monitor dry-run and its sealed evidence.'
echo 'Still enforced: the live per-wave preflight against the same thresholds,'
echo 'durable rehome disabled, the exact selector generation and membership, the'
echo 'reviewed Terraform plan, one cell at a time behind the rollout lease, and'
echo 'single-dispatch mutation.'
} >> "${GITHUB_STEP_SUMMARY}"
- name: Download exact prior canary authority
if: ${{ inputs.mode == 'batch-apply' }}
uses: actions/download-artifact@v4
@@ -136,17 +181,23 @@ jobs:
if: ${{ inputs.mode == 'batch-apply' }}
env:
CANARY_RUN_ID: ${{ inputs.canary-run-id }}
CELL_IDS: ${{ inputs.cell-ids }}
TARGET_DIGEST: ${{ inputs.target-image-digest }}
ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
run: |
node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
--file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
--commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
--target-digest "${{ inputs.target-image-digest }}" \
--rollback-digest "${{ inputs.rollback-image-digest }}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}"
--cell-ids "${CELL_IDS}" \
--target-digest "${TARGET_DIGEST}" \
--rollback-digest "${ROLLBACK_DIGEST}" \
--selector-generation "${SELECTOR_GENERATION}" \
--rehome-generation "${REHOME_GENERATION}"
- name: Reject previously consumed aggregate safety evidence
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
env:
GH_TOKEN: ${{ github.token }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -163,7 +214,7 @@ jobs:
> "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
- name: Consume aggregate safety evidence for this exact wave
if: ${{ inputs.mode != 'verify' }}
if: ${{ inputs.mode != 'verify' && inputs.gate-override-confirmation == '' }}
uses: actions/upload-artifact@v4
with:
name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
@@ -188,6 +239,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '0'
secrets: inherit
@@ -209,6 +262,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '1'
secrets: inherit
@@ -230,6 +285,8 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '2'
secrets: inherit
@@ -251,9 +308,149 @@ jobs:
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '3'
secrets: inherit
cell_5:
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
needs: [gate, cell_4]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '4'
secrets: inherit
cell_6:
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
needs: [gate, cell_5]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '5'
secrets: inherit
cell_7:
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
needs: [gate, cell_6]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '6'
secrets: inherit
cell_8:
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
needs: [gate, cell_7]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '7'
secrets: inherit
cell_9:
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
needs: [gate, cell_8]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '8'
secrets: inherit
cell_10:
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
needs: [gate, cell_9]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '9'
secrets: inherit
seal_canary:
if: ${{ inputs.mode == 'canary-apply' }}
needs: [gate, cell_1]
@@ -266,6 +463,10 @@ jobs:
with: { node-version: 24 }
- name: Seal exact successful canary authority
env:
GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
ACTOR: ${{ github.actor }}
run: |
mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
@@ -276,6 +477,9 @@ jobs:
--commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
--selector-generation "${{ inputs.expected-selector-generation }}" \
--rehome-generation "${{ inputs.expected-rehome-generation }}" \
--gate-override-reason "${GATE_OVERRIDE_REASON}" \
--gate-override-confirmation "${GATE_OVERRIDE_CONFIRMATION}" \
--actor "${ACTOR}" \
> "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
- uses: actions/upload-artifact@v4
@@ -294,6 +498,12 @@ jobs:
- cell_2
- cell_3
- cell_4
- cell_5
- cell_6
- cell_7
- cell_8
- cell_9
- cell_10
- seal_canary
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
.github/workflows/cloud-operate-relay-production-rehome-job.yml
+5 -1
View File
@@ -14,6 +14,7 @@ on:
not-before: { required: true, type: string }
rate-per-minute: { required: true, type: string }
preference-max-age-ms: { required: true, type: string }
host-cooldown-ms: { required: true, type: string }
drain-grace-ms: { required: true, type: string }
confirmation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
@@ -54,6 +55,7 @@ jobs:
NOT_BEFORE: ${{ inputs.not-before }}
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
CONFIRMATION: ${{ inputs.confirmation }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -128,6 +130,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -299,6 +302,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -316,7 +320,7 @@ jobs:
echo '### Regional rehome control'
jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
"${RUNNER_TEMP}/relay-rehome-control.json"
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`"' \
jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`\n- host not arrived (24h): `\(.hostNotArrivedLast24Hours // "not reported")`\n- oldest active age (ms): `\(.oldestActiveAgeMs // "none")`"' \
"${RUNNER_TEMP}/relay-rehome-inventory.json"
} >> "${GITHUB_STEP_SUMMARY}"
.github/workflows/cloud-operate-relay-production-rehome.yml
+6
View File
@@ -52,6 +52,11 @@ on:
required: true
default: '86400000'
type: string
host-cooldown-ms:
description: Minimum gap between two rehomes of the same host
required: true
default: '604800000'
type: string
drain-grace-ms:
description: Per-host source drain grace
required: true
@@ -99,6 +104,7 @@ jobs:
not-before: ${{ inputs.not-before }}
rate-per-minute: ${{ inputs.rate-per-minute }}
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
drain-grace-ms: ${{ inputs.drain-grace-ms }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
.github/workflows/cloud-push-deploy.yml
+566
View File
@@ -0,0 +1,566 @@
name: Deploy Push Gateway Production
on:
workflow_dispatch:
inputs:
source_sha:
description: Full reviewed commit SHA to build (feature may remain unmerged)
required: true
type: string
confirmation:
description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
required: true
type: string
permissions:
contents: read
id-token: write
# Serialize push traffic changes independently of Relay and the shared database.
concurrency:
group: production-push-rollout
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
deploy:
if: >-
${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
github.ref == 'refs/heads/main' }}
runs-on: blacksmith-2vcpu-ubuntu-2204
environment: production
env:
GCP_PROJECT_ID: onorca-cloud
GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
SERVICE_NAME: orca-cloud-push
REPOSITORY_ID: orca-cloud
IMAGE_NAME: push
PUSH_ORIGIN: https://push.onorca.dev
PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
# Scaling the serving revision must already hold, matching push_min_instances and
# push_max_instances. Terraform owns both, and the candidate inherits them from the
# service, so this deploy never passes a scaling flag: doing so would write a
# Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
# `push_max_instances` raise would then be reverted by every deploy. These two values
# are the expected shape, asserted before the candidate is created and again on the
# candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
PUSH_MIN_INSTANCES: 1
PUSH_MAX_INSTANCES: 2
CONFIRMATION: ${{ inputs.confirmation }}
SOURCE_SHA: ${{ inputs.source_sha }}
steps:
- uses: actions/checkout@v4
- name: Require the explicit deploy confirmation
shell: bash
run: |
set -euo pipefail
test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
[[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
# Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
- name: Fetch the immutable gateway source
shell: bash
run: |
set -euo pipefail
git fetch --no-tags origin "${SOURCE_SHA}"
test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
mkdir -p "${RUNNER_TEMP}/push-source"
git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
| tar -x -C "${RUNNER_TEMP}/push-source"
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
- uses: google-github-actions/setup-gcloud@v2
- uses: docker/setup-buildx-action@v3
- name: Configure Docker auth
run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
# Building an image does not need the deployment lease.
- name: Build and publish the immutable gateway image
shell: bash
run: |
set -euo pipefail
image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
-f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
-t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
[[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
>> "${GITHUB_ENV}"
echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
# Refuse older images before they ever boot against production.
- name: Require image support for inert validation
shell: bash
run: |
set -euo pipefail
docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
import { loadPushConfig } from "./apps/push/dist/config.js";
const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation", ORCA_PUSH_FCM_PROJECT_ID: "onorca-cloud" };
if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
let rejected = false;
try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
if (!rejected) throw new Error("validation_mode_not_fail_closed");
'
# Held across the deploy, not just a separate schema step: the gateway opens its pool and
# applies its schema while the new revision starts, so the revision is the schema step.
- uses: ./.github/actions/cloud-sql-rollout-lease
with:
bucket: onorca-cloud-terraform-state
object: terraform/state/push-rollout/production.lock
# Why: the candidate inherits the serving revision's scaling. A serving revision that has
# drifted below the floor would hand the candidate a cold start on every notification, and
# one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
# rollout lease was taken for. Refuse to inherit either rather than latch it.
- name: Record the serving revision and require its Terraform-owned scaling
shell: bash
run: |
set -euo pipefail
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test -n "${serving}"
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
if test "${revisions}" != "${serving}"; then
echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
exit 1
fi
floor="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
"below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
"--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
exit 1
fi
ceiling="$(gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
gcloud run revisions describe "${serving}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-rollback-revision.json"
image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
[[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
"${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
# Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
# isolate background consumers from production.
- name: Deploy the candidate revision with no traffic
shell: bash
run: |
set -euo pipefail
tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
{
echo "VALIDATION_DEPLOY_ATTEMPTED=true"
echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
echo "VALIDATION_TAG=${tag}"
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
} >> "${GITHUB_ENV}"
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--image "${IMAGE}" \
--tag "${tag}" \
--revision-suffix "${tag}" \
--no-traffic \
--update-env-vars ORCA_PUSH_MODE=validation \
--quiet
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er --arg tag "${tag}" \
'[.status.traffic[] | select(.tag == $tag)]
| if length == 1 then .[0] else error("tagged candidate is not unique") end')"
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
# A tagged revision is directly addressable and sits outside the service-wide cap, so the
# candidate and the serving revision each draw up to the ceiling during the probe window.
# Successor creation later requires three revision pools; assert the inherited ceiling.
- name: Require the candidate to serve the exact image and inherited scaling
shell: bash
run: |
set -euo pipefail
served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(spec.containers[0].image)')"
test "${served}" = "${IMAGE}"
test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
- name: Probe the candidate readiness endpoint
shell: bash
run: |
set -euo pipefail
[[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
for attempt in $(seq 1 30); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
--max-time 10 "${CANDIDATE_URL}/ready" || true)"
if test "${code}" = 200; then
jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: /ready returned ${code}"
sleep 5
done
echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
exit 1
# Why: a gateway that boots and answers /ready can still be unable to send. This proves the
# runtime account's FCM grant end to end without delivering anything: validate_only stops
# Google before any push, and the deliberately invalid token means a healthy credential
# answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
#
# Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
# nothing about the credential, so it is retried rather than treated as either answer; a
# denied credential still fails on the first attempt, without burning the retries.
- name: Prove the runtime identity can reach FCM
shell: bash
run: |
set -euo pipefail
token="$(gcloud auth print-access-token \
--impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
test -n "${token}"
echo "::add-mask::${token}"
body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
for attempt in $(seq 1 5); do
code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
-X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
-H "Authorization: Bearer ${token}" \
-H 'Content-Type: application/json' \
--data "${body}" || true)"
status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
test "${code}" = 401 || test "${code}" = 403; then
break
fi
sleep 5
done
if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
echo "the push runtime identity cannot send through FCM" >&2
exit 1
fi
test "${status}" = INVALID_ARGUMENT
# Cloud Run requires a successor before the latest revision can be deleted.
- name: Retire inert validation and activate the verified image
shell: bash
run: |
set -euo pipefail
tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
{
echo "CANDIDATE_TAG=${tag}"
echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
echo "ACTIVATION_ATTEMPTED=true"
} >> "${GITHUB_ENV}"
# This is the production-effect boundary: schema, pruners and workers start here.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
--arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.containers[0].image == $image and .spec.serviceAccountName == $account and
all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
(.spec | shape) == ($prior[0].spec | shape) and
.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
<<< "${revision}" > /dev/null
candidate="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
| if length == 1 then .[0] else error("active candidate is not unique") end')"
test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
url="$(jq -er '.url' <<< "${candidate}")"
[[ "${url}" =~ ^https://[^/]+$ ]]
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${url}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
- name: Shift all traffic to the verified candidate
shell: bash
run: |
set -euo pipefail
echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${CANDIDATE_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${CANDIDATE_REVISION}"
echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
# Why: the summary is written before the origin check, not after it. Once traffic has
# moved, the rollback target is the single thing an operator needs, and a summary that only
# appeared on success would be missing in exactly the run that needs it.
- name: Publish the rollout summary
if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
shell: bash
run: |
set -euo pipefail
{
echo '### Push gateway rollout'
echo
echo "Source: ${SOURCE_SHA}"
echo
echo "Revision: \`${CANDIDATE_REVISION}\`"
echo
echo "Image: \`${IMAGE_DIGEST}\`"
echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
echo
echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
"\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
} >> "${GITHUB_STEP_SUMMARY}"
- name: Verify the public origin after the shift
shell: bash
run: |
set -euo pipefail
for attempt in $(seq 1 30); do
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
"${PUSH_ORIGIN}/ready" || true)"
if test "${code}" = 200; then
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
exit 0
fi
echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
sleep 5
done
echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
exit 1
# Why: everything after the shift runs with production on the candidate. A failure there
# is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
# is undone here rather than left to whoever reads the run.
- name: Roll traffic back to the previous revision
if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
test -n "${ROLLBACK_REVISION:-}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--to-revisions "${ROLLBACK_REVISION}=100" \
--quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
| if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${ROLLBACK_REVISION}"
echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
{
echo
echo '### Push gateway rolled back'
echo
echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
"\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
} >> "${GITHUB_STEP_SUMMARY}"
# Deleting a revision does not restore the service template that Terraform reconciles.
- name: Restore the known-good service template
if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
shell: bash
run: |
set -euo pipefail
test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
test -n "${ROLLBACK_IMAGE}"
# A partial activation may leave validation plus active; free one slot before recovery.
latest="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--format='value(status.latestCreatedRevisionName)')"
test -n "${latest}"
if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${VALIDATION_REVISION}"
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--remove-tags "${VALIDATION_TAG}" --quiet
gcloud run revisions delete "${VALIDATION_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
fi
fi
tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
>> "${GITHUB_STEP_SUMMARY}"
# Known-good schema/workers can run here even though HTTP stays on the old revision.
gcloud run deploy "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
--remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovered-service.json"
gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
> "${RUNNER_TEMP}/push-recovery-revision.json"
jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
--arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
--slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
--slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
def shape: del(.containers[0].image) |
.containers[0].env = ((.containers[0].env // []) |
map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
.spec.template.spec.containers[0].image == $image and
all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
$recovered[0].spec.containers[0].image == $image and
all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
($recovered[0].spec | shape) == ($prior[0].spec | shape) and
.spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
(.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
([.status.traffic[] | select((.percent // 0) > 0)] |
length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Promote and verify the known-good recovery revision
if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
shell: bash
run: |
set -euo pipefail
url="$(jq -er --arg tag "${RECOVERY_TAG}" \
'.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
[[ "${url}" =~ ^https://[^/]+$ ]]
curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${url}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
serving="$(gcloud run services describe "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
| jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
| jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
>> "${GITHUB_STEP_SUMMARY}"
- name: Delete the rejected candidate revision
if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
shell: bash
run: |
set -euo pipefail
if test -z "${CANDIDATE_REVISION:-}"; then
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
exit 0
fi
if test -n "${CANDIDATE_TAG:-}"; then
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
fi
existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
--filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
if test -n "${existing}"; then
test "${existing}" = "${CANDIDATE_REVISION}"
gcloud run revisions delete "${CANDIDATE_REVISION}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
fi
echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
echo "candidate revision ${CANDIDATE_REVISION} is absent"
# Public checks commit the serving revision; cleanup failures must not roll it back.
- name: Retire previous consumers after public checks
if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
shell: bash
run: |
set -euo pipefail
serving="${CANDIDATE_REVISION}"
if test "${RECOVERY_VERIFIED:-false}" = true; then
serving="${TEMPLATE_RECOVERY_REVISION}"
fi
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
while IFS= read -r revision; do
test -n "${revision}" || continue
test "${revision}" != "${serving}" || continue
case "${revision}" in
"${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
*) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
esac
gcloud run revisions delete "${revision}" \
--project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
done <<< "${revisions}"
echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
>> "${GITHUB_STEP_SUMMARY}"
- name: Drop the candidate traffic tag
if: always()
shell: bash
run: |
set -euo pipefail
test -n "${CANDIDATE_TAG:-}" || exit 0
gcloud run services update-traffic "${SERVICE_NAME}" \
--project "${GCP_PROJECT_ID}" \
--region "${GCP_REGION}" \
--remove-tags "${CANDIDATE_TAG}" \
--quiet
.github/workflows/cloud-verify.yml
+1
View File
@@ -90,6 +90,7 @@ jobs:
--health-timeout 5s
--health-retries 10
env:
ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
steps:
- uses: actions/checkout@v4
.github/workflows/computer-e2e.yml
+9
View File
@@ -6,6 +6,9 @@ on:
- '.github/workflows/computer-e2e.yml'
- 'config/electron-builder.config.cjs'
- 'config/scripts/build-computer-macos.mjs'
- 'config/scripts/build-native-for-platform.mjs'
- 'config/scripts/build-native-for-platform.test.mjs'
- 'config/scripts/pnpm-cli-invocation.mjs'
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/build-windows-cli-launcher.test.mjs'
- 'config/scripts/computer-e2e-workflow.test.mjs'
@@ -157,6 +160,12 @@ jobs:
pnpm vitest run
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
# Why: the parallel launcher's cancellation tests are darwin-only and no
# other PR job runs on macOS.
- name: Parallel native build launcher cancellation
run: >-
pnpm vitest run --config config/vitest.config.ts
config/scripts/build-native-for-platform.test.mjs
- name: Authenticated helper owner-loss smoke
run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1
- name: Swift tests and signed universal helper verification
.github/workflows/daemon-relocation-spike.yml
+22 -1
View File
@@ -57,7 +57,28 @@ jobs:
uses: actions/cache@v4
with:
path: dist/win-unpacked
key: win-unpacked-${{ hashFiles('src/**', 'config/**', 'package.json', 'pnpm-lock.yaml') }}
# mobile/ is in the key because beforePack requires out/mobile-web, whose bytes come from
# the mobile install and, once Phase C flips the bundle, from the page trees below; a
# mobile-only change must miss this cache, not reuse a stale installer. src/** and
# config/** already cover src/mobile-web and the two bundle builders.
key: >-
win-unpacked-${{ hashFiles(
'src/**',
'config/**',
'package.json',
'pnpm-lock.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-unpacked.outputs.cache-hit != 'true'
- name: Build unpacked app
if: steps.cache-unpacked.outputs.cache-hit != 'true'
.github/workflows/daily-mac-build.yml
+25 -10
View File
@@ -90,7 +90,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: main
fetch-depth: 0
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the daily repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -155,6 +156,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
if: steps.freshness.outputs.should_build == 'true'
@@ -167,6 +171,8 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
@@ -174,7 +180,12 @@ jobs:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.freshness.outputs.should_build == 'true'
# Why: signing is what makes a daily installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
@@ -206,17 +217,21 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$DAILY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. That dragged hourlies backwards so
# electron-updater stopped offering them; dailies would do the same. A
# separate token because GH_TOKEN above is the App's, scoped to the
# daily repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor so unpublishing a
# buggy main release cannot drag this series below a daily already out.
channel_tags="$(gh release list --repo "$DAILY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_DAILY_RELEASE_NAMES="$names" \
node config/scripts/daily-build-version.mjs \
>"$RUNNER_TEMP/daily-identity.txt"
.github/workflows/dev-channel-win-build.yml
+9
View File
@@ -203,6 +203,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Caches the Electron binary and electron-builder's tool downloads (nsis,
# winCodeSign). Same key shape as release-cut's Windows leg.
@@ -219,6 +222,8 @@ jobs:
# Why retried: pnpm install triggers electron's postinstall, which pulls the
# Electron binary from GitHub release assets, and that CDN returns transient
# 504s often enough to lose a build to it.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
@@ -227,6 +232,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why the packaging check runs before the 20-minute build: it only needs
# node_modules, and a stale config should cost seconds rather than a build.
- name: Verify dev-channel packaging identity
.github/workflows/e2e.yml
+34 -1
View File
@@ -170,8 +170,39 @@ jobs:
# artifact instead of starting five concurrent electron-vite builds.
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
# launches but never creates a BrowserWindow.
- name: Balance E2E shard from timing evidence
env:
ORCA_BACKGROUND_LAUNCH: '1'
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
ORCA_E2E_WEB_CLIENT: '1'
ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay
run: |
mkdir -p ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --list --reporter=json > ci-shards/discovery.json
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
node config/scripts/ci-e2e-shard-plan.mjs ci-shards/discovery.json '${{ matrix.shard }}' ci-shards
pnpm exec playwright test --config tests/playwright.config.ts --project=electron-headless --test-list=ci-shards/selected.txt --list --reporter=json > ci-shards/selected-discovery.json
node config/scripts/ci-e2e-shard-plan.mjs --verify ci-shards/assignment.json ci-shards/selected-discovery.json
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --test-list=ci-shards/selected.txt
- name: Upload E2E shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: e2e-shard-${{ matrix.shard_name }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
# The frame benchmark needs a mapped window, which the headless shards exclude.
- name: Run worktree first-paint benchmark
if: matrix.shard == '1/14'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
@@ -227,6 +258,7 @@ jobs:
mapfile -t TEST_FILES < <(jq -r '.[] | select(
. != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
. != "tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts" and
. != "tests/e2e/local-ssh-browser-routing.spec.ts" and
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
. != "tests/e2e/ssh-localhost.spec.ts" and
@@ -272,6 +304,7 @@ jobs:
if: >-
inputs.test_files == '' ||
inputs.ssh_source_changed == 'true' ||
contains(inputs.test_files, 'tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/local-ssh-browser-routing.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts') ||
contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
.github/workflows/git-command-termination-runtime.yml
+22
View File
@@ -0,0 +1,22 @@
name: Git command termination runtime
on:
pull_request:
paths:
- 'src/main/git/command-runner/spawned-command-tree-kill*'
- '.github/workflows/git-command-termination-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
windows-exit:
runs-on: windows-latest
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Verify exited native child does not trigger taskkill
run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/git/command-runner/spawned-command-tree-kill.test.ts
.github/workflows/hourly-mac-build.yml
+31 -13
View File
@@ -137,7 +137,8 @@ jobs:
uses: actions/checkout@v6
with:
ref: ${{ needs.preflight.outputs.head_sha }}
fetch-depth: 0
# Version helpers only read HEAD; published versions come from git tags.
fetch-depth: 1
# Why: this job only reads stablyai/orca and never pushes; every write
# goes to the hourly repo through a minted App token passed by env.
# Not persisting the checkout credential shrinks the blast radius if a
@@ -163,6 +164,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -174,13 +178,19 @@ jobs:
restore-keys: |
electron-builder-mac-
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
@@ -210,17 +220,25 @@ jobs:
# number free", where a stranded draft still holds one.
names="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json name \
--jq '.[].name // empty')"
# Why the main repo's tags decide the base version rather than
# package.json: main's version only moves on `release:` commits, and
# stable patches are cut from release branches that never merge back, so
# package.json can sit several patches behind what users are running. A
# separate token because GH_TOKEN above is the App's, scoped to the
# hourly repo. Empty on failure — the script then falls back to
# package.json, which is stale but never wrong enough to fail a build.
published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh release list \
--repo "$GITHUB_REPOSITORY" --limit 100 --exclude-drafts \
--json tagName --jq '.[].tagName' || true)"
echo "Highest published tag seen: $(head -1 <<<"$published")"
# Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
# GitHub release and leaves the tag. On 2026-09-14 we deleted v1.4.202's
# release for a bug; hourlies had already climbed to 1.4.203, then
# `gh release list` fell back to v1.4.201 and the next hourlies shipped
# as 1.4.202-hourly — which electron-updater will not install over
# 1.4.203-hourly or over the still-tagged 1.4.202. A separate token
# because GH_TOKEN above is the App's, scoped to the hourly repo. Empty
# on failure — the script then falls back to package.json, which is
# stale but never wrong enough to fail a build.
main_tags="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
"repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
--jq '.[].ref | sub("^refs/tags/"; "")' || true)"
# Already-shipped channel tags are a second floor: even if main's tag
# list is empty this run, a 1.4.203-hourly already out must not be
# followed by a 1.4.202-hourly.
channel_tags="$(gh release list --repo "$HOURLY_REPO" --limit 200 --json tagName \
--jq '.[].tagName' || true)"
published="$main_tags"$'\n'"$channel_tags"
echo "Published version sources: $(grep -c . <<<"$main_tags" || true) main tags, $(grep -c . <<<"$channel_tags" || true) channel tags"
ORCA_PUBLISHED_VERSIONS="$published" ORCA_HOURLY_RELEASE_NAMES="$names" \
node config/scripts/hourly-build-version.mjs \
>"$RUNNER_TEMP/hourly-identity.txt"
.github/workflows/mobile-ios-release.yml
+2
View File
@@ -94,6 +94,8 @@ jobs:
run: node -e 'const fs = require("node:fs"); const { expo } = require("./app.json"); fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${expo.version}\nbuild_number=${expo.ios.buildNumber}\n`)'
- name: Expo prebuild
env:
ORCA_IOS_APS_ENVIRONMENT: production
run: npx expo prebuild --platform ios --no-install
- name: Install CocoaPods
.github/workflows/mobile.yml
+105 -1
View File
@@ -9,21 +9,55 @@ on:
- ready_for_review
paths:
- 'mobile/**'
# Mobile launch contracts exercise the real host dispatcher and durable receipt store.
- 'src/main/agent-launch/**'
- 'src/main/runtime/rpc/**'
- 'src/main/runtime/runtime-rpc/**'
- 'src/main/runtime/runtime-rpc.ts'
- 'src/main/runtime/device-registry.ts'
- 'src/main/runtime/orca-runtime.ts'
- 'src/main/runtime/agent-session-*.ts'
- 'src/main/native-chat/agent-session-wire/**'
- 'src/shared/agent-launch-*.ts'
- 'src/shared/agent-session-*.ts'
- 'src/shared/new-workspace/worktree-create-collision.ts'
# Why: the mobile terminal link parsers are conformance-tested against
# these shared fixtures; desktop-side fixture edits must re-run this suite.
- 'src/shared/terminal-file-link-conformance.ts'
# Why: mobile imports the negotiated capability names directly and records
# the whole capability read verbatim in its goldens, so a capability added
# desktop-side rewrites a mobile fixture and must re-run this suite.
- 'src/shared/protocol-version.ts'
# Why: mobile's rpc-params-contract.ts is a type-only re-export of the
# generated params catalog, and mobile/tsconfig.json includes **/*.ts. A
# schema edit anywhere under here changes mobile's types, so a desktop-only
# change can break mobile's typecheck with no other mobile signal.
- 'src/shared/rpc-contract/**'
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
- '.github/actions/install-node-dependencies/**'
- '.github/workflows/mobile-ios-release.yml'
# Why main too: a behaviour-change branch legitimately pins its own last fenced commit, and that
# commit only stops being reachable when the branch squash-merges. The pull_request run cannot
# see that; this one is where the pin guard finds it.
push:
branches:
- main
paths:
- 'mobile/**'
- '.github/workflows/mobile.yml'
concurrency:
group: mobile-${{ github.event.pull_request.number || github.ref }}
# Per commit on main, not per branch. GitHub cancels any PENDING run in a group when a new one
# queues, whatever `cancel-in-progress` says, so one shared main group drops the middle merge of
# three -- and a pin that breaks there is exactly what this workflow now checks for.
group: mobile-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
jobs:
verify:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
env:
@@ -41,6 +75,10 @@ jobs:
uses: actions/checkout@v6
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
@@ -58,6 +96,19 @@ jobs:
- name: Typecheck
run: pnpm typecheck
# Why a ratchet and not the raw typecheck: mobile/tsconfig.json excludes test files, so until
# tsconfig.test.json existed nothing checked them, and at introduction 127 of the 632 had
# drifted. This fails when a test file that checks today stops checking, when a test leaves
# the program, and on @ts-nocheck; the baseline may only shrink.
- name: Typecheck tests (ratchet)
run: pnpm run check:tests-typecheck
# This includes the bridged replay of the whole recording corpus, which used to be a second
# step of its own behind RPC_FOUNDATION_BRIDGE=1. A gate nobody can forget to set is the point:
# it fails when a divergence class grows, when a divergence lands in no class at all, or when
# one of the 103 goldens inside the C1 page closure changes the verdict it is pinned to. It is
# ~3 min of test time on its own, and Vitest runs it on a worker beside the rest of the suite,
# so folding it in costs a fraction of that in wall time and one step less to skip.
- name: Test
run: pnpm test
@@ -83,3 +134,56 @@ jobs:
- name: Check formatting
run: pnpm format:check
recording-pin:
name: RPC recording pin
runs-on: ubuntu-latest
defaults:
run:
working-directory: mobile
steps:
- name: Checkout
uses: actions/checkout@v6
with:
# The ancestry verdict is read straight off history. On a shallow checkout
# `git merge-base --is-ancestor` answers from grafted parents, so the guard refuses to
# answer at all rather than reporting a pass it has no evidence for -- and the pinned tree
# below has to be checkable out.
fetch-depth: 0
- uses: ./.github/actions/install-node-dependencies
with:
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
# Seconds. No `--ref`, so the pin is judged against the same tree it was read out of. On a
# pull request that is the merge preview, which already carries main's repins; judging the
# branch head instead fails every branch cut before the day's repin, and its instruction would
# tell the author to pin their own head -- creating the break this guard exists to catch. A
# branch that pins its own commit passes here and fails on the push after the squash, which is
# where the pin actually leaves the history.
- name: Check the recording pin is reachable
shell: bash
run: pnpm exec tsx scripts/rpc-recording-pin-guard.mts ancestry
# ~2 min locally for the record itself, so it is gated rather than run twice over. A pull
# request that moves none of the corpus, the manifest or the recorder cannot move this
# verdict away from the one the base commit already published, and `verify` replays the
# corpus against the branch tree in the meantime. A push to main has no `verify` job and is
# where a squash lands a spliced corpus, so there it always runs.
- name: Reproduce the corpus from the pinned tree
shell: bash
env:
PIN_GUARD_BASE: ${{ github.event.pull_request.base.sha }}
run: |
if [ -n "$PIN_GUARD_BASE" ]; then
pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce --if-changed-since "$PIN_GUARD_BASE"
else
pnpm exec tsx scripts/rpc-recording-pin-guard.mts reproduce
fi
.github/workflows/packaged-browser-e2e.yml
+74
View File
@@ -0,0 +1,74 @@
name: Packaged browser compatibility
on:
workflow_dispatch:
inputs:
ref:
description: Commit SHA or ref to validate (defaults to the selected revision)
type: string
required: false
schedule:
- cron: '20 8 * * 1'
workflow_call:
inputs:
ref:
type: string
required: false
permissions:
contents: read
jobs:
compatibility:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Download pinned old release
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca"
python3 - <<'PYVERIFY'
import base64,hashlib,os,pathlib,subprocess
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
package=root/'orca-ide_1.4.188_amd64.deb'
expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g=='
assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected
extracted=root/'extracted'
subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
executable=extracted/'opt'/'Orca'/'orca-ide'
assert executable.is_file() and os.access(executable,os.X_OK)
with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(executable)+'\n')
print('Verified old package:',executable)
PYVERIFY
- name: Build current Electron app
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:relay
pnpm exec electron-vite build --mode e2e
pnpm run build:web-from-renderer
- name: Run both mixed-version directions
env:
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
run: >-
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
pnpm exec playwright test --config tests/playwright.config.ts
tests/e2e/packaged-mixed-version-browser-placement.spec.ts
--project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
- name: Require all six compatibility executions
if: always()
run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
- uses: actions/upload-artifact@v7
if: always()
with:
name: packaged-mixed-version-audit
path: test-results/
retention-days: 3
.github/workflows/pi-owner-runtime.yml
+32
View File
@@ -0,0 +1,32 @@
name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/**'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- 'tests/tools/omp-completion-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
- name: Verify OMP completion over native HTTP
run: node tests/tools/omp-completion-runtime-smoke.mjs
.github/workflows/pi-provider-runtime.yml
+28
View File
@@ -0,0 +1,28 @@
name: Pi extension provider verification
on:
pull_request:
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
.github/workflows/pr.yml
+181 -52
View File
@@ -29,6 +29,7 @@ jobs:
should_run: ${{ steps.filter.outputs.should_run }}
native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
mobile_web_app: ${{ steps.filter.outputs.mobile_web_app }}
static_analysis: ${{ steps.filter.outputs.static_analysis }}
typecheck: ${{ steps.filter.outputs.typecheck }}
git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
@@ -126,34 +127,27 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Lint
run: pnpm exec oxlint --format github
- name: Reject low-evidence patterns
run: pnpm run audit:anti-slop
- name: Enforce focused code-quality plugins
run: pnpm run audit:code-quality:native
- name: Enforce type-aware code-quality baseline
run: pnpm run audit:code-quality:type-aware
# Why: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Mobile is a separate pnpm project,
# so the root install above leaves it empty and every mobile type degrades to
# an `error` type — reported as phantom findings against the changed lines.
# Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
# the gitignored terminal/mermaid webview engine modules that tracked source imports,
# and skipping it degrades those very types the step exists to resolve. The drift
# guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
# lockfile carries patchedDependencies that a silent rewrite would drop.
- name: Install mobile dependencies
# Why here: the changed-code gate lints mobile files too, and its type-aware pass
# resolves types from mobile/node_modules. Without the install every mobile type
# degrades to an `error` type — reported as phantom findings against the changed lines.
- uses: ./.github/actions/install-mobile-dependencies
if: needs.code_paths.outputs.mobile_dependencies == 'true'
working-directory: mobile
run: |
pnpm install --frozen-lockfile
if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
fi
- name: Enforce changed-code quality
run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
@@ -167,6 +161,9 @@ jobs:
- name: Check reliability gate manifest
run: pnpm run check:reliability-gates
- name: Enforce dead design-system classes
run: pnpm run check:dead-classes
- name: Check VM runtime rollback compatibility
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
@@ -203,6 +200,9 @@ jobs:
- name: Boot orcad and round-trip a terminal
run: pnpm run smoke:orcad-terminal
- name: Verify the generated RPC params catalog
run: pnpm run verify:rpc-params-catalog
- name: Verify bundled skill guides
run: pnpm run verify:bundled-skill-guides
@@ -270,6 +270,12 @@ jobs:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node .github/scripts/check-root-directory-entries.mjs "$BASE_SHA" "$HEAD_SHA"
# Why here: the READMEs embed media owned by docs/site and resources/onboarding,
# and the classifier skips static_analysis for docs-only diffs. This job runs
# on every PR and needs no install.
- name: Check README local links
run: node config/scripts/check-readme-local-links.mjs
typecheck:
needs: [code_paths]
if: needs.code_paths.outputs.typecheck == 'true'
@@ -314,40 +320,59 @@ jobs:
# Why: the 2.25.5 lane is a source build of a pinned tarball, so it produced the
# same binary on every PR for minutes of runner time. The key carries the version
# because that is the only input; the sha256 assertion below still guards the
# tarball on the miss path that actually builds.
# tarball on the miss path that actually builds. Only this PR's own later pushes
# can restore it — GitHub scopes a cache written from a pull_request run to that
# ref — so a first push always takes the build path below.
- name: Cache baseline Git build
uses: actions/cache@v5
with:
path: ~/.cache/orca-git-compat/git-2.25.5
key: git-compat-baseline-${{ runner.os }}-${{ runner.arch }}-2.25.5
# Why its own step: this is `make -j$(nproc)` on every core, and the lanes below
# spend their wall clock waiting on container starts, not on Git. Sharing a runner
# with the build stretched one ~1.5s boundary case past Vitest's 30s timeout, so
# the build has to finish before anything timed starts.
- name: Build the baseline Git binary
run: |
archive="$RUNNER_TEMP/git-2.25.5.tar.gz"
source="$HOME/.cache/orca-git-compat/git-2.25.5"
if [ -x "$source/git" ]; then
exit 0
fi
curl -fsSL https://www.kernel.org/pub/software/scm/git/git-2.25.5.tar.gz -o "$archive"
echo "41662c52fc16fec4963bfc41075e71f8ead6b5e386797eb6f9a1111ff95a8ddf $archive" \
| sha256sum --check
mkdir -p "$source"
tar -xzf "$archive" -C "$source" --strip-components=1
make -C "$source" -j"$(nproc)" \
NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PYTHON=YesPlease git
# Why: the linked binaries are what the next run needs; the objects that
# produced them are most of the tree and would bloat the cache entry.
find "$source" -name '*.o' -delete
- name: Verify Git binary compatibility matrix
run: |
specs=(
"alpine/git:edge-2.38.1|2.38.1"
"alpine/git:v2.49.1|2.49.1"
)
# Why pull up front: a lane's first `docker run` otherwise pulls its image
# while the sibling lane is mid-test, and that stall is charged to the test.
for spec in "${specs[@]}"; do
docker pull --quiet "${spec%%|*}"
done
pids=()
(
archive="$RUNNER_TEMP/git-2.25.5.tar.gz"
source="$HOME/.cache/orca-git-compat/git-2.25.5"
if [ ! -x "$source/git" ]; then
curl -fsSL https://www.kernel.org/pub/software/scm/git/git-2.25.5.tar.gz -o "$archive"
echo "41662c52fc16fec4963bfc41075e71f8ead6b5e386797eb6f9a1111ff95a8ddf $archive" \
| sha256sum --check
mkdir -p "$source"
tar -xzf "$archive" -C "$source" --strip-components=1
make -C "$source" -j"$(nproc)" \
NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PYTHON=YesPlease git
# Why: the linked binaries are what the next run needs; the objects that
# produced them are most of the tree and would bloat the cache entry.
find "$source" -name '*.o' -delete
fi
ORCA_GIT_COMPAT_BINARY="$source/git" ORCA_GIT_COMPAT_VERSION="2.25.5" \
ORCA_GIT_COMPAT_BINARY="$HOME/.cache/orca-git-compat/git-2.25.5/git" \
ORCA_GIT_COMPAT_VERSION="2.25.5" \
pnpm exec vitest run --config config/vitest.config.ts \
src/shared/git-binary-compatibility.test.ts
) &
pids+=("$!")
for spec in \
"alpine/git:edge-2.38.1|2.38.1" \
"alpine/git:v2.49.1|2.49.1"; do
for spec in "${specs[@]}"; do
(
image="${spec%%|*}"
version="${spec#*|}"
@@ -624,6 +649,79 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts \
src/main/orcad/external-chromium-browser-process.integration.test.ts
# Why its own job: it needs mobile/node_modules and a real browser, and the sharded `test`
# matrix would pay for both on every shard to run two files. Dark through Phase C: this proves
# `build:mobile-web:app` on every PR that touches the page, and ships nothing -- packaging still
# builds the Phase A bootstrap via build:mobile-web.
mobile_web_app:
name: mobile web app bundle
needs: [code_paths]
if: needs.code_paths.outputs.mobile_web_app == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
# Why no native-runtime: the builder is esbuild and the render check is a browser. Nothing
# in this job loads node-pty.
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# The entry lives in mobile/ so one React resolves; without this every RN import is nothing.
- uses: ./.github/actions/install-mobile-dependencies
# Why the runner's Google Chrome and not a downloaded chromium: same reason as the orcad
# browser job -- Ubuntu 24.04 only ships an AppArmor userns profile for the Chrome .deb.
# Why fail instead of skip: a silently skipped render check is the failure this job exists
# to prevent.
- name: Resolve Chrome for the render check
run: |
set -euo pipefail
chrome="$(command -v google-chrome || command -v google-chrome-stable || true)"
if [ -z "$chrome" ]; then
echo "::error::No Google Chrome on the runner; the render check would silently skip."
exit 1
fi
"$chrome" --version
echo "ORCA_MOBILE_WEB_RENDER_BROWSER=$chrome" >> "$GITHUB_ENV"
# The drawer check runs on WebKit as well as Chrome, because the shell's iOS WebView is
# WebKit and the Chrome above cannot stand in for it. Downloaded rather than resolved from
# the runner: Ubuntu ships no WebKit build to point at.
- name: Install WebKit for the drawer check
run: pnpm exec playwright install --with-deps webkit
- name: Build and verify the app bundle
run: pnpm run build:mobile-web:app
# The bundling tests skip themselves where mobile dependencies are absent, which is how they
# stay green in the sharded `test` job. This is the job that installs them, so here a missing
# install has to fail rather than skip everything the job exists to run.
# Why a prefix and not a file list: the list this replaces had gone stale twice without
# anyone noticing, because a census whose closure block skips without the env flag below is
# green in the sharded `test` job whether or not it ever runs here. The prefix is the same
# one `pr-code-change-scope.mjs` fires this job on, so naming a test into the family is all
# it takes to have it run. Quoted because these are vitest filename filters, matched as
# substrings against the discovered files, and the shell must not touch them.
#
# Cost: 18 files in 25-30s wall, of which the frame-budget sweep is 2.5s. That sweep encodes
# 111 noise JPEGs in Chromium, so it is the one step here whose cost grows with its viewport
# set; adding rows to that set is a decision about this job's runtime.
- name: Builder, override census and render checks
env:
ORCA_MOBILE_WEB_APP_DEPS_REQUIRED: '1'
run: |
pnpm exec vitest run --config config/vitest.config.ts \
'config/scripts/mobile-web-app-' \
'config/scripts/build-mobile-web-app-bundle.test.mjs'
cross-version-wire:
name: cross-version wire compatibility
needs: [code_paths]
@@ -658,6 +756,8 @@ jobs:
tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts
tests/e2e/cross-version-wire/reported-lossy-initial-snapshot.unit.test.ts
tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts
tests/e2e/cross-version-wire/cross-version-worktree-identity-downgrade.unit.test.ts
tests/e2e/cross-version-wire/cross-version-session-tabs-retirement-proof.unit.test.ts
managed_hook_node18:
name: managed hooks on Node 18
@@ -709,6 +809,13 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why --no-file-parallelism: every file here launches a full Electron stack twice, and each
# probe carries its own in-process deadline. Four at once on a 4-vCPU runner starve each other
@@ -743,6 +850,12 @@ jobs:
- name: Project web client from renderer build
run: pnpm run build:web-from-renderer
# Why here and not inside "Build package inputs": this job assembles packaging inputs step by
# step instead of calling build:release, and electron-builder's beforePack guard hard-fails
# without out/mobile-web.
- name: Build mobile web bundle
run: pnpm run build:mobile-web
- name: Build native components
run: pnpm run build:native
@@ -772,18 +885,9 @@ jobs:
[[ "$rpm_marker" == rpm ]] || { echo "Expected rpm marker, got: $rpm_marker"; exit 1; }
- name: Verify headless serve signal shutdown
run: node config/scripts/run-headless-serve-shutdown-docker.mjs --appimage dist/orca-linux.AppImage
- name: Verify extracted launcher serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint launcher
- name: Verify AppImage CLI registration and serve signal shutdown
run: >-
node config/scripts/run-headless-serve-shutdown-docker.mjs
--appimage dist/orca-linux.AppImage --entrypoint appimage
--signal-target serving-electron --int-delivery pid
--appimage dist/orca-linux.AppImage --all-entrypoints
# A default container reproduces the hostile AppImage launch environment.
- name: Verify Linux CLI launch contract
@@ -825,6 +929,13 @@ jobs:
with:
native-runtime: node
persist-native-cache: 'false'
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Save compiled Node native modules
if: steps.deps.outputs.native-cache-hit != 'true'
@@ -832,9 +943,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
# vitest runs here directly rather than through `pnpm test`, so the addon
# assertions only hold once install-node-dependencies has rebuilt natives.
@@ -843,6 +954,16 @@ jobs:
pnpm exec vitest run --config config/vitest.config.ts
config/scripts/rebuild-native-deps.test.mjs
config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
config/scripts/rebuild-native-deps-node-pty.test.mjs
config/scripts/ensure-native-runtime-job-ownership.test.mjs
config/scripts/verify-packaged-node-pty-job-ownership.test.mjs
config/scripts/windows-pe-machine.test.mjs
config/scripts/script-module-dependencies.test.mjs
src/main/windows-registry-addon.test.ts
config/scripts/windows-process-tree-gyp-path.test.mjs
config/scripts/windows-process-tree-gyp-rebuild.test.mjs
config/scripts/package-electron-runtime-contract.test.mjs
config/scripts/electron-builder-runtime-resources.test.mjs
src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts
src/main/browser/browser-route-tcp-egress.electron.test.ts
src/main/browser/browser-route-webrtc-egress.electron.test.ts
@@ -855,9 +976,13 @@ jobs:
src/shared/child-process/windows-cmd-shim-resolution.win32.test.ts
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
src/main/codex/windows-hook-command.test.ts
src/main/codex/windows-hook-upgrade.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-msys-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
src/main/windows/windows-process-tree-command-line-patch.test.ts
src/main/windows/windows-process-table-native-addon.win32.test.ts
src/main/windows-live-tree-kill.win32.test.ts
src/main/wsl/wsl-runner.test.ts
src/main/wsl/wsl-guest-environment.test.ts
@@ -898,9 +1023,9 @@ jobs:
with:
path: |
node_modules/.pnpm/node-pty@*/node_modules/node-pty/build
node_modules/.pnpm/windows-native-registry@*/node_modules/windows-native-registry/build
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
native/windows-registry/build
node_modules/.pnpm/@vscode+windows-process-tre*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-electron-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json') }}
- name: Prepare Electron native runtime
run: node config/scripts/ensure-native-runtime.mjs --runtime=electron
@@ -969,6 +1094,7 @@ jobs:
- shell_contracts
- test
- orcad_browser
- mobile_web_app
- cross-version-wire
- managed_hook_node18
- package
@@ -1005,6 +1131,8 @@ jobs:
TEST_SHOULD_RUN: ${{ needs.code_paths.outputs.test }}
ORCAD_BROWSER: ${{ needs.orcad_browser.result }}
ORCAD_BROWSER_SHOULD_RUN: ${{ needs.code_paths.outputs.orcad_browser }}
MOBILE_WEB_APP: ${{ needs.mobile_web_app.result }}
MOBILE_WEB_APP_SHOULD_RUN: ${{ needs.code_paths.outputs.mobile_web_app }}
CROSS_VERSION_WIRE: ${{ needs.cross-version-wire.result }}
CROSS_VERSION_WIRE_SHOULD_RUN: ${{ needs.code_paths.outputs.cross-version-wire }}
MANAGED_HOOK_NODE18: ${{ needs.managed_hook_node18.result }}
@@ -1047,6 +1175,7 @@ jobs:
check_job shell_contracts "$SHELL_CONTRACTS" "$SHELL_CONTRACTS_SHOULD_RUN"
check_job test "$TEST" "$TEST_SHOULD_RUN"
check_job orcad_browser "$ORCAD_BROWSER" "$ORCAD_BROWSER_SHOULD_RUN"
check_job mobile_web_app "$MOBILE_WEB_APP" "$MOBILE_WEB_APP_SHOULD_RUN"
check_job cross-version-wire "$CROSS_VERSION_WIRE" "$CROSS_VERSION_WIRE_SHOULD_RUN"
check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN"
check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN"
.github/workflows/release-cut.yml
+100 -36
View File
@@ -107,6 +107,10 @@ jobs:
with:
ref: ${{ github.event_name == 'schedule' && 'main' || inputs.ref }}
fetch-depth: 0
# Why: version math recovers unpublished tags; checkout's default
# fetch-tags:false hides them, so a patch cut recreates vX.Y.Z and
# `git push` overwrites the existing tag.
fetch-tags: true
- name: Setup Node.js
uses: actions/setup-node@v6
@@ -805,6 +809,16 @@ jobs:
with:
ref: refs/tags/${{ needs.cut.outputs.tag }}
- name: Restore draft-release scripts from the workflow ref
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- \
config/scripts/create-draft-release.mjs \
config/scripts/assert-github-release-is-draft.mjs
- name: Create draft release with bounded generated notes
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -846,7 +860,8 @@ jobs:
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- \
tests/e2e/golden-source-control-open-diff.spec.ts \
tests/e2e/golden-terminal-file-link.spec.ts
tests/e2e/golden-terminal-file-link.spec.ts \
tests/e2e/golden-worktree-create-switch.spec.ts
- name: Install native build tools
if: runner.os == 'Linux'
@@ -871,8 +886,17 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Why: this install runs lifecycle scripts, so node-gyp rebuilds
# native/windows-registry and fetches that Node version's headers from
# nodejs.org. One `read ECONNRESET` there failed this blocking gate and the
# whole cut. Retry like the release build's install below.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for platform golden
run: npx electron-vite build --mode e2e
@@ -1088,8 +1112,14 @@ jobs:
npm install -g node-gyp@11.5.0
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
# Same node-gyp header fetch as the blocking golden gate above.
- name: Install dependencies
run: pnpm install --frozen-lockfile
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
- name: Build Electron app for terminal rendering evidence
run: npx electron-vite build --mode e2e
@@ -1161,14 +1191,14 @@ jobs:
~\AppData\Local\electron-builder\Cache
- os: ubuntu-latest
platform: linux-x64
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish always
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish always -c.publish.releaseType=draft
unpacked_dir: dist/linux-unpacked
eb_cache_path: |
~/.cache/electron
~/.cache/electron-builder
- os: ubuntu-24.04-arm
platform: linux-arm64
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_LINUX_ARM64_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --arm64 --publish always
release_command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_LINUX_ARM64_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --arm64 --publish always -c.publish.releaseType=draft
unpacked_dir: dist/linux-arm64-unpacked
eb_cache_path: |
~/.cache/electron
@@ -1204,22 +1234,45 @@ jobs:
# ref, so cutting from an older/off-main ref whose tree predates a composite
# action would fail the step with "Can't find 'action.yml'". Restore the
# actions directory from the commit this workflow file itself came from.
- name: Restore composite actions from the workflow ref
if: matrix.platform == 'win' && github.run_attempt == 1
# Not Windows-only: every platform now consumes install-mobile-dependencies, so
# any of them can be the one whose cut ref predates the action.
- name: Restore draft-publish scripts from the workflow ref
# Why: this job checks out the release tag, so a cut from an older SHA
# still has electron-builder releaseType:release and no re-draft helper.
# The workflow YAML is from main; restore the scripts it invokes.
shell: bash
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
action_path=".github/actions/install-signpath-module/action.yml"
if [ -f "$action_path" ]; then
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs
- name: Restore composite actions from the workflow ref
shell: bash
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
PLATFORM: ${{ matrix.platform }}
run: |
set -euo pipefail
required=(.github/actions/install-mobile-dependencies/action.yml)
if [ "$PLATFORM" = win ] && [ "$GITHUB_RUN_ATTEMPT" = 1 ]; then
required+=(.github/actions/install-signpath-module/action.yml)
fi
missing=()
for action_path in "${required[@]}"; do
[ -f "$action_path" ] || missing+=("$action_path")
done
if [ "${#missing[@]}" -eq 0 ]; then
echo "Composite actions already present at the cut ref."
exit 0
fi
echo "Cut ref predates $action_path; restoring it from $WORKFLOW_SHA."
echo "Cut ref predates ${missing[*]}; restoring from $WORKFLOW_SHA."
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- .github/actions
test -f "$action_path"
for action_path in "${required[@]}"; do
test -f "$action_path"
done
# pnpm must be on PATH before setup-node so setup-node can locate the store for caching.
- name: Setup pnpm
@@ -1232,6 +1285,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Why: release builds hit the same native-module postinstall path as
# PR CI, so keep the pinned node-gyp override here too instead of
@@ -1262,6 +1318,8 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
@@ -1270,6 +1328,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: `pnpm build:release` verifies the Linux computer-use provider by
# importing AT-SPI bindings, which are runtime package deps but are not
# present on stock GitHub Ubuntu release runners.
@@ -1309,6 +1371,20 @@ jobs:
echo "identity=$identity" >>"$GITHUB_OUTPUT"
echo "Classified $TAG as $identity"
# Why here and not in build:relay: only a Windows runner can compile it, and
# arm64 cross-compiles from this same x64 agent. Mirrors dev-channel-win-build.yml,
# which had it while release-cut did not — so every stable installer through
# v1.4.203 shipped Windows relays with no windows-process-tree.node, silently
# falling back to the PowerShell scan on every Windows SSH host.
# Why no run_attempt guard, unlike the artifact steps below: Build app is ungated,
# so a rerun would reach the required-addon check with nothing staged and fail.
- name: Build Windows process-table addon for the relay
if: matrix.platform == 'win'
shell: bash
run: |
node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=x64
node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=arm64
# Why ORCA_POSTHOG_WRITE_KEY here: this is the only build that
# produces a published binary, so this is the only place the secret
# needs to be in scope. The key is a PostHog *project* API key, not
@@ -1331,6 +1407,9 @@ jobs:
ORCA_BUILD_IDENTITY: ${{ steps.tag-classify.outputs.identity }}
ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token
ORCA_POSTHOG_WRITE_KEY: ${{ secrets.ORCA_POSTHOG_WRITE_KEY }}
# Fail the release rather than ship a relay that silently falls back to
# the PowerShell scan on every Windows SSH host.
ORCA_REQUIRE_RELAY_NATIVE_ADDONS: ${{ matrix.platform == 'win' && 'x64,arm64' || '' }}
- name: Gate runtime file-watcher process isolation
if: runner.os == 'Linux'
@@ -2141,36 +2220,21 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Verify release remains draft after artifact upload
# Why: the build matrix must never be the actor that exposes a partial
# release. If an uploader or GitHub transition flips draft early, fail
# this platform leg and leave the diagnostic monitor artifact behind.
# Why: electron-builder `--publish always` can create a public release
# as soon as this platform uploads. Re-draft immediately, then fail, so
# /releases/latest never keeps serving a missing Windows exe.
# Why bash: the Windows matrix defaults to pwsh, which does not expand
# "$TAG" into argv.
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ needs.cut.outputs.tag }}
run: |
set -euo pipefail
releases_json="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100")"
# Why: release upload must validate the draft before it is publicly visible.
draft="$(jq -e -r --arg tag "$TAG" '
map(select(.tag_name == $tag))
| if length == 1 and (.[0].draft | type) == "boolean" then (.[0].draft | tostring) else empty end
' <<<"$releases_json")" || {
echo "::error::Release $TAG was not found in the draft-aware releases list, or its draft state was missing."
exit 1
}
if [[ "$draft" != "true" ]]; then
echo "::error::Release $TAG was published during the ${{ matrix.platform }} artifact upload."
exit 1
fi
run: node config/scripts/assert-github-release-is-draft.mjs "${{ needs.cut.outputs.tag }}"
# Why post-publish for Linux: electron-builder packs and uploads in a
# single `--publish always` invocation, so there is no cheap insertion
# point between pack and upload without splitting those steps. Running
# verify last still blocks the bad release: the binary is uploaded to the
# draft, but a failed matrix job blocks `publish-release` from flipping
# the release from draft → published, so users never see it. A human then
# deletes the draft and re-cuts.
# Why post-pack for Linux: electron-builder packs and uploads in one
# `--publish always` invocation. The previous step re-drafts if that
# upload flipped the GitHub release public; this telemetry check still
# blocks `publish-release` from undrafting a bad binary.
#
# Why this guards against: a misconfigured CI run where
# `ORCA_POSTHOG_WRITE_KEY` is unset or the tag fails to classify
.github/workflows/release-mac-build.yml
+38 -20
View File
@@ -37,6 +37,14 @@ jobs:
with:
ref: refs/tags/${{ inputs.tag }}
- name: Restore draft-publish scripts from the workflow ref
env:
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs
- name: Setup pnpm
uses: pnpm/setup@v2
with:
@@ -47,6 +55,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
# Cache the Electron binary + electron-builder tool downloads (notarytool,
# winCodeSign, nsis, squirrel, AppImage). Saves ~30-90s per job, incl. mac.
@@ -64,13 +75,19 @@ jobs:
# Electron binary from GitHub release assets. GitHub's download CDN
# occasionally returns 504s that fail the whole release. Retry on
# failure so transient network errors don't require a manual re-run.
# Why both CPUs: the mac config packages x64 and arm64 from this arm64
# runner, so the install must carry both variants of the native optional deps.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
max_attempts: 3
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
command: pnpm install --frozen-lockfile --cpu=current,x64,arm64
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
- name: Verify macOS signing environment
run: node config/scripts/verify-macos-release-env.mjs
@@ -133,13 +150,28 @@ jobs:
# Kill only its child and require both PTY and watch recovery before packaging.
node config/scripts/relay-watcher-fault-harness.mjs
- name: Abort if the parent release-cut run was cancelled
# Why: this workflow is dispatched separately, so cancelling release-cut
# does not stop mac `--publish always`. A cancelled parent left v1.4.206
# public with only a partial mac upload.
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PARENT_RUN: ${{ inputs.release_run_id }}
run: |
set -euo pipefail
conclusion="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PARENT_RUN" --jq '.conclusion // empty')"
if [[ "$conclusion" == "cancelled" || "$conclusion" == "failure" || "$conclusion" == "timed_out" ]]; then
echo "::error::Parent release-cut run $PARENT_RUN is $conclusion; refusing to publish mac artifacts."
exit 1
fi
- name: Publish release artifacts (macOS)
uses: nick-fields/retry@v4
with:
timeout_minutes: 45
max_attempts: 3
retry_wait_seconds: 30
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always -c.publish.releaseType=draft
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -149,28 +181,14 @@ jobs:
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
- name: Verify release remains draft after artifact upload
# Why: the macOS build must never be the actor that exposes a partial
# release. If an uploader or GitHub transition flips draft early, fail
# this job so release-cut never publishes the release.
# Why: re-draft immediately if electron-builder flipped the GitHub
# release public, then fail. Checking without restoring leaves
# /releases/latest serving a missing Windows exe.
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
releases_json="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100")"
# Why: release upload must validate the draft before it is publicly visible.
draft="$(jq -e -r --arg tag "$TAG" '
map(select(.tag_name == $tag))
| if length == 1 and (.[0].draft | type) == "boolean" then (.[0].draft | tostring) else empty end
' <<<"$releases_json")" || {
echo "::error::Release $TAG was not found in the draft-aware releases list, or its draft state was missing."
exit 1
}
if [[ "$draft" != "true" ]]; then
echo "::error::Release $TAG was published during the mac artifact upload."
exit 1
fi
run: node config/scripts/assert-github-release-is-draft.mjs "${{ inputs.tag }}"
# Why post-publish for macOS: electron-builder packs and uploads in a
# single `--publish always` invocation, so there is no cheap insertion
.github/workflows/terminal-ime-e2e.yml
+37
View File
@@ -70,3 +70,40 @@ jobs:
path: test-results/
retention-days: 7
if-no-files-found: ignore
linux-wayland:
name: Linux Wayland Hangul terminating digit
runs-on: ubuntu-22.04
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install native build, nested compositor and IME tools
run: >-
sudo apt-get update && sudo apt-get install -y
build-essential python3 fonts-noto-cjk dbus-x11 dconf-gsettings-backend
ibus ibus-hangul gnome-shell gnome-settings-daemon libglib2.0-bin
xdotool xvfb x11-utils imagemagick
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- name: Build Electron app for E2E
env:
VITE_EXPOSE_STORE: 'true'
run: |
pnpm run build:relay
pnpm exec electron-vite build --mode e2e
pnpm run build:web-from-renderer
- name: Run native Wayland Hangul terminating digit
env:
SKIP_BUILD: '1'
run: node config/scripts/run-terminal-ibus-hangul-e2e.mjs --nested-wayland
- name: Upload Wayland terminal IME evidence
if: always()
uses: actions/upload-artifact@v7
with:
name: terminal-wayland-ime-evidence
path: test-results/
retention-days: 7
if-no-files-found: error
.github/workflows/unit-tests.yml
+56
View File
@@ -33,18 +33,26 @@ jobs:
native-runtime: node
node-version: ${{ matrix.node }}
cache-electron-package: 'true'
cache-dependency-path: |
pnpm-lock.yaml
cloud/pnpm-lock.yaml
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
- name: Test shard
env:
ORCA_BALANCE_UNIT_SHARDS: '1'
ORCA_BACKGROUND_LAUNCH: '1'
run: |
export ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
pnpm exec vitest run --config config/vitest.config.ts \
--exclude=src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts \
--exclude=src/main/daemon/shell-ready.test.ts \
--exclude=src/main/daemon/node-pty-fd-leak.test.ts \
--exclude=src/main/providers/local-pty-shell-ready-zsh-launch-environment.test.ts \
--exclude=src/main/providers/__tests__/shell-ready-framework-example.test.ts \
--exclude=src/main/pty/omp-shell-wrapper-alias-safety.test.ts \
--exclude=src/main/pty/omp-shell-wrapper.node-pty.test.ts \
--exclude=src/main/shell-startup-feature-channel.test.ts \
--exclude=src/main/terminal-history-fish-session.node-pty.test.ts \
@@ -56,5 +64,53 @@ jobs:
--exclude=src/shared/pty-reply-echo-shapes.node-pty.test.ts \
--exclude=src/shared/startup-shell-portability.live-shell.test.ts \
--exclude=src/shared/posix-command-path-lookup.test.ts \
--exclude=tests/e2e/relay-region-compatibility.unit.test.ts \
--exclude=tests/e2e/relay-region-correction.unit.test.ts \
--exclude=tests/e2e/cross-version-wire/** \
--shard=${{ matrix.shard }}/${{ matrix.shard_total }}
- name: Upload unit shard assignment
if: always()
# Diagnostic upload outages must not change the test verdict.
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: unit-shard-node-${{ matrix.node }}-${{ matrix.shard }}-attempt-${{ github.run_attempt }}
path: ci-shards/
retention-days: 14
if-no-files-found: warn
relay_integration:
name: relay integration node ${{ fromJSON(inputs.node_versions)[0] }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: ${{ fromJSON(inputs.node_versions)[0] }}
cache-electron-package: 'true'
cache-dependency-path: |
pnpm-lock.yaml
cloud/pnpm-lock.yaml
# These two tests import the cloud relay workspace directly. Keeping them in one job
# avoids installing and building the same workspace once per unit-test shard.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test relay integration contracts
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
tests/e2e/relay-region-compatibility.unit.test.ts
tests/e2e/relay-region-correction.unit.test.ts
.github/workflows/win-crash-survival-e2e.yml
+18 -1
View File
@@ -55,6 +55,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
@@ -67,6 +70,9 @@ jobs:
uses: actions/cache@v4
with:
path: dist/orca-windows-setup.exe
# The mobile page trees are in the key because beforePack builds the mobile web bundle
# into the installer; src/** and config/** already cover src/mobile-web and the two
# bundle builders. A mobile-only change must miss this cache, not reuse a stale exe.
key: >-
crash-survival-installer-${{ hashFiles(
'src/**',
@@ -85,7 +91,12 @@ jobs:
'.npmrc',
'package.json',
'pnpm-lock.yaml',
'pnpm-workspace.yaml'
'pnpm-workspace.yaml',
'mobile/package.json',
'mobile/pnpm-lock.yaml',
'mobile/app/**',
'mobile/src/**',
'mobile/web-entry/**'
) }}
# Why: production edits miss the installer cache by design, but Electron
@@ -101,6 +112,12 @@ jobs:
restore-keys: |
crash-survival-electron-builder-
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
.github/workflows/win-update-survival-e2e.yml
+6
View File
@@ -75,6 +75,12 @@ jobs:
path: dist/orca-windows-setup.exe
key: branch-installer-${{ hashFiles('src/**', 'config/**', 'native/**', 'resources/win32/**', 'package.json', 'pnpm-lock.yaml') }}
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules. Gated with the
# build it feeds, so a cache hit does not pay for an install nothing consumes.
- uses: ./.github/actions/install-mobile-dependencies
if: steps.cache-installer.outputs.cache-hit != 'true'
- name: Build Windows installer (unsigned)
if: steps.cache-installer.outputs.cache-hit != 'true'
run: |
.github/workflows/windows-signing-rehearsal.yml
+9
View File
@@ -57,6 +57,9 @@ jobs:
with:
node-version-file: package.json
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
mobile/pnpm-lock.yaml
- name: Cache electron-builder downloads
uses: actions/cache@v5
@@ -68,6 +71,8 @@ jobs:
restore-keys: |
electron-builder-win-
# Why host-only: this job packages only for its own runner OS and
# architecture, so the default host-scoped install is deliberate.
- name: Install dependencies
uses: nick-fields/retry@v4
with:
@@ -76,6 +81,10 @@ jobs:
retry_wait_seconds: 30
command: pnpm install --frozen-lockfile
# Why here: electron-builder's beforePack requires out/mobile-web, and the bundle
# build resolves React Native and Expo from mobile/node_modules.
- uses: ./.github/actions/install-mobile-dependencies
# Why: rehearsal builds are never published, so the official-build
# secrets (telemetry key, diagnostics URL) are intentionally omitted.
- name: Build app
.gitignore
+23
View File
@@ -28,6 +28,9 @@ out/
/build/
release/
native/**/.build/
# node-gyp output for the vendored Windows registry addon; generated per host and ABI.
native/windows-registry/build/
native/windows-registry/bin/
# pnpm
.pnpm-store/
@@ -95,24 +98,40 @@ docs/**
# The deployable docs app is source, not local engineering notes.
!docs/site/
!docs/site/**
!docs/audits/
!docs/audits/closed-editor-model-lifetime/
!docs/audits/closed-editor-model-lifetime/**
!docs/assets/
!docs/assets/**
!docs/audits/
!docs/audits/plugin-uninstall-log-retirement/
!docs/audits/plugin-uninstall-log-retirement/**
!docs/readme/
!docs/readme/**
!docs/STYLEGUIDE.md
!docs/audits/
!docs/audits/crashpad-read-limit/
!docs/audits/crashpad-read-limit/source-hashes.json
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-pty-transcript-capture.md
!docs/reference/agent-session-search-query-tuning.md
!docs/reference/agent-session-search-contract.md
!docs/reference/agent-status-store.md
!docs/reference/antigravity-readiness-evidence.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
!docs/reference/ime-regression-checklist.md
!docs/reference/linux-glibc-compatibility.md
!docs/reference/macos-press-and-hold.md
!docs/reference/orcad-operations.md
!docs/reference/pnpm-install-policy.md
!docs/reference/relay-grace-time-reconfiguration.md
!docs/reference/windows-cmd-shim-resolution.md
!docs/reference/windows-daemon-host-relocation.md
!docs/reference/windows-edr-posture.md
!docs/reference/windows-msys-job-breakaway.md
!docs/reference/windows-process-enumeration.md
!docs/reference/wsl-runner-verification.md
!docs/reference/remote-wire-compatibility.md
@@ -121,6 +140,7 @@ docs/**
!docs/reference/ssh-host-key-verification.md
!docs/reference/ssh-reconnect-source-recovery.md
!docs/reference/windows-setup-shell.md
!docs/reference/windows-terminal-shell-selection.md
!docs/reference/worktree-scan-fingerprint.md
!docs/reference/wsl-command-execution.md
!docs/reference/wsl-probe-failure-semantics.md
@@ -172,3 +192,6 @@ tests/e2e/.cross-version-checkouts/
# IS committed). Also keeps oxfmt/oxlint, which honor this file, from walking
# vendored gems.
/mobile/vendor/
# Generated by config/scripts/sync-anti-slop-plugin.mjs from the pinned oxlint-plugin-anti-slop
.anti-slop-plugin/
.oxfmtrc.json
+5 -1
View File
@@ -4,5 +4,9 @@
"semi": false,
"printWidth": 100,
"trailingComma": "none",
"ignorePatterns": ["cloud/**", ".github/actions/cloud-sql-rollout-lease/**"]
"ignorePatterns": [
"cloud/**",
".github/actions/cloud-sql-rollout-lease/**",
".anti-slop-plugin/**"
]
}
.oxlintrc.json
+1
View File
@@ -180,6 +180,7 @@
}
],
"ignorePatterns": [
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
"**/node_modules",
"**/dist",
"**/out",
AGENTS.md
+35 -1
View File
@@ -1,6 +1,6 @@
# Design System
All UI work — layout, color, typography, spacing, component selection, UX behavior — must follow [`docs/STYLEGUIDE.md`](./docs/STYLEGUIDE.md). Use the tokens defined in `src/renderer/src/assets/main.css` (the canonical source) and the shadcn primitives in `src/renderer/src/components/ui/`. Don't invent new color values, font sizes, or shadow tiers when a documented one already covers the role. When STYLEGUIDE.md is silent, follow the resolution order in its final section.
All UI work — layout, color, typography, spacing, component selection, UX behavior — must follow [`docs/STYLEGUIDE.md`](./docs/STYLEGUIDE.md). Most of it is linted: `pnpm run check:code-quality:changed` fails on new restyles of a `components/ui/` primitive, raw palette colors, and computed `className` strings; `pnpm lint` fails on any class Tailwind cannot generate. See the Enforcement section of the style guide before suppressing either. Use the tokens defined in `src/renderer/src/assets/main.css` (the canonical source) and the shadcn primitives in `src/renderer/src/components/ui/`. Don't invent new color values, font sizes, or shadow tiers when a documented one already covers the role. When STYLEGUIDE.md is silent, follow the resolution order in its final section.
## Electron UI Validation
@@ -33,11 +33,31 @@ Never use vague names like `helpers`, `utils`, `common`, `misc`, or `shared-stuf
## Type Declarations: Prefer `.ts` Over `.d.ts`
## Type Assertions: Prefer Checked Types
Avoid type assertions except `as const`. Unavoidable casts need a line-specific `SAFETY:` explanation:
```ts
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: Explain the verified invariant here.
```
# Verifying Changes
- **Typecheck**: `pnpm tc` (or `tc:node` / `tc:cli` / `tc:web`)
- **Test**: `pnpm test [path/to/file.test.ts]`
- **Lint**: `oxlint`, or `pnpm run check:code-quality:changed` for changed files (full `pnpm lint` is slow); format with `pnpm format`
- **Design system**: `pnpm run lint:design-system` for the full renderer report (not a gate); the changed-lines gate above is what CI enforces
# Writing Pull Requests
Fill in [`.github/pull_request_template.md`](./.github/pull_request_template.md), written for a reviewer who has never seen this code:
- No jargon — plain language, no internal shorthand.
- The before and after as the user experiences it.
- The mechanism you changed, not just the symptom.
- Why this approach over the alternatives you considered.
Cover all four concisely. Don't pad or walk the diff.
# Considerations
@@ -52,14 +72,20 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **Keyboard shortcuts**: Never hardcode `e.metaKey`. Use a platform check (`navigator.userAgent.includes('Mac')`) to pick `metaKey` on Mac and `ctrlKey` on Linux/Windows. Electron menu accelerators should use `CmdOrCtrl`.
- **Shortcut labels in UI**: Display `⌘` / `⇧` on Mac and `Ctrl+` / `Shift+` on other platforms.
- **File paths**: Use `path.join` or Electron/Node path utilities — never assume `/` or `\`.
- **Windows terminal shells**: `--shell` picks the shell a terminal *is*; `--command` is typed into whatever shell the host spawned, so a shell choice routed through `command` silently becomes a child process. See [`docs/reference/windows-terminal-shell-selection.md`](./docs/reference/windows-terminal-shell-selection.md).
- **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import. Recognised npm/pnpm `.cmd` shims are resolved to their real target so the spawn skips `cmd.exe` entirely; see [`docs/reference/windows-cmd-shim-resolution.md`](./docs/reference/windows-cmd-shim-resolution.md) before adding a shim shape or debugging one.
- **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
- **Windows MSYS/Git Bash panes**: their children break away from the per-PTY job unless it is created without `JOB_OBJECT_LIMIT_BREAKAWAY_OK`, and a `conpty.node` built before that fix passes every existing gate. Before changing the per-PTY job or debugging `windows-msys-job.win32.test.ts`, read [`docs/reference/windows-msys-job-breakaway.md`](./docs/reference/windows-msys-job-breakaway.md).
- **Windows daemon-host relocation**: the terminal daemon runs from a copy of the app runtime under `%LOCALAPPDATA%`, which is what survives an auto-update. Before touching that copy, its exe name, or the NSIS uninstall macro, read [`docs/reference/windows-daemon-host-relocation.md`](./docs/reference/windows-daemon-host-relocation.md).
- **Windows EDR signal**: don't add `-ExecutionPolicy Bypass`, `-EncodedCommand`, `cmd.exe /c` with escaped free text, per-operation interpreter spawning, or runtime `Add-Type` compilation without reading [`docs/reference/windows-edr-posture.md`](./docs/reference/windows-edr-posture.md) first — behavioural EDR scores each of those, and being signed does not clear them.
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
- **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.
## Native Dependency Installs
Ordinary `pnpm install` covers the host OS and CPU only. Before packaging for another architecture — including `pnpm build:mac`, which builds x64 and arm64 by default — run `pnpm install:release`. electron-builder only warns on a missing `extraResources` source, so the `beforePack` guard is what turns a thin install into a build failure instead of a silently broken artifact; see [`docs/reference/pnpm-install-policy.md`](./docs/reference/pnpm-install-policy.md).
## SSH Use Case
All changes must consider the SSH use case. Don't assume local-only execution. Before changing anything that reports on, stops, or lists remote work, follow [`docs/reference/ssh-execution-boundary.md`](./docs/reference/ssh-execution-boundary.md): the execution host owns everything that touches execution, and loss of contact is never evidence of process death — the verdict vocabulary is `live` / `unverifiable` / `exited`, with no synonyms.
@@ -68,6 +94,14 @@ All changes must consider the SSH use case. Don't assume local-only execution. B
All changes must consider folder workspaces as well as git worktrees. Don't assume every workspace is a git worktree.
## Agent Status
The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy.
## Agent Terminal Screens
A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md).
## Remote Wire Compatibility
Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change.
README.md
+12 -12
View File
@@ -36,7 +36,7 @@
Monitor and steer your agents from your phone — get notified when an agent finishes and send follow-ups from anywhere.
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.50](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.50/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
</td>
<td width="50%">
@@ -54,7 +54,7 @@ Fan one prompt across five agents, each in its own isolated git worktree — com
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/assets/feature-wall/parallel-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/parallel-worktrees.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/model/worktrees"><picture><source srcset="docs/site/public/docs/tab-split.gif" type="image/gif"><img src="docs/site/public/docs/posters/tab-split.jpg" alt="Parallel worktree orchestration" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -68,7 +68,7 @@ Ghostty-class terminals with WebGL rendering, infinite splits, and scrollback th
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="docs/assets/feature-wall/terminal-splits.gif" type="image/gif"><img src="docs/assets/feature-wall/terminal-splits.jpg" alt="Terminal splits" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/terminal"><picture><source srcset="resources/onboarding/feature-wall/tile-02.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-02.poster.jpg" alt="Terminal splits" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -82,7 +82,7 @@ Click any UI element in a real Chromium window to send its HTML, CSS, and a crop
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/assets/feature-wall/design-mode.gif" type="image/gif"><img src="docs/assets/feature-wall/design-mode.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/browser/design-mode"><picture><source srcset="docs/site/public/docs/orca-design-mode.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-05.poster.jpg" alt="Embedded browser and Design Mode" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -96,7 +96,7 @@ Browse PRs, issues, and project boards in-app — open a worktree from any task
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="docs/assets/feature-wall/github-linear.gif" type="image/gif"><img src="docs/assets/feature-wall/github-linear.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/linear"><picture><source srcset="resources/onboarding/feature-wall/tile-03.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-03.poster.jpg" alt="GitHub and Linear task workflows in Orca" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -110,7 +110,7 @@ Run agents on a beefy remote box with full file editing, git, and terminals —
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="docs/assets/feature-wall/ssh-worktrees.gif" type="image/gif"><img src="docs/assets/feature-wall/ssh-worktrees.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/ssh"><picture><source srcset="resources/onboarding/feature-wall/tile-06.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-06.poster.jpg" alt="Remote worktrees over SSH" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -124,7 +124,7 @@ Drop comments on any diff line and ship them back to the agent — review, edit,
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/assets/feature-wall/annotate-diff.gif" type="image/gif"><img src="docs/assets/feature-wall/annotate-diff.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/review/annotate-ai-diff"><picture><source srcset="docs/site/public/docs/annotate-ai-diff.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-08.poster.jpg" alt="Annotate AI-generated diffs" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -138,7 +138,7 @@ VS Code's editor with autosave everywhere — drag files or images straight into
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="docs/assets/feature-wall/file-drag.gif" type="image/gif"><img src="docs/assets/feature-wall/file-drag.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/editing/file-explorer"><picture><source srcset="resources/onboarding/feature-wall/tile-07.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-07.poster.jpg" alt="Drag files and images into an agent prompt" width="100%" /></picture></a>
</td>
</tr>
<tr>
@@ -152,7 +152,7 @@ Agents drive Orca too — script every workflow with `orca worktree create`, `sn
</td>
<td width="50%">
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="docs/assets/feature-wall/orca-cli.gif" type="image/gif"><img src="docs/assets/feature-wall/orca-cli.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
<a href="https://www.onorca.dev/docs/cli/overview"><picture><source srcset="resources/onboarding/feature-wall/tile-09.gif" type="image/gif"><img src="resources/onboarding/feature-wall/tile-09.poster.jpg" alt="Script Orca from the CLI" width="100%" /></picture></a>
</td>
</tr>
</table>
@@ -230,7 +230,7 @@ yay -S stably-orca-bin
Pair with your desktop app to monitor and steer your agents from your phone.
- **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) or [join TestFlight](https://testflight.apple.com/join/YjeGMQBA)
- **Android:** [Download APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
- **Android:** [Download APK 0.0.50](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.50/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
---
@@ -238,9 +238,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
- **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
- **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
- **WeChat:** Scan to join the Orca community WeChat group 8. Group 8 may be full; if so, scan the Group 9 QR code instead.
- **WeChat:** Scan to join the Orca community WeChat group 9.
<img src="docs/assets/wechat-qr-group8.jpg" alt="WeChat group 8 QR code for the Orca community" width="160" />&nbsp;&nbsp;<img src="docs/assets/wechat-qr-group9.jpg" alt="WeChat group 9 QR code for the Orca community" width="160" />
<img src="docs/assets/wechat-qr-group9.jpg" alt="WeChat group 9 QR code for the Orca community" width="160" />
- **Feedback &amp; Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
- **Privacy:** See the [privacy &amp; telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
cloud/README.md
+43 -7
View File
@@ -24,6 +24,40 @@ the repository's root [MIT license](../LICENSE).
- `apps/relay-ops`: the relay operations console and the incident monitor
behind `pnpm ops:relay`, `pnpm incident:relay`, and
`pnpm incident:relay-preflight`.
- `apps/push` and `packages/push-contract`: the mobile push gateway that holds
the APNs key and sends to phones through APNs and FCM, and its wire contract.
It is deployed and operated from here but is not part of the relay data path;
see [docs/push-gateway.md](docs/push-gateway.md).
## Mobile push gateway
`apps/push` is a separate Cloud Run service from the relay. Phones never hold an
Orca credential for it: the desktop host authenticates with the same X25519
key it uses for the relay, answering an encrypted challenge to mint a 24 hour
session, then registers each paired phone's native push token and asks the
gateway to push. The gateway queues each event as its own notification,
enforces per-host quotas and request limits, and retires a
registration as soon as Apple or Google reports the token unregistered.
Provider push is the only ordinary mobile OS-banner path. The notification
socket is retained only for live dismissal and reconnect tray reconciliation;
it never creates or recovers banners. Desktop notification categories remain
authoritative.
Each delivery is persisted as one notification event. Before deploying an
incompatible queue format, stop all older push gateway revisions and clear only
unpublished push delivery fixtures; no queue preservation or migration is required.
FCM notification messages are inherently collapsible while offline and have a
small concurrent collapse-key budget, so every pending alert is not guaranteed.
Storage follows the relay pattern: PostgreSQL in production, SQLite for tests
and local development. Configure it with `ORCA_PUSH_PUBLIC_URL`, `ORCA_PUSH_FCM_PROJECT_ID`,
`ORCA_PUSH_DATABASE_URL`, the three APNs variables (`ORCA_PUSH_APNS_KEY`,
`ORCA_PUSH_APNS_KEY_ID`, `ORCA_PUSH_APPLE_TEAM_ID`, all three or none), and
optionally `ORCA_PUSH_APNS_TOPIC`. The FCM credential comes from
the runtime service account, so no key material is configured for Android. See
[push gateway operations](docs/push-gateway.md) for deployment and recovery.
Logging is aggregate counters only. Tokens, notification titles, notification
bodies, and full host fingerprints never reach a log line.
## Infrastructure and operations
@@ -38,16 +72,18 @@ the repository's root [MIT license](../LICENSE).
- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests
read, including the Terraform root partition.
- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor
reference, and the workflow variable reference in `docs/relay-workflows.md`.
reference, the workflow variable reference in `docs/relay-workflows.md`, and
the push gateway runbook in `docs/push-gateway.md`.
## Workflows
The 24 `.github/workflows/cloud-*.yml` workflows are the relay's deploy and
operate surface: publish and deploy the director, roll GCE cell capacity,
operate Asia admission and regional rehoming, prove staging capacity, monitor
production, and power staging up and down. `.github/actions/cloud-sql-rollout-lease`
is the compare-and-swap lease that serializes every rollout against the shared
Cloud SQL instance.
The 25 `.github/workflows/cloud-*.yml` workflows are the deploy and operate
surface: publish and deploy the director, roll GCE cell capacity, operate Asia
admission and regional rehoming, prove staging capacity, monitor production,
power staging up and down, and deploy the mobile push gateway.
`.github/actions/cloud-sql-rollout-lease` is the compare-and-swap lease that
serializes rollouts against the shared Cloud SQL instance. Push reuses that
action with its own lease object and deployment concurrency group.
Every one of them is inert. Each top-level job is gated on
`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is
cloud/apps/push/Dockerfile
+29
View File
@@ -0,0 +1,29 @@
FROM node:24-alpine AS build
WORKDIR /app
RUN corepack enable
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
COPY packages/push-contract/package.json packages/push-contract/package.json
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
COPY apps/push/package.json apps/push/package.json
RUN pnpm install --frozen-lockfile
COPY packages/push-contract packages/push-contract
COPY apps/push apps/push
COPY packages/postgres-schema packages/postgres-schema
RUN pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build && pnpm --filter @orca-cloud/push build
FROM node:24-alpine AS runtime
ENV NODE_ENV=production
ENV PORT=8080
WORKDIR /app
RUN corepack enable
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY packages/push-contract/package.json packages/push-contract/package.json
COPY packages/postgres-schema/package.json packages/postgres-schema/package.json
COPY apps/push/package.json apps/push/package.json
COPY --from=build /app/packages/push-contract/dist packages/push-contract/dist
COPY --from=build /app/packages/postgres-schema/dist packages/postgres-schema/dist
COPY --from=build /app/apps/push/dist apps/push/dist
RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/push...
USER node
EXPOSE 8080
CMD ["node", "apps/push/dist/index.js"]
cloud/apps/push/package.json
+35
View File
@@ -0,0 +1,35 @@
{
"name": "@orca-cloud/push",
"private": true,
"version": "0.0.0",
"type": "module",
"main": "dist/index.js",
"scripts": {
"build": "pnpm clean && tsc -p tsconfig.build.json",
"clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
"dev": "tsx watch src/index.ts",
"lint": "tsc -p tsconfig.json --noEmit",
"pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/push-contract build",
"start": "node dist/index.js",
"test": "vitest run",
"typecheck": "tsc -p tsconfig.json --noEmit"
},
"dependencies": {
"@hono/node-server": "^1.19.17",
"@orca-cloud/postgres-schema": "workspace:*",
"@orca-cloud/push-contract": "workspace:*",
"google-auth-library": "^10.5.0",
"hono": "^4.13.7",
"pg": "^8.22.0",
"pg-connection-string": "2.14.0",
"tweetnacl": "^1.0.3",
"zod": "^3.25.76"
},
"devDependencies": {
"@types/node": "^24.10.0",
"@types/pg": "^8.20.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.1.11"
}
}
cloud/apps/push/src/apns-authentication-token.ts
+42
View File
@@ -0,0 +1,42 @@
import { createPrivateKey, type KeyObject, sign } from 'node:crypto'
import type { ApnsCredentials } from './config.js'
// Apple rejects a provider token older than an hour and throttles reissue
// under about 20 minutes, so 50 minutes is the safe rotation point.
export const APNS_TOKEN_ROTATION_MS = 50 * 60 * 1000
function base64UrlJson(value: Record<string, unknown>): string {
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url')
}
export class ApnsAuthenticationToken {
private readonly privateKey: KeyObject
private cached: { token: string; issuedAtMs: number } | null = null
constructor(
private readonly credentials: ApnsCredentials,
private readonly now: () => number = Date.now,
private readonly rotationMs: number = APNS_TOKEN_ROTATION_MS
) {
this.privateKey = createPrivateKey(credentials.keyPem)
}
value(): string {
const nowMs = this.now()
if (this.cached && nowMs - this.cached.issuedAtMs < this.rotationMs) return this.cached.token
const header = base64UrlJson({ alg: 'ES256', kid: this.credentials.keyId })
const payload = base64UrlJson({
iss: this.credentials.teamId,
iat: Math.floor(nowMs / 1000)
})
const signingInput = `${header}.${payload}`
// ES256 requires the raw r||s pair; Node emits DER unless asked otherwise.
const signature = sign('sha256', Buffer.from(signingInput, 'utf8'), {
key: this.privateKey,
dsaEncoding: 'ieee-p1363'
}).toString('base64url')
const token = `${signingInput}.${signature}`
this.cached = { token, issuedAtMs: nowMs }
return token
}
}
cloud/apps/push/src/apns-client.test.ts
+220
View File
@@ -0,0 +1,220 @@
import { generateKeyPairSync } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { ApnsAuthenticationToken, APNS_TOKEN_ROTATION_MS } from './apns-authentication-token.js'
import { ApnsClient } from './apns-client.js'
import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js'
import type { ApnsCredentials } from './config.js'
import { buildPushDelivery } from './push-delivery-message.js'
const HOST = 'abcdefghijklmnop'
function credentials(): ApnsCredentials {
const { privateKey } = generateKeyPairSync('ec', {
namedCurve: 'P-256',
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
publicKeyEncoding: { type: 'spki', format: 'pem' }
})
return { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' }
}
function delivery(now = Date.now()) {
return buildPushDelivery({
expiresAt: now + 300_000,
registrationId: 'reg-1',
hostFingerprint: HOST,
notification: {
notificationId: 'note-1',
notificationSeq: 7,
notificationEpoch: 'epoch-1',
source: 'agent-task-complete',
agentState: 'needs-input',
title: 'Agent needs input',
body: 'Waiting on your answer',
worktreeId: 'wt-1'
}
})
}
function fakeTransport(response: ApnsResponse) {
const requests: ApnsRequest[] = []
return {
requests,
transport: async (request: ApnsRequest): Promise<ApnsResponse> => {
requests.push(request)
return response
}
}
}
describe('apns authentication token', () => {
it('signs an ES256 provider token and caches it until the rotation point', () => {
let clock = 1_700_000_000_000
const authentication = new ApnsAuthenticationToken(credentials(), () => clock)
const first = authentication.value()
const [header, payload, signature] = first.split('.')
expect(JSON.parse(Buffer.from(header!, 'base64url').toString('utf8'))).toEqual({
alg: 'ES256',
kid: 'ABCDE12345'
})
expect(JSON.parse(Buffer.from(payload!, 'base64url').toString('utf8'))).toEqual({
iss: 'TEAM123456',
iat: Math.floor(clock / 1000)
})
expect(Buffer.from(signature!, 'base64url').byteLength).toBe(64)
clock += APNS_TOKEN_ROTATION_MS - 1
expect(authentication.value()).toBe(first)
clock += 1
expect(authentication.value()).not.toBe(first)
})
})
describe('apns client', () => {
it('sends the specified headers, path, and alert body', async () => {
const clock = 1_700_000_000_000
const fake = fakeTransport({ status: 200, body: '' })
const client = new ApnsClient({
topic: 'com.stably.orca.mobile',
credentials: credentials(),
transport: fake.transport,
now: () => clock
})
await expect(
client.send(delivery(clock), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
).resolves.toEqual({ status: 'sent' })
const request = fake.requests[0]!
expect(request.host).toBe('api.push.apple.com')
expect(request.path).toBe(`/3/device/${'a'.repeat(64)}`)
expect(request.headers).toMatchObject({
'apns-topic': 'com.stably.orca.mobile',
'apns-push-type': 'alert',
'apns-priority': '10',
'apns-expiration': String(Math.floor(clock / 1000) + 5 * 60),
'apns-collapse-id': expect.stringMatching(/^[a-f0-9]{64}$/)
})
expect(request.headers.authorization).toMatch(/^bearer /)
expect(JSON.parse(request.body)).toEqual({
aps: {
alert: { title: 'Agent needs input', body: 'Waiting on your answer' },
sound: 'default',
'thread-id': HOST
},
orca: {
hostFingerprint: HOST,
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: 7,
notificationEpoch: 'epoch-1',
source: 'agent-task-complete',
agentState: 'needs-input'
}
})
})
it('targets the sandbox host and keeps the individual collapse id', async () => {
const fake = fakeTransport({ status: 200, body: '' })
const client = new ApnsClient({
topic: 'com.stably.orca.mobile',
credentials: credentials(),
transport: fake.transport
})
await client.send(delivery(), { token: 'b'.repeat(64), apnsEnvironment: 'sandbox' })
expect(fake.requests[0]?.host).toBe('api.sandbox.push.apple.com')
expect(fake.requests[0]?.headers['apns-collapse-id']).toMatch(/^[a-f0-9]{64}$/)
})
it.each([
[410, 'Unregistered'],
[400, 'BadDeviceToken'],
[400, 'Unregistered']
])('classifies %i %s as a dead token', async (status, reason) => {
const fake = fakeTransport({ status, body: JSON.stringify({ reason }) })
const client = new ApnsClient({
topic: 'com.stably.orca.mobile',
credentials: credentials(),
transport: fake.transport
})
await expect(
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
).resolves.toEqual({ status: 'dead', reason })
})
it.each([
[400, 'PayloadTooLarge'],
[400, 'DeviceTokenNotForTopic'],
[429, 'TooManyRequests'],
[500, 'InternalServerError']
])('treats %i %s with the appropriate retry policy', async (status, reason) => {
const fake = fakeTransport({ status, body: JSON.stringify({ reason }) })
const client = new ApnsClient({
topic: 'com.stably.orca.mobile',
credentials: credentials(),
transport: fake.transport
})
await expect(
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
).resolves.toEqual({ status: 'error', reason, retryable: status === 429 || status >= 500 })
})
it('reports a transport failure as an error rather than throwing', async () => {
const client = new ApnsClient({
topic: 'com.stably.orca.mobile',
credentials: credentials(),
transport: async () => {
throw new Error('socket hang up')
}
})
await expect(
client.send(delivery(), { token: 'a'.repeat(64), apnsEnvironment: 'production' })
).resolves.toEqual({ status: 'error', reason: 'Error', retryable: true })
})
})
it('does not collapse background dismissals with visible alerts', async () => {
const fake = fakeTransport({ status: 200, body: '' })
const apns = new ApnsClient({
topic: 'test',
credentials: credentials(),
transport: fake.transport
})
const alert = delivery()
await apns.send(
{ ...alert, orca: { ...alert.orca, kind: 'dismiss' } },
{
token: 'test',
apnsEnvironment: 'sandbox'
}
)
expect(fake.requests[0]?.headers).not.toHaveProperty('apns-collapse-id')
expect(fake.requests[0]?.headers).toMatchObject({
'apns-push-type': 'background',
'apns-priority': '5'
})
expect(JSON.parse(fake.requests[0]!.body).aps).toEqual({ 'content-available': 1 })
})
it('keeps the absolute deadline across retries and refuses expired delivery', async () => {
let now = 1_700_000_000_000
const fake = fakeTransport({ status: 503, body: '{}' })
const client = new ApnsClient({
topic: 'test',
credentials: credentials(),
now: () => now,
transport: fake.transport
})
const pending = delivery(now)
const device = { token: 'test', apnsEnvironment: 'sandbox' as const }
await client.send(pending, device)
now += 60_000
await client.send(pending, device)
expect(fake.requests.map((request) => request.headers['apns-expiration'])).toEqual([
String(pending.expiresAt / 1000),
String(pending.expiresAt / 1000)
])
now = pending.expiresAt
await expect(client.send(pending, device)).resolves.toEqual({
status: 'error',
reason: 'expired'
})
expect(fake.requests).toHaveLength(2)
})
cloud/apps/push/src/apns-client.ts
+95
View File
@@ -0,0 +1,95 @@
import type { ApnsEnvironment } from '@orca-cloud/push-contract'
import { ApnsAuthenticationToken } from './apns-authentication-token.js'
import type { ApnsTransport } from './apns-http2-transport.js'
import type { ApnsCredentials } from './config.js'
import type { PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
const APNS_HOSTS: Record<ApnsEnvironment, string> = {
production: 'api.push.apple.com',
sandbox: 'api.sandbox.push.apple.com'
}
const DEAD_TOKEN_REASONS = new Set(['BadDeviceToken', 'Unregistered'])
export type ApnsClientOptions = {
topic: string
credentials: ApnsCredentials
transport: ApnsTransport
now?: () => number
}
function readReason(body: string): string {
try {
const parsed = JSON.parse(body) as { reason?: unknown }
return typeof parsed.reason === 'string' ? parsed.reason : 'unknown'
} catch {
return 'unparseable'
}
}
export function apnsBody(delivery: PushDelivery): string {
return JSON.stringify({
aps:
delivery.orca.kind === 'dismiss'
? { 'content-available': 1 }
: {
alert: { title: delivery.title, body: delivery.body },
...(delivery.sound === false ? {} : { sound: 'default' }),
'thread-id': delivery.hostFingerprint
},
orca: delivery.orca
})
}
export class ApnsClient {
private readonly authentication: ApnsAuthenticationToken
private readonly now: () => number
constructor(private readonly options: ApnsClientOptions) {
this.now = options.now ?? Date.now
this.authentication = new ApnsAuthenticationToken(options.credentials, this.now)
}
async send(
delivery: PushDelivery,
device: { token: string; apnsEnvironment: ApnsEnvironment }
): Promise<PushProviderOutcome> {
const expiration = Math.floor(delivery.expiresAt / 1000)
if (expiration * 1000 <= this.now()) return { status: 'error', reason: 'expired' }
let response
try {
response = await this.options.transport({
host: APNS_HOSTS[device.apnsEnvironment],
path: `/3/device/${device.token}`,
headers: {
authorization: `bearer ${this.authentication.value()}`,
'apns-topic': this.options.topic,
'apns-push-type': delivery.orca.kind === 'dismiss' ? 'background' : 'alert',
'apns-priority': delivery.orca.kind === 'dismiss' ? '5' : '10',
'apns-expiration': String(expiration),
...(delivery.orca.kind === 'dismiss' ? {} : { 'apns-collapse-id': delivery.collapseId })
},
body: apnsBody(delivery)
})
} catch (error) {
return {
status: 'error',
reason: error instanceof Error ? error.name : 'transport_failed',
retryable: true
}
}
if (response.status === 200) return { status: 'sent' }
const reason = readReason(response.body)
if (response.status === 410) return { status: 'dead', reason }
if (response.status === 400 && DEAD_TOKEN_REASONS.has(reason)) {
return { status: 'dead', reason }
}
return {
status: 'error',
reason,
retryable: response.status === 429 || response.status >= 500,
...(response.retryAfterMs === undefined ? {} : { retryAfterMs: response.retryAfterMs })
}
}
}
cloud/apps/push/src/apns-http2-transport.ts
+50
View File
@@ -0,0 +1,50 @@
import { connect, constants, type ClientHttp2Session } from 'node:http2'
import { readApnsStreamResponse, type ApnsResponse } from './apns-stream-response.js'
export type ApnsRequest = {
host: string
path: string
headers: Record<string, string>
body: string
}
export type { ApnsResponse }
export type ApnsTransport = (request: ApnsRequest) => Promise<ApnsResponse>
// APNs requires HTTP/2 and rewards a long-lived session per host, so sessions
// are cached and only dropped when the socket itself goes away.
export function createApnsHttp2Transport(): ApnsTransport & { close(): void } {
const sessions = new Map<string, ClientHttp2Session>()
const sessionFor = (host: string): ClientHttp2Session => {
const existing = sessions.get(host)
if (existing && !existing.closed && !existing.destroyed) return existing
const session = connect(`https://${host}`)
const forget = (): void => {
if (sessions.get(host) === session) sessions.delete(host)
}
session.on('error', forget)
session.on('close', forget)
sessions.set(host, session)
return session
}
const transport = async (request: ApnsRequest): Promise<ApnsResponse> => {
const stream = sessionFor(request.host).request({
...request.headers,
[constants.HTTP2_HEADER_METHOD]: 'POST',
[constants.HTTP2_HEADER_PATH]: request.path,
[constants.HTTP2_HEADER_AUTHORITY]: request.host,
'content-type': 'application/json',
'content-length': String(Buffer.byteLength(request.body))
})
return await readApnsStreamResponse(stream, request.body)
}
return Object.assign(transport, {
close(): void {
for (const session of sessions.values()) session.close()
sessions.clear()
}
})
}
cloud/apps/push/src/apns-session-replacement.test.ts
+45
View File
@@ -0,0 +1,45 @@
import { EventEmitter } from 'node:events'
import { expect, it, vi } from 'vitest'
const mocks = vi.hoisted(() => ({
connect: vi.fn(),
read: vi.fn(async () => ({ status: 200, body: '' }))
}))
vi.mock('node:http2', async (original) => ({
...(await original<typeof import('node:http2')>()),
connect: mocks.connect
}))
vi.mock('./apns-stream-response.js', () => ({ readApnsStreamResponse: mocks.read }))
import { createApnsHttp2Transport } from './apns-http2-transport.js'
it('keeps the replacement cached when the draining session closes later', async () => {
const sessions: Array<
EventEmitter & {
closed: boolean
destroyed: boolean
request: ReturnType<typeof vi.fn>
close: ReturnType<typeof vi.fn>
}
> = []
mocks.connect.mockImplementation(() => {
const session = Object.assign(new EventEmitter(), {
closed: false,
destroyed: false,
request: vi.fn(() => ({})),
close: vi.fn()
})
sessions.push(session)
return session
})
const transport = createApnsHttp2Transport()
const request = { host: 'api.push.apple.com', path: '/synthetic', headers: {}, body: '{}' }
await transport(request)
sessions[0]!.closed = true
await transport(request)
sessions[0]!.emit('close')
sessions[0]!.emit('error', new Error('old-session'))
await transport(request)
expect(sessions).toHaveLength(2)
expect(sessions[1]!.request).toHaveBeenCalledTimes(2)
transport.close()
expect(sessions[1]!.close).toHaveBeenCalledOnce()
})
cloud/apps/push/src/apns-stream-response.test.ts
+82
View File
@@ -0,0 +1,82 @@
import { EventEmitter } from 'node:events'
import { describe, expect, it } from 'vitest'
import { readApnsStreamResponse, type ApnsResponseStream } from './apns-stream-response.js'
type FakeStream = ApnsResponseStream & {
sentBody: string | null
destroyedWith: Error | null
fireTimeout(): void
}
function fakeApnsStream(): FakeStream {
const emitter = new EventEmitter() as FakeStream
emitter.sentBody = null
emitter.destroyedWith = null
let onTimeout: (() => void) | null = null
emitter.setTimeout = (_ms, callback) => {
onTimeout = callback
}
emitter.destroy = (error?: Error) => {
emitter.destroyedWith = error ?? null
if (error) emitter.emit('error', error)
}
emitter.end = (body: string) => {
emitter.sentBody = body
}
emitter.fireTimeout = () => onTimeout?.()
return emitter
}
describe('apns stream response', () => {
it('resolves with the status and the concatenated body', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, '{"aps":{}}')
expect(stream.sentBody).toBe('{"aps":{}}')
stream.emit('response', { ':status': '200' })
stream.emit('data', Buffer.from('{"re'))
stream.emit('data', Buffer.from('ason":"ok"}'))
stream.emit('end')
await expect(pending).resolves.toEqual({ status: 200, body: '{"reason":"ok"}' })
})
it('rejects when the peer resets the stream without an end or an error', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, 'body')
stream.emit('response', { ':status': '200' })
// NGHTTP2_NO_ERROR: node emits only 'close', so nothing else would settle.
stream.emit('close')
await expect(pending).rejects.toThrow('apns_stream_closed')
})
it('keeps the resolved response when close follows a completed end', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, 'body')
stream.emit('response', { ':status': '410' })
stream.emit('end')
stream.emit('close')
await expect(pending).resolves.toEqual({ status: 410, body: '' })
})
it('keeps the original error when close follows a stream error', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, 'body')
stream.emit('error', new Error('socket_hang_up'))
stream.emit('close')
await expect(pending).rejects.toThrow('socket_hang_up')
})
it('destroys the stream on timeout and surfaces the timeout error', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, 'body', 10)
stream.fireTimeout()
await expect(pending).rejects.toThrow('apns_timeout')
expect(stream.destroyedWith?.message).toBe('apns_timeout')
})
it('reports a missing status header as zero rather than NaN', async () => {
const stream = fakeApnsStream()
const pending = readApnsStreamResponse(stream, 'body')
stream.emit('end')
await expect(pending).resolves.toEqual({ status: 0, body: '' })
})
})
cloud/apps/push/src/apns-stream-response.ts
+53
View File
@@ -0,0 +1,53 @@
import type { EventEmitter } from 'node:events'
import { providerRetryAfter } from './provider-retry-delay.js'
import { constants } from 'node:http2'
export type ApnsResponse = { status: number; body: string; retryAfterMs?: number }
// The subset of ClientHttp2Stream this module drives, so a fake emitter can
// stand in for a real APNs stream in tests.
export type ApnsResponseStream = EventEmitter & {
setTimeout(ms: number, callback: () => void): void
destroy(error?: Error): void
end(body: string): void
}
export const APNS_REQUEST_TIMEOUT_MS = 10_000
export function readApnsStreamResponse(
stream: ApnsResponseStream,
body: string,
timeoutMs = APNS_REQUEST_TIMEOUT_MS
): Promise<ApnsResponse> {
return new Promise<ApnsResponse>((resolve, reject) => {
let settled = false
const settle = (run: () => void): void => {
if (settled) return
settled = true
run()
}
let status = 0
let retryAfterMs: number | undefined
const chunks: Buffer[] = []
stream.setTimeout(timeoutMs, () => stream.destroy(new Error('apns_timeout')))
stream.on('response', (headers: Record<string, unknown>) => {
status = Number(headers[constants.HTTP2_HEADER_STATUS] ?? 0)
retryAfterMs = providerRetryAfter(String(headers['retry-after'] ?? ''))
})
stream.on('data', (chunk: Buffer) => chunks.push(chunk))
stream.on('error', (error: Error) => settle(() => reject(error)))
stream.on('end', () =>
settle(() =>
resolve({
status,
body: Buffer.concat(chunks).toString('utf8'),
...(retryAfterMs === undefined ? {} : { retryAfterMs })
})
)
)
// A peer reset with NGHTTP2_NO_ERROR emits neither 'end' nor 'error', which
// would leave the worker's delivery pending for the life of the process.
stream.on('close', () => settle(() => reject(new Error('apns_stream_closed'))))
stream.end(body)
})
}
cloud/apps/push/src/apns-topic-recovery.test.ts
+48
View File
@@ -0,0 +1,48 @@
import { expect, it } from 'vitest'
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
import {
APNS_TOKEN,
createPushServerHarness,
notification
} from './push-server-harness.test-fixture.js'
it('delivers again after a topic error without re-registering the phone', async () => {
const h = await createPushServerHarness()
try {
const token = await h.signIn(createPushHostKeypair(71))
const registered = await h.post(
'/v1/devices',
{
v: 1,
deviceId: 'phone',
platform: 'ios',
token: APNS_TOKEN,
apnsEnvironment: 'sandbox'
},
token
)
const { registrationId } = (await registered.json()) as { registrationId: string }
h.setApnsResponse({ status: 400, body: JSON.stringify({ reason: 'DeviceTokenNotForTopic' }) })
for (const seq of [1, 2]) {
const sent = await h.post(
'/v1/send',
{
v: 1,
registrationIds: [registrationId],
notification: notification({ notificationId: `topic-${seq}`, notificationSeq: seq })
},
token
)
expect(await sent.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
await h.flushDeliveries()
if (seq === 1) {
expect(h.server.observability.consume().delivery_error).toBe(1)
h.setApnsResponse({ status: 200, body: '' })
}
}
expect(h.apnsRequests).toHaveLength(2)
expect(h.server.observability.consume().delivery_sent).toBe(1)
} finally {
await h.close()
}
})
cloud/apps/push/src/canonical-base64.ts
+9
View File
@@ -0,0 +1,9 @@
// Rejects the many base64 spellings of the same bytes: a non-canonical
// encoding would change the transcript the host signs without changing the key.
export function decodeCanonicalBase64(value: string, expectedBytes: number): Buffer | null {
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) return null
const decoded = Buffer.from(value, 'base64')
return decoded.byteLength === expectedBytes && decoded.toString('base64') === value
? decoded
: null
}
cloud/apps/push/src/client-ip-rate-limit.test.ts
+169
View File
@@ -0,0 +1,169 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { Hono } from 'hono'
import { describe, expect, it, vi } from 'vitest'
import { ClientIpRateLimiter, clientIpRateLimit } from './client-ip-rate-limit.js'
const CAPACITY = PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp
function limiterApp(limiter: ClientIpRateLimiter, trustedProxyHops = 0): Hono {
const app = new Hono()
app.post('/probe', clientIpRateLimit(limiter, { trustedProxyHops }), (context) =>
context.json({ ok: true })
)
return app
}
describe('client ip rate limiter', () => {
it('admits exactly the per-minute allowance and refuses the next request', () => {
const limiter = new ClientIpRateLimiter({ now: () => 1_000 })
for (let index = 0; index < CAPACITY; index++) {
expect(limiter.allow('203.0.113.7')).toBe(true)
}
expect(limiter.allow('203.0.113.7')).toBe(false)
})
it('keeps one client ip from spending another one budget', () => {
const limiter = new ClientIpRateLimiter({ now: () => 1_000 })
for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7')
expect(limiter.allow('203.0.113.7')).toBe(false)
expect(limiter.allow('198.51.100.9')).toBe(true)
})
it('refills over the window rather than resetting on a boundary', () => {
let clock = 1_000
const limiter = new ClientIpRateLimiter({ now: () => clock })
for (let index = 0; index < CAPACITY; index++) limiter.allow('203.0.113.7')
expect(limiter.allow('203.0.113.7')).toBe(false)
// Half a window buys back half the allowance, no more.
clock += 30_000
for (let index = 0; index < CAPACITY / 2; index++) {
expect(limiter.allow('203.0.113.7')).toBe(true)
}
expect(limiter.allow('203.0.113.7')).toBe(false)
})
it('bounds what it remembers when a flood of distinct ips arrives', () => {
let clock = 1_000
const limiter = new ClientIpRateLimiter({ now: () => clock, maxTrackedIps: 8 })
for (let index = 0; index < 200; index++) {
clock += 1
limiter.allow(`198.51.100.${index}`)
}
expect(limiter.trackedIpCount()).toBeLessThanOrEqual(8)
})
it('evicts the least recently used bucket without scanning the map', () => {
const limiter = new ClientIpRateLimiter({ capacity: 1, maxTrackedIps: 2, now: () => 1_000 })
limiter.allow('old')
limiter.allow('recent')
expect(limiter.allow('old')).toBe(false)
const entries = vi.spyOn(Map.prototype, 'entries')
const iterator = vi.spyOn(Map.prototype, Symbol.iterator)
try {
limiter.allow('new')
expect(entries).not.toHaveBeenCalled()
expect(iterator).not.toHaveBeenCalled()
} finally {
entries.mockRestore()
iterator.mockRestore()
}
expect(limiter.available('old')).toBe(false)
expect(limiter.available('recent')).toBe(true)
expect(limiter.trackedIpCount()).toBe(2)
})
it('answers 429 with a rate_limited body once the bucket is empty', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
const headers = { 'x-forwarded-for': '10.0.0.1, 10.0.0.2, 203.0.113.7' }
for (let index = 0; index < CAPACITY; index++) {
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
}
const limited = await app.request('/probe', { method: 'POST', headers })
expect(limited.status).toBe(429)
expect(await limited.json()).toEqual({ error: 'rate_limited' })
})
it('buckets on the last forwarded hop, the only one the platform appended', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
for (let index = 0; index < CAPACITY; index++) {
await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': `10.0.0.${index}, 203.0.113.7` }
})
}
const sameClient = await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': '10.9.9.9, 203.0.113.7' }
})
expect(sameClient.status).toBe(429)
const otherClient = await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': '10.0.0.1, 198.51.100.9' }
})
expect(otherClient.status).toBe(200)
})
it('gives a spoofed left-most hop no escape from the caller own bucket', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000 }))
// A caller that rewrites its own x-forwarded-for on every request still ends
// up behind the one value Cloud Run appended.
for (let index = 0; index < CAPACITY; index++) {
const allowed = await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': `198.51.100.${index}, 203.0.113.7` }
})
expect(allowed.status).toBe(200)
}
const spoofed = await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': '198.51.100.250, 10.1.1.1, 203.0.113.7' }
})
expect(spoofed.status).toBe(429)
})
it('skips the configured trusted proxies when counting from the right', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
// <client>, <cloud run>, <load balancer>: one trusted hop after the client.
const headers = { 'x-forwarded-for': '203.0.113.7, 10.0.0.1' }
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(429)
expect(
(
await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': '198.51.100.9, 10.0.0.1' }
})
).status
).toBe(200)
})
it('trusts nothing when the header is shorter than the configured depth', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }), 1)
// Only one hop, so the client value the depth points at does not exist.
const headers = { 'x-forwarded-for': '203.0.113.7' }
expect((await app.request('/probe', { method: 'POST', headers })).status).toBe(200)
expect(
(
await app.request('/probe', {
method: 'POST',
headers: { 'x-forwarded-for': '198.51.100.9' }
})
).status
).toBe(429)
})
it('ignores spoofable x-real-ip and uses a single shared bucket', async () => {
const app = limiterApp(new ClientIpRateLimiter({ now: () => 1_000, capacity: 1 }))
expect(
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '198.51.100.9' } }))
.status
).toBe(200)
expect(
(await app.request('/probe', { method: 'POST', headers: { 'x-real-ip': '203.0.113.7' } }))
.status
).toBe(429)
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
expect((await app.request('/probe', { method: 'POST' })).status).toBe(429)
})
})
cloud/apps/push/src/client-ip-rate-limit.ts
+95
View File
@@ -0,0 +1,95 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import type { Context, MiddlewareHandler } from 'hono'
const REFILL_WINDOW_MS = 60_000
const MAX_TRACKED_IPS = 10_000
const UNKNOWN_CLIENT_IP = 'unknown'
export type ClientIpRateLimiterOptions = {
capacity?: number
windowMs?: number
maxTrackedIps?: number
now?: () => number
}
type Bucket = { tokens: number; updatedAt: number }
// Read x-forwarded-for from the right. Cloud Run appends the connecting peer,
// so the last value is the only one it wrote; everything to its left is
// whatever the caller sent and can be a fresh forgery on every request.
// trustedProxyHops is how many appenders sit between Cloud Run and the client
// (0 today, 1 once a load balancer fronts it). A header too short for that
// depth is not trusted at all and falls through to the shared bucket, which
// throttles rather than opens.
export function readClientIp(context: Context, trustedProxyHops = 0): string {
const hops =
context.req
.header('x-forwarded-for')
?.split(',')
.map((hop) => hop.trim())
.filter((hop) => hop.length > 0) ?? []
const client = hops[hops.length - 1 - trustedProxyHops]
return client ?? UNKNOWN_CLIENT_IP
}
// Per-instance admission avoids a database round trip; capacity scales with instance count.
export class ClientIpRateLimiter {
private readonly buckets = new Map<string, Bucket>()
private readonly capacity: number
private readonly windowMs: number
private readonly maxTrackedIps: number
private readonly now: () => number
constructor(options: ClientIpRateLimiterOptions = {}) {
this.capacity = options.capacity ?? PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp
this.windowMs = options.windowMs ?? REFILL_WINDOW_MS
this.maxTrackedIps = options.maxTrackedIps ?? MAX_TRACKED_IPS
this.now = options.now ?? Date.now
}
available(clientIp: string): boolean {
return this.tokensAt(this.buckets.get(clientIp), this.now()) >= 1
}
allow(clientIp: string): boolean {
const now = this.now()
const tokens = this.tokensAt(this.buckets.get(clientIp), now)
this.buckets.delete(clientIp)
this.buckets.set(clientIp, { tokens: tokens < 1 ? tokens : tokens - 1, updatedAt: now })
if (this.buckets.size > this.maxTrackedIps) {
const oldest = this.buckets.keys().next().value
if (oldest !== undefined) this.buckets.delete(oldest)
}
return tokens >= 1
}
trackedIpCount(): number {
return this.buckets.size
}
private tokensAt(bucket: Bucket | undefined, now: number): number {
if (!bucket) return this.capacity
const refilled = ((now - bucket.updatedAt) * this.capacity) / this.windowMs
return Math.min(this.capacity, bucket.tokens + Math.max(0, refilled))
}
}
export type ClientIpRateLimitOptions = {
trustedProxyHops?: number
onLimited?: () => void
}
export function clientIpRateLimit(
limiter: ClientIpRateLimiter,
options: ClientIpRateLimitOptions = {}
): MiddlewareHandler {
const trustedProxyHops = options.trustedProxyHops ?? 0
return async (context, next) => {
if (!limiter.allow(readClientIp(context, trustedProxyHops))) {
options.onLimited?.()
return context.json({ error: 'rate_limited' }, 429)
}
await next()
return
}
}
cloud/apps/push/src/config.test.ts
+109
View File
@@ -0,0 +1,109 @@
import { generateKeyPairSync } from 'node:crypto'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { describe, expect, it } from 'vitest'
import { loadPushConfig, PUSH_DATABASE_POOL_MAX } from './config.js'
function apnsKeyPem(): string {
return generateKeyPairSync('ec', {
namedCurve: 'P-256',
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
publicKeyEncoding: { type: 'spki', format: 'pem' }
}).privateKey
}
const MINIMAL = {
ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev',
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-cloud'
}
describe('push gateway config', () => {
it('applies the documented defaults', () => {
expect(loadPushConfig(MINIMAL)).toEqual({
mode: 'active',
port: 8080,
publicUrl: 'https://push.onorca.dev',
databaseUrl: undefined,
dataDir: './data/push',
databasePoolMax: PUSH_DATABASE_POOL_MAX,
apns: undefined,
apnsTopic: PUSH_DEFAULTS.apnsTopic,
fcmProjectId: 'onorca-cloud',
trustedProxyHops: 0
})
})
it('reads a full APNs credential and the overridable knobs', () => {
const keyPem = apnsKeyPem()
const config = loadPushConfig({
...MINIMAL,
PORT: '9090',
ORCA_PUSH_DATABASE_URL: 'postgres://localhost/orca_push',
ORCA_PUSH_DATA_DIR: '/var/lib/push',
ORCA_PUSH_APNS_KEY: keyPem,
ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345',
ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456',
ORCA_PUSH_APNS_TOPIC: 'com.stably.orca.mobile.dev',
ORCA_PUSH_FCM_PROJECT_ID: 'onorca-staging',
ORCA_PUSH_TRUSTED_PROXY_HOPS: '1'
})
expect(config).toMatchObject({
port: 9090,
databaseUrl: 'postgres://localhost/orca_push',
dataDir: '/var/lib/push',
apns: { keyPem, keyId: 'ABCDE12345', teamId: 'TEAM123456' },
apnsTopic: 'com.stably.orca.mobile.dev',
trustedProxyHops: 1,
fcmProjectId: 'onorca-staging'
})
})
it('requires an explicit FCM project instead of silently targeting production', () => {
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: undefined })).toThrow()
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_FCM_PROJECT_ID: ' ' })).toThrow()
})
it('refuses a partial APNs credential', () => {
expect(() => loadPushConfig({ ...MINIMAL, ORCA_PUSH_APNS_KEY: apnsKeyPem() })).toThrow(
'configured together'
)
expect(() =>
loadPushConfig({
...MINIMAL,
ORCA_PUSH_APNS_KEY: 'not-a-pem',
ORCA_PUSH_APNS_KEY_ID: 'ABCDE12345',
ORCA_PUSH_APPLE_TEAM_ID: 'TEAM123456'
})
).toThrow('PEM text')
})
it('requires a canonical HTTPS origin outside loopback', () => {
expect(() =>
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'https://push.onorca.dev/v1' })
).toThrow('must be an origin')
expect(() =>
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://push.onorca.dev' })
).toThrow('must use HTTPS')
expect(
loadPushConfig({ ...MINIMAL, ORCA_PUSH_PUBLIC_URL: 'http://localhost:8080' }).publicUrl
).toBe('http://localhost:8080')
})
it('treats an empty optional variable as unset', () => {
expect(
loadPushConfig({ ...MINIMAL, ORCA_PUSH_DATABASE_URL: '', ORCA_PUSH_APNS_KEY_ID: '' })
).toMatchObject({ databaseUrl: undefined, apns: undefined })
})
})
it('treats blank defaulted environment settings as absent', () => {
const blanks = Object.fromEntries(
[
'PORT',
'ORCA_PUSH_DATA_DIR',
'ORCA_PUSH_APNS_TOPIC',
'ORCA_PUSH_DATABASE_POOL_MAX',
'ORCA_PUSH_TRUSTED_PROXY_HOPS'
].map((key) => [key, ' '])
)
expect(loadPushConfig({ ...MINIMAL, ...blanks })).toEqual(loadPushConfig(MINIMAL))
})
cloud/apps/push/src/config.ts
+108
View File
@@ -0,0 +1,108 @@
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { z } from 'zod'
export const PUSH_DATABASE_POOL_MAX = 10
const OptionalTextSchema = z.preprocess(
(value) => (value === '' ? undefined : value),
z.string().min(1).optional()
)
const EnvSchema = z.object({
ORCA_PUSH_MODE: z.enum(['active', 'validation']).default('active'),
PORT: z.coerce.number().int().positive().default(8080),
ORCA_PUSH_PUBLIC_URL: z.string().url(),
ORCA_PUSH_DATABASE_URL: OptionalTextSchema,
ORCA_PUSH_DATA_DIR: z.string().min(1).default('./data/push'),
ORCA_PUSH_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_PUSH_APNS_KEY: OptionalTextSchema,
ORCA_PUSH_APNS_KEY_ID: z.preprocess(
(value) => (value === '' ? undefined : value),
z
.string()
.regex(/^[A-Z0-9]{10}$/)
.optional()
),
ORCA_PUSH_APPLE_TEAM_ID: z.preprocess(
(value) => (value === '' ? undefined : value),
z
.string()
.regex(/^[A-Z0-9]{10}$/)
.optional()
),
ORCA_PUSH_APNS_TOPIC: z.string().min(1).max(255).default(PUSH_DEFAULTS.apnsTopic),
ORCA_PUSH_FCM_PROJECT_ID: z.string().regex(/^[a-z0-9-]{4,64}$/),
// How many proxies append to x-forwarded-for after the client. 0 is Cloud Run
// alone; raise it to 1 when a load balancer fronts the service.
ORCA_PUSH_TRUSTED_PROXY_HOPS: z.coerce.number().int().nonnegative().max(8).default(0)
})
export type ApnsCredentials = { keyPem: string; keyId: string; teamId: string }
export type PushConfig = {
mode: 'active' | 'validation'
port: number
publicUrl: string
databaseUrl?: string
dataDir: string
databasePoolMax: number
apns?: ApnsCredentials
apnsTopic: string
fcmProjectId: string
trustedProxyHops: number
}
function canonicalOrigin(value: string, name: string): string {
const url = new URL(value)
if (url.origin !== value || url.pathname !== '/') throw new Error(`${name} must be an origin`)
const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
if (url.protocol !== 'https:' && !(loopback && url.protocol === 'http:')) {
throw new Error(`${name} must use HTTPS outside loopback development`)
}
return value
}
// The APNs key, key id, and team id are one credential; a partial set would
// pass startup and then fail every iOS send at runtime.
function readApnsCredentials(parsed: z.infer<typeof EnvSchema>): ApnsCredentials | undefined {
const parts = [
parsed.ORCA_PUSH_APNS_KEY,
parsed.ORCA_PUSH_APNS_KEY_ID,
parsed.ORCA_PUSH_APPLE_TEAM_ID
]
const present = parts.filter((value) => value !== undefined).length
if (present === 0) return undefined
if (present !== parts.length) {
throw new Error('APNs key, key id, and team id must be configured together')
}
const keyPem = parsed.ORCA_PUSH_APNS_KEY!
if (!keyPem.includes('-----BEGIN')) throw new Error('ORCA_PUSH_APNS_KEY must be PEM text')
return {
keyPem,
keyId: parsed.ORCA_PUSH_APNS_KEY_ID!,
teamId: parsed.ORCA_PUSH_APPLE_TEAM_ID!
}
}
export function loadPushConfig(env: NodeJS.ProcessEnv = process.env): PushConfig {
const parsed = EnvSchema.parse(
Object.fromEntries(
Object.entries(env).map(([key, value]) => [
key,
key !== 'ORCA_PUSH_MODE' && value?.trim() === '' ? undefined : value
])
)
)
return {
mode: parsed.ORCA_PUSH_MODE,
port: parsed.PORT,
publicUrl: canonicalOrigin(parsed.ORCA_PUSH_PUBLIC_URL, 'ORCA_PUSH_PUBLIC_URL'),
databaseUrl: parsed.ORCA_PUSH_DATABASE_URL,
dataDir: parsed.ORCA_PUSH_DATA_DIR,
databasePoolMax: parsed.ORCA_PUSH_DATABASE_POOL_MAX ?? PUSH_DATABASE_POOL_MAX,
apns: readApnsCredentials(parsed),
apnsTopic: parsed.ORCA_PUSH_APNS_TOPIC,
fcmProjectId: parsed.ORCA_PUSH_FCM_PROJECT_ID,
trustedProxyHops: parsed.ORCA_PUSH_TRUSTED_PROXY_HOPS
}
}
cloud/apps/push/src/desktop-host-proof-interop.test.ts
+47
View File
@@ -0,0 +1,47 @@
import { describe, expect, it } from 'vitest'
import { createHmac } from 'node:crypto'
import vector from '../../../packages/push-contract/src/push-host-proof-vector.json' with { type: 'json' }
import { answerPushHostChallenge, createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
import { PushHostChallengeStore } from './host-challenge-store.js'
import { deriveHostFingerprint } from './host-fingerprint.js'
import { openInMemoryPushDatabase } from './push-database.js'
// Why: the desktop answers challenges in a workspace this one cannot import.
// Both sides replay the same checked-in vector, so a transcript drift on
// either side fails in that side's own suite.
describe('desktop host proof interop', () => {
it('the checked-in vector answers to the same proof the fixture host computes', () => {
const secretKey = new Uint8Array(Buffer.from(vector.hostSecretKeyB64, 'base64'))
const keypair = { publicKey: new Uint8Array(Buffer.from(vector.hostPublicKeyB64, 'base64')), secretKey }
expect(deriveHostFingerprint(keypair.publicKey)).toBe(vector.hostFingerprint)
const proof = answerPushHostChallenge(vector.challenge, {
gatewayOrigin: vector.gatewayOrigin,
keypair,
now: () => vector.issuedAt + 1_000
})
const expected = createHmac('sha256', Buffer.from(vector.challengeSecretB64, 'base64'))
.update(Buffer.from('orca-push-host-proof/v1\0ack\0'))
.update(Buffer.from(vector.transcriptB64, 'base64'))
.digest('base64')
expect(proof).toBe(expected)
})
it('a live challenge from the store round-trips through the fixture host once', async () => {
const database = await openInMemoryPushDatabase()
const store = new PushHostChallengeStore(database, vector.gatewayOrigin)
const keypair = createPushHostKeypair(11)
const challenge = await store.issue(Buffer.from(keypair.publicKey).toString('base64'))
expect(challenge).not.toBeNull()
const proof = answerPushHostChallenge(challenge!, { gatewayOrigin: vector.gatewayOrigin, keypair })
expect(proof).not.toBeNull()
expect(await store.verify(challenge!.challengeId, proof!)).toEqual({
ok: true,
hostFingerprint: deriveHostFingerprint(keypair.publicKey)
})
expect(await store.verify(challenge!.challengeId, proof!)).toEqual({
ok: false,
reason: 'already_consumed'
})
await database.close()
})
})
cloud/apps/push/src/device-registration-delete-race.test.ts
+88
View File
@@ -0,0 +1,88 @@
import { expect, it, vi } from 'vitest'
import { openPushDatabase, type PushDatabase } from './push-database.js'
import { PushDeviceRegistryStore } from './device-registry-store.js'
const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL
it.skipIf(!databaseUrl)(
'serializes deletion with a registration that has already read its row',
async () => {
if (!process.env.CI && new URL(databaseUrl!).port !== '55440')
throw new Error('isolated_postgres_port_required')
const database = await openPushDatabase({
databaseUrl,
dataDir: '',
poolMax: 4,
applicationName: 'push-delete-race'
})
let release!: () => void
const paused = new Promise<void>((resolve) => {
release = resolve
})
let read = false
let pause = false
const wrapped: PushDatabase = {
dialect: database.dialect,
query: database.query.bind(database),
close: database.close.bind(database),
lockQuotaScope: database.lockQuotaScope.bind(database),
transaction: (run) =>
database.transaction((tx) =>
run({
dialect: tx.dialect,
close: tx.close.bind(tx),
transaction: tx.transaction.bind(tx),
lockQuotaScope: tx.lockQuotaScope.bind(tx),
query: async (sql, params) => {
const rows = await tx.query(sql, params)
if (pause && sql.startsWith('SELECT registration_id FROM push_devices')) {
read = true
await paused
}
return rows
}
})
)
}
const devices = new PushDeviceRegistryStore(wrapped)
const input = {
hostFingerprint: 'delete-race-host',
deviceId: 'phone',
platform: 'android' as const,
token: 'synthetic'
}
let registration: Promise<unknown> | undefined
let deletion: Promise<boolean> | undefined
try {
await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [
input.hostFingerprint
])
const first = await devices.upsert(input)
if (!first.ok) throw new Error('registration refused')
pause = true
registration = devices.upsert(input)
await vi.waitFor(() => expect(read).toBe(true))
let deleted = false
deletion = devices.deleteOwned(input.hostFingerprint, first.registrationId).then((value) => {
deleted = true
return value
})
await vi.waitFor(async () => {
const rows = await database.query(
"SELECT 1 FROM pg_stat_activity WHERE application_name = 'push-delete-race' AND wait_event_type = 'Lock'"
)
expect(deleted || rows.length > 0).toBe(true)
})
expect(deleted).toBe(false)
release()
expect(await registration).toEqual(first)
expect(await deletion).toBe(true)
} finally {
release()
await Promise.allSettled([registration, deletion])
await database.query('DELETE FROM push_devices WHERE host_fingerprint = ?', [
input.hostFingerprint
])
await database.close()
}
}
)
cloud/apps/push/src/device-registry-store.test.ts
+191
View File
@@ -0,0 +1,191 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { PushDeviceRegistryStore, type PushDeviceUpsert } from './device-registry-store.js'
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
const OWNER = 'abcdefghijklmnop'
const OTHER = 'ponmlkjihgfedcba'
describe('push device registry store', () => {
let database: PushDatabase
let clock = 1_700_000_000_000
let devices: PushDeviceRegistryStore
beforeEach(async () => {
database = await openInMemoryPushDatabase()
clock = 1_700_000_000_000
devices = new PushDeviceRegistryStore(database, () => clock)
})
afterEach(async () => {
await database.close()
})
async function upsertOk(input: PushDeviceUpsert): Promise<string> {
const result = await devices.upsert(input)
if (!result.ok) throw new Error(`unexpected upsert refusal: ${result.reason}`)
return result.registrationId
}
function androidDevice(deviceId: string): PushDeviceUpsert {
return {
hostFingerprint: OWNER,
deviceId,
platform: 'android',
token: `token-${deviceId}`
}
}
it('keeps one registration per host and device while replacing the token', async () => {
const first = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'ios',
token: 'a'.repeat(64),
apnsEnvironment: 'sandbox'
})
clock += 1_000
const second = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'ios',
token: 'b'.repeat(64),
apnsEnvironment: 'production'
})
expect(second).toBe(first)
const registration = await devices.findById(first)
expect(registration).toMatchObject({
token: 'b'.repeat(64),
apnsEnvironment: 'production',
dead: false
})
expect(await devices.list(OWNER)).toHaveLength(1)
})
it('revives a registration that a re-registered token replaces', async () => {
const registrationId = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'android',
token: 'token-one'
})
await devices.markDead((await devices.findById(registrationId))!)
expect((await devices.findById(registrationId))?.dead).toBe(true)
await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'android',
token: 'token-two'
})
expect(await devices.findById(registrationId)).toMatchObject({
token: 'token-two',
dead: false
})
})
it('lets only the owning host delete a registration', async () => {
const registrationId = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'android',
token: 'token-one'
})
expect(await devices.deleteOwned(OTHER, registrationId)).toBe(false)
expect(await devices.findById(registrationId)).not.toBeNull()
expect(await devices.deleteOwned(OWNER, registrationId)).toBe(true)
expect(await devices.findById(registrationId)).toBeNull()
})
it('scopes lookups and listings to the owning host', async () => {
const owned = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'device-1',
platform: 'android',
token: 'token-one'
})
const foreign = await upsertOk({
hostFingerprint: OTHER,
deviceId: 'device-2',
platform: 'android',
token: 'token-two'
})
const found = await devices.findOwned(OWNER, [owned, foreign])
expect([...found.keys()]).toEqual([owned])
expect(await devices.list(OTHER)).toEqual([
{ registrationId: foreign, deviceId: 'device-2', platform: 'android', dead: false }
])
expect(await devices.findOwned(OWNER, [])).toEqual(new Map())
})
it('refuses a new device once the host reaches its registration cap', async () => {
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
await upsertOk(androidDevice(`device-${index}`))
}
expect(await devices.upsert(androidDevice('one-too-many'))).toEqual({
ok: false,
reason: 'too_many_devices'
})
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
})
it('still lets a capped host re-register a device it already owns', async () => {
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
await upsertOk(androidDevice(`device-${index}`))
}
const rotated = await devices.upsert({ ...androidDevice('device-0'), token: 'rotated-token' })
expect(rotated.ok).toBe(true)
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
})
it('frees a slot when a registration is deleted', async () => {
const first = await upsertOk(androidDevice('device-0'))
for (let index = 1; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
await upsertOk(androidDevice(`device-${index}`))
}
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false)
expect(await devices.deleteOwned(OWNER, first)).toBe(true)
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(true)
})
it('counts the cap per host, not across the whole table', async () => {
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
await upsertOk(androidDevice(`device-${index}`))
}
expect((await devices.upsert(androidDevice('extra'))).ok).toBe(false)
expect(
(await devices.upsert({ ...androidDevice('device-0'), hostFingerprint: OTHER })).ok
).toBe(true)
})
it('bounds list reads to the host device allowance', async () => {
// Straight past the per-host cap, so only the query LIMIT can bound this.
const rows = PUSH_LIMITS.maxDevicesPerHost + 5
for (let index = 0; index < rows; index++) {
await database.query(
`INSERT INTO push_devices (registration_id, host_fingerprint, device_id, platform, token,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[`reg-${index}`, OWNER, `device-${index}`, 'android', 'token', clock + index, clock]
)
}
expect(await devices.list(OWNER)).toHaveLength(PUSH_LIMITS.maxDevicesPerHost)
})
it('separates the same device id registered against two hosts', async () => {
const first = await upsertOk({
hostFingerprint: OWNER,
deviceId: 'shared-device',
platform: 'ios',
token: 'a'.repeat(64),
apnsEnvironment: 'sandbox'
})
const second = await upsertOk({
hostFingerprint: OTHER,
deviceId: 'shared-device',
platform: 'ios',
token: 'c'.repeat(64),
apnsEnvironment: 'sandbox'
})
expect(first).not.toBe(second)
})
})
cloud/apps/push/src/device-registry-store.ts
+177
View File
@@ -0,0 +1,177 @@
import { randomUUID } from 'node:crypto'
import {
PUSH_LIMITS,
type ApnsEnvironment,
type PushDeviceSummary,
type PushPlatform
} from '@orca-cloud/push-contract'
import type { PushDatabase, SqlRow } from './push-database.js'
const DEVICE_CAP_LOCK_PREFIX = 'orca-push-device-cap:'
export type PushDeviceRegistration = {
registrationId: string
hostFingerprint: string
deviceId: string
platform: PushPlatform
token: string
apnsEnvironment?: ApnsEnvironment
dead: boolean
}
export type PushDeviceUpsertResult =
| { ok: true; registrationId: string }
| { ok: false; reason: 'too_many_devices' }
export type PushDeviceUpsert = {
hostFingerprint: string
deviceId: string
platform: PushPlatform
token: string
apnsEnvironment?: ApnsEnvironment
}
function toRegistration(row: SqlRow): PushDeviceRegistration {
const apnsEnvironment = row.apns_environment
return {
registrationId: String(row.registration_id),
hostFingerprint: String(row.host_fingerprint),
deviceId: String(row.device_id),
platform: String(row.platform) as PushPlatform,
token: String(row.token),
...(apnsEnvironment === null || apnsEnvironment === undefined
? {}
: { apnsEnvironment: String(apnsEnvironment) as ApnsEnvironment }),
dead: row.dead_at !== null && row.dead_at !== undefined
}
}
export class PushDeviceRegistryStore {
constructor(
private readonly database: PushDatabase,
private readonly now: () => number = Date.now
) {}
// The registration id is stable for a (host, device) pair so a re-registered
// phone keeps the id the desktop already persisted; only the token rotates.
async upsert(input: PushDeviceUpsert): Promise<PushDeviceUpsertResult> {
const now = this.now()
return await this.database.transaction<PushDeviceUpsertResult>(async (transaction) => {
// deviceId is caller-chosen, so counting and inserting must not interleave
// or a burst of new ids would walk straight past the cap.
await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${input.hostFingerprint}`)
const [existing] = await transaction.query(
'SELECT registration_id FROM push_devices WHERE host_fingerprint = ? AND device_id = ?',
[input.hostFingerprint, input.deviceId]
)
if (existing) {
const registrationId = String(existing.registration_id)
await transaction.query(
`UPDATE push_devices
SET platform = ?, token = ?, apns_environment = ?,
dead_at = NULL, updated_at = ?
WHERE registration_id = ?`,
[input.platform, input.token, input.apnsEnvironment ?? null, now, registrationId]
)
return { ok: true, registrationId }
}
const [countRow] = await transaction.query(
'SELECT COUNT(*) AS devices FROM push_devices WHERE host_fingerprint = ?',
[input.hostFingerprint]
)
if (Number(countRow?.devices ?? 0) >= PUSH_LIMITS.maxDevicesPerHost) {
return { ok: false, reason: 'too_many_devices' }
}
const registrationId = randomUUID()
await transaction.query(
`INSERT INTO push_devices
(registration_id, host_fingerprint, device_id, platform, token, apns_environment,
dead_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)`,
[
registrationId,
input.hostFingerprint,
input.deviceId,
input.platform,
input.token,
input.apnsEnvironment ?? null,
now,
now
]
)
return { ok: true, registrationId }
})
}
async deleteOwned(hostFingerprint: string, registrationId: string): Promise<boolean> {
return this.database.transaction(async (transaction) => {
await transaction.lockQuotaScope(`${DEVICE_CAP_LOCK_PREFIX}${hostFingerprint}`)
const [result] = await transaction.query(
'DELETE FROM push_devices WHERE registration_id = ? AND host_fingerprint = ?',
[registrationId, hostFingerprint]
)
return Number(result?.changes ?? 0) > 0
})
}
async list(hostFingerprint: string): Promise<PushDeviceSummary[]> {
const rows = await this.database.query(
// Bounded by the device-list response limit, so an
// oversized table degrades to a truncated list instead of a 500.
`SELECT registration_id, device_id, platform, dead_at
FROM push_devices WHERE host_fingerprint = ? ORDER BY created_at ASC LIMIT ?`,
[hostFingerprint, PUSH_LIMITS.maxDevicesPerHost]
)
return rows.map((row) => ({
registrationId: String(row.registration_id),
deviceId: String(row.device_id),
platform: String(row.platform) as PushPlatform,
dead: row.dead_at !== null && row.dead_at !== undefined
}))
}
async findOwned(
hostFingerprint: string,
registrationIds: readonly string[]
): Promise<Map<string, PushDeviceRegistration>> {
if (registrationIds.length === 0) return new Map()
const placeholders = registrationIds.map(() => '?').join(', ')
const rows = await this.database.query(
`SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment,
dead_at
FROM push_devices
WHERE host_fingerprint = ? AND registration_id IN (${placeholders})`,
[hostFingerprint, ...registrationIds]
)
return new Map(
rows.map((row) => {
const registration = toRegistration(row)
return [registration.registrationId, registration]
})
)
}
async findById(registrationId: string): Promise<PushDeviceRegistration | null> {
const [row] = await this.database.query(
`SELECT registration_id, host_fingerprint, device_id, platform, token, apns_environment,
dead_at
FROM push_devices WHERE registration_id = ?`,
[registrationId]
)
return row ? toRegistration(row) : null
}
async markDead(observed: PushDeviceRegistration): Promise<void> {
await this.database.query(
`UPDATE push_devices SET dead_at = ?, updated_at = ? WHERE registration_id = ? AND token = ? AND platform = ? AND COALESCE(apns_environment, '') = ?`,
[
this.now(),
this.now(),
observed.registrationId,
observed.token,
observed.platform,
observed.apnsEnvironment ?? ''
]
)
}
}
cloud/apps/push/src/durable-push-schema.ts
+45
View File
@@ -0,0 +1,45 @@
export const DURABLE_PUSH_SCHEMA = `
CREATE TABLE IF NOT EXISTS push_dismissed_events (
host_fingerprint TEXT NOT NULL,
notification_epoch TEXT NOT NULL,
notification_id TEXT NOT NULL,
notification_seq BIGINT NOT NULL,
created_at BIGINT NOT NULL,
PRIMARY KEY(host_fingerprint, notification_epoch, notification_id)
);
CREATE INDEX IF NOT EXISTS push_dismissed_retention ON push_dismissed_events(created_at);
CREATE TABLE IF NOT EXISTS push_events (
event_id TEXT PRIMARY KEY,
host_fingerprint TEXT NOT NULL,
kind TEXT NOT NULL,
fingerprint TEXT NOT NULL,
created_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL
);
CREATE INDEX IF NOT EXISTS push_events_quota ON push_events(host_fingerprint, kind, created_at);
CREATE TABLE IF NOT EXISTS push_event_recipients (
event_id TEXT NOT NULL,
registration_id TEXT NOT NULL,
created_at BIGINT NOT NULL,
PRIMARY KEY(event_id, registration_id)
);
CREATE TABLE IF NOT EXISTS push_delivery_batches (
batch_id TEXT PRIMARY KEY,
host_fingerprint TEXT NOT NULL,
registration_id TEXT NOT NULL,
kind TEXT NOT NULL,
payload_json TEXT NOT NULL,
state TEXT NOT NULL,
due_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
lease_token TEXT,
lease_until BIGINT NOT NULL,
attempts BIGINT NOT NULL,
created_at BIGINT NOT NULL
);
CREATE INDEX IF NOT EXISTS push_events_retention ON push_events(created_at);
CREATE INDEX IF NOT EXISTS push_recipients_retention ON push_event_recipients(created_at);
CREATE INDEX IF NOT EXISTS push_batches_expiry ON push_delivery_batches(expires_at);
CREATE INDEX IF NOT EXISTS push_batches_due ON push_delivery_batches(state, due_at);
CREATE INDEX IF NOT EXISTS push_batches_registration ON push_delivery_batches(registration_id, state);
`
cloud/apps/push/src/durable-push-store.test.ts
+289
View File
@@ -0,0 +1,289 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { afterEach, describe, expect, it } from 'vitest'
import { openInMemoryPushDatabase, openPushDatabase, type PushDatabase } from './push-database.js'
import { DurablePushStore, DELIVERY_LEASE_MS } from './durable-push-store.js'
import type { PushNotification } from '@orca-cloud/push-contract'
const cleanups: (() => Promise<void>)[] = []
afterEach(async () => {
await Promise.all(cleanups.splice(0).map((cleanup) => cleanup()))
})
const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'): PushNotification => ({
notificationId: `notification-${seq}`,
notificationEpoch: 'epoch',
notificationSeq: seq,
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
kind
})
async function fixture() {
const databaseUrl =
process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440')
throw new Error('isolated_postgres_port_required')
let db: PushDatabase
if (databaseUrl) {
const admin = new pg.Client({ connectionString: databaseUrl })
await admin.connect()
const schema = `durable_${randomUUID().replaceAll('-', '')}`
let scoped: PushDatabase | undefined
cleanups.push(async () => {
try {
await scoped?.close()
} finally {
try {
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await admin.end()
}
}
})
await admin.query(`CREATE SCHEMA ${schema}`)
const url = new URL(databaseUrl)
url.searchParams.set('options', `-c search_path=${schema}`)
db = scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
} else {
db = await openInMemoryPushDatabase()
cleanups.push(() => db.close())
}
let now = 1_000_000
const clock = () => now
return {
db,
store: new DurablePushStore(db, clock),
clock,
advance: (ms: number) => {
now += ms
}
}
}
describe('durable push acceptance', () => {
it('counts a logical event once across phones and separates the 300/15min dismissal budget', async () => {
const { store, advance } = await fixture()
for (let i = 0; i < 300; i++) {
expect(await store.accept('host', 'phone1', notification(i))).toBe('queued')
expect(await store.accept('host', 'phone2', notification(i))).toBe('queued')
expect(await store.accept('host', 'phone1', notification(i, 'dismiss'))).toBe('queued')
}
expect(await store.accept('host', 'phone1', notification(300))).toBe('rate_limited')
expect(await store.accept('host', 'phone1', notification(300, 'dismiss'))).toBe('rate_limited')
expect(await store.accept('another-host', 'phone3', notification(300))).toBe('queued')
advance(15 * 60_000)
expect(await store.accept('host', 'phone1', notification(301))).toBe('queued')
})
it('queues one delivery per event and recovers work across service instances', async () => {
const { db, store, clock, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
const restarted = new DurablePushStore(db, clock)
await restarted.accept('host', 'phone', notification(1))
advance(1)
await restarted.accept('host', 'phone', notification(2))
const rows = await db.query(
"SELECT payload_json, due_at, created_at FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending' ORDER BY created_at, batch_id",
['phone']
)
expect(rows).toHaveLength(2)
expect(rows.map((row) => JSON.parse(String(row.payload_json)))).toEqual([
notification(1),
notification(2)
])
expect(rows.every((row) => Number(row.due_at) >= Number(row.created_at))).toBe(true)
const delivery = await restarted.claim()
expect(delivery?.notification.notificationSeq).toBe(1)
expect(await store.claim()).toBeNull()
advance(DELIVERY_LEASE_MS)
const reclaimed = await store.claim()
expect(reclaimed?.id).toBe(delivery?.id)
expect(reclaimed?.lease).not.toBe(delivery?.lease)
await restarted.finish(delivery!)
expect(await store.claim()).toBeNull()
await store.finish(reclaimed!)
const second = await restarted.claim()
expect(second?.notification.notificationSeq).toBe(2)
await restarted.finish(second!)
expect(await restarted.claim()).toBeNull()
})
it('never extends expiry and refuses conflicting duplicate content', async () => {
const { store, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
expect(await store.accept('host', 'phone', { ...notification(1), body: 'changed' })).toBe(
'error'
)
const delivery = (await store.claim())!
await store.finish(delivery, 10 * 60_000)
advance(60_000)
expect(await store.claim()).toBeNull()
advance(5 * 60_000)
expect(await store.accept('host', 'phone', notification(1))).toBe('error')
})
it('orders a due retry before a fresh first attempt without delaying the retry', async () => {
const { store, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
const first = (await store.claim())!
await store.finish(first, 1000)
expect(await store.claim()).toBeNull()
advance(1000)
await store.accept('host', 'phone', notification(2))
const retry = (await store.claim())!
expect(retry.notification.notificationSeq).toBe(1)
await store.finish(retry)
const fresh = (await store.claim())!
expect(fresh?.notification.notificationSeq).toBe(2)
await store.finish(fresh!)
})
it('orders an expired first-attempt lease by creation time after a retry becomes due', async () => {
const { db, store, clock, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
const retry = (await store.claim())!
await store.finish(retry, 1000)
advance(2000)
await db.query(
`INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at)
VALUES ('crashed-singleton', 'host', 'phone', 'alert', ?, 'pending', ?, ?, 0, 1, ?)`,
[JSON.stringify(notification(2)), clock() - 1, clock() + 300_000, clock()]
)
const reclaimedRetry = (await store.claim())!
expect(reclaimedRetry.notification.notificationSeq).toBe(1)
await store.finish(reclaimedRetry)
const reclaimedCrash = (await store.claim())!
expect(reclaimedCrash.notification.notificationSeq).toBe(2)
await store.finish(reclaimedCrash)
})
it('rolls quota and payload back together if persistence fails', async () => {
const { db, store } = await fixture()
await db.query('ALTER TABLE push_delivery_batches RENAME TO push_delivery_batches_unavailable')
try {
const databaseUrl =
process.env.ORCA_PUSH_DURABLE_TEST_POSTGRES_URL ?? process.env.ORCA_PUSH_TEST_DATABASE_URL
if (databaseUrl) {
const concurrent = await openPushDatabase({ databaseUrl, dataDir: '' })
try {
await expect(
concurrent.query('SELECT COUNT(*) FROM push_delivery_batches')
).resolves.toHaveLength(1)
} finally {
await concurrent.close()
}
}
await expect(store.accept('host', 'phone', notification(1))).rejects.toThrow()
expect(await db.query('SELECT * FROM push_events')).toEqual([])
expect(await db.query('SELECT * FROM push_event_recipients')).toEqual([])
} finally {
await db.query(
'ALTER TABLE push_delivery_batches_unavailable RENAME TO push_delivery_batches'
)
}
})
})
it('serializes concurrent instances at the quota boundary', async () => {
const { db, store, clock } = await fixture()
for (let seq = 0; seq < 299; seq++) await store.accept('host', 'phone', notification(seq))
const second = new DurablePushStore(db, clock)
const results = await Promise.all(
Array.from({ length: 6 }, (_, index) =>
(index % 2 ? store : second).accept('host', 'phone', notification(400 + index))
)
)
expect(results.filter((result) => result === 'queued')).toHaveLength(1)
expect(results.filter((result) => result === 'rate_limited')).toHaveLength(5)
})
it('cancels unsent alerts and prevents an older replay after dismissal', async () => {
const { store } = await fixture()
const alert = notification(1)
await store.accept('host', 'phone', alert)
await store.accept('host', 'phone', {
...notification(2, 'dismiss'),
notificationId: alert.notificationId
})
const delivery = (await store.claim())!
expect(delivery.notification.kind).toBe('dismiss')
await store.finish(delivery)
expect(await store.claim()).toBeNull()
await store.accept('host', 'another-phone', alert)
expect(await store.claim()).toBeNull()
})
it('does not resurrect an in-flight alert after a dismissal and transient provider failure', async () => {
const { store, advance } = await fixture()
await store.accept('host', 'phone', notification(1))
const inFlight = (await store.claim())!
await store.accept('host', 'phone', {
...notification(2, 'dismiss'),
notificationId: notification(1).notificationId
})
await store.finish(inFlight, 1000)
const dismissal = (await store.claim())!
expect(dismissal.notification.kind).toBe('dismiss')
await store.finish(dismissal)
advance(1000)
expect(await store.claim()).toBeNull()
expect(await store.pendingCount('phone')).toBe(0)
})
it.each([false, true])(
'normalizes default alert kind (explicit first: %s)',
async (explicitFirst) => {
const { db, store } = await fixture()
const { kind: _kind, ...implicit } = notification(1)
const explicit = { kind: 'alert' as const, ...implicit }
for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) {
expect(await store.accept('host', 'phone', event)).toBe('queued')
}
expect(await store.pendingCount('phone')).toBe(1)
expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(1)
expect(await store.accept('host', 'phone', { ...explicit, body: 'changed' })).toBe('error')
expect(await store.accept('host', 'phone', { ...implicit, kind: 'dismiss' })).toBe('queued')
expect(await db.query('SELECT event_id FROM push_events')).toHaveLength(2)
}
)
it('fences late renew and finish after an expired claim is dismissed', async () => {
const { db, store, advance } = await fixture()
const alert = notification(1)
await store.accept('host', 'phone', alert)
const stale = (await store.claim())!
advance(DELIVERY_LEASE_MS)
await store.accept('host', 'phone', {
...notification(2, 'dismiss'),
notificationId: alert.notificationId
})
const read = async () =>
(
await db.query(
'SELECT state, payload_json, lease_until FROM push_delivery_batches WHERE batch_id = ?',
[stale.id]
)
)[0]
const cancelled = await read()
expect(cancelled).toMatchObject({ state: 'dismissed', payload_json: '{}' })
await store.renew(stale)
expect(await read()).toEqual(cancelled)
await store.finish(stale, 1000)
expect(await read()).toEqual(cancelled)
await store.finish(stale)
expect(await read()).toEqual(cancelled)
const dismissal = (await store.claim())!
expect(dismissal.notification.kind).toBe('dismiss')
await store.finish(dismissal)
await store.accept('host', 'phone', notification(3))
const fresh = (await store.claim())!
await store.finish(fresh, 1000)
advance(1000)
const retry = (await store.claim())!
expect(retry.id).toBe(fresh.id)
await store.finish(retry)
expect(await store.claim()).toBeNull()
})
cloud/apps/push/src/durable-push-store.ts
+194
View File
@@ -0,0 +1,194 @@
import { isDismissedAlert, reconcileQueuedDismissal } from './push-queued-dismissal.js'
import { parsePushDeliveryPayload } from './push-delivery-payload.js'
import { createHash, randomUUID } from 'node:crypto'
import { PUSH_LIMITS, type PushNotification } from '@orca-cloud/push-contract'
import type { PushDatabase, SqlRow } from './push-database.js'
const RETENTION_MS = 24 * 60 * 60_000
export const DELIVERY_LEASE_MS = 30_000
export type QueuedPushDelivery = {
id: string
registrationId: string
hostFingerprint: string
notification: PushNotification
expiresAt: number
lease: string
attempts: number
}
export class DurablePushStore {
constructor(
private readonly database: PushDatabase,
private readonly now = Date.now
) {}
async accept(
host: string,
registrationId: string,
notification: PushNotification
): Promise<'queued' | 'rate_limited' | 'error'> {
const now = this.now()
const kind = notification.kind ?? 'alert'
const eventId = createHash('sha256')
.update(
JSON.stringify([host, kind, notification.notificationEpoch, notification.notificationSeq])
)
.digest('hex')
const { sound: _sound, kind: _kind, ...content } = notification
const fingerprint = createHash('sha256')
.update(JSON.stringify({ kind, ...content }))
.digest('hex')
return this.database.transaction(async (tx) => {
await tx.lockQuotaScope(`push-events:${host}`)
const [existing] = await tx.query('SELECT * FROM push_events WHERE event_id = ?', [eventId])
if (existing && existing.fingerprint !== fingerprint) return 'error'
const expiresAt = existing
? Number(existing.expires_at)
: Math.min(
notification.expiresAt ?? Infinity,
now + PUSH_LIMITS.notificationTtlSeconds * 1000
)
if (expiresAt <= now) return 'error'
if (!existing) {
const [count] = await tx.query(
'SELECT COUNT(*) AS total FROM push_events WHERE host_fingerprint = ? AND kind = ? AND created_at > ?',
[host, kind, now - PUSH_LIMITS.eventQuotaWindowMs]
)
if (Number(count?.total ?? 0) >= PUSH_LIMITS.hostEventsPerWindow) return 'rate_limited'
await tx.query(
'INSERT INTO push_events(event_id, host_fingerprint, kind, fingerprint, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)',
[eventId, host, kind, fingerprint, now, expiresAt]
)
}
const [recipient] = await tx.query(
'SELECT event_id FROM push_event_recipients WHERE event_id = ? AND registration_id = ?',
[eventId, registrationId]
)
if (recipient) return 'queued'
if (await reconcileQueuedDismissal(tx, host, registrationId, notification, now))
return 'queued'
await tx.query(
`INSERT INTO push_delivery_batches(batch_id, host_fingerprint, registration_id, kind, payload_json, state, due_at, expires_at, lease_until, attempts, created_at)
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, 0, 0, ?)`,
[
randomUUID(),
host,
registrationId,
kind,
JSON.stringify(notification),
now,
expiresAt,
now
]
)
await tx.query(
'INSERT INTO push_event_recipients(event_id, registration_id, created_at) VALUES (?, ?, ?)',
[eventId, registrationId, now]
)
return 'queued'
})
}
async claim(): Promise<QueuedPushDelivery | null> {
return this.database.transaction(async (tx) => {
await tx.lockQuotaScope('push-worker-claim')
const now = this.now()
const params = [now, now, now, now]
const predicate =
"state = 'pending' AND lease_until <= ? AND expires_at > ? AND due_at <= ? AND NOT EXISTS (SELECT 1 FROM push_delivery_batches busy WHERE busy.registration_id = push_delivery_batches.registration_id AND busy.lease_until > ?)"
let [row] = await tx.query(
`SELECT * FROM push_delivery_batches WHERE ${predicate} ORDER BY due_at, created_at, batch_id LIMIT 1`,
params
)
if (!row) return null
await tx.lockQuotaScope(`push-events:${String(row.host_fingerprint)}`)
;[row] = await tx.query('SELECT * FROM push_delivery_batches WHERE batch_id = ?', [
row.batch_id
])
if (!row || row.state !== 'pending' || Number(row.expires_at) <= now) return null
const notification = parsePushDeliveryPayload(String(row.payload_json))
if (await isDismissedAlert(tx, String(row.host_fingerprint), notification)) {
await tx.query(
"UPDATE push_delivery_batches SET state = 'dismissed', payload_json = '{}' WHERE batch_id = ?",
[row.batch_id]
)
return null
}
const lease = randomUUID()
await tx.query(
'UPDATE push_delivery_batches SET lease_token = ?, lease_until = ?, attempts = attempts + 1 WHERE batch_id = ?',
[lease, now + DELIVERY_LEASE_MS, row.batch_id]
)
return this.delivery(row, lease)
})
}
private delivery(row: SqlRow, lease: string): QueuedPushDelivery {
return {
id: String(row.batch_id),
registrationId: String(row.registration_id),
hostFingerprint: String(row.host_fingerprint),
notification: parsePushDeliveryPayload(String(row.payload_json)),
expiresAt: Number(row.expires_at),
lease,
attempts: Number(row.attempts) + 1
}
}
async renew(delivery: QueuedPushDelivery): Promise<void> {
await this.database.query(
"UPDATE push_delivery_batches SET lease_until = ? WHERE batch_id = ? AND lease_token = ? AND state = 'pending'",
[this.now() + DELIVERY_LEASE_MS, delivery.id, delivery.lease]
)
}
async finish(
delivery: QueuedPushDelivery,
retryAfterMs?: number,
outcome = 'done'
): Promise<void> {
const now = this.now()
const retryAt = retryAfterMs === undefined ? Infinity : now + Math.max(1000, retryAfterMs)
const retry = retryAt < delivery.expiresAt
await this.database.query(
`UPDATE push_delivery_batches SET state = ?, payload_json = ?, due_at = ?, lease_until = 0, lease_token = NULL
WHERE batch_id = ? AND lease_token = ? AND state = 'pending'`,
[
retry ? 'pending' : retryAfterMs !== undefined ? 'expired' : outcome,
retry ? JSON.stringify(delivery.notification) : '{}',
retry ? retryAt : now,
delivery.id,
delivery.lease
]
)
}
async pendingCount(registrationId: string): Promise<number> {
const [row] = await this.database.query(
"SELECT COUNT(*) AS total FROM push_delivery_batches WHERE registration_id = ? AND state = 'pending'",
[registrationId]
)
return Number(row?.total ?? 0)
}
async prune(): Promise<number> {
const now = this.now()
await this.database.query(
"UPDATE push_delivery_batches SET state = 'expired', payload_json = '{}' WHERE expires_at <= ? AND state = 'pending'",
[now]
)
await this.database.query('DELETE FROM push_dismissed_events WHERE created_at < ?', [
now - RETENTION_MS
])
await this.database.query('DELETE FROM push_delivery_batches WHERE expires_at < ?', [
now - RETENTION_MS
])
await this.database.query('DELETE FROM push_event_recipients WHERE created_at < ?', [
now - RETENTION_MS
])
const [result] = await this.database.query('DELETE FROM push_events WHERE created_at < ?', [
now - RETENTION_MS
])
return Number(result?.changes ?? 0)
}
}
cloud/apps/push/src/durable-push-worker.test.ts
+221
View File
@@ -0,0 +1,221 @@
import { afterEach, expect, it, vi } from 'vitest'
import type { PushNotification } from '@orca-cloud/push-contract'
import { DurablePushStore } from './durable-push-store.js'
import { DurablePushWorker } from './durable-push-worker.js'
import { PushDispatcher } from './push-dispatcher.js'
import { PushDeviceRegistryStore } from './device-registry-store.js'
import { openInMemoryPushDatabase } from './push-database.js'
import type { PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
const cleanups: (() => Promise<void>)[] = []
afterEach(async () => {
for (const cleanup of cleanups.splice(0)) await cleanup()
vi.useRealTimers()
vi.restoreAllMocks()
})
const note = (seq: number, overrides: Partial<PushNotification> = {}): PushNotification => ({
notificationId: `note-${seq}`,
notificationSeq: seq,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: 'Finished task',
...overrides
})
async function fixture() {
const db = await openInMemoryPushDatabase()
let time = 1_000_000
const now = () => time
const store = new DurablePushStore(db, now)
const devices = new PushDeviceRegistryStore(db, now)
const device = await devices.upsert({
hostFingerprint: 'host',
deviceId: 'phone',
platform: 'android',
token: 'test-token'
})
if (!device.ok) throw new Error('registration failed')
const send = vi.fn(async (_delivery: PushDelivery): Promise<PushProviderOutcome> => ({
status: 'sent'
}))
const onRetry = vi.fn()
const dispatcher = new PushDispatcher({ devices, fcm: { send } as never })
const worker = new DurablePushWorker(store, dispatcher, { now, onRetry })
cleanups.push(async () => {
await worker.stop()
await db.close()
})
vi.spyOn(console, 'warn').mockImplementation(() => {})
return {
db,
store,
devices,
worker,
dispatcher,
send,
onRetry,
now,
registrationId: device.registrationId,
accept: (notification: PushNotification) =>
store.accept('host', device.registrationId, notification),
advance: (ms: number) => {
time += ms
}
}
}
it('sends every burst event immediately with its original content and identity', async () => {
const h = await fixture()
await h.accept(note(1))
await h.accept(
note(2, { agentState: 'needs-input', title: 'Answer needed', body: 'Please respond' })
)
await h.worker.runDue()
expect(h.send).toHaveBeenCalledTimes(2)
const first = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 1)![0]
const second = h.send.mock.calls.find(([delivery]) => delivery.orca.notificationSeq === 2)![0]
expect(first).toMatchObject({
title: 'Done',
body: 'Finished task',
orca: {
notificationId: 'note-1',
notificationSeq: 1
}
})
expect(second).toMatchObject({
title: 'Answer needed',
body: 'Please respond',
orca: { notificationId: 'note-2', notificationSeq: 2 }
})
expect(first.collapseId).not.toBe(second.collapseId)
expect(h.send.mock.calls.every(([delivery]) => !('coalescedCount' in delivery.orca))).toBe(true)
expect(h.send.mock.calls.every(([delivery]) => !('summaryMembers' in delivery.orca))).toBe(true)
expect(h.onRetry).not.toHaveBeenCalled()
})
it('keeps untrackable bells and per-phone deliveries individually replaceable', async () => {
const h = await fixture()
const other = await h.devices.upsert({
hostFingerprint: 'host',
deviceId: 'phone2',
platform: 'android',
token: 'other-token'
})
if (!other.ok) throw new Error('registration failed')
await h.accept(note(1, { notificationId: undefined, source: 'terminal-bell', agentState: null }))
await h.accept(note(2))
await h.accept(note(3))
await h.store.accept('host', other.registrationId, note(2))
await h.worker.runDue()
expect(h.send).toHaveBeenCalledTimes(4)
const deliveries = h.send.mock.calls.map(([delivery]) => delivery)
const primary = deliveries.filter((delivery) => delivery.registrationId === h.registrationId)
expect(primary).toHaveLength(3)
expect(new Set(primary.map((delivery) => delivery.collapseId)).size).toBe(3)
expect(
deliveries.find((delivery) => delivery.registrationId === other.registrationId)
).toMatchObject({ orca: { notificationId: 'note-2', notificationSeq: 2 } })
})
it('persists provider retry delay and resumes it through a new worker', async () => {
const h = await fixture()
h.send.mockResolvedValueOnce({
status: 'error',
reason: 'busy',
retryable: true,
retryAfterMs: 10000
})
await h.accept(note(1))
await h.worker.runDue()
expect(h.send).toHaveBeenCalledOnce()
await h.worker.stop()
const restarted = new DurablePushWorker(h.store, h.dispatcher, { now: h.now, onRetry: h.onRetry })
h.advance(9999)
await restarted.runDue()
expect(h.send).toHaveBeenCalledOnce()
h.advance(1)
await restarted.runDue()
expect(h.send).toHaveBeenCalledTimes(2)
expect(h.send.mock.calls.map(([delivery]) => delivery.expiresAt)).toEqual([1_300_000, 1_300_000])
expect(h.onRetry).toHaveBeenCalledOnce()
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
await restarted.stop()
})
it('expires instead of shortening a provider delay beyond the delivery lifetime', async () => {
const h = await fixture()
h.send.mockResolvedValue({
status: 'error',
reason: 'busy',
retryable: true,
retryAfterMs: 600000
})
await h.accept(note(1))
await h.worker.runDue()
h.advance(600000)
await h.worker.runDue()
expect(h.send).toHaveBeenCalledOnce()
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
expect(h.onRetry).not.toHaveBeenCalled()
})
it('rechecks the device before a persisted retry and does not send after unregistration', async () => {
const h = await fixture()
h.send.mockResolvedValue({ status: 'error', reason: 'timeout', retryable: true })
await h.accept(note(1))
await h.worker.runDue()
await h.devices.deleteOwned('host', h.registrationId)
h.advance(3000)
await h.worker.runDue()
expect(h.send).toHaveBeenCalledOnce()
expect(await h.store.pendingCount(h.registrationId)).toBe(0)
})
it('joins active work on shutdown and leaves unclaimed work for the next instance', async () => {
const h = await fixture()
let finish!: (outcome: PushProviderOutcome) => void
let started!: () => void
const entered = new Promise<void>((resolve) => {
started = resolve
})
h.send.mockImplementationOnce(() => {
started()
return new Promise((resolve) => {
finish = resolve
})
})
await h.accept(note(1))
const pending = h.worker.runDue()
await entered
await h.accept(note(2))
let stopped = false
const stopping = h.worker.stop().then(() => {
stopped = true
})
await Promise.resolve()
expect(stopped).toBe(false)
finish({ status: 'sent' })
await Promise.all([pending, stopping])
expect(stopped).toBe(true)
expect(h.send).toHaveBeenCalledOnce()
const resumed = new DurablePushWorker(h.store, h.dispatcher, { now: h.now })
await resumed.runDue()
expect(h.send).toHaveBeenCalledTimes(2)
await resumed.stop()
})
it('runs due work on its timer and releases the timer on stop', async () => {
const h = await fixture()
vi.useFakeTimers()
await h.accept(note(1))
h.worker.start()
h.worker.start()
expect(vi.getTimerCount()).toBe(1)
await vi.advanceTimersByTimeAsync(1000)
await h.worker.runDue()
expect(h.send).toHaveBeenCalledOnce()
await h.worker.stop()
expect(vi.getTimerCount()).toBe(0)
})
cloud/apps/push/src/durable-push-worker.ts
+89
View File
@@ -0,0 +1,89 @@
import { buildPushDelivery } from './push-delivery-message.js'
import type { PushDispatcher } from './push-dispatcher.js'
import type { DurablePushStore } from './durable-push-store.js'
export class DurablePushWorker {
private timer?: NodeJS.Timeout
private running: Promise<void> | null = null
private stopped = false
constructor(
private readonly store: DurablePushStore,
private readonly dispatcher: PushDispatcher,
private readonly options: { now?: () => number; onRetry?: () => void } = {}
) {}
start(): void {
if (this.timer) return
this.stopped = false
this.timer = setInterval(() => {
void this.runDue().catch(() => {
console.warn(JSON.stringify({ event: 'orca_push_worker_failed' }))
})
}, 1000)
this.timer.unref()
}
async runDue(): Promise<void> {
if (this.running) {
await this.running
return
}
if (this.stopped) return
const pending = Promise.allSettled(Array.from({ length: 4 }, () => this.drain())).then(
(results) => {
const failure = results.find((result) => result.status === 'rejected')
if (failure?.status === 'rejected') throw failure.reason
}
)
this.running = pending
try {
await pending
} finally {
this.running = null
}
}
private async drain(): Promise<void> {
for (let count = 0; count < 25 && !this.stopped; count++) {
const queued = await this.store.claim()
if (!queued) return
const delivery = buildPushDelivery({
expiresAt: queued.expiresAt,
registrationId: queued.registrationId,
hostFingerprint: queued.hostFingerprint,
notification: queued.notification
})
if ((this.options.now ?? Date.now)() >= queued.expiresAt) {
await this.store.finish(queued)
continue
}
const heartbeat = setInterval(() => {
void this.store.renew(queued).catch(() => {})
}, 10_000)
heartbeat.unref()
try {
if (queued.attempts > 1) this.options.onRetry?.()
const outcome = await this.dispatcher.sendOnce(delivery)
const retryAfterMs =
outcome.status === 'error' && outcome.retryable
? Math.max(
outcome.retryAfterMs ?? 0,
Math.min(30_000, 1000 * 2 ** Math.min(queued.attempts, 5))
)
: undefined
await this.store.finish(queued, retryAfterMs, outcome.status)
} catch {
await this.store.finish(queued, 5000)
} finally {
clearInterval(heartbeat)
}
}
}
async stop(): Promise<void> {
this.stopped = true
if (this.timer) clearInterval(this.timer)
this.timer = undefined
await this.running
}
}
cloud/apps/push/src/fcm-access-token.ts
+15
View File
@@ -0,0 +1,15 @@
import { GoogleAuth } from 'google-auth-library'
import { FCM_SCOPE } from './fcm-client.js'
// Resolves the runtime service account credential from the GCE metadata server
// in Cloud Run and from GOOGLE_APPLICATION_CREDENTIALS locally; the library
// caches and refreshes the token itself.
export function createFcmAccessTokenProvider(): () => Promise<string> {
const auth = new GoogleAuth({ scopes: [FCM_SCOPE] })
return async () => {
const client = await auth.getClient()
const token = await client.getAccessToken()
if (!token.token) throw new Error('fcm_access_token_unavailable')
return token.token
}
}
cloud/apps/push/src/fcm-client.test.ts
+229
View File
@@ -0,0 +1,229 @@
import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
const NOW = 1_700_000_000_000
const HOST = 'abcdefghijklmnop'
const TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz'
function delivery(agentState: 'needs-input' | null = 'needs-input') {
return buildPushDelivery({
expiresAt: NOW + 300_000,
registrationId: 'reg-1',
hostFingerprint: HOST,
notification: {
notificationId: 'note-1',
notificationSeq: 7,
notificationEpoch: 'epoch-1',
source: 'agent-task-complete',
agentState,
title: 'Agent needs input',
body: 'Waiting on your answer',
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1'
}
})
}
function fakeTransport(response: FcmResponse) {
const requests: FcmRequest[] = []
return {
requests,
transport: async (request: FcmRequest): Promise<FcmResponse> => {
requests.push(request)
return response
}
}
}
function client(response: FcmResponse) {
const fake = fakeTransport(response)
return {
fake,
client: new FcmClient({
projectId: 'onorca-cloud',
now: () => NOW,
accessToken: async () => 'access-token',
transport: fake.transport
})
}
}
describe('fcm client', () => {
it('posts the v1 send payload for the configured project', async () => {
const { fake, client: fcm } = client({ status: 200, body: '{"name":"projects/x/messages/1"}' })
await expect(fcm.send(delivery(), { token: TOKEN })).resolves.toEqual({ status: 'sent' })
const request = fake.requests[0]!
expect(request.url).toBe('https://fcm.googleapis.com/v1/projects/onorca-cloud/messages:send')
expect(request.accessToken).toBe('access-token')
expect(JSON.parse(request.body)).toEqual({
message: {
token: TOKEN,
android: { priority: 'HIGH', ttl: '300s' },
data: {
title: 'Agent needs input',
message: 'Waiting on your answer',
tag: delivery().collapseId,
channelId: 'orca-desktop',
hostFingerprint: HOST,
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: '7',
notificationEpoch: 'epoch-1',
source: 'agent-task-complete',
agentState: 'needs-input'
}
}
})
})
it('carries every data value as a string and omits a null agent state', async () => {
const { fake, client: fcm } = client({ status: 200, body: '{}' })
await fcm.send(delivery(null), { token: TOKEN })
const message = JSON.parse(fake.requests[0]!.body) as {
message: {
android: Record<string, unknown>
data: Record<string, string>
}
}
expect(Object.values(message.message.data).every((value) => typeof value === 'string')).toBe(
true
)
expect(message.message.data.agentState).toBeUndefined()
const tag = createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
expect(message.message.data.coalescedCount).toBeUndefined()
expect(message.message.data.tag).toBe(tag)
expect(message.message.android).not.toHaveProperty('collapse_key')
expect(message.message).not.toHaveProperty('notification')
expect(message.message.data).not.toHaveProperty('body')
})
it('marks an unregistered token dead from the status or the error detail', async () => {
const byStatus = client({
status: 404,
body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'not registered' } })
})
await expect(byStatus.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'dead',
reason: 'UNREGISTERED'
})
const byDetail = client({
status: 404,
body: JSON.stringify({
error: {
status: 'NOT_FOUND',
message: 'Requested entity was not found.',
details: [{ errorCode: 'UNREGISTERED' }]
}
})
})
await expect(byDetail.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'dead',
reason: 'UNREGISTERED'
})
})
it('marks an invalid-argument that names the token dead, and others an error', async () => {
const named = client({
status: 400,
body: JSON.stringify({
error: { status: 'INVALID_ARGUMENT', message: 'The registration token is not valid.' }
})
})
await expect(named.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'dead',
reason: 'INVALID_ARGUMENT'
})
const unnamed = client({
status: 400,
body: JSON.stringify({
error: { status: 'INVALID_ARGUMENT', message: 'Invalid value at message.android.ttl' }
})
})
await expect(unnamed.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'error',
reason: 'INVALID_ARGUMENT',
retryable: false,
retryAfterMs: 10000
})
})
it('treats a server fault and a transport failure as errors', async () => {
const faulted = client({
status: 503,
body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } })
})
await expect(faulted.client.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'error',
reason: 'UNAVAILABLE',
retryable: true,
retryAfterMs: 10000
})
const broken = new FcmClient({
projectId: 'onorca-cloud',
now: () => NOW,
accessToken: async () => 'access-token',
transport: async () => {
throw new Error('ECONNRESET')
}
})
await expect(broken.send(delivery(), { token: TOKEN })).resolves.toEqual({
status: 'error',
reason: 'Error',
retryable: true
})
})
})
it('does not send when credential refresh crosses the absolute expiry', async () => {
let now = 1000
const fake = fakeTransport({ status: 200, body: '{}' })
const fcm = new FcmClient({
projectId: 'test',
now: () => now,
accessToken: async () => {
now = 3000
return 'test-token'
},
transport: fake.transport
})
await expect(fcm.send({ ...delivery(), expiresAt: 2000 }, { token: TOKEN })).resolves.toEqual({
status: 'error',
reason: 'expired'
})
expect(fake.requests).toHaveLength(0)
})
it('decreases retry TTL and refuses expired delivery before refreshing credentials', async () => {
let now = NOW
let refreshes = 0
const fake = fakeTransport({ status: 503, body: '{}' })
const fcm = new FcmClient({
projectId: 'test',
now: () => now,
accessToken: async () => {
refreshes++
return 'test-token'
},
transport: fake.transport
})
const pending = delivery()
await fcm.send(pending, { token: TOKEN })
now += 60_000
await fcm.send(pending, { token: TOKEN })
expect(fake.requests.map((request) => JSON.parse(request.body).message.android.ttl)).toEqual([
'300s',
'240s'
])
now = pending.expiresAt
await expect(fcm.send(pending, { token: TOKEN })).resolves.toEqual({
status: 'error',
reason: 'expired'
})
expect(fake.requests).toHaveLength(2)
expect(refreshes).toBe(2)
})
cloud/apps/push/src/fcm-client.ts
+139
View File
@@ -0,0 +1,139 @@
import { providerRetryAfter } from './provider-retry-delay.js'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
export const FCM_SCOPE = 'https://www.googleapis.com/auth/firebase.messaging'
export type FcmRequest = { url: string; accessToken: string; body: string }
export type FcmResponse = { status: number; body: string; retryAfterMs?: number }
export type FcmTransport = (request: FcmRequest) => Promise<FcmResponse>
export type FcmClientOptions = {
projectId: string
accessToken: () => Promise<string>
transport: FcmTransport
channelId?: string
now?: () => number
}
type FcmErrorBody = {
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
}
export function fcmMessageBody(input: {
delivery: PushDelivery
token: string
channelId: string
now?: number
}): string {
const { delivery } = input
const now = input.now ?? Date.now()
return JSON.stringify({
message: {
token: input.token,
android: {
priority: 'HIGH',
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`
},
// Notification payloads collapse offline; Expo renders these data messages natively.
data: {
...orcaDataStrings(delivery.orca),
...(delivery.orca.kind === 'dismiss'
? {}
: {
title: delivery.title,
message: delivery.body,
tag: delivery.collapseId,
channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
...(delivery.sound === false ? { sound: '' } : {})
})
}
}
})
}
function readFcmError(body: string): { status: string; message: string; errorCodes: string[] } {
try {
const parsed = JSON.parse(body) as FcmErrorBody
return {
status: typeof parsed.error?.status === 'string' ? parsed.error.status : 'unknown',
message: typeof parsed.error?.message === 'string' ? parsed.error.message : '',
errorCodes: (parsed.error?.details ?? [])
.map((detail) => detail.errorCode)
.filter((code): code is string => typeof code === 'string')
}
} catch {
return { status: 'unparseable', message: '', errorCodes: [] }
}
}
export class FcmClient {
private readonly channelId: string
constructor(private readonly options: FcmClientOptions) {
this.channelId = options.channelId ?? PUSH_DEFAULTS.androidChannelId
}
async send(delivery: PushDelivery, device: { token: string }): Promise<PushProviderOutcome> {
if (delivery.expiresAt <= (this.options.now ?? Date.now)())
return { status: 'error', reason: 'expired' }
let response: FcmResponse
try {
const accessToken = await this.options.accessToken()
const now = (this.options.now ?? Date.now)()
if (delivery.expiresAt <= now) return { status: 'error', reason: 'expired' }
response = await this.options.transport({
url: `https://fcm.googleapis.com/v1/projects/${this.options.projectId}/messages:send`,
accessToken,
body: fcmMessageBody({
delivery,
token: device.token,
channelId: this.channelId,
now
})
})
} catch (error) {
return {
status: 'error',
reason: error instanceof Error ? error.name : 'transport_failed',
retryable: true
}
}
if (response.status >= 200 && response.status < 300) return { status: 'sent' }
const failure = readFcmError(response.body)
if (failure.status === 'UNREGISTERED' || failure.errorCodes.includes('UNREGISTERED')) {
return { status: 'dead', reason: 'UNREGISTERED' }
}
// A revoked token also surfaces as INVALID_ARGUMENT naming the token field.
if (failure.status === 'INVALID_ARGUMENT' && /\btoken\b/i.test(failure.message)) {
return { status: 'dead', reason: 'INVALID_ARGUMENT' }
}
return {
status: 'error',
reason: failure.status,
retryable: response.status === 429 || response.status >= 500,
retryAfterMs: Math.max(response.status === 429 ? 60_000 : 10_000, response.retryAfterMs ?? 0)
}
}
}
export function createFcmFetchTransport(fetchImpl: typeof fetch = fetch): FcmTransport {
return async (request) => {
const response = await fetchImpl(request.url, {
method: 'POST',
headers: {
authorization: `Bearer ${request.accessToken}`,
'content-type': 'application/json'
},
body: request.body,
redirect: 'error',
signal: AbortSignal.timeout(10_000)
})
return {
status: response.status,
body: await response.text(),
retryAfterMs: providerRetryAfter(response.headers.get('retry-after') ?? undefined)
}
}
}
cloud/apps/push/src/host-challenge-answering.test-fixture.ts
+163
View File
@@ -0,0 +1,163 @@
import { createHmac, timingSafeEqual } from 'node:crypto'
import {
PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN,
PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN,
PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT,
PUSH_LIMITS
} from '@orca-cloud/push-contract'
import nacl from 'tweetnacl'
import { decodeCanonicalBase64 } from './canonical-base64.js'
import { deriveHostFingerprint } from './host-fingerprint.js'
// The desktop side of the push challenge, written the way the shipped host
// will answer it, so the gateway is exercised against a real box-opening peer.
const textEncoder = new TextEncoder()
const textDecoder = new TextDecoder()
export type PushHostKeypair = { publicKey: Uint8Array; secretKey: Uint8Array }
export type PushChallengeWire = {
challengeId: string
gatewayEphemeralPublicKeyB64: string
nonceB64: string
ciphertextB64: string
expiresAt: number
}
export function createPushHostKeypair(seed?: number): PushHostKeypair {
const pair =
seed === undefined
? nacl.box.keyPair()
: nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(seed))
return { publicKey: pair.publicKey, secretKey: pair.secretKey }
}
export function hostPublicKeyB64(keypair: PushHostKeypair): string {
return Buffer.from(keypair.publicKey).toString('base64')
}
function equal(left: Uint8Array | undefined, right: Uint8Array): boolean {
return Boolean(left && left.byteLength === right.byteLength && timingSafeEqual(left, right))
}
function uint64(value: number): Uint8Array {
const bytes = new Uint8Array(8)
new DataView(bytes.buffer).setBigUint64(0, BigInt(value), false)
return bytes
}
function parseTranscript(transcript: Uint8Array): Map<string, Uint8Array> | null {
const fields = new Map<string, Uint8Array>()
const view = new DataView(transcript.buffer, transcript.byteOffset, transcript.byteLength)
let offset = 0
try {
while (offset < transcript.byteLength) {
const nameLength = view.getUint32(offset, false)
offset += 4
const name = textDecoder.decode(transcript.slice(offset, offset + nameLength))
offset += nameLength
const valueLength = view.getUint32(offset, false)
offset += 4
if (fields.has(name) || offset + valueLength > transcript.byteLength) return null
fields.set(name, transcript.slice(offset, offset + valueLength))
offset += valueLength
}
} catch {
return null
}
return offset === transcript.byteLength ? fields : null
}
function readUint64(value: Uint8Array | undefined): number | null {
if (!value || value.byteLength !== 8) return null
const parsed = new DataView(value.buffer, value.byteOffset, value.byteLength).getBigUint64(0, false)
return parsed <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(parsed) : null
}
export type PushHostProofContext = {
gatewayOrigin: string
keypair: PushHostKeypair
now?: () => number
onInvalid?: (reason: string) => void
}
function validateTranscript(
transcript: Uint8Array,
challenge: PushChallengeWire,
context: PushHostProofContext,
gatewayKey: Uint8Array,
nonce: Uint8Array
): boolean {
const fields = parseTranscript(transcript)
if (!fields || fields.size !== PUSH_HOST_PROOF_TRANSCRIPT_FIELD_COUNT) {
context.onInvalid?.('transcript-structure')
return false
}
const now = (context.now ?? Date.now)()
const issuedAt = readUint64(fields.get('issuedAt'))
const expiresAt = readUint64(fields.get('expiresAt'))
const fingerprint = deriveHostFingerprint(context.keypair.publicKey)
const checks: [string, boolean][] = [
['issuedAt-readable', issuedAt !== null],
[
'issuedAt-not-future',
issuedAt === null || issuedAt - PUSH_LIMITS.clockSkewToleranceMs <= now
],
['not-expired', now - PUSH_LIMITS.clockSkewToleranceMs <= challenge.expiresAt],
['issuedAt-before-expiry', issuedAt === null || issuedAt <= challenge.expiresAt],
[
'window',
issuedAt === null || challenge.expiresAt - issuedAt <= PUSH_LIMITS.challengeTtlMs
],
['expiry-consistent', expiresAt === challenge.expiresAt],
['protocol', equal(fields.get('protocol'), textEncoder.encode(PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN))],
['version', equal(fields.get('version'), new Uint8Array([1]))],
['gatewayOrigin', equal(fields.get('gatewayOrigin'), textEncoder.encode(context.gatewayOrigin))],
['gatewayEphemeralPublicKey', equal(fields.get('gatewayEphemeralPublicKey'), gatewayKey)],
['challengeNonce', equal(fields.get('challengeNonce'), nonce)],
['challengeId', equal(fields.get('challengeId'), textEncoder.encode(challenge.challengeId))],
['hostFingerprint', equal(fields.get('hostFingerprint'), textEncoder.encode(fingerprint))],
['hostPublicKey', equal(fields.get('hostPublicKey'), context.keypair.publicKey)],
['issuedAt-value', issuedAt === null || uint64(issuedAt).byteLength === 8]
]
const failed = checks.filter(([, ok]) => !ok).map(([name]) => name)
if (failed.length === 0) return true
context.onInvalid?.(`transcript:${failed.join('+')}`)
return false
}
export function answerPushHostChallenge(
challenge: PushChallengeWire,
context: PushHostProofContext
): string | null {
const gatewayKey = decodeCanonicalBase64(challenge.gatewayEphemeralPublicKeyB64, 32)
const nonce = decodeCanonicalBase64(challenge.nonceB64, 24)
const ciphertext = Buffer.from(challenge.ciphertextB64, 'base64')
if (!gatewayKey || !nonce || ciphertext.toString('base64') !== challenge.ciphertextB64) return null
const plaintext = nacl.box.open(ciphertext, nonce, gatewayKey, context.keypair.secretKey)
if (!plaintext) {
context.onInvalid?.('challenge-box-open')
return null
}
const domain = textEncoder.encode(`${PUSH_HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`)
if (
!equal(plaintext.slice(0, domain.byteLength), domain) ||
plaintext.byteLength < domain.byteLength + 36
) {
return null
}
const transcriptLength = new DataView(
plaintext.buffer,
plaintext.byteOffset + domain.byteLength,
4
).getUint32(0, false)
const transcriptStart = domain.byteLength + 4
const secretStart = transcriptStart + transcriptLength
if (secretStart + 32 !== plaintext.byteLength) return null
const transcript = plaintext.slice(transcriptStart, secretStart)
if (!validateTranscript(transcript, challenge, context, gatewayKey, nonce)) return null
return createHmac('sha256', plaintext.slice(secretStart))
.update(textEncoder.encode(`${PUSH_HOST_PROOF_TRANSCRIPT_DOMAIN}\0ack\0`))
.update(transcript)
.digest('base64')
}
cloud/apps/push/src/host-challenge-store.test.ts
+191
View File
@@ -0,0 +1,191 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
answerPushHostChallenge,
createPushHostKeypair,
hostPublicKeyB64
} from './host-challenge-answering.test-fixture.js'
import { PushHostChallengeStore } from './host-challenge-store.js'
import { deriveHostFingerprint } from './host-fingerprint.js'
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
const GATEWAY_ORIGIN = 'https://push.onorca.dev'
describe('push host challenge store', () => {
let database: PushDatabase
let clock = 1_700_000_000_000
let store: PushHostChallengeStore
beforeEach(async () => {
database = await openInMemoryPushDatabase()
clock = 1_700_000_000_000
store = new PushHostChallengeStore(database, GATEWAY_ORIGIN, () => clock)
})
afterEach(async () => {
await database.close()
})
it('completes a challenge, proof, and consume round trip', async () => {
const host = createPushHostKeypair(1)
const challenge = await store.issue(hostPublicKeyB64(host))
expect(challenge).not.toBeNull()
expect(challenge!.expiresAt).toBe(clock + PUSH_LIMITS.challengeTtlMs)
expect(challenge!.hostFingerprint).toBe(deriveHostFingerprint(host.publicKey))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})
expect(proof).not.toBeNull()
await expect(store.verify(challenge!.challengeId, proof!)).resolves.toEqual({
ok: true,
hostFingerprint: deriveHostFingerprint(host.publicKey)
})
})
it('never stores material that reproduces the proof', async () => {
const host = createPushHostKeypair(2)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})
const [row] = await database.query('SELECT secret_hash FROM push_challenges')
expect(String(row?.secret_hash)).not.toBe(proof)
expect(Buffer.from(String(row?.secret_hash), 'base64url').byteLength).toBe(32)
})
it('rejects a replayed challenge', async () => {
const host = createPushHostKeypair(3)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})!
await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true })
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
ok: false,
reason: 'already_consumed'
})
})
it('rejects a challenge the moment its own ttl elapses', async () => {
const host = createPushHostKeypair(4)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})!
clock += PUSH_LIMITS.challengeTtlMs + 1
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
ok: false,
reason: 'expired'
})
})
it('spends no skew tolerance on its own expiry, so the ttl is the whole window', async () => {
const host = createPushHostKeypair(5)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})!
// A proof that the host would still consider in-window is refused here: the
// gateway issued expires_at against this clock and needs no allowance.
clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs - 1
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
ok: false,
reason: 'expired'
})
})
it('accepts a proof that lands just inside the ttl', async () => {
const host = createPushHostKeypair(26)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})!
clock += PUSH_LIMITS.challengeTtlMs
await expect(store.verify(challenge!.challengeId, proof)).resolves.toMatchObject({ ok: true })
})
it('keeps an expired row long enough to answer expired rather than unknown', async () => {
const host = createPushHostKeypair(27)
const challenge = await store.issue(hostPublicKeyB64(host))
const proof = answerPushHostChallenge(challenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: host,
now: () => clock
})!
clock += PUSH_LIMITS.challengeTtlMs + 1
expect(await store.pruneExpired()).toBe(0)
await expect(store.verify(challenge!.challengeId, proof)).resolves.toEqual({
ok: false,
reason: 'expired'
})
})
it('refuses a wrong host: the box will not open and a foreign proof will not match', async () => {
const owner = createPushHostKeypair(6)
const intruder = createPushHostKeypair(7)
const ownerChallenge = await store.issue(hostPublicKeyB64(owner))
expect(
answerPushHostChallenge(ownerChallenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: intruder,
now: () => clock
})
).toBeNull()
const intruderChallenge = await store.issue(hostPublicKeyB64(intruder))
const intruderProof = answerPushHostChallenge(intruderChallenge!, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair: intruder,
now: () => clock
})!
await expect(store.verify(ownerChallenge!.challengeId, intruderProof)).resolves.toEqual({
ok: false,
reason: 'proof_mismatch'
})
})
it('rejects a proof bound to a different gateway origin', async () => {
const host = createPushHostKeypair(8)
const challenge = await store.issue(hostPublicKeyB64(host))
const reasons: string[] = []
expect(
answerPushHostChallenge(challenge!, {
gatewayOrigin: 'https://push.example.test',
keypair: host,
now: () => clock,
onInvalid: (reason) => reasons.push(reason)
})
).toBeNull()
expect(reasons.join()).toContain('gatewayOrigin')
})
it('rejects an unknown challenge id and a malformed public key', async () => {
await expect(store.verify('missing', Buffer.alloc(32, 9).toString('base64'))).resolves.toEqual({
ok: false,
reason: 'unknown_challenge'
})
await expect(store.issue('not-base64!!')).resolves.toBeNull()
await expect(store.issue(Buffer.alloc(31, 1).toString('base64'))).resolves.toBeNull()
})
it('prunes challenges that fell out of the skew window', async () => {
const host = createPushHostKeypair(9)
await store.issue(hostPublicKeyB64(host))
expect(await store.pruneExpired()).toBe(0)
clock += PUSH_LIMITS.challengeTtlMs + PUSH_LIMITS.clockSkewToleranceMs + 1
expect(await store.pruneExpired()).toBe(1)
})
})
cloud/apps/push/src/host-challenge-store.ts
+135
View File
@@ -0,0 +1,135 @@
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
import {
buildPushHostChallengePlaintext,
buildPushHostProofMacInput,
buildPushHostProofTranscript,
PUSH_LIMITS
} from '@orca-cloud/push-contract'
import nacl from 'tweetnacl'
import { decodeCanonicalBase64 } from './canonical-base64.js'
import { deriveHostFingerprint } from './host-fingerprint.js'
import type { PushDatabase } from './push-database.js'
export type IssuedPushChallenge = {
challengeId: string
gatewayEphemeralPublicKeyB64: string
nonceB64: string
ciphertextB64: string
expiresAt: number
hostFingerprint: string
}
export type PushProofVerification =
| { ok: true; hostFingerprint: string }
| { ok: false; reason: 'unknown_challenge' | 'already_consumed' | 'expired' | 'proof_mismatch' }
function sha256(value: Uint8Array): string {
return createHash('sha256').update(value).digest('base64url')
}
function equalDigest(left: string, right: string): boolean {
const leftBytes = Buffer.from(left)
const rightBytes = Buffer.from(right)
return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes)
}
export class PushHostChallengeStore {
constructor(
private readonly database: PushDatabase,
private readonly gatewayOrigin: string,
private readonly now: () => number = Date.now
) {}
async issue(hostPublicKeyB64: string): Promise<IssuedPushChallenge | null> {
const hostPublicKey = decodeCanonicalBase64(hostPublicKeyB64, 32)
if (!hostPublicKey) return null
const hostFingerprint = deriveHostFingerprint(hostPublicKey)
const ephemeral = nacl.box.keyPair()
const challengeNonce = randomBytes(nacl.box.nonceLength)
const challengeSecret = randomBytes(32)
const challengeId = randomUUID()
const issuedAt = this.now()
const expiresAt = issuedAt + PUSH_LIMITS.challengeTtlMs
const transcript = buildPushHostProofTranscript({
gatewayOrigin: this.gatewayOrigin,
gatewayEphemeralPublicKey: ephemeral.publicKey,
challengeNonce,
challengeId,
issuedAt,
expiresAt,
hostFingerprint,
hostPublicKey
})
const ciphertext = nacl.box(
buildPushHostChallengePlaintext(transcript, challengeSecret),
challengeNonce,
hostPublicKey,
ephemeral.secretKey
)
const expectedProof = createHmac('sha256', challengeSecret)
.update(buildPushHostProofMacInput(transcript))
.digest()
await this.database.query(
`INSERT INTO push_challenges
(challenge_id, host_fingerprint, secret_hash, expires_at,
consumed_at)
VALUES (?, ?, ?, ?, NULL)`,
[
challengeId,
hostFingerprint,
// The stored digest is of the ack the secret produces, never of the
// secret itself: a database reader must not be able to forge a proof.
sha256(expectedProof),
expiresAt
]
)
return {
challengeId,
gatewayEphemeralPublicKeyB64: Buffer.from(ephemeral.publicKey).toString('base64'),
nonceB64: Buffer.from(challengeNonce).toString('base64'),
ciphertextB64: Buffer.from(ciphertext).toString('base64'),
expiresAt,
hostFingerprint
}
}
async verify(challengeId: string, proofB64: string): Promise<PushProofVerification> {
const proof = decodeCanonicalBase64(proofB64, 32)
return await this.database.transaction<PushProofVerification>(async (transaction) => {
const [row] = await transaction.query(
`SELECT host_fingerprint, secret_hash, expires_at, consumed_at
FROM push_challenges WHERE challenge_id = ?`,
[challengeId]
)
if (!row) return { ok: false, reason: 'unknown_challenge' }
if (row.consumed_at !== null && row.consumed_at !== undefined) {
return { ok: false, reason: 'already_consumed' }
}
const now = this.now()
// No skew allowance here: the gateway set expires_at from this same clock.
// The tolerance belongs to the host, which validates a foreign timestamp.
if (now > Number(row.expires_at)) return { ok: false, reason: 'expired' }
if (!proof || !equalDigest(sha256(proof), String(row.secret_hash))) {
return { ok: false, reason: 'proof_mismatch' }
}
// Consume under the same predicate the read used, so two concurrent
// proofs for one challenge cannot both mint a session.
const [consumed] = await transaction.query(
'UPDATE push_challenges SET consumed_at = ? WHERE challenge_id = ? AND consumed_at IS NULL',
[now, challengeId]
)
if (Number(consumed?.changes ?? 0) !== 1) return { ok: false, reason: 'already_consumed' }
return { ok: true, hostFingerprint: String(row.host_fingerprint) }
})
}
// Rows outlive the expiry check by the skew tolerance so a late proof reads
// as 'expired' rather than as an unknown challenge.
async pruneExpired(): Promise<number> {
const cutoff = this.now() - PUSH_LIMITS.clockSkewToleranceMs
const [result] = await this.database.query('DELETE FROM push_challenges WHERE expires_at < ?', [
cutoff
])
return Number(result?.changes ?? 0)
}
}
cloud/apps/push/src/host-fingerprint.ts
+16
View File
@@ -0,0 +1,16 @@
import { createHash } from 'node:crypto'
import { PUSH_HOST_FINGERPRINT_LENGTH } from '@orca-cloud/push-contract'
// Identical derivation to deriveRelayHostId on the desktop, so a host and a
// phone reach the same fingerprint from the same X25519 public key.
export function deriveHostFingerprint(hostPublicKey: Uint8Array): string {
return createHash('sha256')
.update(hostPublicKey)
.digest('base64url')
.slice(0, PUSH_HOST_FINGERPRINT_LENGTH)
}
// Logs may carry at most this much of a fingerprint.
export function fingerprintLogPrefix(hostFingerprint: string): string {
return hostFingerprint.slice(0, 4)
}
cloud/apps/push/src/host-proof-concurrency.test.ts
+47
View File
@@ -0,0 +1,47 @@
import { expect, it } from 'vitest'
import { openInMemoryPushDatabase, openPushDatabase } from './push-database.js'
import { PushHostChallengeStore } from './host-challenge-store.js'
import {
answerPushHostChallenge,
createPushHostKeypair,
hostPublicKeyB64
} from './host-challenge-answering.test-fixture.js'
it('accepts independent proofs but consumes each challenge only once under concurrency', async () => {
const databaseUrl = process.env.ORCA_PUSH_TEST_DATABASE_URL
if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') {
throw new Error('isolated_postgres_port_required')
}
const db = databaseUrl
? await openPushDatabase({ databaseUrl, dataDir: '', poolMax: 4 })
: await openInMemoryPushDatabase()
const host = createPushHostKeypair()
const origin = 'https://push.onorca.dev'
const store = new PushHostChallengeStore(db, origin)
const challenges = await Promise.all([
store.issue(hostPublicKeyB64(host)),
store.issue(hostPublicKeyB64(host))
])
try {
const proofs = challenges.map((challenge) =>
answerPushHostChallenge(challenge!, { gatewayOrigin: origin, keypair: host })!
)
const results = await Promise.all(
challenges.flatMap((challenge, index) =>
Array.from({ length: 5 }, () => store.verify(challenge!.challengeId, proofs[index]!))
)
)
expect(results.filter((result) => result.ok)).toEqual([
{ ok: true, hostFingerprint: challenges[0]!.hostFingerprint },
{ ok: true, hostFingerprint: challenges[0]!.hostFingerprint }
])
expect(results.filter((result) => !result.ok)).toEqual(
Array.from({ length: 8 }, () => ({ ok: false, reason: 'already_consumed' }))
)
} finally {
for (const challenge of challenges) {
await db.query('DELETE FROM push_challenges WHERE challenge_id = ?', [challenge!.challengeId])
}
await db.close()
}
})
cloud/apps/push/src/host-session-store.test.ts
+70
View File
@@ -0,0 +1,70 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { PushHostSessionStore } from './host-session-store.js'
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
const HOST = 'abcdefghijklmnop'
describe('push host session store', () => {
let database: PushDatabase
let clock = 1_700_000_000_000
let sessions: PushHostSessionStore
beforeEach(async () => {
database = await openInMemoryPushDatabase()
clock = 1_700_000_000_000
sessions = new PushHostSessionStore(database, () => clock)
})
afterEach(async () => {
await database.close()
})
it('mints a 24 hour session and stores only its hash', async () => {
const session = await sessions.create(HOST)
expect(session.expiresAt).toBe(clock + PUSH_LIMITS.sessionTtlMs)
expect(Buffer.from(session.sessionToken, 'base64url').byteLength).toBe(32)
const [row] = await database.query('SELECT token_hash FROM push_sessions')
expect(String(row?.token_hash)).not.toBe(session.sessionToken)
await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({
ok: true,
hostFingerprint: HOST
})
})
it('reports expiry separately from an unknown token', async () => {
const session = await sessions.create(HOST)
clock += PUSH_LIMITS.sessionTtlMs + 1
await expect(sessions.resolve(session.sessionToken)).resolves.toEqual({
ok: false,
reason: 'session_expired'
})
await expect(sessions.resolve('not-a-session')).resolves.toEqual({
ok: false,
reason: 'unknown_session'
})
})
it('accepts a session on its final millisecond', async () => {
const session = await sessions.create(HOST)
clock += PUSH_LIMITS.sessionTtlMs
await expect(sessions.resolve(session.sessionToken)).resolves.toMatchObject({ ok: true })
})
it('keeps one live session per host and prunes it once expired', async () => {
const first = await sessions.create(HOST)
const second = await sessions.create(HOST)
// The earlier session is gone the moment its host proves again, so a flood
// of proofs leaves one row per host rather than one per proof.
await expect(sessions.resolve(first.sessionToken)).resolves.toEqual({
ok: false,
reason: 'unknown_session'
})
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
const other = await sessions.create('ponmlkjihgfedcba')
await expect(sessions.resolve(second.sessionToken)).resolves.toMatchObject({ ok: true })
clock += PUSH_LIMITS.sessionTtlMs + 1
expect(await sessions.pruneExpired()).toBe(2)
await expect(sessions.resolve(other.sessionToken)).resolves.toMatchObject({ ok: false })
})
})
cloud/apps/push/src/host-session-store.ts
+65
View File
@@ -0,0 +1,65 @@
import { createHash, randomBytes } from 'node:crypto'
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import type { PushDatabase } from './push-database.js'
export type IssuedPushSession = {
sessionToken: string
expiresAt: number
hostFingerprint: string
}
export type PushSessionLookup =
| { ok: true; hostFingerprint: string; expiresAt: number }
| { ok: false; reason: 'unknown_session' | 'session_expired' }
function hashSessionToken(sessionToken: string): string {
return createHash('sha256').update(sessionToken).digest('base64url')
}
export class PushHostSessionStore {
constructor(
private readonly database: PushDatabase,
private readonly now: () => number = Date.now
) {}
async create(hostFingerprint: string): Promise<IssuedPushSession> {
const sessionToken = randomBytes(32).toString('base64url')
const createdAt = this.now()
const expiresAt = createdAt + PUSH_LIMITS.sessionTtlMs
await this.database.transaction(async (transaction) => {
// Why: a desktop holds one session at a time and only re-proves once it is
// gone, so an earlier row is dead weight. It also bounds the table to one
// row per host however many proofs a self-minted identity answers.
await transaction.lockQuotaScope(`orca-push-session:${hostFingerprint}`)
await transaction.query('DELETE FROM push_sessions WHERE host_fingerprint = ?', [
hostFingerprint
])
await transaction.query(
`INSERT INTO push_sessions (token_hash, host_fingerprint, expires_at, created_at)
VALUES (?, ?, ?, ?)`,
[hashSessionToken(sessionToken), hostFingerprint, expiresAt, createdAt]
)
})
return { sessionToken, expiresAt, hostFingerprint }
}
async resolve(sessionToken: string): Promise<PushSessionLookup> {
const [row] = await this.database.query(
'SELECT host_fingerprint, expires_at FROM push_sessions WHERE token_hash = ?',
[hashSessionToken(sessionToken)]
)
if (!row) return { ok: false, reason: 'unknown_session' }
const expiresAt = Number(row.expires_at)
// No skew grace here: a 24h session that just expired should be re-minted
// through the challenge, which is cheap and already handled by the host.
if (this.now() > expiresAt) return { ok: false, reason: 'session_expired' }
return { ok: true, hostFingerprint: String(row.host_fingerprint), expiresAt }
}
async pruneExpired(): Promise<number> {
const [result] = await this.database.query('DELETE FROM push_sessions WHERE expires_at < ?', [
this.now()
])
return Number(result?.changes ?? 0)
}
}
cloud/apps/push/src/index.ts
+55
View File
@@ -0,0 +1,55 @@
import { startPushBackground } from './push-background.js'
import { loadPushConfig } from './config.js'
import { openPushDatabase } from './push-database.js'
import { createPushServer } from './push-server.js'
const config = loadPushConfig()
const database = await openPushDatabase({
...(config.databaseUrl === undefined ? {} : { databaseUrl: config.databaseUrl }),
dataDir: config.dataDir,
poolMax: config.databasePoolMax,
applicationName: 'orca-push',
readOnly: config.mode === 'validation'
})
const {
server,
challenges,
sessions,
deliveryStore,
worker,
observability,
closeTransports,
requestDrain
} = createPushServer(config, database)
const stopBackground = startPushBackground(config, { challenges, sessions, deliveryStore, worker })
observability.start()
server.listen(config.port, () => {
console.log(`[orca-push] listening on ${config.publicUrl} (port ${config.port})`)
})
let stopping = false
const shutdown = (): void => {
if (stopping) return
stopping = true
// Cloud Run sends SIGKILL after ten seconds; leave time for explicit cleanup.
const deadline = setTimeout(() => process.exit(1), 9_000)
deadline.unref()
const requests = requestDrain.begin()
const deliveries = stopBackground()
const connections = new Promise<void>((resolve) => server.close(() => resolve()))
void Promise.all([requests, connections, deliveries])
.then(async () => {
closeTransports()
await database.close()
observability.stop()
clearTimeout(deadline)
})
.catch(() => {
console.warn(JSON.stringify({ event: 'orca_push_shutdown_failed' }))
process.exitCode = 1
})
}
process.once('SIGTERM', shutdown)
process.once('SIGINT', shutdown)
cloud/apps/push/src/provider-retry-delay.ts
+9
View File
@@ -0,0 +1,9 @@
export function providerRetryAfter(
value: string | undefined,
now = Date.now()
): number | undefined {
if (!value) return undefined
const seconds = Number(value)
const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(value) - now
return Number.isFinite(delay) ? Math.max(0, delay) : undefined
}
cloud/apps/push/src/push-auth-admission.ts
+21
View File
@@ -0,0 +1,21 @@
// Bound unauthenticated database lookups independently of Cloud Run HTTP concurrency.
export class PushAuthAdmission {
private active = 0
private readonly waiting: (() => void)[] = []
async run<T>(operation: () => Promise<T>): Promise<T | null> {
if (this.active >= 4) {
if (this.waiting.length >= 32) return null
await new Promise<void>((resolve) => this.waiting.push(resolve))
} else {
this.active++
}
try {
return await operation()
} finally {
const next = this.waiting.shift()
if (next) next()
else this.active--
}
}
}
cloud/apps/push/src/push-authenticated-ip-limit.test.ts
+50
View File
@@ -0,0 +1,50 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { expect, it } from 'vitest'
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
import { createPushServerHarness, FCM_TOKEN } from './push-server-harness.test-fixture.js'
it('bounds valid hosts together across routes without letting key rotation reset the IP budget', async () => {
const harness = await createPushServerHarness()
const headers = { 'x-forwarded-for': '203.0.113.7' }
try {
const hostCount =
PUSH_LIMITS.authenticatedRequestsPerMinutePerIp /
PUSH_LIMITS.authenticatedRequestsPerMinutePerHost
for (let host = 0; host < hostCount; host++) {
const token = await harness.signIn(createPushHostKeypair(host + 1))
for (
let request = 0;
request < PUSH_LIMITS.authenticatedRequestsPerMinutePerHost;
request++
) {
expect((await harness.authorized('/v1/devices', { headers }, token)).status).toBe(200)
}
}
const token = await harness.signIn(createPushHostKeypair(99))
const registration = {
method: 'POST',
headers: { ...headers, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, deviceId: 'phone', platform: 'android', token: FCM_TOKEN })
}
expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(429)
expect((await harness.authorized('/v1/send', { method: 'POST', headers }, token)).status).toBe(
429
)
expect(
(
await harness.authorized(
'/v1/devices',
{
...registration,
headers: { ...registration.headers, 'x-forwarded-for': '198.51.100.9' }
},
token
)
).status
).toBe(200)
harness.advanceClock(60_000)
expect((await harness.authorized('/v1/devices', registration, token)).status).toBe(200)
} finally {
await harness.close()
}
}, 30_000)
cloud/apps/push/src/push-background.ts
+43
View File
@@ -0,0 +1,43 @@
import type { PushConfig } from './config.js'
import type { createPushServer } from './push-server.js'
const CHALLENGE_PRUNE_INTERVAL_MS = 60_000
const SESSION_PRUNE_INTERVAL_MS = 10 * 60_000
const DELIVERY_PRUNE_INTERVAL_MS = 60_000
function prune(label: string, run: () => Promise<number>, intervalMs: number): NodeJS.Timeout {
const timer = setInterval(() => {
void run().catch((error: unknown) => {
console.warn(
JSON.stringify({
event: 'orca_push_prune_failed',
target: label,
error: error instanceof Error ? error.name : 'unknown'
})
)
})
}, intervalMs)
timer.unref()
return timer
}
export function startPushBackground(
config: Pick<PushConfig, 'mode'>,
runtime: Pick<
ReturnType<typeof createPushServer>,
'challenges' | 'sessions' | 'deliveryStore' | 'worker'
>
): () => Promise<void> {
if (config.mode === 'validation') return async () => {}
const { challenges, sessions, deliveryStore, worker } = runtime
const timers = [
prune('challenges', () => challenges.pruneExpired(), CHALLENGE_PRUNE_INTERVAL_MS),
prune('sessions', () => sessions.pruneExpired(), SESSION_PRUNE_INTERVAL_MS),
prune('deliveries', () => deliveryStore.prune(), DELIVERY_PRUNE_INTERVAL_MS)
]
worker.start()
return async () => {
for (const timer of timers) clearInterval(timer)
await worker.stop()
}
}
cloud/apps/push/src/push-database-postgres-startup.test.ts
+121
View File
@@ -0,0 +1,121 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
const fakes = vi.hoisted(() => ({
configs: [] as Array<Record<string, unknown>>,
lifecycle: [] as string[],
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
release: vi.fn()
}))
vi.mock('pg', () => ({
default: {
Pool: class {
on = vi.fn()
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
private readonly label: string
constructor(config: Record<string, unknown>) {
fakes.configs.push(config)
this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
fakes.lifecycle.push(`open ${this.label}`)
}
async end(): Promise<void> {
fakes.lifecycle.push(`end ${this.label}`)
}
}
}
}))
import { openPushDatabase } from './push-database.js'
import { pushSchemaStatements } from './push-schema.js'
describe('PostgreSQL push gateway startup', () => {
beforeEach(() => {
fakes.configs.length = 0
fakes.lifecycle.length = 0
fakes.query.mockClear()
})
afterEach(() => {
vi.restoreAllMocks()
})
const socketPassword = `${randomUUID()}@/`
const socketUrl = `postgresql://push:${encodeURIComponent(socketPassword)}@/orca_push?host=/cloudsql/test:region:instance`
it('passes the Terraform socket URL unchanged to both active pools', async () => {
const database = await openPushDatabase({ databaseUrl: socketUrl, dataDir: '/unused' })
expect(fakes.configs.map((config) => config.connectionString)).toEqual([socketUrl, socketUrl])
await database.close()
})
it('parses socket credentials and query options before enforcing validation read-only', async () => {
const options = '-c search_path=validation -c default_transaction_read_only=off'
const database = await openPushDatabase({
databaseUrl: `${socketUrl}&port=5433&sslmode=disable&options=${encodeURIComponent(options)}`,
dataDir: '/unused',
readOnly: true
})
expect(fakes.configs).toHaveLength(1)
expect(fakes.configs[0]).toMatchObject({
host: '/cloudsql/test:region:instance',
user: 'push',
password: socketPassword,
database: 'orca_push',
port: 5433,
ssl: false,
options: `${options} -c default_transaction_read_only=on`
})
expect(fakes.configs[0]).not.toHaveProperty('connectionString')
expect(fakes.query).not.toHaveBeenCalled()
await database.close()
})
// Why: a CREATE INDEX on a grown table can outlive the 5s request deadline,
// and a schema that inherits it fails every startup at the same statement.
it('applies the schema on an untimed pool that is gone before the serving pool opens', async () => {
const database = await openPushDatabase({
databaseUrl: 'postgresql://push@localhost:55440/orca_push',
dataDir: '/unused',
poolMax: 2,
applicationName: 'orca-push'
})
expect(fakes.lifecycle).toEqual([
'open max=1 statement_timeout=0',
'end max=1 statement_timeout=0',
'open max=2 statement_timeout=5000'
])
expect(fakes.configs[0]).toMatchObject({
application_name: 'orca-push/schema',
lock_timeout: 1_000,
idle_in_transaction_session_timeout: 5_000
})
expect(
fakes.query.mock.calls.map(([sql]) => sql).slice(0, pushSchemaStatements().length)
).toEqual(pushSchemaStatements())
await database.close()
})
it('retries a transaction the pool statement_timeout aborted', async () => {
const database = await openPushDatabase({
databaseUrl: 'postgresql://push@localhost:55440/orca_push',
dataDir: '/unused'
})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
let attempts = 0
const result = await database.transaction(async () => {
attempts += 1
if (attempts === 1) throw Object.assign(new Error('canceling statement'), { code: '57014' })
return 'done'
})
expect(result).toBe('done')
expect(attempts).toBe(2)
expect(warn.mock.calls.map(([line]) => String(line))).toEqual([
expect.stringContaining('"code":"57014"')
])
warn.mockRestore()
await database.close()
})
})
cloud/apps/push/src/push-database.ts
+286
View File
@@ -0,0 +1,286 @@
import { mkdirSync } from 'node:fs'
import { join } from 'node:path'
import { DatabaseSync } from 'node:sqlite'
import pg from 'pg'
import { parseIntoClientConfig } from 'pg-connection-string'
import { applyPostgresSchema } from '@orca-cloud/postgres-schema'
import { pushSchemaStatements } from './push-schema.js'
const POSTGRES_LOCK_TIMEOUT_MS = 1_000
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
const POSTGRES_TRANSACTION_ATTEMPTS = 3
const POSTGRES_RETRY_MAX_DELAY_MS = 25
export type SqlRow = Record<string, unknown>
export interface PushDatabase {
readonly dialect: 'sqlite' | 'postgres'
query(sql: string, params?: unknown[]): Promise<SqlRow[]>
transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T>
// Serializes every transaction that reads then writes the same identity's
// quota rows. Must be called inside a transaction; it releases at commit.
lockQuotaScope(key: string): Promise<void>
close(): Promise<void>
}
function postgresSql(sql: string): string {
let index = 0
return sql.replace(/\?/g, () => `$${++index}`)
}
function returnsRows(sql: string): boolean {
return /^\s*(select|with)/i.test(sql) || /returning/i.test(sql)
}
class SqliteTransaction implements PushDatabase {
readonly dialect = 'sqlite' as const
constructor(protected readonly database: DatabaseSync) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
const statement = this.database.prepare(sql)
const bound = params.map((value) => (value === undefined ? null : value)) as never[]
if (returnsRows(sql)) return statement.all(...bound) as SqlRow[]
const result = statement.run(...bound)
return [{ changes: Number(result.changes) }]
}
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
return await operation(this)
}
// BEGIN IMMEDIATE already holds the single writer lock for the whole
// transaction, so there is nothing narrower left to take.
async lockQuotaScope(): Promise<void> {}
async close(): Promise<void> {}
}
class SqliteDatabase extends SqliteTransaction {
// node:sqlite is synchronous and has no nested transactions, so overlapping
// callers are serialized behind one tail promise instead of racing BEGIN.
private tail: Promise<void> = Promise.resolve()
override async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
await this.tail
return await super.query(sql, params)
}
override async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
const previous = this.tail
let release!: () => void
this.tail = new Promise((resolve) => (release = resolve))
await previous
this.database.exec('BEGIN IMMEDIATE')
const transaction = new SqliteTransaction(this.database)
try {
const result = await operation(transaction)
this.database.exec('COMMIT')
return result
} catch (error) {
this.database.exec('ROLLBACK')
throw error
} finally {
release()
}
}
override async close(): Promise<void> {
await this.tail
this.database.close()
}
}
class PostgresTransaction implements PushDatabase {
readonly dialect = 'postgres' as const
constructor(private readonly client: pg.PoolClient) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
const result = await this.client.query(postgresSql(sql), params)
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
}
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
return await operation(this)
}
// READ COMMITTED lets a concurrent count-then-insert read the same
// under-quota total, so the identity is serialized for the whole transaction.
async lockQuotaScope(key: string): Promise<void> {
await this.query('SELECT pg_advisory_xact_lock(hashtext(?::text))', [key])
}
async close(): Promise<void> {}
}
function retryablePostgresTransactionError(error: unknown): boolean {
const code = String((error as { code?: unknown }).code)
// 57014 is the pool statement_timeout firing. It aborts the transaction the
// same way a lock timeout does, so it takes the bounded retry path too.
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
}
async function waitForPostgresRetry(): Promise<void> {
const delayMs = Math.floor(Math.random() * (POSTGRES_RETRY_MAX_DELAY_MS + 1))
await new Promise((resolve) => setTimeout(resolve, delayMs))
}
class PostgresDatabase implements PushDatabase {
readonly dialect = 'postgres' as const
constructor(private readonly pool: pg.Pool) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
const client = await this.pool.connect()
try {
const result = await client.query(postgresSql(sql), params)
return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
} finally {
client.release()
}
}
async transaction<T>(operation: (transaction: PushDatabase) => Promise<T>): Promise<T> {
for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
const client = await this.pool.connect()
try {
await client.query('BEGIN')
const result = await operation(new PostgresTransaction(client))
await client.query('COMMIT')
return result
} catch (error) {
await client.query('ROLLBACK').catch(() => undefined)
if (
!retryablePostgresTransactionError(error) ||
attempt === POSTGRES_TRANSACTION_ATTEMPTS
) {
throw error
}
console.warn(
JSON.stringify({
event: 'orca_push_postgres_transaction_retry',
code: String((error as { code?: unknown }).code),
attempt
})
)
} finally {
client.release()
}
// A PostgreSQL transaction is unusable after an abort, so retry all work
// on a fresh pooled client with a small full-jitter delay.
await waitForPostgresRetry()
}
throw new Error('postgres_transaction_retry_exhausted')
}
// An advisory transaction lock taken outside a transaction is released by the
// implicit commit before the caller reads anything, which protects nothing.
async lockQuotaScope(): Promise<void> {
throw new Error('lock_quota_scope_requires_transaction')
}
async close(): Promise<void> {
await this.pool.end()
}
}
async function applySchema(database: PushDatabase): Promise<void> {
for (const statement of pushSchemaStatements()) await database.query(statement)
}
// Why: DDL is not a request. A CREATE INDEX on a grown table can legitimately
// outlive the request statement_timeout, and inheriting it would fail every
// startup at the same statement instead of finishing once. One connection of
// its own, closed before the serving pool opens, keeps the untimed session off
// the request path entirely.
async function applySchemaOnUntimedPool(
databaseUrl: string,
applicationName: string | undefined
): Promise<void> {
const pool = new pg.Pool({
connectionString: databaseUrl,
max: 1,
application_name: applicationName ? `${applicationName}/schema` : undefined,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: 0,
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
absorbPostgresIdleClientErrors(pool)
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(pushSchemaStatements(), (statement) => database.query(statement), {
eventPrefix: 'orca_push_postgres_schema',
// Push has no catalog pre-check, so a lock timeout here says nothing about whether the
// object already exists and the old bounded retry is still the right answer.
retryLockTimeout: true
})
} finally {
await database.close().catch(() => undefined)
}
}
export function absorbPostgresIdleClientErrors(pool: Pick<pg.Pool, 'on'>): void {
pool.on('error', () => {
// node-postgres removes failed idle clients itself; an unhandled 'error'
// would crash the service and turn a SQL blip into a restart loop.
console.warn('[orca-push] idle PostgreSQL client failed')
})
}
export async function openPushDatabase(input: {
databaseUrl?: string
dataDir: string
poolMax?: number
applicationName?: string
readOnly?: boolean
}): Promise<PushDatabase> {
let database: PushDatabase
if (input.databaseUrl) {
if (!input.readOnly) await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
let connection: pg.ClientConfig = { connectionString: input.databaseUrl }
if (input.readOnly) {
connection = parseIntoClientConfig(input.databaseUrl)
// A URL parameter must not trigger a second parse that overrides read-only options.
delete connection.connectionString
connection.options = `${connection.options ?? ''} -c default_transaction_read_only=on`.trim()
}
const pool = new pg.Pool({
...connection,
max: input.poolMax ?? 10,
application_name: input.applicationName,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
absorbPostgresIdleClientErrors(pool)
database = new PostgresDatabase(pool)
} else {
mkdirSync(input.dataDir, { recursive: true })
const sqlite = new DatabaseSync(join(input.dataDir, 'orca-push.sqlite'), {
readOnly: input.readOnly ?? false
})
if (!input.readOnly) sqlite.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;')
database = new SqliteDatabase(sqlite)
}
if (database.dialect === 'postgres' || input.readOnly) return database
try {
await applySchema(database)
return database
} catch (error) {
await database.close().catch(() => undefined)
throw error
}
}
export async function openInMemoryPushDatabase(): Promise<PushDatabase> {
const sqlite = new DatabaseSync(':memory:')
sqlite.exec('PRAGMA foreign_keys = ON;')
const database = new SqliteDatabase(sqlite)
await applySchema(database)
return database
}
cloud/apps/push/src/push-delivery-lifecycle.test.ts
+85
View File
@@ -0,0 +1,85 @@
import { afterEach, expect, it, vi } from 'vitest'
import { Hono } from 'hono'
import { PushRequestDrain } from './push-request-drain.js'
import { PushDispatcher } from './push-dispatcher.js'
import { PushDeviceRegistryStore } from './device-registry-store.js'
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
import { buildPushDelivery } from './push-delivery-message.js'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { notification } from './push-server-harness.test-fixture.js'
const databases: PushDatabase[] = []
afterEach(async () => {
await Promise.all(databases.splice(0).map((db) => db.close()))
vi.restoreAllMocks()
})
const note = PushNotificationSchema.parse(notification())
const tick = () => new Promise((resolve) => setImmediate(resolve))
function deferred() {
let resolve!: () => void
const promise = new Promise<void>((done) => {
resolve = done
})
return { promise, resolve }
}
async function registered() {
const db = await openInMemoryPushDatabase()
databases.push(db)
const devices = new PushDeviceRegistryStore(db)
const input = {
hostFingerprint: 'abcdefghijklmnop',
deviceId: 'device',
platform: 'android' as const,
token: 'old-token'
}
const row = await devices.upsert(input)
if (!row.ok) throw new Error('registration failed')
const delivery = buildPushDelivery({
expiresAt: Date.now() + 300_000,
registrationId: row.registrationId,
hostFingerprint: input.hostFingerprint,
notification: note
})
return { db, devices, input, delivery }
}
it('does not retire a refreshed token after the old token fails', async () => {
const h = await registered()
const gate = deferred()
const send = vi.fn(async () => {
await gate.promise
return { status: 'dead', reason: 'UNREGISTERED' }
})
vi.spyOn(console, 'warn').mockImplementation(() => {})
const dispatcher = new PushDispatcher({ devices: h.devices, fcm: { send } as never })
const pending = dispatcher.sendOnce(h.delivery)
await tick()
await h.devices.upsert({ ...h.input, token: 'replacement-token' })
gate.resolve()
await pending
expect(await h.devices.findById(h.delivery.registrationId)).toMatchObject({
token: 'replacement-token',
dead: false
})
})
it('rejects new requests during drain and waits for an admitted handler', async () => {
const gate = deferred()
const requests = new PushRequestDrain()
const app = new Hono().use('*', requests.middleware).post('/send', async (c) => {
await gate.promise
return c.json({ queued: true })
})
const pending = app.request('/send', { method: 'POST' })
await tick()
let drained = false
const drain = requests.begin().then(() => {
drained = true
})
expect((await app.request('/send', { method: 'POST' })).status).toBe(503)
expect(drained).toBe(false)
gate.resolve()
expect((await pending).status).toBe(200)
await drain
expect(drained).toBe(true)
})
cloud/apps/push/src/push-delivery-message.ts
+77
View File
@@ -0,0 +1,77 @@
import { createHash } from 'node:crypto'
import { type PushNotification } from '@orca-cloud/push-contract'
export type PushOrcaData = {
kind?: 'alert' | 'dismiss'
hostFingerprint: string
worktreeId?: string
paneKey?: string
notificationId?: string
notificationSeq: number
notificationEpoch: string
source: string
agentState: string | null
}
export type PushDelivery = {
expiresAt: number
sound?: boolean
registrationId: string
hostFingerprint: string
title: string
body: string
collapseId: string
orca: PushOrcaData
}
export function collapseIdFor(notification: PushNotification, hostFingerprint: string): string {
const identity =
notification.notificationId === undefined
? [notification.notificationEpoch, notification.notificationSeq]
: notification.notificationId
return createHash('sha256')
.update(JSON.stringify([hostFingerprint, identity]))
.digest('hex')
}
export function buildPushDelivery(input: {
expiresAt: number
registrationId: string
hostFingerprint: string
notification: PushNotification
}): PushDelivery {
const { notification, hostFingerprint } = input
return {
...(notification.sound === false ? { sound: false } : {}),
expiresAt: input.expiresAt,
registrationId: input.registrationId,
hostFingerprint,
title: notification.title,
body: notification.body,
collapseId: collapseIdFor(notification, hostFingerprint),
orca: {
...(notification.kind ? { kind: notification.kind } : {}),
hostFingerprint,
...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }),
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
...(notification.notificationId === undefined
? {}
: { notificationId: notification.notificationId }),
notificationSeq: notification.notificationSeq,
notificationEpoch: notification.notificationEpoch,
source: notification.source,
agentState: notification.agentState
}
}
}
export function orcaDataStrings(orca: PushOrcaData): Record<string, string> {
return Object.fromEntries(
Object.entries(orca)
.filter(([, value]) => value !== undefined && value !== null)
.map(([key, value]) => [
key,
typeof value === 'object' ? JSON.stringify(value) : String(value)
])
)
}
cloud/apps/push/src/push-delivery-payload.ts
+8
View File
@@ -0,0 +1,8 @@
import type { PushNotification } from '@orca-cloud/push-contract'
export function parsePushDeliveryPayload(payload: string): PushNotification {
const value: unknown = JSON.parse(payload)
if (!value || typeof value !== 'object' || Array.isArray(value))
throw new Error('invalid_push_delivery_payload')
return value as PushNotification
}
cloud/apps/push/src/push-deploy-preflight.test.ts
+17
View File
@@ -0,0 +1,17 @@
import { readFileSync } from 'node:fs'
import { expect, it } from 'vitest'
import { loadPushConfig } from './config.js'
it('runs the deployment image preflight against the actual config loader', () => {
const workflow = readFileSync(
new URL('../../../../.github/workflows/cloud-push-deploy.yml', import.meta.url),
'utf8'
)
const step = workflow.split('- name: Require image support for inert validation')[1]!
const script = step.match(/--input-type=module -e '([\s\S]*?)'/)?.[1]
expect(script).toBeDefined()
const run = new Function('loadPushConfig', script!.replace(/import .*?;/, ''))
expect(() => run(loadPushConfig)).not.toThrow()
expect(() => run(() => ({ mode: 'active' }))).toThrow('validation_mode_unsupported')
expect(() => run(() => ({ mode: 'validation' }))).toThrow('validation_mode_not_fail_closed')
})
cloud/apps/push/src/push-dismissal-provider.test.ts
+31
View File
@@ -0,0 +1,31 @@
import { expect, it } from 'vitest'
import { apnsBody } from './apns-client.js'
import { fcmMessageBody } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
it('dismissal provider payloads cannot display a new alert or play a sound', () => {
const delivery = buildPushDelivery({
expiresAt: Date.now() + 300_000,
registrationId: 'reg',
hostFingerprint: 'host',
notification: {
kind: 'dismiss',
notificationId: 'note',
notificationSeq: 2,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: null,
title: 'Orca',
body: ''
}
})
expect(JSON.parse(apnsBody(delivery)).aps).toEqual({ 'content-available': 1 })
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
expect(android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('collapse_key')
expect(android.data).not.toHaveProperty('title')
expect(android.data).not.toHaveProperty('message')
expect(android.data).not.toHaveProperty('sound')
expect(android.data.kind).toBe('dismiss')
})
cloud/apps/push/src/push-dispatcher.ts
+52
View File
@@ -0,0 +1,52 @@
import type { ApnsClient } from './apns-client.js'
import type { PushDeviceRegistryStore } from './device-registry-store.js'
import type { FcmClient } from './fcm-client.js'
import { fingerprintLogPrefix } from './host-fingerprint.js'
import type { PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
export type PushDispatcherOptions = {
devices: PushDeviceRegistryStore
apns?: ApnsClient
fcm?: FcmClient
onOutcome?: (outcome: PushProviderOutcome['status']) => void
}
// Retires the registration when the provider says the token is gone.
export class PushDispatcher {
constructor(private readonly options: PushDispatcherOptions) {}
async sendOnce(delivery: PushDelivery): Promise<PushProviderOutcome> {
const device = await this.options.devices.findById(delivery.registrationId)
if (!device || device.dead) return { status: 'dead', reason: 'registration_unavailable' }
let outcome: PushProviderOutcome
if (device.platform === 'ios') {
outcome = this.options.apns
? await this.options.apns.send(delivery, {
token: device.token,
apnsEnvironment: device.apnsEnvironment ?? 'production'
})
: { status: 'error', reason: 'apns_not_configured' }
} else {
outcome = this.options.fcm
? await this.options.fcm.send(delivery, { token: device.token })
: { status: 'error', reason: 'fcm_not_configured' }
}
this.options.onOutcome?.(outcome.status)
if (outcome.status === 'dead') {
await this.options.devices.markDead(device)
}
if (outcome.status !== 'sent') {
console.warn(
JSON.stringify({
event: 'orca_push_delivery_failed',
platform: device.platform,
status: outcome.status,
reason: outcome.reason,
host: fingerprintLogPrefix(delivery.hostFingerprint)
})
)
}
return outcome
}
}
cloud/apps/push/src/push-notification-sound.test.ts
+33
View File
@@ -0,0 +1,33 @@
import { expect, it } from 'vitest'
import { apnsBody } from './apns-client.js'
import { fcmMessageBody } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
it('carries a silent preference through validation to APNs and Android payloads', () => {
const notification = PushNotificationSchema.parse({
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'terminal-bell',
agentState: null,
title: 'Bell',
body: '',
sound: false
})
const delivery = buildPushDelivery({
expiresAt: Date.now() + 300_000,
registrationId: 'reg',
hostFingerprint: 'host',
notification
})
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.channelId
).toBe('orca-desktop-silent')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.sound
).toBe('')
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
})
cloud/apps/push/src/push-observability.ts
+56
View File
@@ -0,0 +1,56 @@
const COUNTER_NAMES = [
'ip_rate_limited',
'request_error',
'challenge_issued',
'challenge_rejected',
'session_issued',
'session_rejected',
'device_registered',
'device_rejected',
'device_deleted',
'send_queued',
'send_dead',
'send_rate_limited',
'send_error',
'delivery_sent',
'delivery_dead',
'delivery_error',
'delivery_retry'
] as const
type PushCounterName = (typeof COUNTER_NAMES)[number]
// Aggregate counters only. Nothing here may accept a token, a title, a body,
// or more than the first four characters of a host fingerprint.
export class PushObservability {
private counters = new Map<PushCounterName, number>()
private timer: NodeJS.Timeout | null = null
record(name: PushCounterName, delta = 1): void {
this.counters.set(name, (this.counters.get(name) ?? 0) + delta)
}
consume(): Record<PushCounterName, number> {
const snapshot = Object.fromEntries(
COUNTER_NAMES.map((name) => [name, this.counters.get(name) ?? 0])
) as Record<PushCounterName, number>
this.counters = new Map()
return snapshot
}
start(intervalMs = 60_000): void {
if (this.timer) return
this.timer = setInterval(() => {
const counters = this.consume()
if (Object.values(counters).every((value) => value === 0)) return
console.warn(JSON.stringify({ event: 'orca_push_counters', ...counters }))
}, intervalMs)
this.timer.unref()
}
stop(): void {
if (!this.timer) return
clearInterval(this.timer)
this.timer = null
}
}
cloud/apps/push/src/push-pane-routing.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { expect, it } from 'vitest'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js'
it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => {
const base = {
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
worktreeId: 'folder:/work'
}
const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111'
for (const extra of [{}, { paneKey }]) {
const notification = PushNotificationSchema.parse({ ...base, ...extra })
const delivery = buildPushDelivery({
notification,
hostFingerprint: 'host',
registrationId: 'phone',
expiresAt: Date.now() + 300000
})
expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined)
expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined)
}
})
cloud/apps/push/src/push-provider-outcome.ts
+6
View File
@@ -0,0 +1,6 @@
// What a provider send resolved to, before the send route maps it onto the
// contract's queued / dead / rate_limited / error statuses.
export type PushProviderOutcome =
| { status: 'sent' }
| { status: 'dead'; reason: string }
| { status: 'error'; reason: string; retryable?: boolean; retryAfterMs?: number }
cloud/apps/push/src/push-queued-dismissal.ts
+57
View File
@@ -0,0 +1,57 @@
import type { PushNotification } from '@orca-cloud/push-contract'
import type { PushDatabase } from './push-database.js'
import { parsePushDeliveryPayload } from './push-delivery-payload.js'
export async function reconcileQueuedDismissal(
tx: PushDatabase,
host: string,
registrationId: string,
notification: PushNotification,
now: number
): Promise<boolean> {
if (!notification.notificationId) return false
const key = [host, notification.notificationEpoch, notification.notificationId]
const [dismissed] = await tx.query(
'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?',
key
)
if (notification.kind !== 'dismiss')
return Number(dismissed?.notification_seq ?? -1) >= notification.notificationSeq
await tx.query(
`INSERT INTO push_dismissed_events(host_fingerprint, notification_epoch, notification_id, notification_seq, created_at)
VALUES (?, ?, ?, ?, ?) ON CONFLICT(host_fingerprint, notification_epoch, notification_id)
DO UPDATE SET notification_seq = CASE WHEN push_dismissed_events.notification_seq > excluded.notification_seq THEN push_dismissed_events.notification_seq ELSE excluded.notification_seq END, created_at = excluded.created_at`,
[...key, notification.notificationSeq, now]
)
const deliveries = await tx.query(
"SELECT batch_id, payload_json FROM push_delivery_batches WHERE host_fingerprint = ? AND registration_id = ? AND kind = 'alert' AND state = 'pending' AND lease_until <= ?",
[host, registrationId, now]
)
for (const delivery of deliveries) {
const queued = parsePushDeliveryPayload(String(delivery.payload_json))
if (
queued.notificationEpoch !== notification.notificationEpoch ||
queued.notificationId !== notification.notificationId ||
queued.notificationSeq > notification.notificationSeq
)
continue
await tx.query(
'UPDATE push_delivery_batches SET payload_json = ?, state = ? WHERE batch_id = ?',
['{}', 'dismissed', delivery.batch_id]
)
}
return false
}
export async function isDismissedAlert(
tx: PushDatabase,
host: string,
notification: PushNotification
): Promise<boolean> {
if (notification.kind === 'dismiss' || !notification.notificationId) return false
const rows = await tx.query(
'SELECT notification_seq FROM push_dismissed_events WHERE host_fingerprint = ? AND notification_epoch = ? AND notification_id = ?',
[host, notification.notificationEpoch, notification.notificationId]
)
return Number(rows[0]?.notification_seq ?? -1) >= notification.notificationSeq
}
cloud/apps/push/src/push-readiness.test.ts
+26
View File
@@ -0,0 +1,26 @@
import { expect, it, vi } from 'vitest'
import { createPushReadiness } from './push-readiness.js'
import type { PushDatabase } from './push-database.js'
it('shares a slow check and caches failures before retrying', async () => {
let clock = 0
let reject!: (error: Error) => void
const query = vi.fn(
() =>
new Promise<never>((_, fail) => {
reject = fail
})
)
const ready = createPushReadiness({ query } as unknown as PushDatabase, { now: () => clock })
const checks = Array.from({ length: 100 }, () => ready())
expect(query).toHaveBeenCalledTimes(1)
reject(new Error('offline'))
expect(await Promise.all(checks)).toEqual(Array(100).fill(false))
expect(await ready()).toBe(false)
expect(query).toHaveBeenCalledTimes(1)
clock = 10_000
const retry = ready()
expect(query).toHaveBeenCalledTimes(2)
reject(new Error('offline'))
expect(await retry).toBe(false)
})
cloud/apps/push/src/push-readiness.ts
+39
View File
@@ -0,0 +1,39 @@
import type { PushDatabase } from './push-database.js'
export type PushReadinessOptions = {
cacheMs?: number
now?: () => number
}
// The gateway holds no JWKS dependency, so readiness is exactly "can we reach
// the database": /health stays unconditional for the container probe.
export function createPushReadiness(
database: PushDatabase,
options: PushReadinessOptions = {}
): () => Promise<boolean> {
const cacheMs = options.cacheMs ?? 10_000
const now = options.now ?? Date.now
let cachedAt = Number.NEGATIVE_INFINITY
let cached = false
let pending: Promise<boolean> | null = null
async function check(): Promise<boolean> {
try {
await database.query('SELECT 1 AS ready')
cached = true
} catch {
cached = false
}
cachedAt = now()
return cached
}
return async () => {
if (now() - cachedAt < cacheMs) return cached
pending ??= check().finally(() => {
pending = null
})
return pending
}
}
cloud/apps/push/src/push-request-drain.ts
+28
View File
@@ -0,0 +1,28 @@
import type { MiddlewareHandler } from 'hono'
export class PushRequestDrain {
private draining = false
private active = 0
private readonly waiters = new Set<() => void>()
readonly middleware: MiddlewareHandler = async (context, next) => {
if (this.draining) return context.json({ error: 'shutting_down' }, 503)
this.active++
try {
await next()
} finally {
this.active--
if (this.active === 0) {
for (const resolve of this.waiters) resolve()
this.waiters.clear()
}
}
}
begin(): Promise<void> {
this.draining = true
return this.active === 0
? Promise.resolve()
: new Promise((resolve) => this.waiters.add(resolve))
}
}
cloud/apps/push/src/push-schema.ts
+46
View File
@@ -0,0 +1,46 @@
import { DURABLE_PUSH_SCHEMA } from './durable-push-schema.js'
// Applied at startup for both dialects, including additive queue tables,
// so every column type has to read the same in SQLite and PostgreSQL.
const PUSH_SCHEMA = `
CREATE TABLE IF NOT EXISTS push_challenges (
challenge_id TEXT PRIMARY KEY,
host_fingerprint TEXT NOT NULL,
secret_hash TEXT NOT NULL,
expires_at BIGINT NOT NULL,
consumed_at BIGINT
);
CREATE INDEX IF NOT EXISTS push_challenges_expires_at ON push_challenges(expires_at);
CREATE TABLE IF NOT EXISTS push_sessions (
token_hash TEXT PRIMARY KEY,
host_fingerprint TEXT NOT NULL,
expires_at BIGINT NOT NULL,
created_at BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS push_sessions_host ON push_sessions(host_fingerprint);
CREATE INDEX IF NOT EXISTS push_sessions_expires_at ON push_sessions(expires_at);
CREATE TABLE IF NOT EXISTS push_devices (
registration_id TEXT PRIMARY KEY,
host_fingerprint TEXT NOT NULL,
device_id TEXT NOT NULL,
platform TEXT NOT NULL,
token TEXT NOT NULL,
apns_environment TEXT,
dead_at BIGINT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS push_devices_host_device
ON push_devices(host_fingerprint, device_id);
`
export function pushSchemaStatements(): string[] {
// Comments are stripped before the split so a ';' inside one cannot cut a
// statement in half and hand SQLite an "incomplete input" fragment.
return (PUSH_SCHEMA + DURABLE_PUSH_SCHEMA)
.replace(/--[^\n]*/g, '')
.split(';')
.map((statement) => statement.trim())
.filter((statement) => statement.length > 0)
}
cloud/apps/push/src/push-send-idempotency.test.ts
+64
View File
@@ -0,0 +1,64 @@
import { afterEach, expect, it } from 'vitest'
import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
const harnesses: Awaited<ReturnType<typeof createPushServerHarness>>[] = []
afterEach(async () => {
await Promise.all(harnesses.splice(0).map((h) => h.close()))
})
it('returns queued for concurrent retries without double quota or delivery', async () => {
const h = await createPushServerHarness()
harnesses.push(h)
const token = await h.signIn(createPushHostKeypair(2))
const registrationId = await h.registerAndroid(token)
const body = { v: 1, registrationIds: [registrationId], notification: notification() }
const responses = await Promise.all(
Array.from({ length: 10 }, () => h.post('/v1/send', body, token))
)
for (const response of responses)
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
expect(await h.server.deliveryStore.pendingCount(registrationId)).toBe(1)
await h.flushDeliveries()
await h.post('/v1/send', body, token)
await h.flushDeliveries()
expect(h.fcmRequests).toHaveLength(1)
expect(JSON.parse(h.fcmRequests[0]!.body).message.data.coalescedCount).toBeUndefined()
expect(
Number((await h.database.query('SELECT COUNT(*) AS count FROM push_events'))[0]?.count)
).toBe(1)
await h.post(
'/v1/send',
{ ...body, notification: notification({ notificationEpoch: 'new-epoch' }) },
token
)
await h.flushDeliveries()
expect(h.fcmRequests).toHaveLength(2)
})
it.each([false, true])(
'accepts default alert kind equivalently through the API (explicit first: %s)',
async (explicitFirst) => {
const h = await createPushServerHarness()
harnesses.push(h)
const token = await h.signIn(createPushHostKeypair(3))
const registrationId = await h.registerAndroid(token)
const implicit = notification()
const explicit = { kind: 'alert', ...implicit }
for (const event of explicitFirst ? [explicit, implicit] : [implicit, explicit]) {
const response = await h.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: event },
token
)
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
}
await h.flushDeliveries()
expect(h.fcmRequests).toHaveLength(1)
const changed = await h.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: { ...explicit, body: 'changed' } },
token
)
expect(await changed.json()).toEqual({ results: [{ registrationId, status: 'error' }] })
}
)
cloud/apps/push/src/push-server-auth.test.ts
+162
View File
@@ -0,0 +1,162 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
import type { PushDatabase } from './push-database.js'
import { createPushServer } from './push-server.js'
import {
createPushServerHarness,
testPushConfig
} from './push-server-harness.test-fixture.js'
describe('push gateway authentication and device routes', () => {
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
beforeEach(async () => {
harness = await createPushServerHarness()
})
afterEach(async () => {
await harness.close()
})
it('answers health unconditionally and ready from the database', async () => {
expect((await harness.server.app.request('/health')).status).toBe(200)
expect((await harness.server.app.request('/ready')).status).toBe(200)
})
it('reports not ready when the database is unreachable', async () => {
const unreachable: PushDatabase = {
dialect: 'sqlite',
query: async () => {
throw new Error('no connection')
},
transaction: async (operation) => await operation(unreachable),
lockQuotaScope: async () => undefined,
close: async () => undefined
}
const broken = createPushServer(testPushConfig(), unreachable, {
fcmAccessToken: async () => 'token',
fcmTransport: async () => ({ status: 200, body: '{}' })
})
expect((await broken.app.request('/health')).status).toBe(200)
expect((await broken.app.request('/ready')).status).toBe(503)
await broken.worker.stop()
})
it('completes challenge, session, register, list, delete', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(11))
const registrationId = await harness.registerAndroid(sessionToken)
const list = await harness.authorized('/v1/devices', {}, sessionToken)
expect(await list.json()).toEqual({
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: false }]
})
const deleted = await harness.authorized(
`/v1/devices/${registrationId}`,
{ method: 'DELETE' },
sessionToken
)
expect(deleted.status).toBe(204)
expect(await harness.server.devices.findById(registrationId)).toBeNull()
})
it('refuses a request with no bearer, a bogus bearer, and an expired session', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(12))
expect((await harness.server.app.request('/v1/devices')).status).toBe(401)
const bogus = await harness.authorized('/v1/devices', {}, 'nonsense')
expect(bogus.status).toBe(401)
expect(await bogus.json()).toEqual({ error: 'invalid_token' })
harness.advanceClock(PUSH_LIMITS.sessionTtlMs + 1)
const expired = await harness.authorized('/v1/devices', {}, sessionToken)
expect(expired.status).toBe(401)
expect(await expired.json()).toEqual({ error: 'session_expired' })
})
it('refuses a replayed proof and an unknown challenge', async () => {
const host = createPushHostKeypair(13)
const challenge = await harness.issueChallenge(host)
const proof = harness.answer(challenge, host)
expect(
(
await harness.post('/v1/host/session', {
v: 1,
challengeId: challenge.challengeId,
proofB64: proof
})
).status
).toBe(200)
const replay = await harness.post('/v1/host/session', {
v: 1,
challengeId: challenge.challengeId,
proofB64: proof
})
expect(replay.status).toBe(401)
expect(await replay.json()).toEqual({ error: 'invalid_proof' })
const unknown = await harness.post('/v1/host/session', {
v: 1,
challengeId: 'no-such-challenge',
proofB64: proof
})
expect(await unknown.json()).toEqual({ error: 'invalid_challenge' })
})
it('never returns the host fingerprint on the challenge itself', async () => {
const challenge = await harness.issueChallenge(createPushHostKeypair(22))
expect(Object.keys(challenge).sort()).toEqual([
'challengeId',
'ciphertextB64',
'expiresAt',
'gatewayEphemeralPublicKeyB64',
'nonceB64'
])
})
it('lets only the owning host delete a registration', async () => {
const ownerToken = await harness.signIn(createPushHostKeypair(14))
const intruderToken = await harness.signIn(createPushHostKeypair(15))
const registrationId = await harness.registerAndroid(ownerToken)
const forbidden = await harness.authorized(
`/v1/devices/${registrationId}`,
{ method: 'DELETE' },
intruderToken
)
expect(forbidden.status).toBe(404)
expect(await forbidden.json()).toEqual({ error: 'not_found' })
expect(await harness.server.devices.findById(registrationId)).not.toBeNull()
})
it('replaces the token on a re-registration and keeps one registration id', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(23))
const first = await harness.registerAndroid(sessionToken)
const again = await harness.post(
'/v1/devices',
{
v: 1,
deviceId: 'device-1',
platform: 'android',
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
},
sessionToken
)
expect(await again.json()).toEqual({ registrationId: first })
expect(await harness.server.devices.findById(first)).toMatchObject({
token: 'rotated_token:APA91b-newnewnewnewnewnewnewnewnewnew'
})
})
it('rejects a malformed registration body', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(16))
const bad = await harness.post(
'/v1/devices',
{ v: 1, deviceId: 'device-1', platform: 'ios', token: 'not-hex' },
sessionToken
)
expect(bad.status).toBe(400)
expect(await bad.json()).toEqual({ error: 'invalid_request' })
})
})
cloud/apps/push/src/push-server-harness.test-fixture.ts
+162
View File
@@ -0,0 +1,162 @@
import { generateKeyPairSync } from 'node:crypto'
import { expect } from 'vitest'
import type { ApnsRequest, ApnsResponse } from './apns-http2-transport.js'
import type { PushConfig } from './config.js'
import type { FcmRequest, FcmResponse } from './fcm-client.js'
import {
answerPushHostChallenge,
hostPublicKeyB64,
type PushHostKeypair
} from './host-challenge-answering.test-fixture.js'
import { openInMemoryPushDatabase, type PushDatabase } from './push-database.js'
import { createPushServer } from './push-server.js'
export const GATEWAY_ORIGIN = 'https://push.onorca.dev'
export const APNS_TOKEN = 'a'.repeat(64)
export const FCM_TOKEN = 'cQ1abcDEF_gh:APA91bZZ-zz0123456789abcdefghijklmnopqrstuvwxyz'
export function notification(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
notificationId: 'note-1',
notificationSeq: 1,
notificationEpoch: 'epoch-1',
source: 'agent-task-complete',
agentState: 'needs-input',
title: 'Agent needs input',
body: 'Waiting on your answer',
worktreeId: 'wt-1',
...overrides
}
}
export function testPushConfig(): PushConfig {
const { privateKey } = generateKeyPairSync('ec', {
namedCurve: 'P-256',
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
publicKeyEncoding: { type: 'spki', format: 'pem' }
})
return {
mode: 'active',
port: 0,
publicUrl: GATEWAY_ORIGIN,
dataDir: './data/push-test',
databasePoolMax: 10,
apns: { keyPem: privateKey, keyId: 'ABCDE12345', teamId: 'TEAM123456' },
apnsTopic: 'com.stably.orca.mobile',
fcmProjectId: 'onorca-cloud',
trustedProxyHops: 0
}
}
type ChallengeWire = {
challengeId: string
gatewayEphemeralPublicKeyB64: string
nonceB64: string
ciphertextB64: string
expiresAt: number
}
export async function createPushServerHarness() {
const database: PushDatabase = await openInMemoryPushDatabase()
let clock = 1_700_000_000_000
const apnsRequests: ApnsRequest[] = []
const fcmRequests: FcmRequest[] = []
let apnsResponse: ApnsResponse = { status: 200, body: '' }
let fcmResponse: FcmResponse = { status: 200, body: '{}' }
const server = createPushServer(testPushConfig(), database, {
now: () => clock,
apnsTransport: async (request) => {
apnsRequests.push(request)
return apnsResponse
},
fcmTransport: async (request) => {
fcmRequests.push(request)
return fcmResponse
},
fcmAccessToken: async () => 'access-token'
})
const post = async (path: string, body: unknown, token?: string): Promise<Response> =>
await server.app.request(path, {
method: 'POST',
headers: {
'content-type': 'application/json',
...(token ? { authorization: `Bearer ${token}` } : {})
},
body: JSON.stringify(body)
})
const issueChallenge = async (keypair: PushHostKeypair): Promise<ChallengeWire> => {
const response = await post('/v1/host/challenge', {
v: 1,
hostPublicKeyB64: hostPublicKeyB64(keypair)
})
expect(response.status).toBe(200)
return (await response.json()) as ChallengeWire
}
const answer = (challenge: ChallengeWire, keypair: PushHostKeypair): string => {
const proof = answerPushHostChallenge(challenge, {
gatewayOrigin: GATEWAY_ORIGIN,
keypair,
now: () => clock
})
expect(proof).not.toBeNull()
return proof!
}
return {
server,
database,
apnsRequests,
fcmRequests,
post,
issueChallenge,
answer,
now: () => clock,
flushDeliveries: async (): Promise<void> => {
await server.worker.runDue()
},
advanceClock: (deltaMs: number): void => {
clock += deltaMs
},
setApnsResponse: (response: ApnsResponse): void => {
apnsResponse = response
},
setFcmResponse: (response: FcmResponse): void => {
fcmResponse = response
},
authorized: async (path: string, init: RequestInit = {}, token?: string): Promise<Response> =>
await server.app.request(path, {
...init,
headers: {
...(init.headers as Record<string, string> | undefined),
...(token ? { authorization: `Bearer ${token}` } : {})
}
}),
signIn: async (keypair: PushHostKeypair): Promise<string> => {
const challenge = await issueChallenge(keypair)
const response = await post('/v1/host/session', {
v: 1,
challengeId: challenge.challengeId,
proofB64: answer(challenge, keypair)
})
expect(response.status).toBe(200)
return ((await response.json()) as { sessionToken: string }).sessionToken
},
registerAndroid: async (token: string, deviceId = 'device-1'): Promise<string> => {
const response = await post(
'/v1/devices',
{ v: 1, deviceId, platform: 'android', token: FCM_TOKEN },
token
)
expect(response.status).toBe(200)
return ((await response.json()) as { registrationId: string }).registrationId
},
close: async (): Promise<void> => {
await server.worker.stop()
// A test may close the database itself to provoke a route failure.
await database.close().catch(() => undefined)
}
}
}
cloud/apps/push/src/push-server-limits.test.ts
+273
View File
@@ -0,0 +1,273 @@
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createPushHostKeypair, hostPublicKeyB64 } from './host-challenge-answering.test-fixture.js'
import {
createPushServerHarness,
FCM_TOKEN,
notification
} from './push-server-harness.test-fixture.js'
const CLIENT_IP = '203.0.113.7'
const OTHER_CLIENT_IP = '198.51.100.9'
function oversizedChallengeBody(): string {
return JSON.stringify({ v: 1, filler: 'x'.repeat(PUSH_LIMITS.maxHttpBodyBytes) })
}
function chunkedRequest(path: string, body: string): Request {
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(body))
controller.close()
}
})
return new Request(`http://push.test${path}`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: stream,
duplex: 'half'
} as RequestInit)
}
describe('push gateway request limits', () => {
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
beforeEach(async () => {
harness = await createPushServerHarness()
})
afterEach(async () => {
await harness.close()
})
it('refuses an oversized chunked body that declares no content length', async () => {
const request = chunkedRequest('/v1/host/challenge', oversizedChallengeBody())
expect(request.headers.get('content-length')).toBeNull()
const response = await harness.server.app.request(request)
expect(response.status).toBe(413)
expect(await response.json()).toEqual({ error: 'request_too_large' })
})
it('still refuses an oversized body that declares a content length', async () => {
const body = oversizedChallengeBody()
const response = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers: {
'content-type': 'application/json',
'content-length': String(Buffer.byteLength(body))
},
body
})
expect(response.status).toBe(413)
expect(await response.json()).toEqual({ error: 'request_too_large' })
})
it('lets a chunked body under the cap through to schema validation', async () => {
const response = await harness.server.app.request(
chunkedRequest(
'/v1/host/challenge',
JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(60)) })
)
)
expect(response.status).toBe(200)
})
it('caps an authenticated oversized send as well', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(61))
const response = await harness.server.app.request(
new Request('http://push.test/v1/send', {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${sessionToken}`
},
body: new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(oversizedChallengeBody()))
controller.close()
}
}),
duplex: 'half'
} as RequestInit)
)
expect(response.status).toBe(413)
expect(await response.json()).toEqual({ error: 'request_too_large' })
})
it('rate limits one client ip across both unauthenticated routes', async () => {
const body = JSON.stringify({
v: 1,
hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(62))
})
// Cloud Run appends the peer, so the caller's own IP is the last value.
const headers = {
'content-type': 'application/json',
'x-forwarded-for': `10.0.0.1, ${CLIENT_IP}`
}
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
const allowed = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers,
body
})
expect(allowed.status).toBe(200)
}
const limited = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers,
body
})
expect(limited.status).toBe(429)
expect(await limited.json()).toEqual({ error: 'rate_limited' })
// The session route draws on the same bucket, so a flood cannot simply move.
const session = await harness.server.app.request('/v1/host/session', {
method: 'POST',
headers,
body: JSON.stringify({ v: 1, challengeId: 'anything', proofB64: 'x'.repeat(44) })
})
expect(session.status).toBe(429)
const other = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers: { ...headers, 'x-forwarded-for': `10.0.0.1, ${OTHER_CLIENT_IP}` },
body
})
expect(other.status).toBe(200)
// A caller rewriting the left of the chain lands in its own bucket anyway.
const spoofed = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers: { ...headers, 'x-forwarded-for': `198.51.100.250, ${CLIENT_IP}` },
body
})
expect(spoofed.status).toBe(429)
})
it('lets a throttled client back in once the window refills', async () => {
const body = JSON.stringify({
v: 1,
hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(63))
})
const headers = { 'content-type': 'application/json', 'x-forwarded-for': CLIENT_IP }
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body })
}
expect(
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
.status
).toBe(429)
harness.advanceClock(60_000)
expect(
(await harness.server.app.request('/v1/host/challenge', { method: 'POST', headers, body }))
.status
).toBe(200)
})
it('limits authenticated hosts independently behind the same IP', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(64))
const headers = { 'x-forwarded-for': CLIENT_IP }
for (let index = 0; index < 600; index++) {
const listed = await harness.authorized('/v1/devices', { headers }, sessionToken)
expect(listed.status).toBe(200)
}
const limited = await harness.authorized('/v1/devices', { headers }, sessionToken)
expect(limited.status).toBe(429)
const otherToken = await harness.signIn(createPushHostKeypair(68))
expect((await harness.authorized('/v1/devices', { headers }, otherToken)).status).toBe(200)
// The handshake bucket is untouched by any of that.
const challenge = await harness.server.app.request('/v1/host/challenge', {
method: 'POST',
headers: { ...headers, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, hostPublicKeyB64: hostPublicKeyB64(createPushHostKeypair(67)) })
})
expect(challenge.status).toBe(200)
})
it('stops repeated forged bearers after the invalid-auth budget is exhausted', async () => {
const headers = { 'x-forwarded-for': CLIENT_IP }
const [before] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions')
for (let index = 0; index < PUSH_LIMITS.unauthenticatedRequestsPerMinutePerIp; index++) {
const refused = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged')
expect(refused.status).toBe(401)
}
const limited = await harness.authorized('/v1/send', { method: 'POST', headers }, 'forged')
expect(limited.status).toBe(429)
expect(await limited.json()).toEqual({ error: 'rate_limited' })
expect(harness.server.unauthenticatedIps.trackedIpCount()).toBe(0)
const [after] = await harness.database.query('SELECT COUNT(*) AS sessions FROM push_sessions')
expect(Number(after?.sessions)).toBe(Number(before?.sessions))
})
it('answers 409 once a host has registered its device allowance', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(66))
for (let index = 0; index < PUSH_LIMITS.maxDevicesPerHost; index++) {
const accepted = await harness.post(
'/v1/devices',
{
v: 1,
deviceId: `device-${index}`,
platform: 'android',
token: FCM_TOKEN
},
sessionToken
)
expect(accepted.status).toBe(200)
}
const refused = await harness.post(
'/v1/devices',
{ v: 1, deviceId: 'one-too-many', platform: 'android', token: FCM_TOKEN },
sessionToken
)
expect(refused.status).toBe(409)
expect(await refused.json()).toEqual({ error: 'too_many_devices' })
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
expect(((await listed.json()) as { devices: unknown[] }).devices).toHaveLength(
PUSH_LIMITS.maxDevicesPerHost
)
})
// Why: a database error carries the failing row in its message. The response
// and the log must both stop at the error's name.
it('answers an unexpected route failure with a bare 500 and logs only the name', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(66))
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
await harness.database.close()
const response = await harness.authorized('/v1/devices', {}, sessionToken)
expect(response.status).toBe(500)
expect(await response.json()).toEqual({ error: 'internal' })
const logged = warn.mock.calls.map((call) => String(call[0])).join('\n')
expect(logged).toContain('"event":"orca_push_request_failed"')
expect(logged).not.toContain('SELECT')
expect(logged).not.toContain('push_devices')
expect(harness.server.observability.consume().request_error).toBe(1)
} finally {
warn.mockRestore()
}
})
it('charges a repeated registration id once and returns one result', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(65))
const registrationId = await harness.registerAndroid(sessionToken)
const response = await harness.post(
'/v1/send',
{
v: 1,
registrationIds: [registrationId, registrationId, registrationId],
notification: notification()
},
sessionToken
)
expect(await response.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(1)
const [row] = await harness.database.query('SELECT COUNT(*) AS sends FROM push_events')
expect(Number(row?.sends)).toBe(1)
})
})
cloud/apps/push/src/push-server-send.test.ts
+229
View File
@@ -0,0 +1,229 @@
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { PUSH_LIMITS } from '@orca-cloud/push-contract'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
import {
APNS_TOKEN,
createPushServerHarness,
FCM_TOKEN,
notification
} from './push-server-harness.test-fixture.js'
describe('push gateway send route', () => {
let harness: Awaited<ReturnType<typeof createPushServerHarness>>
beforeEach(async () => {
harness = await createPushServerHarness()
})
afterEach(async () => {
await harness.close()
})
it('rejects a batch over the registration cap', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(16))
const oversized = await harness.post(
'/v1/send',
{
v: 1,
registrationIds: Array.from(
{ length: PUSH_LIMITS.maxRegistrationIdsPerSend + 1 },
(_, index) => `reg-${index}`
),
notification: notification()
},
sessionToken
)
expect(oversized.status).toBe(400)
expect(await oversized.json()).toEqual({ error: 'invalid_request' })
})
it('queues a send, delivers it to fcm, and reports a dead token on the next send', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(17))
const registrationId = await harness.registerAndroid(sessionToken)
const queued = await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: notification() },
sessionToken
)
expect(await queued.json()).toEqual({ results: [{ registrationId, status: 'queued' }] })
harness.setFcmResponse({
status: 404,
body: JSON.stringify({ error: { status: 'UNREGISTERED', message: 'gone' } })
})
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } }
})
const afterDeath = await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: notification() },
sessionToken
)
expect(await afterDeath.json()).toEqual({ results: [{ registrationId, status: 'dead' }] })
const listed = await harness.authorized('/v1/devices', {}, sessionToken)
expect(await listed.json()).toEqual({
devices: [{ registrationId, deviceId: 'device-1', platform: 'android', dead: true }]
})
})
it('leaves a live registration alone when the provider reports a transient failure', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(24))
const registrationId = await harness.registerAndroid(sessionToken)
await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: notification() },
sessionToken
)
harness.setFcmResponse({
status: 503,
body: JSON.stringify({ error: { status: 'UNAVAILABLE', message: 'backend busy' } })
})
await harness.flushDeliveries()
expect(await harness.server.devices.findById(registrationId)).toMatchObject({ dead: false })
})
it('reports retries from the durable worker after the provider delay', async () => {
const token = await harness.signIn(createPushHostKeypair(26))
const registrationId = await harness.registerAndroid(token)
await harness.post(
'/v1/send',
{
v: 1,
registrationIds: [registrationId],
notification: notification()
},
token
)
harness.setFcmResponse({ status: 503, body: '{}' })
await harness.flushDeliveries()
expect(harness.server.observability.consume()).toMatchObject({
delivery_error: 1,
delivery_retry: 0
})
harness.advanceClock(10_000)
harness.setFcmResponse({ status: 200, body: '{}' })
await harness.server.worker.runDue()
expect(harness.server.observability.consume()).toMatchObject({
delivery_sent: 1,
delivery_retry: 1
})
})
it('sends a burst as individual APNs alerts grouped by the host thread', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(18))
const registration = await harness.post(
'/v1/devices',
{
v: 1,
deviceId: 'iphone-1',
platform: 'ios',
token: APNS_TOKEN,
apnsEnvironment: 'sandbox'
},
sessionToken
)
const { registrationId } = (await registration.json()) as { registrationId: string }
for (const seq of [1, 2, 3]) {
await harness.post(
'/v1/send',
{
v: 1,
registrationIds: [registrationId],
notification: notification({ notificationId: `note-${seq}`, notificationSeq: seq })
},
sessionToken
)
}
await harness.flushDeliveries()
expect(harness.apnsRequests).toHaveLength(3)
const bodies = harness.apnsRequests.map(
(request) =>
JSON.parse(request.body) as {
aps: { alert: { title: string; body: string }; 'thread-id': string }
orca: Record<string, unknown> & { notificationSeq: number }
}
)
expect(
harness.apnsRequests.every((request) => request.host === 'api.sandbox.push.apple.com')
).toBe(true)
expect(bodies.map((body) => body.aps.alert)).toEqual(
Array.from({ length: 3 }, () => ({
title: 'Agent needs input',
body: 'Waiting on your answer'
}))
)
expect(new Set(bodies.map((body) => body.aps['thread-id'])).size).toBe(1)
expect(bodies.map((body) => body.orca.notificationSeq).sort((a, b) => a - b)).toEqual([1, 2, 3])
expect(bodies.every((body) => !('coalescedCount' in body.orca))).toBe(true)
expect(bodies.every((body) => !('summaryMembers' in body.orca))).toBe(true)
expect(
new Set(harness.apnsRequests.map((request) => request.headers['apns-collapse-id'])).size
).toBe(3)
})
it('sends a lone event through unchanged with its own collapse id', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(25))
const registrationId = await harness.registerAndroid(sessionToken)
await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: notification() },
sessionToken
)
await harness.flushDeliveries()
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
message: { android: { notification: { tag: string } }; data: Record<string, string> }
}
expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.coalescedCount).toBeUndefined()
})
it('reports an error for a registration the host does not own', async () => {
const ownerToken = await harness.signIn(createPushHostKeypair(19))
const intruderToken = await harness.signIn(createPushHostKeypair(20))
const registrationId = await harness.registerAndroid(ownerToken)
const foreign = await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId, 'made-up'], notification: notification() },
intruderToken
)
expect(await foreign.json()).toEqual({
results: [
{ registrationId, status: 'error' },
{ registrationId: 'made-up', status: 'error' }
]
})
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(0)
})
it('rate limits a host that exhausted its 15-minute allowance', async () => {
const sessionToken = await harness.signIn(createPushHostKeypair(21))
const registrationId = await harness.registerAndroid(sessionToken)
const hostFingerprint = (await harness.server.devices.findById(registrationId))!.hostFingerprint
for (let index = 0; index < PUSH_LIMITS.hostEventsPerWindow; index++) {
expect(
await harness.server.deliveryStore.accept(
hostFingerprint,
registrationId,
PushNotificationSchema.parse(
notification({ notificationId: `note-${index + 1000}`, notificationSeq: index + 1000 })
)
)
).toBe('queued')
}
const limited = await harness.post(
'/v1/send',
{ v: 1, registrationIds: [registrationId], notification: notification() },
sessionToken
)
expect(limited.status).toBe(200)
expect(await limited.json()).toEqual({ results: [{ registrationId, status: 'rate_limited' }] })
expect(await harness.server.deliveryStore.pendingCount(registrationId)).toBe(300)
})
})

Some files were not shown because too many files have changed in this diff Show More